[med-svn] [r-cran-future] 06/10: New upstream version 1.6.1
Sébastien Villemot
sebastien at debian.org
Sat Sep 30 11:46:53 UTC 2017
This is an automated email from the git hooks/post-receive script.
sebastien pushed a commit to branch master
in repository r-cran-future.
commit a4affbf31641485c86e7f2e21e2a29c11583d2ac
Author: Sébastien Villemot <sebastien at debian.org>
Date: Sat Sep 30 13:30:30 2017 +0200
New upstream version 1.6.1
---
.Rbuildignore | 58 --
.Rinstignore | 5 -
.covr.R | 27 -
.gitignore | 19 -
.make/Makefile | 475 ---------
.travis.yml | 39 -
CONDUCT.md | 74 --
DESCRIPTION | 60 +-
MD5 | 186 ++++
Makefile | 9 -
NAMESPACE | 29 +-
NEWS | 1062 ++++++++++++++------
OVERVIEW.md | 14 -
R/ClusterFuture-class.R | 243 +++--
R/ClusterRegistry.R | 6 +-
R/ConstantFuture-class.R | 7 +-
R/DEPRECATED.R | 32 +
R/Future-class.R | 279 +++--
R/FutureError-class.R | 16 +-
R/FutureGlobals-class.R | 139 +++
R/FutureRegistry.R | 22 +-
R/MulticoreFuture-class.R | 54 +-
R/MultiprocessFuture-class.R | 13 +-
R/MultisessionFuture-class.R | 10 +-
R/UniprocessFuture-class.R | 68 +-
R/availableCores.R | 103 +-
R/availableWorkers.R | 287 ++++++
R/backtrace.R | 7 +-
R/cluster.R | 17 +-
R/constant.R | 2 +-
R/flapply.R | 36 -
R/future.R | 101 +-
R/futureAssign.R | 50 +-
R/futureAssign_OP.R | 41 +-
R/futureCall.R | 15 +-
R/futureOf.R | 36 +-
R/future_lapply.R | 398 ++++++++
R/futures.R | 4 +-
R/globals.R | 198 ++--
R/globals_OP.R | 12 +-
R/label_OP.R | 12 +-
R/lazy_OP.R | 20 +
R/makeClusterPSOCK.R | 427 ++++++--
R/mandelbrot.R | 94 +-
R/multicore.R | 77 +-
R/multiprocess.R | 8 +-
R/multisession.R | 26 +-
R/nbrOfWorkers.R | 18 +-
R/options.R | 23 +-
R/plan_OP.R | 6 +-
R/remote.R | 18 +-
R/resolve.R | 191 ++--
R/resolved.R | 4 +-
R/seed_OP.R | 20 +
R/sequential.R | 52 +
R/sessionDetails.R | 2 +-
R/signalEarly.R | 21 +-
R/tweak.R | 70 +-
R/tweakExpression.R | 52 +-
R/tweak_OP.R | 10 +-
R/uniprocess.R | 81 --
R/utils.R | 435 ++++++--
R/values.R | 2 +-
R/whichIndex.R | 22 +-
R/zzz.R | 93 +-
R/zzz.plan.R | 154 ++-
appveyor.yml | 72 --
build/vignette.rds | Bin 0 -> 425 bytes
cran-comments.md | 49 -
demo/fibonacci.R | 4 +-
demo/mandelbrot.R | 41 +-
incl/cluster.R | 33 -
incl/future.R | 64 --
incl/futureOf.R | 33 -
incl/lazy.R | 60 --
incl/makeClusterPSOCK.R | 119 ---
incl/mandelbrot.R | 6 -
incl/multicore.R | 22 -
incl/multiprocess.R | 29 -
incl/multisession.R | 29 -
incl/nbrOfWorkers.R | 8 -
incl/plan.R | 59 --
incl/remote.R | 11 -
incl/uniprocess.R | 23 -
inst/doc/future-1-overview.html | 808 +++++++++++++++
README.md => inst/doc/future-1-overview.md.rsp | 293 ++----
inst/doc/future-2-issues.html | 454 +++++++++
{vignettes => inst/doc}/future-2-issues.md.rsp | 18 +-
inst/doc/future-3-topologies.html | 369 +++++++
{vignettes => inst/doc}/future-3-topologies.md.rsp | 44 +-
inst/doc/future-4-startup.html | 268 +++++
{vignettes => inst/doc}/future-4-startup.md.rsp | 12 +-
inst/vignettes-static/future-1-overview.md.rsp.rsp | 148 ++-
.../incl/future-1-overview-example3.R | 2 +-
man/ClusterFuture-class.Rd | 28 +-
man/ConstantFuture-class.Rd | 16 +-
man/Future-class.Rd | 18 +-
man/FutureError.Rd | 1 -
man/FutureGlobals.Rd | 36 +
man/MulticoreFuture-class.Rd | 15 +-
man/MultiprocessFuture-class.Rd | 14 +-
man/UniprocessFuture-class.Rd | 40 +-
man/as.cluster.Rd | 5 +-
man/availableCores.Rd | 51 +-
man/availableWorkers.Rd | 61 ++
man/backtrace.Rd | 1 -
man/cluster.Rd | 30 +-
man/flapply.Rd | 23 -
man/future.Rd | 146 +--
man/future.options.Rd | 35 +-
man/futureOf.Rd | 10 +-
man/future_lapply.Rd | 112 +++
man/futures.Rd | 1 -
man/getExpression.Rd | 14 +-
man/getGlobalsAndPackages.Rd | 1 -
man/grapes-globals-grapes.Rd | 4 -
man/grapes-label-grapes.Rd | 4 -
man/grapes-lazy-grapes.Rd | 18 +
man/grapes-plan-grapes.Rd | 1 -
man/grapes-seed-grapes.Rd | 17 +
man/grapes-tweak-grapes.Rd | 1 -
man/makeClusterPSOCK.Rd | 235 +++--
man/mandelbrot.Rd | 9 +-
man/multicore.Rd | 30 +-
man/multiprocess.Rd | 23 +-
man/multisession.Rd | 29 +-
man/nbrOfWorkers.Rd | 7 +-
man/plan.Rd | 48 +-
man/private_length.Rd | 25 +
man/remote.Rd | 31 +-
man/requestCore.Rd | 9 +-
man/resolve.Rd | 42 +-
man/resolved.Rd | 3 +-
man/run.Rd | 4 +-
man/{eager.Rd => sequential.Rd} | 73 +-
man/sessionDetails.Rd | 3 +-
man/supportsMulticore.Rd | 1 -
man/tweak.Rd | 7 +-
man/usedCores.Rd | 5 +-
man/value.Rd | 3 +-
man/values.Rd | 1 -
revdep/README.md | 404 --------
revdep/check.R | 20 -
revdep/checks.rds | Bin 8651 -> 0 bytes
revdep/problems.md | 67 --
revdep/timing.md | 25 -
tests/000.sessionDetails.R | 10 +
tests/ClusterRegistry.R | 29 +-
tests/DEPRECATED_DEFUNCT.R | 91 ++
tests/Future-class.R | 27 +-
tests/FutureError.R | 21 +-
tests/FutureGlobals.R | 38 +
tests/FutureRegistry.R | 54 +-
tests/as.cluster.R | 54 +-
tests/availableCores.R | 40 +-
tests/availableWorkers.R | 154 +++
tests/backtrace.R | 20 +-
tests/cluster.R | 511 ++++++----
tests/constant.R | 4 +-
tests/demo.R | 14 +-
tests/deprecated.R | 64 --
tests/dotdotdot.R | 22 +-
tests/eager.R | 62 --
tests/early-signaling.R | 122 ++-
tests/flapply.R | 93 --
tests/future,labels.R | 9 +-
tests/future.R | 37 +-
tests/futureAssign.R | 71 +-
tests/futureAssign_OP.R | 64 +-
tests/futureAssign_OP_with_environment.R | 11 +-
tests/futureAssign_OP_with_listenv.R | 21 +-
tests/futureCall.R | 13 +-
tests/futureOf.R | 8 +-
tests/futureOf_with_environment.R | 38 +-
tests/futureOf_with_listenv.R | 67 +-
tests/future_lapply,RNG.R | 160 +++
tests/future_lapply,globals.R | 146 +++
tests/future_lapply.R | 72 ++
tests/futures.R | 18 +-
tests/globals,NSE.R | 19 +-
tests/globals,formulas.R | 192 ++--
tests/globals,manual.R | 172 +++-
tests/globals,resolve.R | 14 +-
tests/globals,subassignment.R | 74 +-
tests/globals,toolarge.R | 16 +-
tests/globals,tricky.R | 132 ++-
tests/globals,tricky_recursive.R | 117 +++
tests/globalsOf,tweaks.R | 14 +-
tests/incl/end.R | 26 +-
tests/incl/start,load-only.R | 56 +-
tests/invalid-owner.R | 10 +-
tests/lazy.R | 112 ---
tests/makeClusterPSOCK.R | 104 ++
tests/mandelbrot.R | 2 +-
tests/multicore.R | 42 +-
tests/multiprocess.R | 32 +-
tests/multisession.R | 55 +-
tests/nbrOfWorkers.R | 42 +-
tests/nested_futures,mc.cores.R | 37 +-
tests/nested_futures.R | 28 +-
tests/objectSize.R | 66 ++
tests/plan.R | 128 ++-
tests/remote.R | 8 +-
tests/requestCore.R | 14 +-
tests/requestNode.R | 17 +-
tests/resolve.R | 203 ++--
tests/rng.R | 106 +-
tests/sequential.R | 28 +
tests/startup.R | 144 +--
tests/transparent.R | 8 +-
tests/tweak.R | 117 ++-
tests/utils.R | 68 +-
tests/uuid.R | 32 +
tests/whichIndex.R | 52 +-
vignettes/future-1-overview.md.rsp | 248 ++---
vignettes/future-2-issues.md.rsp | 18 +-
vignettes/future-3-topologies.md.rsp | 44 +-
vignettes/future-4-startup.md.rsp | 12 +-
218 files changed, 9992 insertions(+), 5858 deletions(-)
diff --git a/.Rbuildignore b/.Rbuildignore
deleted file mode 100644
index cc3088f..0000000
--- a/.Rbuildignore
+++ /dev/null
@@ -1,58 +0,0 @@
-#----------------------------
-# Git and SVN related
-#----------------------------
-^.svn
-^.git
-^.make
-INSTALL[.]md
-OVERVIEW[.]md
-README[.]md
-CONDUCT[.]md
-
-#----------------------------
-# devtools
-#----------------------------
-^revdep
-
-#----------------------------
-# Travis-CI et al.
-#----------------------------
-^[.]travis[.]yml$
-^travis-tool[.]sh$
-^pkg-build[.]sh$
-^appveyor[.]yml$
-^covr-utils.R$
-^[.]covr[.]R$
-^[.]covr[.]rds$
-
-#----------------------------
-# R related
-#----------------------------
-Rplots.pdf$
-^cran-comments[.].*$
-^vignettes/.*[.](pdf|PDF)$
-^vignettes/.*[.](r|R)$
-^vignettes/[.]install_extras$
-^Makefile$
-^incl
-^NAMESPACE,.*[.]txt$
-^nohup.*$
-^[.]R
-^[.]benchmark
-^[.]devel
-^[.]test
-^[.]check
-^.*[.]tar[.]gz$
-
-#----------------------------
-# Package specific
-#----------------------------
-^[.]BatchJobs[.]R$
-[.]future
-
-#----------------------------
-# Miscellaneous
-#----------------------------
-^.ghi
-^.issues
-
diff --git a/.Rinstignore b/.Rinstignore
deleted file mode 100644
index da2aee9..0000000
--- a/.Rinstignore
+++ /dev/null
@@ -1,5 +0,0 @@
-# Certain LaTeX files (e.g. bib, bst, sty) must be part of the build
-# such that they are available for R CMD check. These are excluded
-# from the install using .Rinstignore in the top-level directory
-# such as this one.
-doc/.*[.](bib|bst|sty)$
diff --git a/.covr.R b/.covr.R
deleted file mode 100644
index c9fa22a..0000000
--- a/.covr.R
+++ /dev/null
@@ -1,27 +0,0 @@
-#################################################################
-# Test coverage
-#
-# * covr-utils: https://github.com/HenrikBengtsson/covr-utils
-# * covr: https://github.com/jimhester/covr
-# * Coveralls: https://coveralls.io/
-#
-# Henrik Bengtsson
-#################################################################
-if (!file_test("-f", "covr-utils.R")) {
- source("http://callr.org/install#R.utils[u]")
- R.utils::downloadFile("https://raw.githubusercontent.com/HenrikBengtsson/covr-utils/master/covr-utils.R")
-}
-
-source("covr-utils.R")
-
-# Exclusion rules
-excl <- exclusions(
- filter(r_files(), covr_lines), # Apply 'covr:' rules in source code
- filter(r_files(), stop_lines) # Skip lines with stop().
-)
-str(excl)
-
-# Run through tests, record source code coverage, and
-# publish to Coveralls
-covr <- covr_package(exclusions=excl, quiet=FALSE, use_try=FALSE)
-saveRDS(covr, file=".covr.rds")
diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index 86ab9cf..0000000
--- a/.gitignore
+++ /dev/null
@@ -1,19 +0,0 @@
-.Rhistory
-*~
-**/*~
-.R
-.benchmark
-.check
-.devel
-.test
-*.o
-*.dll
-*.Rout
-.RData
-*.Rproj*
-*.swp
-.covr.rds
-.future
-.ghi
-.issues
-
diff --git a/.make/Makefile b/.make/Makefile
deleted file mode 100644
index ce331ea..0000000
--- a/.make/Makefile
+++ /dev/null
@@ -1,475 +0,0 @@
-# Makefile for R packages
-
-# CORE MACROS
-ifeq ($(OS), Windows_NT)
-CD=cd
-CURDIR=$(subst \,/,$(shell cmd.exe /C cd))
-FSEP=;
-else
-CD=cd -P "$(CURDIR)"; cd # This handles the case when CURDIR is a softlink
-FSEP=:
-endif
-CP=cp
-MAKE=make
-MV=mv
-RM=rm -f
-MKDIR=mkdir -p
-RMDIR=$(RM) -r
-GIT=git
-
-# PACKAGE MACROS
-PKG_VERSION := $(shell grep -i ^version DESCRIPTION | cut -d : -d \ -f 2)
-PKG_NAME := $(shell grep -i ^package DESCRIPTION | cut -d : -d \ -f 2)
-PKG_DIR := $(shell basename "$(CURDIR)")
-PKG_DIR := $(CURDIR)
-PKG_TARBALL := $(PKG_NAME)_$(PKG_VERSION).tar.gz
-PKG_ZIP := $(PKG_NAME)_$(PKG_VERSION).zip
-PKG_TGZ := $(PKG_NAME)_$(PKG_VERSION).tgz
-
-# FILE MACROS
-FILES_R := $(wildcard R/*.R)
-FILES_DATA := $(wildcard data/*)
-FILES_MAN := $(wildcard man/*.Rd)
-FILES_INCL := $(wildcard incl/*)
-FILES_INST := $(wildcard inst/* inst/*/* inst/*/*/* inst/*/*/*/*)
-FILES_VIGNETTES := $(wildcard vignettes/* vignettes/.install_extras)
-FILES_SRC := $(wildcard src/* src/*/* src/*/*/* src/*/*/*/* src/*/*/*/*/* src/*/*/*/*/*/* src/*/*/*/*/*/*/* src/*/*/*/*/*/*/*/*)
-FILES_SRC_C := $(wildcard src/*.c)
-FILES_SRC_H := $(wildcard src/*.h)
-FILES_TESTS := $(wildcard tests/*.R)
-FILES_NEWS := $(wildcard NEWS inst/NEWS)
-FILES_MISC := $(wildcard README.md)
-FILES_ROOT := DESCRIPTION NAMESPACE $(wildcard .Rbuildignore .Rinstignore)
-PKG_FILES := $(FILES_ROOT) $(FILES_NEWS) $(FILES_R) $(FILES_DATA) $(FILES_MAN) $(FILES_INST) $(FILES_VIGNETTES) $(FILES_SRC) $(FILES_TESTS) $(FILES_MISC)
-FILES_MAKEFILE := $(wildcard ../../Makefile)
-
-# Has vignettes in 'vignettes/' or 'inst/doc/'?
-DIR_VIGNS := $(wildcard vignettes inst/doc)
-
-# R MACROS
-R = R
-R_SCRIPT = Rscript
-R_HOME := $(shell $(R_SCRIPT) -e "cat(R.home())")
-
-## R_USE_CRAN := $(shell $(R_SCRIPT) -e "cat(Sys.getenv('R_USE_CRAN', 'FALSE'))")
-R_NO_INIT := --no-init-file
-R_VERSION_STATUS := $(shell $(R_SCRIPT) -e "status <- tolower(R.version[['status']]); if (regexpr('unstable', status) != -1L) status <- 'devel'; cat(status)")
-R_VERSION_X_Y := $(shell $(R_SCRIPT) -e "cat(gsub('[.][0-9]+$$', '', getRversion()))")
-R_VERSION := $(shell $(R_SCRIPT) -e "cat(as.character(getRversion()))")
-R_VERSION_FULL := $(R_VERSION)$(R_VERSION_STATUS)
-R_LIBS_USER_X := $(shell $(R_SCRIPT) -e "cat(.libPaths()[1])")
-R_INCLUDE := $(shell $(R_SCRIPT) -e "cat(R.home('include'))")
-R_OUTDIR := ../_R-$(R_VERSION_FULL)
-## R_BUILD_OPTS :=
-## R_BUILD_OPTS := $(R_BUILD_OPTS) --no-build-vignettes
-R_CHECK_OUTDIR := $(R_OUTDIR)/$(PKG_NAME).Rcheck
-_R_CHECK_CRAN_INCOMING_ = $(shell $(R_SCRIPT) -e "cat(Sys.getenv('_R_CHECK_CRAN_INCOMING_', 'FALSE'))")
-_R_CHECK_XREFS_REPOSITORIES_ = $(shell if test "$(_R_CHECK_CRAN_INCOMING_)" = "TRUE"; then echo ""; else echo "invalidURL"; fi)
-_R_CHECK_FULL_ = $(shell $(R_SCRIPT) -e "cat(Sys.getenv('_R_CHECK_FULL_', ''))")
-R_CHECK_OPTS = --as-cran --timings $(shell if test "$(_R_CHECK_USE_VALGRIND_)" = "TRUE"; then echo "--use-valgrind"; fi)
-R_RD4PDF = $(shell $(R_SCRIPT) -e "if (getRversion() < 3) cat('times,hyper')")
-R_CRAN_OUTDIR := $(R_OUTDIR)/$(PKG_NAME)_$(PKG_VERSION).CRAN
-
-HAS_ASPELL := $(shell $(R_SCRIPT) -e "cat(Sys.getenv('HAS_ASPELL', !inherits(try(aspell('DESCRIPTION', control=c('--master=en_US', '--add-extra-dicts=en_GB'), dictionaries='en_stats', program='aspell'), silent=TRUE), 'try-error')))")
-
-## Git related
-GIT_BRANCH := $(shell $(GIT) symbolic-ref --short HEAD)
-GIT_BRANCH := $(subst /,-,$(GIT_BRANCH))
-GIT_COMMIT := $(shell $(GIT) log -1 --format="%h")
-R_LIBS_BRANCH := $(CURDIR)/.R/$(GIT_BRANCH)
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Main
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-all: build install check
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Displays macros
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-debug:
- @echo CURDIR=\'$(CURDIR)\'
- @echo R_HOME=\'$(R_HOME)\'
- @echo
- @echo PKG_DIR=\'$(PKG_DIR)\'
- @echo PKG_NAME=\'$(PKG_NAME)\'
- @echo PKG_VERSION=\'$(PKG_VERSION)\'
- @echo PKG_TARBALL=\'$(PKG_TARBALL)\'
- @echo
- @echo HAS_ASPELL=\'$(HAS_ASPELL)\'
- @echo
- @echo R=\'$(R)\'
-## @echo R_USE_CRAN=\'$(R_USE_CRAN)\'
- @echo R_NO_INIT=\'$(R_NO_INIT)\'
- @echo R_SCRIPT=\'$(R_SCRIPT)\'
- @echo R_VERSION_X_Y=\'$(R_VERSION_X_Y)\'
- @echo R_VERSION=\'$(R_VERSION)\'
- @echo R_VERSION_STATUS=\'$(R_VERSION_STATUS)\'
- @echo R_VERSION_FULL=\'$(R_VERSION_FULL)\'
- @echo R_LIBS_USER_X=\'$(R_LIBS_USER_X)\'
- @echo R_INCLUDE=\'$(R_INCLUDE)\'
- @echo R_OUTDIR=\'$(R_OUTDIR)\'
- @echo
- @echo "Default packages:" $(shell $(R) --slave -e "cat(paste(getOption('defaultPackages'), collapse=', '))")
- @echo
- @echo R_BUILD_OPTS=\'$(R_BUILD_OPTS)\'
- @echo
- @echo R_CHECK_OUTDIR=\'$(R_CHECK_OUTDIR)\'
- @echo _R_CHECK_CRAN_INCOMING_=\'$(_R_CHECK_CRAN_INCOMING_)\'
- @echo _R_CHECK_XREFS_REPOSITORIES_=\'$(_R_CHECK_XREFS_REPOSITORIES_)\'
- @echo _R_CHECK_FULL_=\'$(_R_CHECK_FULL_)\'
- @echo R_CHECK_OPTS=\'$(R_CHECK_OPTS)\'
- @echo R_RD4PDF=\'$(R_RD4PDF)\'
- @echo
- @echo R_CRAN_OUTDIR=\'$(R_CRAN_OUTDIR)\'
- @echo
-
-
-debug_full: debug
- @echo
- @echo FILES_ROOT=\'$(FILES_ROOT)\'
- @echo FILES_R=\'$(FILES_R)\'
- @echo FILES_DATA=\'$(FILES_DATA)\'
- @echo FILES_MAN=\'$(FILES_MAN)\'
- @echo FILES_INST=\'$(FILES_INST)\'
- @echo FILES_VIGNETTES=\'$(FILES_VIGNETTES)\'
- @echo FILES_SRC=\'$(FILES_SRC)\'
- @echo FILES_TESTS=\'$(FILES_TESTS)\'
- @echo FILES_INCL=\'$(FILES_INCL)\'
- @echo FILES_MISC=\'$(FILES_MISC)\'
- @echo
- @echo DIR_VIGNS=\'$(DIR_VIGNS)\'
- @echo dirname\(DIR_VIGNS\)=\'$(shell dirname $(DIR_VIGNS))\'
- @echo
- @echo GIT_BRANCH=\'$(GIT_BRANCH)\'
- @echo GIT_COMMIT=\'$(GIT_COMMIT)\'
- @echo R_LIBS_BRANCH=\'$(R_LIBS_BRANCH)\'
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Update / install
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Update existing packages
-update:
- $(R_SCRIPT) -e "try(update.packages(ask=FALSE)); source('http://bioconductor.org/biocLite.R'); biocLite(ask=FALSE);"
-
-# Install missing dependencies
-deps: DESCRIPTION
- $(MAKE) update
- $(R_SCRIPT) -e "x <- unlist(strsplit(read.dcf('DESCRIPTION',fields=c('Depends', 'Imports', 'Suggests')),',')); x <- gsub('([[:space:]]*|[(].*[)])', '', x); libs <- .libPaths()[file.access(.libPaths(), mode=2) == 0]; x <- unique(setdiff(x, c('R', rownames(installed.packages(lib.loc=libs))))); if (length(x) > 0) { try(install.packages(x)); x <- unique(setdiff(x, c('R', rownames(installed.packages(lib.loc=libs))))); source('http://bioconductor.org/biocLite.R'); biocLite(x); }"
-
-setup: update deps
- $(R_SCRIPT) -e "source('http://aroma-project.org/hbLite.R'); hbLite('R.oo')"
-
-ns:
- $(R_SCRIPT) -e "library('$(PKG_NAME)'); source('X:/devtools/NAMESPACE.R'); writeNamespaceSection('$(PKG_NAME)'); writeNamespaceImports('$(PKG_NAME)');"
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Build source tarball
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-$(R_OUTDIR)/$(PKG_TARBALL): $(PKG_FILES)
- $(MKDIR) $(R_OUTDIR)
- $(RM) $@
- $(CD) $(R_OUTDIR);\
- $(R) $(R_NO_INIT) CMD build $(R_BUILD_OPTS) $(PKG_DIR)
-
-build: $(R_OUTDIR)/$(PKG_TARBALL)
-
-build_force:
- $(RM) $(R_OUTDIR)/$(PKG_TARBALL)
- $(MAKE) install
-
-# Make sure the tarball is readable
-build_fix: $(R_OUTDIR)/$(PKG_TARBALL)
-ifeq ($(OS), Windows_NT)
- ifeq ($(USERNAME), hb)
- $(MKDIR) X:/tmp/$(R_VERSION_FULL)
- $(CP) -f $< X:/tmp/$(R_VERSION_FULL)/
- $(RM) $<
- $(MV) X:/tmp/$(R_VERSION_FULL)/$(<F) $<
- endif
-endif
-
-build_fast: $(PKG_FILES)
- $(MKDIR) $(R_OUTDIR)
- $(RM) $@
- $(CD) $(R_OUTDIR);\
- $(R) $(R_NO_INIT) CMD build --keep-empty-dirs --no-build-vignettes --no-manual --no-resave-data --compact-vignettes="no" $(R_BUILD_OPTS) $(PKG_DIR)
-
-build: $(R_OUTDIR)/$(PKG_TARBALL)
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Install on current system
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-$(R_LIBS_USER_X)/$(PKG_NAME)/DESCRIPTION: $(R_OUTDIR)/$(PKG_TARBALL) build_fix
- $(CD) $(R_OUTDIR);\
- $(R) --no-init-file CMD INSTALL $(PKG_TARBALL)
-
-install: $(R_LIBS_USER_X)/$(PKG_NAME)/DESCRIPTION
-
-install_force:
- $(RM) $(R_LIBS_USER_X)/$(PKG_NAME)/DESCRIPTION
- $(MAKE) install
-
-install_fast:
- $(CD) $(R_OUTDIR);\
- $(R) --no-init-file CMD INSTALL --no-docs --no-multiarch --no-byte-compile --no-test-load $(PKG_TARBALL)
-
-
-$(R_LIBS_BRANCH)/$(PKG_NAME)/DESCRIPTION: $(R_OUTDIR)/$(PKG_TARBALL) build_fix
- $(CD) $(R_OUTDIR);\
- $(MKDIR) $(R_LIBS_BRANCH);\
- $(R) --no-init-file CMD INSTALL --library=$(R_LIBS_BRANCH) $(PKG_TARBALL)
-
-install_branch: $(R_LIBS_BRANCH)/$(PKG_NAME)/DESCRIPTION
-
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Check source tarball
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-$(R_CHECK_OUTDIR)/.check.complete: $(R_OUTDIR)/$(PKG_TARBALL) build_fix
- $(CD) $(R_OUTDIR);\
- $(RM) -r $(PKG_NAME).Rcheck;\
- export _R_CHECK_CRAN_INCOMING_=$(_R_CHECK_CRAN_INCOMING_);\
- export _R_CHECK_CRAN_INCOMING_USE_ASPELL_=$(HAS_ASPELL);\
- export _R_CHECK_XREFS_REPOSITORIES_=$(_R_CHECK_XREFS_REPOSITORIES_);\
- export _R_CHECK_DOT_INTERNAL_=1;\
- export _R_CHECK_USE_CODETOOLS_=1;\
- export _R_CHECK_FORCE_SUGGESTS_=0;\
- export R_RD4PDF=$(R_RD4PDF);\
- export _R_CHECK_FULL_=$(_R_CHECK_FULL_);\
- $(R) --no-init-file CMD check $(R_CHECK_OPTS) $(PKG_TARBALL);\
- echo done > $(PKG_NAME).Rcheck/.check.complete
-
-check: $(R_CHECK_OUTDIR)/.check.complete
-
-check_force:
- $(RM) -r $(R_CHECK_OUTDIR)
- $(MAKE) check
-
-clang:
- clang -c -pedantic -Wall -I$(R_INCLUDE) src/*.c
- $(RM) *.o
-
-clang-ubsan:
- clang -fsanitize=undefined -I$(R_INCLUDE) -c src/*.c
- $(RM) *.o
-
-valgrind_scan:
- grep -E "^==.*==[ ]+(at|by) 0x" $(R_CHECK_OUTDIR)/tests*/*.Rout | cat
- grep "^==.* ERROR SUMMARY:" $(R_CHECK_OUTDIR)/tests*/*.Rout | grep -v -F "ERROR SUMMARY: 0 errors" | cat
-
-valgrind:
- export _R_CHECK_USE_VALGRIND_=TRUE;\
- $(MAKE) check_force
- $(MAKE) valgrind_scan
-
-# Check the line width of incl/*.(R|Rex) files [max 100 chars in R devel]
-check_Rex:
- $(R_SCRIPT) -e "if (!file.exists('incl')) quit(status=0); setwd('incl/'); fs <- dir(pattern='[.](R|Rex)$$'); ns <- sapply(fs, function(f) max(nchar(readLines(f)))); ns <- ns[ns > 100]; print(ns); if (length(ns) > 0L) quit(status=1)"
-
-
-covr:
- $(R_SCRIPT) -e "source('.covr.R'); covr::shine(covr)"
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Install and build binaries
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-$(R_OUTDIR)/$(PKG_ZIP): $(R_OUTDIR)/$(PKG_TARBALL) build_fix
- $(CD) $(R_OUTDIR);\
- $(R) --no-init-file CMD INSTALL --build --merge-multiarch $(PKG_TARBALL)
-
-binary: $(R_OUTDIR)/$(PKG_ZIP)
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Build Rd help files from Rdoc comments
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-rox:
- $(R_SCRIPT) -e "roxygen2::roxygenize()"
-
-Rd: check_Rex
- $(R_SCRIPT) -e "setwd('..'); Sys.setlocale(locale='C'); R.oo::compileRdoc('$(PKG_NAME)', path='$(PKG_DIR)')"
-
-%.Rd:
- $(R_SCRIPT) -e "setwd('..'); Sys.setlocale(locale='C'); R.oo::compileRdoc('$(PKG_NAME)', path='$(PKG_DIR)', '$*.R')"
-
-missing_Rd:
- $(R_SCRIPT) -e "x <- readLines('$(R_CHECK_OUTDIR)/00check.log'); from <- grep('Undocumented code objects:', x)+1; if (length(from) > 0L) { to <- grep('All user-level objects', x)-1; x <- x[from:to]; x <- gsub('^[ ]*', '', x); x <- gsub('[\']', '', x); cat(x, sep='\n', file='999.missingdocs.txt'); }"
-
-spell_Rd:
- $(R_SCRIPT) -e "f <- list.files('man', pattern='[.]Rd$$', full.names=TRUE); utils::aspell(f, filter='Rd')"
-
-
-spell_NEWS:
- $(R_SCRIPT) -e "utils::aspell('$(FILES_NEWS)')"
-
-spell:
- $(R_SCRIPT) -e "utils::aspell('DESCRIPTION', filter='dcf')"
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Build package vignettes
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-$(R_OUTDIR)/vigns: install
- $(MKDIR) $(R_OUTDIR)/vigns/$(shell dirname $(DIR_VIGNS))
- $(CP) DESCRIPTION $(R_OUTDIR)/vigns/
- $(CP) -r $(DIR_VIGNS) $(R_OUTDIR)/vigns/$(shell dirname $(DIR_VIGNS))
- $(CD) $(R_OUTDIR)/vigns;\
- $(R_SCRIPT) -e "v <- tools::buildVignettes(dir='.'); file.path(getwd(), v[['outputs']])"
-
-vignettes: $(R_OUTDIR)/vigns
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Run package tests
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-$(R_OUTDIR)/tests/%.R: $(FILES_TESTS)
- $(RMDIR) $(R_OUTDIR)/tests
- $(MKDIR) $(R_OUTDIR)/tests
- $(CP) $? $(R_OUTDIR)/tests
-
-test_files: $(R_OUTDIR)/tests/*.R
-
-test: $(R_OUTDIR)/tests/%.R
- $(CD) $(R_OUTDIR)/tests;\
- $(R_SCRIPT) -e "for (f in list.files(pattern='[.]R$$')) { print(f); source(f, echo=TRUE) }"
-
-test_full: $(R_OUTDIR)/tests/%.R
- $(CD) $(R_OUTDIR)/tests;\
- export _R_CHECK_FULL_=TRUE;\
- $(R_SCRIPT) -e "for (f in list.files(pattern='[.]R$$')) { print(f); source(f, echo=TRUE) }"
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Benchmarking
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-R_branch:
- R_LIBS="$(R_LIBS_USER_X)$(FSEP)$(R_LIBS)" R_LIBS_USER="$(R_LIBS_BRANCH)" $(R)
-
-benchmark:
- $(MKDIR) .benchmark/$(PKG_VERSION);\
- $(CD) .benchmark/$(PKG_VERSION);\
- $(R_SCRIPT) -e "$(PKG_NAME):::benchmark('index')"
-
-benchmark_branch:
- $(MKDIR) .benchmark/$(PKG_VERSION)_$(GIT_BRANCH)_$(GIT_COMMIT);\
- $(CD) .benchmark/$(PKG_VERSION)_$(GIT_BRANCH)_$(GIT_COMMIT);\
- R_LIBS="$(R_LIBS_USER_X)$(FSEP)$(R_LIBS)" R_LIBS_USER="$(R_LIBS_BRANCH)" $(R_SCRIPT) -e "$(PKG_NAME):::benchmark('index')"
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Miscellaneous development tools
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-## Fully expanded src/*.c files
-.devel/src/%: src/% $(FILES_SRC)
- $(MKDIR) .devel/src/;\
- gcc -I$(R_INCLUDE) -E $< | sed -e '/./b' -e :n -e 'N;s/\n$$//;tn' > .devel/$<
-
-.devel/src/all: $(FILES_SRC)
- for f in $(FILES_SRC_C); do\
- echo $$f;\
- $(MAKE) .devel/$$f;\
- done
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Run extensive CRAN submission checks
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-$(R_CRAN_OUTDIR)/$(PKG_TARBALL): $(R_OUTDIR)/$(PKG_TARBALL) build_fix
- $(MKDIR) $(R_CRAN_OUTDIR)
- $(CP) $(R_OUTDIR)/$(PKG_TARBALL) $(R_CRAN_OUTDIR)
-
-$(R_CRAN_OUTDIR)/$(PKG_NAME),EmailToCRAN.txt: $(R_CRAN_OUTDIR)/$(PKG_TARBALL)
- $(CD) $(R_CRAN_OUTDIR);\
- $(R_SCRIPT) -e "RCmdCheckTools::testPkgsToSubmit(delta=2/3)"
-
-cran_setup: $(R_CRAN_OUTDIR)/$(PKG_TARBALL)
- $(R_SCRIPT) -e "if (!nzchar(system.file(package='RCmdCheckTools'))) { source('http://aroma-project.org/hbLite.R'); hbLite('RCmdCheckTools', devel=TRUE); }"
-
-cran: cran_setup $(R_CRAN_OUTDIR)/$(PKG_NAME),EmailToCRAN.txt
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Send to win-builder server
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-WIN_BUILDER = win-builder.r-project.org
-win-builder-devel: $(R_OUTDIR)/$(PKG_TARBALL)
- curl -v -T $? ftp://anonymous@$(WIN_BUILDER)/R-devel/
-
-win-builder-release: $(R_OUTDIR)/$(PKG_TARBALL)
- curl -v -T $? ftp://anonymous@$(WIN_BUILDER)/R-release/
-
-win-builder: win-builder-devel win-builder-release
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Local repositories
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-ifeq ($(OS), Windows_NT)
-REPOS_PATH = T:/My\ Repositories/braju.com/R
-else
-REPOS_PATH = /tmp/hb/repositories/braju.com/R
-endif
-REPOS_SRC := $(REPOS_PATH)/src/contrib
-
-$(REPOS_SRC):
- $(MKDIR) "$@"
-
-$(REPOS_SRC)/$(PKG_TARBALL): $(R_OUTDIR)/$(PKG_TARBALL) $(REPOS_SRC)
- $(CP) $(R_OUTDIR)/$(PKG_TARBALL) $(REPOS_SRC)
-
-repos: $(REPOS_SRC)/$(PKG_TARBALL)
-
-Makefile: $(FILES_MAKEFILE)
- $(R_SCRIPT) -e "d <- 'Makefile'; s <- '../../Makefile'; if (file_test('-nt', s, d) && (regexpr('Makefile for R packages', readLines(s, n=1L)) != -1L)) file.copy(s, d, overwrite=TRUE)"
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Refresh
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-.make/Makefile:
- $(R_SCRIPT) -e "R.utils::downloadFile('https://raw.githubusercontent.com/HenrikBengtsson/r-package-files/master/.make/Makefile', path='.make/')"
-
-.make/.travis.yml.rsp:
- $(R_SCRIPT) -e "R.utils::downloadFile('https://raw.githubusercontent.com/HenrikBengtsson/r-package-files/master/.make/.travis.yml.rsp', path='.make/')"
-
-.make/appveyor.yml.rsp:
- $(R_SCRIPT) -e "R.utils::downloadFile('https://raw.githubusercontent.com/HenrikBengtsson/r-package-files/master/.make/appveyor.yml.rsp', path='.make/')"
-
-.make/README.md.rsp:
- $(R_SCRIPT) -e "R.utils::downloadFile('https://raw.githubusercontent.com/HenrikBengtsson/r-package-files/master/.make/README.md.rsp', path='.make/')"
-
-.covr.R:
- $(R_SCRIPT) -e "R.utils::downloadFile('https://raw.githubusercontent.com/HenrikBengtsson/covr-utils/master/.covr.R')"
-
-clean:
- $(RM) .make/.travis.yml.rsp .make/appveyor.yml.rsp .make/README.md.rsp .covr.R
- $(RM) covr-utils.R
-
-refresh: clean
- $(MAKE) --silent .make/.travis.yml.rsp
- $(MAKE) --silent .make/appveyor.yml.rsp
- $(MAKE) --silent .make/README.md.rsp
- $(MAKE) --silent .covr.R
- $(R_SCRIPT) -e "R.utils::downloadFile('https://raw.githubusercontent.com/HenrikBengtsson/r-package-files/master/.make/Makefile', path='.make/', skip=FALSE)"
-
-
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# GitHub, Travis CI, ...
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-README.md: .make/README.md.rsp
- $(R_SCRIPT) -e "R.rsp::rfile('$<', postprocess=FALSE)"
-
-.travis.yml: .make/.travis.yml.rsp
- $(R_SCRIPT) -e "R.rsp::rfile('$<', postprocess=FALSE)"
-
-appveyor.yml: .make/appveyor.yml.rsp
- $(R_SCRIPT) -e "R.rsp::rfile('$<', postprocess=FALSE)"
-
-config: .travis.yml appveyor.yml README.md
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 35447e1..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-#----------------------------------------------------------------
-# Travis-CI configuration for R packages
-#
-# REFERENCES:
-# * Travis CI: https://docs.travis-ci.com/user/languages/r
-# * covr: https://github.com/jimhester/covr
-#
-# Validate your .travis.yml file at http://lint.travis-ci.org/
-#----------------------------------------------------------------
-language: r
-sudo: false
-cache: packages
-warnings_are_errors: false
-r_check_args: --as-cran
-latex: false
-
-matrix:
- include:
- - os: linux
- r: oldrel
- - os: linux
- r: release
- r_github_packages:
- - jimhester/covr
- after_success:
- - Rscript -e 'covr::codecov(quiet=FALSE)'
- env: NB='w/ covr' ## Just a label
- - os: linux
- r: devel
- - os: osx
- r: oldrel
- - os: osx
- r: release
-
-
-notifications:
- email:
- on_success: change
- on_failure: change
diff --git a/CONDUCT.md b/CONDUCT.md
deleted file mode 100644
index 9c1c621..0000000
--- a/CONDUCT.md
+++ /dev/null
@@ -1,74 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to making participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, gender identity and expression, level of experience,
-nationality, personal appearance, race, religion, or sexual identity and
-orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or
-advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
- address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
- professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at [http://contributor-covenant.org/version/1/4][version]
-
-[homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/
diff --git a/DESCRIPTION b/DESCRIPTION
index aafa39b..9020688 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,40 +1,36 @@
Package: future
-Version: 1.2.0
-Title: A Future API for R
-Imports:
- digest,
- globals (>= 0.7.1),
- listenv (>= 0.6.0),
- parallel,
- utils
-Suggests:
- R.rsp,
- markdown
+Version: 1.6.1
+Title: Unified Parallel and Distributed Processing in R for Everyone
+Imports: digest, globals (>= 0.10.2), listenv (>= 0.6.0), parallel,
+ utils
+Suggests: R.rsp, markdown
VignetteBuilder: R.rsp
Authors at R: c(person("Henrik", "Bengtsson", role=c("aut", "cre", "cph"),
email = "henrikb at braju.com"))
-Description: A Future API for R is provided. In programming, a future is an
- abstraction for a value that may be available at some point in the future.
- The state of a future can either be unresolved or resolved. As soon as it
- is resolved, the value is available. Futures are useful constructs in for
- instance concurrent evaluation, e.g. parallel processing and distributed
- processing on compute clusters. The purpose of this package is to provide
- a lightweight interface for using futures in R. Functions 'future()' and
- 'value()' exist for creating futures and requesting their values, e.g.
- 'f <- future({ mandelbrot(-0.75, 0, side=3) })' and 'v <- value(f)'. The
- 'resolved()' function can be used to check if a future is resolved or
- not. An infix assignment operator '%<-%' exists for creating futures whose
- values are accessible by the assigned variables (as promises), e.g. 'v %<-%
- { mandelbrot(-0.75, 0, side=3) }'. This package implements synchronous "lazy"
- and "eager" futures, and asynchronous "multicore", "multisession" and ad hoc
- "cluster" futures. Globals variables and functions are automatically identified
- and exported. Required packages are attached in external R sessions whenever
- needed. All types of futures are designed to behave the same such that the
- exact same code work regardless of futures used or number of cores, background
- sessions or cluster nodes available. Additional types of futures are provided by
- other packages enhancing this package.
+Description: The purpose of this package is to provide a lightweight and
+ unified Future API for sequential and parallel processing of R
+ expression via futures. The simplest way to evaluate an expression
+ in parallel is to use `x %<-% { expression }` with `plan(multiprocess)`.
+ This package implements sequential, multicore, multisession, and
+ cluster futures. With these, R expressions can be evaluated on the
+ local machine, on in parallel a set of local machines, or distributed
+ on a mix of local and remote machines.
+ Extensions to this package implements additional backends for
+ processing futures via compute cluster schedulers etc.
+ Because of its unified API, there is no need to modify code in order
+ switch from sequential on the local machine to, say, distributed
+ processing on a remote compute cluster.
+ Another strength of this package is that global variables and functions
+ are automatically identified and exported as needed, making it
+ straightforward to tweak existing code to make use of futures.
License: LGPL (>= 2.1)
LazyLoad: TRUE
URL: https://github.com/HenrikBengtsson/future
BugReports: https://github.com/HenrikBengtsson/future/issues
-RoxygenNote: 5.0.1
+RoxygenNote: 6.0.1
+NeedsCompilation: no
+Packaged: 2017-09-09 03:46:22 UTC; hb
+Author: Henrik Bengtsson [aut, cre, cph]
+Maintainer: Henrik Bengtsson <henrikb at braju.com>
+Repository: CRAN
+Date/Publication: 2017-09-09 14:52:37 UTC
diff --git a/MD5 b/MD5
new file mode 100644
index 0000000..194d4a4
--- /dev/null
+++ b/MD5
@@ -0,0 +1,186 @@
+88ab6edf52c8321e82435363248e7794 *DESCRIPTION
+136c0763e91559628c93dd45a7005ebb *NAMESPACE
+30c6ca2cc1c6dbf9b51938b7c0d34746 *NEWS
+5c414ed9302ea14ce03113003c6cdb2f *R/ClusterFuture-class.R
+25eb4e4c173145fa8dc98ffe50a352b0 *R/ClusterRegistry.R
+5330c624a1a8e1a17da56d44c3c71c72 *R/ConstantFuture-class.R
+57612b9c4a5965bbd55226b149f75735 *R/DEPRECATED.R
+6bb33f2db9231dfe8aea1daca60a66a3 *R/Future-class.R
+3a7018fa40706bbbc80e1972f8c5ef20 *R/FutureError-class.R
+c885fb70124ad447ee40f790ebfa1454 *R/FutureGlobals-class.R
+619d6434d29e79e468fcbde6a887817a *R/FutureRegistry.R
+f98906e8997bd1474af9d5d1e140fe77 *R/MulticoreFuture-class.R
+f8cb8f189a38870c91a256b033004acf *R/MultiprocessFuture-class.R
+6e29de0405dd268237a2ea6e39b0332b *R/MultisessionFuture-class.R
+30c1a369ae47d0886a6e192dd16f4eea *R/UniprocessFuture-class.R
+f1c08954d0788f9b790aaea1fa0cb4b6 *R/as.cluster.R
+83b5826ba5553a8334a4728b8f096d5a *R/availableCores.R
+d957edb0e324e83a1bf957a661eb0a23 *R/availableWorkers.R
+a44339cfd90f7d2ca7525446ed35b62d *R/backtrace.R
+dd0d3d02a95c675d93bd30d8138cffff *R/cluster.R
+eea4e0cb9e96f0cb0a510476eee7ef08 *R/constant.R
+21e1147f4add566ba1cea876b5d3109c *R/future.R
+387de461d906643e5aee16f74fd2059a *R/futureAssign.R
+972d04513096f8557db7457ac0610d51 *R/futureAssign_OP.R
+5f13d03960f0cb2abcb211f2b4689505 *R/futureCall.R
+8b3afbe7637c9598d98839f307006226 *R/futureOf.R
+ea1a65548338dd7c276a4beca2bac675 *R/future_lapply.R
+804a9b4c18552018ab2a0139554ce438 *R/futures.R
+ca2f1ba5d5cd8d9d3e31a9651e3b1854 *R/globals.R
+83a2c09868e19da2f570e7c129f5c155 *R/globals_OP.R
+66a2509dc9ecf98e8210767300bd2b7a *R/label_OP.R
+c6323caf62e03e994a883f5065c836f8 *R/lazy_OP.R
+57e3803588f6415cd26cde2cc8a57905 *R/makeClusterPSOCK.R
+48d751f2868f8bb0c024df9d39e326ef *R/mandelbrot.R
+b5d17e298ed318968c3b45a754e3152b *R/multicore.R
+3144b7092008bf757f97984c7018f1e1 *R/multiprocess.R
+5b4aea4d72b8c9bd3a05b784dde704b4 *R/multisession.R
+0ca5e9c5c788b38246c025d1650c29bd *R/nbrOfWorkers.R
+b59289bd556933c03b2e994d961a92ec *R/options.R
+96b0a255d72b99278999aa8fad8ab955 *R/plan_OP.R
+f7d31d5b8bc089af79373dd8b9021c9e *R/remote.R
+2af8d22fdd25ec5700a8a8aaafe258ca *R/resolve.R
+db685ac795c66040d7f820fefcf5e5a5 *R/resolved.R
+2f8d331845a4e5b34591c96443f6fdb9 *R/seed_OP.R
+225f984078d92ece8d6f56dc2a928158 *R/sequential.R
+825b96f24ad1a79e1f8d02dd4f921fbd *R/sessionDetails.R
+3ed78632adcd9e9e37fbb2388adf4ddd *R/signalEarly.R
+b4573b7471cbc764e97798e63be5ce94 *R/tweak.R
+cb7361a19698a4c77d0e51d5b80bd081 *R/tweakExpression.R
+5be8fd50a0eb136864afc69519066c4d *R/tweak_OP.R
+ed16e04164a37b2e606008ad51bf99fe *R/utils.R
+73be516a4466ee3d21f721fd4e40cc4f *R/values.R
+16c3ca1d6edf4ed7e9a191755531cd44 *R/whichIndex.R
+ddc48c6f16b7d85f3a3abfb7938afd6f *R/zzz.R
+94bcafc3d8f1703feba7b7f42e232e5c *R/zzz.plan.R
+ff9ac56d90c3279a89a8e2b755ef7601 *build/vignette.rds
+f615957051c16ab204dab55cf45ec170 *demo/00Index
+47f62f9433fb92c1e8245c6e919336e2 *demo/fibonacci.R
+68470d445b899ca10d30fef6e4638d26 *demo/mandelbrot.R
+c0f7eae3869f6ccadaeee885d48411d7 *inst/doc/future-1-overview.html
+830efe4418dd98d2c322ca6910820c77 *inst/doc/future-1-overview.md.rsp
+5be8acaa8cb677a3f8ccf725eeb8c1b2 *inst/doc/future-2-issues.html
+a7627bebdea7629c7c87f941fdc6e26f *inst/doc/future-2-issues.md.rsp
+5f28dc8a09460fa467a4b23a9200d91b *inst/doc/future-3-topologies.html
+4e8983ff33bd203c056ab316e5195a9d *inst/doc/future-3-topologies.md.rsp
+a27bbc4688cb3c2c68268075f85a8561 *inst/doc/future-4-startup.html
+69c2c9f7f41deee3a7a57829ba33dcf6 *inst/doc/future-4-startup.md.rsp
+e29337493dcfacd899c8e275863d584c *inst/vignettes-static/future-1-overview.md.rsp.rsp
+d15bee49d2cd9d3c1f2725015a53d7de *inst/vignettes-static/incl/future-1-overview-example2.R
+e4dd9732b6549419d8fd2a5fb715879c *inst/vignettes-static/incl/future-1-overview-example3.R
+e08a0ddcb2ec3cba933e4f348f272994 *man/ClusterFuture-class.Rd
+1e5dfc08f1d4adf32e9d0ce2a4942f0e *man/ConstantFuture-class.Rd
+e8a15baeaa232c993e3cc3e5a0bd0bec *man/Future-class.Rd
+e1d14a933fec573266e274f442959a74 *man/FutureError.Rd
+8828cc1f630a8df785fa3cd9344167af *man/FutureGlobals.Rd
+41e8ec142b60cdf98bc59b9fe32bf035 *man/MulticoreFuture-class.Rd
+da7b70a3783bc0999e6cece743734c5d *man/MultiprocessFuture-class.Rd
+96264e84d618b3ec3b26cf4818ff5939 *man/UniprocessFuture-class.Rd
+b7e1fd4c37a4b32c2dfc90a49aacd0eb *man/as.cluster.Rd
+016abe77fbfcdfa275699c45a5c50243 *man/availableCores.Rd
+e2f5ebe12ea08dde1d29ac54a5cd3ea8 *man/availableWorkers.Rd
+577342e28a4397af240077cfe079fa4c *man/backtrace.Rd
+5c1b80859b14f145354c875942ebe3bf *man/cluster.Rd
+8a8dd6a0ae32aba2f50e4b438cb60e4b *man/future.Rd
+037abe36c9eddbb29d5dca48b5a89bd3 *man/future.options.Rd
+bdb5ed06a66c5d2d26ac207cb007a628 *man/futureOf.Rd
+b9a5c0fb77d6be3564179b25ac1b122e *man/future_lapply.Rd
+4d365d5b4666ea7f1e32402fc4175f5e *man/futures.Rd
+82b2cf8a14f00c200139ac784b76e8ea *man/getExpression.Rd
+f6077e148c1cf766854a48d0f920ab2d *man/getGlobalsAndPackages.Rd
+54efb38d37b902dd2d54de1745b73a21 *man/grapes-globals-grapes.Rd
+f164c1bcbcc990b53b3c3f4e60f7a5b3 *man/grapes-label-grapes.Rd
+61c3e6e56c0db61860a815e7ba1aceb9 *man/grapes-lazy-grapes.Rd
+79b895306c6aaf39a6ae03baba51095f *man/grapes-plan-grapes.Rd
+085ad705358be49cf39ddd5b9db251ee *man/grapes-seed-grapes.Rd
+29b2c36f9cc59f1688c466520250304f *man/grapes-tweak-grapes.Rd
+67a6a43854080e2b22cde6967bb57308 *man/makeClusterPSOCK.Rd
+8879d39a6a3cd8dd3d7f6801024c12d8 *man/mandelbrot.Rd
+600faec2eb9b3da2e0d57010d6c252cb *man/multicore.Rd
+91e9d5428ba21326efd68481ec09bf35 *man/multiprocess.Rd
+a68ddbf53293739aae4c407bc4c450ae *man/multisession.Rd
+1b46683e11efe3a19ec538271bd96a9c *man/nbrOfWorkers.Rd
+b54b71bcd0856dc0a3ef2b11861d69a5 *man/plan.Rd
+0e9265cff7b5a98e4ccd4fe831d84a95 *man/private_length.Rd
+b695ad86222f10f461b382957678e025 *man/remote.Rd
+4c3f8a7e398b842641b73234978fc8f6 *man/requestCore.Rd
+c1e625bb13c3477aed98dd0677e33758 *man/resolve.Rd
+9ba07382a74cef10cbd6e66920816e3b *man/resolved.Rd
+0994628d61f17c629563efc7a0d23862 *man/run.Rd
+c45560b62e5ea3b3f79ba0faaa401dde *man/sequential.Rd
+9a2d320ac796727e9058224529a74c04 *man/sessionDetails.Rd
+a82637479f44058888880867ad9438ac *man/supportsMulticore.Rd
+4f735c3de606eb211cc9def983ddcd6e *man/tweak.Rd
+d97c4e26070f71a511056729b8489201 *man/usedCores.Rd
+291364834088c957e781ef5ed408b4d1 *man/value.Rd
+7d894263cbd967f5f16183bc1f9a0116 *man/values.Rd
+3f47fce2aaf0690984e78f5e477b67cd *tests/000.sessionDetails.R
+e2e04aaf80df5ccd004298336fb36141 *tests/ClusterRegistry.R
+5f4294364e0bd60c27c855e8b3ba12d1 *tests/DEPRECATED_DEFUNCT.R
+683635644c22df08b5afe879b614cda6 *tests/Future-class.R
+e1da5ef72bea7e5d83049719b4f83249 *tests/FutureError.R
+a81713c8b714cff38af45ba68a733932 *tests/FutureGlobals.R
+84cfb2e7d1fe0d31c479f1d2224617d9 *tests/FutureRegistry.R
+88c743d5b06f005e5b22b32ffc34f729 *tests/as.cluster.R
+68a584d21d55d9380702d3fc9e9a1225 *tests/availableCores.R
+7e776c43a35e5a4c3ec33f91e324414d *tests/availableWorkers.R
+665abf9fc09c8dbcaae4dbf9f2515a66 *tests/backtrace.R
+27bcb4f9f9fc0fe7f589e7e2c985df5d *tests/cluster.R
+748ce966bb4fd9af41b2544ea74fe4ba *tests/constant.R
+98ffc71da89ce76e29f4378eb1da30e0 *tests/demo.R
+409d578e83c7f09a74838dadd544055c *tests/dotdotdot.R
+cd9775d6c25454533500a1ed108bec75 *tests/early-signaling.R
+952b774fd9a5773ed9f3a7bfb9db8ab3 *tests/future,labels.R
+6178f7436f0e03d729465d69cb3cb314 *tests/future.R
+8115b3c847c327a2c5a789c5a4e08122 *tests/futureAssign.R
+2669ff40c188a5aa3c7e49c7017535c7 *tests/futureAssign_OP.R
+db5236b009a9892ab4e8238ae8deb773 *tests/futureAssign_OP_with_environment.R
+2909df83606e68e7babfe83d40d3b578 *tests/futureAssign_OP_with_listenv.R
+847abb75deeba4925b103cbc2ebedabb *tests/futureCall.R
+7f8f114fc8c70a8758f6d2047dbdc475 *tests/futureOf.R
+5d7298e1d2c5b36ae62113db05740759 *tests/futureOf_with_environment.R
+ae940cdf10148bb14bdf0a76cf114651 *tests/futureOf_with_listenv.R
+1a08fea5e3d22e812fdb0e94975e9177 *tests/future_lapply,RNG.R
+aac9a7dd9a9ef33483bf3af12b6642d0 *tests/future_lapply,globals.R
+63624fc183cb9b0edacce3d856f72215 *tests/future_lapply.R
+add2d75eca2645715ca2425fb5c74fb7 *tests/futures.R
+c8c9dc2c00a017999e7a9930ba0fc2a1 *tests/globals,NSE.R
+cfce274f3ad575fcfd234262ca54abbd *tests/globals,formulas.R
+9947a36a8e5bccd73cd3215be5c8d240 *tests/globals,manual.R
+ba683a3268e1fc5d2ac1a22803324d9b *tests/globals,resolve.R
+500698e378c2bcf52df97d0860823bf5 *tests/globals,subassignment.R
+ecb7c2de0dbc697ce9c30ed6d4c5f1d7 *tests/globals,toolarge.R
+e3a11683dbeb7f2e01ab65d2b304049e *tests/globals,tricky.R
+1b2f3d68a7cb0f470abf80dbb08d14f7 *tests/globals,tricky_recursive.R
+c3217a852bee297aba7da4acd4834954 *tests/globalsOf,tweaks.R
+b511d966121a42d8d02459046bf834bc *tests/incl/end.R
+7d0c0567817576226a155ab15376620b *tests/incl/start,load-only.R
+3fe5b5d1e7981baf93db9ed2fca8ff2c *tests/incl/start.R
+e2e7182115c52a25761781717ed4da26 *tests/invalid-owner.R
+d333e0e8f65a1143f84e8c7cd0d62319 *tests/makeClusterPSOCK.R
+274f8c93e1fc5f64d0f2ca186ddd828f *tests/mandelbrot.R
+7cd7d37e62ac9c0ad9be9021ae1d0698 *tests/multicore.R
+0c4f8afcb8789fde66607da92e4a2bcc *tests/multiprocess.R
+dd6c1a242f70de57c8b4cfae8708f28f *tests/multisession.R
+7912309132cdd7676fbb4a072047733f *tests/nbrOfWorkers.R
+c521abb9142baf89e4cdf1a13fd9fcdd *tests/nested_futures,mc.cores.R
+a2e6432c332b4a7dfebeef93b521085a *tests/nested_futures.R
+8d116f6bac49ac2ed64b92f7a1c941b6 *tests/objectSize.R
+2e0dcced84c825950da3f109790a3410 *tests/plan.R
+ef3c27448710c6e2f57f2d6a3b3fc622 *tests/remote.R
+f97146e6891c43e50671382dc0fb250a *tests/requestCore.R
+03197f27e44cfa4a217efa9f22523d41 *tests/requestNode.R
+e92b81a321852bb88dc44d86d8eaf559 *tests/resolve.R
+1c3614936849e92cac4beaaa2e2244af *tests/rng.R
+4ba7ac669cb13265dc50a7aaf735d627 *tests/sequential.R
+0a5550cb91c0f2f0a964671414aa59dd *tests/sessionDetails.R
+2039fd0201b583dc625c2d938c041957 *tests/startup.R
+11f3f8007dd49fbb886b5928ed799823 *tests/transparent.R
+a61d28dc4231679167ba9e1934412cc9 *tests/tweak.R
+a60f478eeb974dea72fe008d33d6e453 *tests/utils.R
+9ad5052290b053b8d5891e1a1ca637ad *tests/uuid.R
+ed8892a82f5133d60c146afa2646feea *tests/whichIndex.R
+830efe4418dd98d2c322ca6910820c77 *vignettes/future-1-overview.md.rsp
+a7627bebdea7629c7c87f941fdc6e26f *vignettes/future-2-issues.md.rsp
+4e8983ff33bd203c056ab316e5195a9d *vignettes/future-3-topologies.md.rsp
+69c2c9f7f41deee3a7a57829ba33dcf6 *vignettes/future-4-startup.md.rsp
diff --git a/Makefile b/Makefile
deleted file mode 100644
index bfaeda6..0000000
--- a/Makefile
+++ /dev/null
@@ -1,9 +0,0 @@
-include .make/Makefile
-
-vignettes/future-1-overview.md.rsp: inst/vignettes-static/future-1-overview.md.rsp.rsp
- $(CD) $(@D); \
- $(R_SCRIPT) -e "R.rsp::rfile" ../$< --postprocess=FALSE
- $(RM) README.md
- $(MAKE) README.md
-
-vigs: vignettes/future-1-overview.md.rsp
diff --git a/NAMESPACE b/NAMESPACE
index 1546bac..bda046e 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -1,11 +1,16 @@
# Generated by roxygen2: do not edit by hand
+S3method("[",FutureGlobals)
S3method("[",sessionDetails)
+S3method(as.FutureGlobals,FutureGlobals)
+S3method(as.FutureGlobals,Globals)
+S3method(as.FutureGlobals,list)
S3method(as.cluster,SOCK0node)
S3method(as.cluster,SOCKnode)
S3method(as.cluster,cluster)
S3method(as.cluster,list)
S3method(as.raster,Mandelbrot)
+S3method(c,FutureGlobals)
S3method(c,cluster)
S3method(futures,environment)
S3method(futures,list)
@@ -17,6 +22,7 @@ S3method(getOutput,FutureError)
S3method(mandelbrot,matrix)
S3method(mandelbrot,numeric)
S3method(nbrOfWorkers,"NULL")
+S3method(nbrOfWorkers,cluster)
S3method(nbrOfWorkers,future)
S3method(nbrOfWorkers,multiprocess)
S3method(nbrOfWorkers,uniprocess)
@@ -28,12 +34,14 @@ S3method(print,FutureStrategyList)
S3method(print,future)
S3method(print,sessionDetails)
S3method(resolve,Future)
+S3method(resolve,FutureGlobals)
S3method(resolve,default)
S3method(resolve,environment)
S3method(resolve,list)
S3method(resolve,listenv)
S3method(resolved,ClusterFuture)
S3method(resolved,Future)
+S3method(resolved,FutureGlobals)
S3method(resolved,MulticoreFuture)
S3method(resolved,UniprocessFuture)
S3method(resolved,default)
@@ -43,8 +51,10 @@ S3method(run,ClusterFuture)
S3method(run,Future)
S3method(run,MulticoreFuture)
S3method(run,UniprocessFuture)
+S3method(tweak,"function")
S3method(tweak,character)
S3method(tweak,future)
+S3method(unique,FutureGlobals)
S3method(value,ClusterFuture)
S3method(value,Future)
S3method(value,MulticoreFuture)
@@ -54,24 +64,28 @@ S3method(values,list)
S3method(values,listenv)
export("%->%")
export("%<-%")
-export("%<=%")
-export("%=>%")
export("%globals%")
export("%label%")
+export("%lazy%")
export("%plan%")
+export("%seed%")
export("%tweak%")
export(ClusterFuture)
export(ConstantFuture)
export(EagerFuture)
export(Future)
export(FutureError)
+export(FutureGlobals)
export(LazyFuture)
export(MulticoreFuture)
export(MultiprocessFuture)
export(MultisessionFuture)
+export(SequentialFuture)
export(UniprocessFuture)
+export(as.FutureGlobals)
export(as.cluster)
export(availableCores)
+export(availableWorkers)
export(backtrace)
export(cluster)
export(eager)
@@ -79,14 +93,16 @@ export(future)
export(futureAssign)
export(futureCall)
export(futureOf)
+export(future_lapply)
export(futures)
export(getExpression)
+export(getGlobalsAndPackages)
export(getOutput)
export(lazy)
export(makeClusterPSOCK)
export(makeNodePSOCK)
export(mandelbrot)
-export(mandelbrotTiles)
+export(mandelbrot_tiles)
export(multicore)
export(multiprocess)
export(multisession)
@@ -96,6 +112,7 @@ export(remote)
export(resolve)
export(resolved)
export(run)
+export(sequential)
export(sessionDetails)
export(supportsMulticore)
export(transparent)
@@ -103,6 +120,7 @@ export(tweak)
export(value)
export(values)
importFrom(digest,digest)
+importFrom(globals,Globals)
importFrom(globals,as.Globals)
importFrom(globals,cleanup)
importFrom(globals,globalsByName)
@@ -119,9 +137,14 @@ importFrom(listenv,map)
importFrom(listenv,parse_env_subset)
importFrom(parallel,clusterCall)
importFrom(parallel,clusterExport)
+importFrom(parallel,nextRNGStream)
+importFrom(parallel,nextRNGSubStream)
+importFrom(parallel,splitIndices)
importFrom(parallel,stopCluster)
importFrom(utils,capture.output)
importFrom(utils,file_test)
importFrom(utils,head)
importFrom(utils,object.size)
+importFrom(utils,read.table)
importFrom(utils,sessionInfo)
+importFrom(utils,str)
diff --git a/NEWS b/NEWS
index 20fc4a2..cf4ca63 100644
--- a/NEWS
+++ b/NEWS
@@ -1,361 +1,831 @@
Package: future
===============
+Version: 1.6.1 [2017-09-08]
+
+NEW FEATURES:
+
+ o Now exporting getGlobalsAndPackages().
+
+BUG FIXES:
+
+ o future_lapply() would give "Error in objectSize.env(x, depth = depth - 1L):
+ object 'nnn' not found" when for instance 'nnn' is part of an unresolved
+ expression that is an argument value.
+
+SOFTWARE QUALITY:
+
+ o FIX: Some of the package assertion tests made too precise assumptions about
+ the object sizes, which fails with the introduction of ALTREP in R-devel
+ which causes the R's SEXP header size to change.
+
+
+Version: 1.6.0 [2017-08-11]
+
+NEW FEATURES:
+
+ o Now tweak(), and hence plan(), generates a more informative error message if
+ a non-future function is specified by mistake, e.g. calling plan(cluster)
+ with the 'survival' package attached after 'future' is equivalent to calling
+ plan(survival::cluster) when plan(future::cluster) was intended.
+
+BUG FIXES:
+
+ o nbrOfWorkers() gave an error with plan(remote). Fixed by making the 'remote'
+ future inherit 'cluster' (as it should).
+
+SOFTWARE QUALITY:
+
+ o TESTS: No longer testing forced termination of forked cluster workers when
+ running on Solaris. The termination was done by launching a future that
+ called quit(), but that appeared to have corrupted the main R session when
+ running on Solaris.
+
+DEPRECATED AND DEFUNCT:
+
+ o Formally defunct 'eager' and 'lazy' futures; use 'sequential' instead.
+
+ o Dropped previously defunct %<=% and %=>% operators.
+
+
+Version: 1.5.0 [2017-05-24]
+
+SIGNIFICANT CHANGES:
+
+ o Multicore and multisession futures no longer reserve one core for the
+ main R process, which was done to lower the risk for producing a higher
+ CPU load than the number of cores available for the R session.
+
+NEW FEATURES:
+
+ o makeClusterPSOCK() now defaults to use the Windows PuTTY software's SSH
+ client 'plink -ssh', if 'ssh' is not found.
+
+ o Argument 'homogeneous' of makeNodePSOCK(), a helper function of
+ makeClusterPSOCK(), will default to FALSE also if the hostname is a
+ fully qualified domain name (FQDN), that is, it "contains periods".
+ For instance, c('node1', 'node2.server.org') will use homogeneous = TRUE
+ for the first worker and homogeneous = FALSE for the second.
+
+ o makeClusterPSOCK() now asserts that each cluster node is functioning by
+ retrieving and recording the node's session information including the
+ process ID of the corresponding R process.
+
+ o Nested futures sets option 'mc.cores' to prevent spawning of recursive
+ parallel processes by mistake. Because 'mc.cores' controls _additional_
+ processes, it was previously set to zero. However, since some functions
+ such as mclapply() does not support that, it is now set to one instead.
+
+DOCUMENTATION:
+
+ o Help on makeClusterPSOCK() gained more detailed descriptions on arguments
+ and what their defaults are.
+
+DEPRECATED AND DEFUNCT:
+
+ o Formally deprecated eager futures; use sequential instead.
+
+BUG FIXES:
+
+ o future_lapply() with multicore / multisession futures, would use a
+ suboptimal workload balancing where it split up the data in one chunk too
+ many. This is no longer a problem because of how argument 'workers' is
+ now defined for those type of futures (see note on top).
+
+ o future_lapply(), as well as lazy multicore and lazy sequential futures, did
+ not respect option 'future.globals.resolve', but was hardcoded to always
+ resolve globals (future.globals.resolve = TRUE).
+
+ o When globals larger than the allowed size (option 'future.globals.maxSize')
+ are detected an informative error message is generated. Previous version
+ introduced a bug causing the error to produce another error.
+
+ o Lazy sequential futures would produce an error when resolved if required
+ packages had been detached.
+
+ o print() would not display globals gathered for lazy sequential futures.
+
+SOFTWARE QUALITY:
+
+ o Added package tests for globals part of formulas part of other globals,
+ e.g. purrr::map(x, ~ rnorm(.)), which requires globals (>= 0.10.0).
+
+ o Now package tests with parallel::makeCluster() not only test for
+ type = 'PSOCK' clusters but also 'FORK' (when supported).
+
+ o TESTS: Cleaned up test scripts such that the overall processing time for
+ the tests was roughly halved, while preserving the same test coverage.
+
+
+Version: 1.4.0 [2017-03-12]
+
+SIGNIFICANT CHANGES:
+
+ o The default for future_lapply() is now to _not_ generate RNG seeds
+ (future.seed = FALSE). If proper random number generation is needed,
+ use future.seed = TRUE. For more details, see help page.
+
+NEW FEATURES:
+
+ o future() and future_lapply() gained argument 'packages' for explicitly
+ specifying packages to be attached when the futures are evaluated.
+ Note that the default throughout the future package is that all globals
+ and all required packages are automatically identified and gathered, so
+ in most cases those do not have to be specified manually.
+
+ o SPEED: future_lapply() scans global variables for non-resolved futures
+ (to resolve them) and calculate their total size once. Previously, each
+ chunk (a future) would redo this.
+
+ o The default values for arguments 'connectTimeout' and 'timeout' of
+ makeNodePSOCK() can now be controlled via global options.
+
+GLOBALS:
+
+ o Now future_lapply(x, FUN, ...) identifies global objects among 'x', 'FUN'
+ and '...' recursively until no new globals are found. Previously, only
+ the first level of globals were scanned. This is mostly thanks to a bug
+ fix in globals 0.9.0.
+
+RANDOM NUMBER GENERATION:
+
+ o Now future_lapply() guarantees that the RNG state of the calling R process
+ after returning is updated compared to what it was before and in the exact
+ same way regardless of 'future.seed' (except FALSE), 'future.scheduling'
+ and future strategy used. This is done in order to guarantee that an R
+ script calling future_lapply() multiple times should be numerically
+ reproducible given the same initial seed.
+
+ o It is now possible to specify a pre-generated sequence of .Random.seed
+ seeds to be used for each FUN(x[i], ...) call in future_lapply(x, FUN, ...).
+
+BUG FIXES:
+
+ o A future that used a global object 'x' of a class that overrides length()
+ would produce an error if length(x) reports more elements than what can
+ be subsetted.
+
+ o nbrOfWorkers() gave an error with plan(cluster, workers = cl) where
+ 'cl' is a cluster object created by parallel::makeCluster() etc.
+ This prevented for instance future_lapply() to work with such setups.
+
+ o plan(cluster, workers = cl) where cl <- makeCluster(..., type = MPI")
+ would give an instant error due to an invalid internal assertion.
+
+DEPRECATED AND DEFUNCT:
+
+ o Previously deprecated arguments 'maxCores' and 'cluster' are now defunct.
+
+ o Previously deprecated assignment operators %<=% and %=>% are now defunct.
+
+ o availableCores(method = "mc.cores") is now defunct in favor of "mc.cores+1".
+
+
+Version: 1.3.0 [2017-01-18]
+
+SIGNIFICANT CHANGES:
+
+ o Where applicable, workers are now initiated when calling plan(), e.g.
+ plan(cluster) will set up workers on all cluster nodes. Previously,
+ this only happened when the first future was created.
+
+NEW FEATURES:
+
+ o Renamed 'eager' futures to 'sequential', e.g. plan(sequential).
+ The 'eager' futures will be deprecated in an upcoming release.
+
+ o Added support for controlling whether a future is resolved eagerly
+ or lazily when creating the future, e.g. future(..., lazy = TRUE)
+ futureAssign(..., lazy = TRUE), and x %<-% { ... } %lazy% TRUE.
+
+ o future(), futureAssign() and futureCall() gained argument 'seed',
+ which specifies a L'Ecuyer-CMRG random seed to be used by the future.
+ The seed for future assignment can be specified via %seed%.
+
+ o futureAssign() now passes all additional arguments to future().
+
+ o Added future_lapply() which supports load balancing ("chunking")
+ and perfect reproducibility (regardless of type of load balancing
+ and how futures are resolved) via initial random seed.
+
+ o Added availableWorkers(). By default it returns localhost workers
+ according to availableCores(). In addition, it detects common
+ HPC allocations given in environment variables set by the HPC
+ scheduler.
+
+ o The default for plan(cluster) is now workers = availableWorkers().
+
+ o Now plan() stops any clusters that were implicitly created. For
+ instance, a multisession cluster created by plan(multisession) will
+ be stopped when plan(eager) is called.
+
+ o makeClusterPSOCK() treats workers that refer to a local machine by
+ its local or canonical hostname as "localhost". This avoids having
+ to launch such workers over SSH, which may not be supported on all
+ systems / compute cluster.
+
+ o Option 'future.debug' = TRUE also reports on total size of
+ globals identified and for cluster futures also the size of the
+ individual global variables exported.
+
+ o Option 'future.wait.timeout' (replaces 'future.wait.times') specifies
+ the maximum waiting time for a free workers (e.g. a core or a
+ compute node) before generating a timeout error.
+
+ o Option 'future.availableCores.fallback', which defaults to environment
+ variable 'R_FUTURE_AVAILABLECORES_FALLBACK' can now be used to specify
+ the default number of cores / workers returned by availableCores()
+ and availableWorkers() when no other settings are available. For
+ instance, if R_FUTURE_AVAILABLECORES_FALLBACK=1 is set system wide
+ in an HPC environment, then all R processes that uses availableCores()
+ to detect how many cores can be used will run as single-core processes.
+ Without this fallback setting, and without other core-specifying settings,
+ the default will be to use all cores on the machine, which does not play
+ well on multi-user systems.
+
+GLOBALS:
+
+ o Globals part of locally defined functions are now also identified
+ thanks to globals (>= 0.8.0) updates.
+
+DEPRECATED AND DEFUNCT:
+
+ o Lazy futures and plan(lazy) are now deprecated. Instead, use plan(eager)
+ and then f <- future(..., lazy = TRUE) or x %<-% { ... } %lazy% TRUE.
+ The reason behind this is that in some cases code that uses futures
+ only works under eager evaluation (lazy = FALSE; the default), or vice
+ verse. By removing the "lazy" future strategy, the user can no longer
+ override the lazy = TRUE / FALSE that the developer is using.
+
+BUG FIXES:
+
+ o Creation of cluster futures (including multisession ones) would
+ time out already after 40 seconds if all workers were busy.
+ New default timeout is 30 days (option 'future.wait.timeout').
+
+ o nbrOfWorkers() gave an error for plan(cluster, workers) where 'workers'
+ was a character vector or a 'cluster' object of the parallel package.
+ Because of this, future_lapply() gave an error with such setups.
+
+ o availableCores(methods = "_R_CHECK_LIMIT_CORES_") would give an error
+ if not running R CMD check.
+
+
Version: 1.2.0 [2016-11-12]
-o Added makeClusterPSOCK() - a version of parallel::makePSOCKcluster()
- that allows for more flexible control of how PSOCK cluster workers
- are set up and how they are launched and communicated with if running
- on external machines.
-o Added generic as.cluster() for coercing objects to cluster objects
- to be used as in plan(cluster, workers = as.cluster(x)). Also added
- a c() implementation for cluster objects such that multiple cluster
- objects can be combined into a single one.
-o plan() and plan("list") now prints more user-friendly output.
-o Added sessionDetails() for gathering details of the current R session.
-o ROBUSTNESS: On Unix, internal myInternalIP() tries more alternatives
- for finding the local IP number.
-o DEPRECATED: %<=% is deprecated. Use %<-% instead. Same for %=>%.
-o BUG FIX: values() for lists and list environments of futures where
- one or more of the futures resolved to NULL would give an error.
-o BUG FIX: value() for ClusterFuture would give cryptic error message
- "Error in stop(ex) : bad error message" if the cluster worker had
- crashed / terminated. Now it will instead give an error message like
- "Failed to retrieve the value of ClusterFuture from cluster node #1 on
- 'localhost'. The reason reported was ‘error reading from connection".
-o BUG FIX: Argument 'user' to remote() was ignored (since 1.1.0).
+NEW FEATURES:
+
+ o Added makeClusterPSOCK() - a version of parallel::makePSOCKcluster()
+ that allows for more flexible control of how PSOCK cluster workers
+ are set up and how they are launched and communicated with if running
+ on external machines.
+ o Added generic as.cluster() for coercing objects to cluster objects
+ to be used as in plan(cluster, workers = as.cluster(x)). Also added
+ a c() implementation for cluster objects such that multiple cluster
+ objects can be combined into a single one.
+
+ o Added sessionDetails() for gathering details of the current R session.
+
+ o plan() and plan("list") now prints more user-friendly output.
+
+ o On Unix, internal myInternalIP() tries more alternatives for finding
+ the local IP number.
+
+DEPRECATED AND DEFUNCT:
+
+ o %<=% is deprecated. Use %<-% instead. Same for %=>%.
+
+BUG FIXES:
+
+ o values() for lists and list environments of futures where one or more
+ of the futures resolved to NULL would give an error.
+
+ o value() for ClusterFuture would give cryptic error message
+ "Error in stop(ex) : bad error message" if the cluster worker had
+ crashed / terminated. Now it will instead give an error message like
+ "Failed to retrieve the value of ClusterFuture from cluster node #1 on
+ 'localhost'. The reason reported was "error reading from connection".
+
+ o Argument 'user' to remote() was ignored (since 1.1.0).
+
+
Version: 1.1.1 [2016-10-10]
-o FIX: For the special case where 'remote' futures use
- workers = "localhost" they (again) use the exact same R executable
- as the main / calling R session (in all other cases it uses
- whatever 'Rscript' is found in the PATH). This was already indeed
- implemented in 1.0.1, but with the added support for reverse SSH
- tunnels in 1.1.0 this default behavior was lost.
+BUG FIXES:
+ o For the special case where 'remote' futures use workers = "localhost"
+ they (again) use the exact same R executable as the main / calling R
+ session (in all other cases it uses whatever 'Rscript' is found in the
+ PATH). This was already indeed implemented in 1.0.1, but with the
+ added support for reverse SSH tunnels in 1.1.0 this default behavior
+ was lost.
+
+
Version: 1.1.0 [2016-10-09]
-o REMOTE CLUSTERS: It is now very simple to use cluster() and remote()
- to connect to remote clusters / machines. As long as you can connect
- via ssh to those machines, it works also with these future. The new
- code completely avoids incoming firewall and incoming port forwarding
- issues previously needed. This is done by using reverse SSH tunneling.
- There is also no need to worry about internal or external IP numbers.
-o GLOBALS: Global variables part of subassignments in future expressions
- are recognized and exported (iff found), e.g. x$a <- value,
- x[["a"]] <- value, and x[1,2,3] <- value.
-o GLOBALS: Global variables part of formulae in future expressions are
- recognized and exported (iff found), e.g. y ~ x | z.
-o GLOBALS: As an alternative to the default automatic identification of
- globals, it is now also possible to explicitly specify them either
- by their names (as a character vector) or by their names and values
- (as a named list), e.g. f <- future({ 2*a }, globals=c("a")) or
- f <- future({ 2*a }, globals=list(a=42)). For future assignments one
- can use the %globals% operator, e.g. y %<-% { 2*a } %globals% c("a").
-o Now all Future classes supports run() for launching the future and
- value() calls run() if the future has not been launched.
-o Added optional argument 'label' to all futures, e.g.
- f <- future(42, label="answer") and v %<-% { 42 } %label% "answer".
-o Added argument 'user' to cluster() and remote().
-o ROBUSTNESS: Now the default future strategy is explicitly set when
- no strategies are set, e.g. when used nested futures. Previously,
- only mc.cores was set so that only a single core was used, but now
- also plan("default") set.
-o WORKAROUND: resolved() on cluster futures would block on Linux until
- future was resolved. This is due to a bug in R. The workaround is
- to use round the timeout (in seconds) to an integer, which seems to
- always work / be respected.
-o DOCUMENTATION: Added vignette on command-line options and other
- methods for controlling the default type of futures to use.
-o MEMORY: Now plan(cluster, gc=TRUE) causes the background
- R session to be garbage collected immediately after the value
- is collected. Since multisession and remote futures are special
- cases of cluster futures, the same is true for these as well.
+NEW FEATURES:
+
+ o REMOTE CLUSTERS: It is now very simple to use cluster() and remote()
+ to connect to remote clusters / machines. As long as you can connect
+ via ssh to those machines, it works also with these future. The new
+ code completely avoids incoming firewall and incoming port forwarding
+ issues previously needed. This is done by using reverse SSH tunneling.
+ There is also no need to worry about internal or external IP numbers.
+
+ o Added optional argument 'label' to all futures, e.g.
+ f <- future(42, label="answer") and v %<-% { 42 } %label% "answer".
+
+ o Added argument 'user' to cluster() and remote().
+
+ o Now all Future classes supports run() for launching the future and
+ value() calls run() if the future has not been launched.
+
+ o MEMORY: Now plan(cluster, gc=TRUE) causes the background
+ R session to be garbage collected immediately after the value
+ is collected. Since multisession and remote futures are special
+ cases of cluster futures, the same is true for these as well.
+
+ o ROBUSTNESS: Now the default future strategy is explicitly set when
+ no strategies are set, e.g. when used nested futures. Previously,
+ only mc.cores was set so that only a single core was used, but now
+ also plan("default") set.
+
+ o WORKAROUND: resolved() on cluster futures would block on Linux until
+ future was resolved. This is due to a bug in R. The workaround is
+ to use round the timeout (in seconds) to an integer, which seems to
+ always work / be respected.
+
+GLOBALS:
+
+ o Global variables part of subassignments in future expressions
+ are recognized and exported (iff found), e.g. x$a <- value,
+ x[["a"]] <- value, and x[1,2,3] <- value.
+
+ o Global variables part of formulae in future expressions are
+ recognized and exported (iff found), e.g. y ~ x | z.
+
+ o As an alternative to the default automatic identification of
+ globals, it is now also possible to explicitly specify them either
+ by their names (as a character vector) or by their names and values
+ (as a named list), e.g. f <- future({ 2*a }, globals=c("a")) or
+ f <- future({ 2*a }, globals=list(a=42)). For future assignments one
+ can use the %globals% operator, e.g. y %<-% { 2*a } %globals% c("a").
+DOCUMENTATION:
+
+ o Added vignette on command-line options and other methods for
+ controlling the default type of futures to use.
+
+
Version: 1.0.1 [2016-07-04]
-o DOCUMENTATION: Adding section to vignette on globals in formulas
- describing how they are currently not automatically detected
- and how to explicitly export them.
-o ROBUSTNESS: For the special case where 'remote' futures use
- workers = "localhost" they now use the exact same R executable
- as the main / calling R session (in all other cases it uses
- whatever 'Rscript' is found in the PATH).
-o FutureError now extends simpleError and no longer the error
- class of captured errors.
-o BUG FIX: Since future 0.13.0, a global 'pkg' would be
- overwritten by the name of the last package attached in future.
-o BUG FIX: Futures that generated R.oo::Exception errors, they
- triggered another internal error.
+
+ o ROBUSTNESS: For the special case where 'remote' futures use
+ workers = "localhost" they now use the exact same R executable
+ as the main / calling R session (in all other cases it uses
+ whatever 'Rscript' is found in the PATH).
+
+ o FutureError now extends simpleError and no longer the error
+ class of captured errors.
+
+DOCUMENTATION:
+
+ o Adding section to vignette on globals in formulas describing how
+ they are currently not automatically detected and how to explicitly
+ export them.
+
+BUG FIXES:
+ o Since future 0.13.0, a global 'pkg' would be overwritten by the
+ name of the last package attached in future.
+ o Futures that generated R.oo::Exception errors, they triggered
+ another internal error.
+
+
Version: 1.0.0 [2016-06-24]
-o GLOBALS: Falsely identified global variables no longer generate
- an error when the future is created. Instead, we leave it to R
- and the evaluation of the individual futures to throw an error
- if the a global variable is truly missing. This was done in order
- to automatically handle future expressions that use non-standard
- evaluation (NSE), e.g. subset(df, x < 3) where 'x' is falsely
- identified as a global variable.
-o Add support for remote(..., myip="<external>"), which now
- queries a set of external lookup services in case one of them
- fails.
-o DEMO: Now the Mandelbrot demo tiles a single Mandelbrot region
- with one future per tile. This better illustrates parallelism.
-o Add mandelbrot() function used in demo to the API for convenience.
-o DOCUMENTATION: Documented R options used by the future package.
-o ROBUSTNESS: If .future.R script, which is sourced when the future
- package is attached, gives an error, then the error is ignored
- with a warning.
-o CLEANUP: Dropped support for system environment variable
- 'R_FUTURE_GLOBALS_MAXSIZE'.
-o TROUBLESHOOTING: If the future requires attachment of packages,
- then each namespace is loaded separately and before attaching
- the package. This is done in order to see the actual error
- message in case there is a problem while loading the namespace.
- With require()/library() this error message is otherwise suppressed
- and replaced with a generic one.
-o BUG FIX: Custom futures based on a constructor function that
- is defined outside a package gave an error.
-o BUG FIX: plan("default") assumed that the 'future.plan' option
- was a string; gave an error if it was a function.
-o BUG FIX: Various future options were not passed on to futures.
-o BUG FIX: A startup .future.R script is no longer sourced if the
- future package is attached by a future expression.
+NEW FEATURES:
-Version: 0.15.0 [2016-06-13]
-o Now .future.R (if found in the current directory or otherwise in
- the user's home directory) is sourced when the future package is
- attach (but not loaded). This helps separating scripts from
- configuration of futures.
-o Added remote futures, which are cluster futures with convenient
- default arguments for simple remote access to R, e.g.
- plan(remote, workers="login.my-server.org").
-o Added support for plan(cluster, workers=c("n1", "n2", "n2", "n4")),
- where 'workers' (also for ClusterFuture()) is a set of host names
- passed to parallel::makeCluster(workers). It can also be the number
- of localhost workers.
-o Added command line option --parallel=<p>, which is long for -p <p>.
-o Now command line option -p <p> also set the default future strategy
- to multiprocessing (if p >= 2 and eager otherwise), unless another
- strategy is already specified via option 'future.plan' or system
- environment variable R_FUTURE_PLAN.
-o Now availableCores() also acknowledges environment variable NSLOTS
- set by Sun/Oracle Grid Engine (SGE).
-o MEMORY: Added argument 'gc=FALSE' to all futures. When TRUE, the
- garbage collector will run at the very end in the process that
- evaluated the future (just before returning the value). This may
- help lowering the overall memory footprint when running multiple
- parallel R processes. The user can enable this by specifying
- plan(multiprocess, gc=TRUE). The developer can control this using
- future(expr, gc=TRUE) or v %<-% { expr } %tweak% list(gc=TRUE).
-o SPEEDUP: Significantly decreased the overhead of creating a future,
- particularly multicore futures.
-o BUG FIX: Future would give an error with plan(list("eager")),
- whereas it did work with plan("eager") and plan(list(eager)).
+ o Add support for remote(..., myip="<external>"), which now
+ queries a set of external lookup services in case one of them
+ fails.
+
+ o Add mandelbrot() function used in demo to the API for convenience.
+
+ o ROBUSTNESS: If .future.R script, which is sourced when the future
+ package is attached, gives an error, then the error is ignored
+ with a warning.
+
+ o TROUBLESHOOTING: If the future requires attachment of packages,
+ then each namespace is loaded separately and before attaching
+ the package. This is done in order to see the actual error
+ message in case there is a problem while loading the namespace.
+ With require()/library() this error message is otherwise suppressed
+ and replaced with a generic one.
+GLOBALS:
+
+ o Falsely identified global variables no longer generate
+ an error when the future is created. Instead, we leave it to R
+ and the evaluation of the individual futures to throw an error
+ if the a global variable is truly missing. This was done in order
+ to automatically handle future expressions that use non-standard
+ evaluation (NSE), e.g. subset(df, x < 3) where 'x' is falsely
+ identified as a global variable.
+
+ o Dropped support for system environment variable 'R_FUTURE_GLOBALS_MAXSIZE'.
+
+DOCUMENTATION:
+
+ o DEMO: Now the Mandelbrot demo tiles a single Mandelbrot region
+ with one future per tile. This better illustrates parallelism.
+
+ o Documented R options used by the future package.
+
+BUG FIXES:
+
+ o Custom futures based on a constructor function that
+ is defined outside a package gave an error.
+
+ o plan("default") assumed that the 'future.plan' option
+ was a string; gave an error if it was a function.
+
+ o Various future options were not passed on to futures.
+
+ o A startup .future.R script is no longer sourced if the
+ future package is attached by a future expression.
+
+
+Version: 0.15.0 [2016-06-13]
+NEW FEATURES:
+
+ o Added remote futures, which are cluster futures with convenient
+ default arguments for simple remote access to R, e.g.
+ plan(remote, workers="login.my-server.org").
+
+ o Now .future.R (if found in the current directory or otherwise in
+ the user's home directory) is sourced when the future package is
+ attach (but not loaded). This helps separating scripts from
+ configuration of futures.
+
+ o Added support for plan(cluster, workers=c("n1", "n2", "n2", "n4")),
+ where 'workers' (also for ClusterFuture()) is a set of host names
+ passed to parallel::makeCluster(workers). It can also be the number
+ of localhost workers.
+
+ o Added command line option --parallel=<p>, which is long for -p <p>.
+
+ o Now command line option -p <p> also set the default future strategy
+ to multiprocessing (if p >= 2 and eager otherwise), unless another
+ strategy is already specified via option 'future.plan' or system
+ environment variable R_FUTURE_PLAN.
+
+ o Now availableCores() also acknowledges environment variable NSLOTS
+ set by Sun/Oracle Grid Engine (SGE).
+
+ o MEMORY: Added argument 'gc=FALSE' to all futures. When TRUE, the
+ garbage collector will run at the very end in the process that
+ evaluated the future (just before returning the value). This may
+ help lowering the overall memory footprint when running multiple
+ parallel R processes. The user can enable this by specifying
+ plan(multiprocess, gc=TRUE). The developer can control this using
+ future(expr, gc=TRUE) or v %<-% { expr } %tweak% list(gc=TRUE).
+
+ o SPEED: Significantly decreased the overhead of creating a future,
+ particularly multicore futures.
+
+BUG FIXES:
+
+ o Future would give an error with plan(list("eager")),
+ whereas it did work with plan("eager") and plan(list(eager)).
+
+
Version: 0.14.0 [2016-05-16]
-o Renamed arguments 'maxCores' and 'cluster' to 'workers'. If using
- the old argument names a deprecation warning will be generated, but
- it will still work until made defunct in a future release.
-o Added nbrOfWorkers().
-o values() passes arguments '...' to value() of each Future.
-o Added FutureError class.
-o Added informative print() method for the Future class.
-o BUG FIX: resolve() for lists and environments did not work
- properly when the set of futures was not resolved in order,
- which could happen with asynchronous futures.
+NEW FEATURES:
+ o Added nbrOfWorkers().
+
+ o Added informative print() method for the Future class.
+
+ o values() passes arguments '...' to value() of each Future.
+
+ o Added FutureError class.
+
+DEPRECATED AND DEFUNCT:
+
+ o Renamed arguments 'maxCores' and 'cluster' to 'workers'. If using
+ the old argument names a deprecation warning will be generated, but
+ it will still work until made defunct in a future release.
+
+BUG FIXES:
+
+ o resolve() for lists and environments did not work
+ properly when the set of futures was not resolved in order,
+ which could happen with asynchronous futures.
+
+
Version: 0.13.0 [2016-04-13]
-o Add support to plan() for specifying different future strategies for
- the different levels of nested futures.
-o Add backtrace() for listing the trace the expressions evaluated (the
- calls made) before a condition was caught.
-o Add transparent futures, which are eager futures with early signaling
- of conditioned enabled and whose expression is evaluated in the calling
- environment. This makes the evaluation of such futures as similar
- as possible to how R evaluates expressions, which in turn simplifies
- troubleshooting errors etc.
-o Add support for early signaling of conditions. The default is
- (as before) to signal conditions when the value is queried.
- In addition, they may be signals as soon as possible, e.g. when
- checking whether a future is resolved or not.
-o Signaling of conditions when calling value() is now controlled by
- argument 'signal' (previously 'onError').
-o Now UniprocessFuture:s captures the call stack for errors occurring
- while resolving futures.
-o ClusterFuture gained argument 'persistent=FALSE'. With persistent=TRUE,
- any objects in the cluster R session that was created during the
- evaluation of a previous future is available for succeeding futures
- that are evaluated in the same session. Moreover, globals are
- still identified and exported but "missing" globals will not give
- an error - instead it is assumed such globals are available in the
- environment where the future is evaluated.
-o OVERHEAD: Utility functions exported by ClusterFuture are now much
- smaller; previously they would export all of the package environment.
-o BUG FIX: f <- multicore(NA, maxCores=2) would end up in an endless
- waiting loop for a free core if availableCores() returned one.
-o BUG FIX: ClusterFuture would ignore local=TRUE.
+NEW FEATURES:
+
+ o Add support to plan() for specifying different future strategies for
+ the different levels of nested futures.
+
+ o Add backtrace() for listing the trace the expressions evaluated (the
+ calls made) before a condition was caught.
+
+ o Add transparent futures, which are eager futures with early signaling
+ of conditioned enabled and whose expression is evaluated in the calling
+ environment. This makes the evaluation of such futures as similar
+ as possible to how R evaluates expressions, which in turn simplifies
+ troubleshooting errors etc.
+
+ o Add support for early signaling of conditions. The default is
+ (as before) to signal conditions when the value is queried.
+ In addition, they may be signals as soon as possible, e.g. when
+ checking whether a future is resolved or not.
+ o Signaling of conditions when calling value() is now controlled by
+ argument 'signal' (previously 'onError').
+
+ o Now UniprocessFuture:s captures the call stack for errors occurring
+ while resolving futures.
+
+ o ClusterFuture gained argument 'persistent=FALSE'. With persistent=TRUE,
+ any objects in the cluster R session that was created during the
+ evaluation of a previous future is available for succeeding futures
+ that are evaluated in the same session. Moreover, globals are
+ still identified and exported but "missing" globals will not give
+ an error - instead it is assumed such globals are available in the
+ environment where the future is evaluated.
+
+ o OVERHEAD: Utility functions exported by ClusterFuture are now much
+ smaller; previously they would export all of the package environment.
+
+BUG FIXES:
+
+ o f <- multicore(NA, maxCores=2) would end up in an endless
+ waiting loop for a free core if availableCores() returned one.
+
+ o ClusterFuture would ignore local=TRUE.
+
+
Version: 0.12.0 [2016-02-23]
-o Added multiprocess futures, which are multicore futures if supported,
- otherwise multisession futures. This makes it possible to use
- plan(multiprocess) everywhere regardless of operating system.
-o Future strategy functions gained class attributes such that it is
- possible to test what type of future is currently used, e.g.
- inherits(plan(), "multicore").
-o ROBUSTNESS: It is only the R process that created a future that can
- resolve it. If a non-resolved future is queried by another R process,
- then an informative error is generated explaining that this is not
- possible.
-o ROBUSTNESS: Now value() for multicore futures detects if the underlying
- forked R process was terminated before completing and if so generates
- an informative error messages.
-o SPEED: Adjusted the parameters for the schema used to wait for next
- available cluster node such that nodes are polled more frequently.
-o GLOBALS: resolve() gained argument 'recursive'.
-o GLOBALS: Added option 'future.globals.resolve' for controlling whether
- global variables should be resolved for futures or not. If TRUE, then
- globals are searched recursively for any futures and if found such
- "global" futures are resolved. If FALSE, global futures are not
- located, but if they are later trying to be resolved by the parent
- future, then an informative error message is generated clarifying
- that only the R process that created the future can resolve it.
- The default is currently FALSE.
-o FIX: Exports of objects available in packages already attached
- by the future were still exported.
-o FIX: Now availableCores() returns 3L (=2L+1L) instead of 2L
- if _R_CHECK_LIMIT_CORES_ is set.
+NEW FEATURES:
+
+ o Added multiprocess futures, which are multicore futures if supported,
+ otherwise multisession futures. This makes it possible to use
+ plan(multiprocess) everywhere regardless of operating system.
+
+ o Future strategy functions gained class attributes such that it is
+ possible to test what type of future is currently used, e.g.
+ inherits(plan(), "multicore").
+
+ o ROBUSTNESS: It is only the R process that created a future that can
+ resolve it. If a non-resolved future is queried by another R process,
+ then an informative error is generated explaining that this is not
+ possible.
+
+ o ROBUSTNESS: Now value() for multicore futures detects if the underlying
+ forked R process was terminated before completing and if so generates
+ an informative error messages.
+ o SPEED: Adjusted the parameters for the schema used to wait for next
+ available cluster node such that nodes are polled more frequently.
+
+GLOBALS:
+
+ o resolve() gained argument 'recursive'.
+
+ o Added option 'future.globals.resolve' for controlling whether
+ global variables should be resolved for futures or not. If TRUE, then
+ globals are searched recursively for any futures and if found such
+ "global" futures are resolved. If FALSE, global futures are not
+ located, but if they are later trying to be resolved by the parent
+ future, then an informative error message is generated clarifying
+ that only the R process that created the future can resolve it.
+ The default is currently FALSE.
+
+BUG FIX:
+
+ o FIX: Exports of objects available in packages already attached
+ by the future were still exported.
+
+ o FIX: Now availableCores() returns 3L (=2L+1L) instead of 2L
+ if _R_CHECK_LIMIT_CORES_ is set.
+
+
Version: 0.11.0 [2016-01-15]
-o GLOBALS: All futures now validates globals by default (globals=TRUE).
-o Add multisession futures, which analogously to multicore ones,
- use multiple cores on the local machine with the difference
- that they are evaluated in separate R session running in the
- background rather than separate forked R processes.
- A multisession future is a special type of cluster futures that
- do not require explicit setup of cluster nodes.
-o Add support for cluster futures, which can make use of a cluster
- of nodes created by parallel::makeCluster().
-o Add futureCall(), which is for futures what do.call() is otherwise.
-o Standardized how options are named, i.e. 'future.<option>'.
- If you used any future options previously, make sure to check
- they follow the above format.
+NEW FEATURES:
+
+ o Add multisession futures, which analogously to multicore ones,
+ use multiple cores on the local machine with the difference
+ that they are evaluated in separate R session running in the
+ background rather than separate forked R processes.
+ A multisession future is a special type of cluster futures that
+ do not require explicit setup of cluster nodes.
+
+ o Add support for cluster futures, which can make use of a cluster
+ of nodes created by parallel::makeCluster().
+
+ o Add futureCall(), which is for futures what do.call() is otherwise.
+
+ o Standardized how options are named, i.e. 'future.<option>'.
+ If you used any future options previously, make sure to check
+ they follow the above format.
+GLOBALS:
+
+ o All futures now validates globals by default (globals=TRUE).
+
+
Version: 0.10.0 [2015-12-30]
-o Now %<=% can also assign to multi-dimensional list environments.
-o Add futures(), values() and resolved().
-o Add resolve() to resolve futures in lists and environments.
-o Now availableCores() also acknowledges the number of CPUs
- allotted by Slurm.
-o CLEANUP: Now the internal future variable created by %<=% is
- removed when the future variable is resolved.
-o BUG FIX: futureOf(envir=x) did not work properly when 'x' was
- a list environment.
+NEW FEATURES:
+
+ o Now %<=% can also assign to multi-dimensional list environments.
+
+ o Add futures(), values() and resolved().
+
+ o Add resolve() to resolve futures in lists and environments.
+
+ o Now availableCores() also acknowledges the number of CPUs
+ allotted by Slurm.
+ o CLEANUP: Now the internal future variable created by %<=% is
+ removed when the future variable is resolved.
+
+BUG FIXES:
+
+ o futureOf(envir=x) did not work properly when 'x' was
+ a list environment.
+
+
Version: 0.9.0 [2015-12-11]
-o GLOBALS: Now globals ("unknown" variables) are identified
- using the new findGlobals(..., method="ordered") in
- globals (> 0.5.0) such that a global variable preceding
- a local variable with the same name is properly identified
- and exported/frozen.
-o DOCUMENTATION: Updated vignette on common issues with the
- case where a global variable is not identified because it
- is hidden by an element assignment in the future expression.
-o ROBUSTNESS: Now values of environment variables are trimmed
- before being parsed.
-o ROBUSTNESS: Add reproducibility test for random number
- generation using Pierre L'Ecuyer's RNG stream regardless
- of how futures are evaluated, e.g. eager, lazy and multicore.
-o BUG FIX: Errors occurring in multicore futures could prevent
- further multicore futures from being created.
+NEW FEATURES:
-Version: 0.8.2 [2015-10-14]
-o BUG FIX: Globals that were copies of package objects
- were not exported to the future environments.
-o BUG FIX: The future package had to be attached or
- future::future() had to be imported, if %<=% was used
- internally in another package. Similarly, it also had
- to be attached if multicore futures where used.
+ o ROBUSTNESS: Now values of environment variables are trimmed
+ before being parsed.
+
+ o ROBUSTNESS: Add reproducibility test for random number
+ generation using Pierre L'Ecuyer's RNG stream regardless
+ of how futures are evaluated, e.g. eager, lazy and multicore.
+
+GLOBALS:
+ o Now globals ("unknown" variables) are identified using the
+ new findGlobals(..., method="ordered") in globals (> 0.5.0)
+ such that a global variable preceding a local variable with
+ the same name is properly identified and exported/frozen.
+DOCUMENTATION:
+
+ o Updated vignette on common issues with the case where a global
+ variable is not identified because it is hidden by an element
+ assignment in the future expression.
+
+BUG FIXES:
+
+ o Errors occurring in multicore futures could prevent
+ further multicore futures from being created.
+
+
+Version: 0.8.2 [2015-10-14]
+
+BUG FIXES:
+
+ o Globals that were copies of package objects were not exported to the
+ future environments.
+
+ o The future package had to be attached or future::future() had to be
+ imported, if %<=% was used internally in another package.
+ Similarly, it also had to be attached if multicore futures where used.
+
+
Version: 0.8.1 [2015-10-05]
-o eager() and multicore() gained argument 'globals', where
- globals=TRUE will validate that all global variables
- identified can be located already before the future is
- created. This provides the means for providing the same
- tests on global variables with eager and multicore futures
- as with lazy futures.
-o lazy(sum(x, ...), globals=TRUE) now properly passes `...`
- from the function from which the future is setup. If not
- called within a function or called within a function without
- `...` arguments, an informative error message is thrown.
-o Added vignette 'Futures in R: Common issues with solutions'.
+DOCUMENTATION:
+
+ o Added vignette 'Futures in R: Common issues with solutions'.
+
+GLOBALS:
+ o eager() and multicore() gained argument 'globals', where
+ globals=TRUE will validate that all global variables
+ identified can be located already before the future is
+ created. This provides the means for providing the same
+ tests on global variables with eager and multicore futures
+ as with lazy futures.
+
+BUG FIXES:
+
+ o lazy(sum(x, ...), globals=TRUE) now properly passes `...`
+ from the function from which the future is setup. If not
+ called within a function or called within a function without
+ `...` arguments, an informative error message is thrown.
+
+
Version: 0.8.0 [2015-09-06]
-o plan("default") resets to the default strategy, which is
- synchronous eager evaluation unless option 'future_plan'
- or environment variable 'R_FUTURE_PLAN' has been set.
-o availableCores("mc.cores") returns getOption("mc.cores") + 1L,
- because option 'mc.cores' specifies "allowed number of _additional_
- R processes" to be used in addition to the main R process.
-o BUG FIX: plan(future::lazy) and similar gave errors.
+NEW FEATURES:
+
+ o plan("default") resets to the default strategy, which is
+ synchronous eager evaluation unless option 'future_plan'
+ or environment variable 'R_FUTURE_PLAN' has been set.
+ o availableCores("mc.cores") returns getOption("mc.cores") + 1L,
+ because option 'mc.cores' specifies "allowed number of _additional_
+ R processes" to be used in addition to the main R process.
+
+BUG FIXES:
+
+ o plan(future::lazy) and similar gave errors.
+
+
Version: 0.7.0 [2015-07-13]
-o ROBUSTNESS: multicore() blocks until one of the CPU cores
- is available, iff all are currently occupied by other
- multicore futures.
-o multicore() gained argument 'maxCores', which makes it
- possible to use for instance plan(multicore, maxCores=4L).
-o Add availableMulticore() [from (in-house) 'async' package].
-o More colorful demo("mandelbrot", package="future").
-o BUG FIX: old <- plan(new) now returns the old plan/strategy
- (was the newly set one).
+NEW FEATURES:
+
+ o multicore() gained argument 'maxCores', which makes it
+ possible to use for instance plan(multicore, maxCores=4L).
+ o Add availableMulticore() [from (in-house) 'async' package].
+
+DOCUMENTATION:
+
+ o More colorful demo("mandelbrot", package="future").
+
+BUG FIXES:
+
+ o ROBUSTNESS: multicore() blocks until one of the CPU cores
+ is available, iff all are currently occupied by other
+ multicore futures.
+
+ o old <- plan(new) now returns the old plan/strategy
+ (was the newly set one).
+
+
Version: 0.6.0 [2015-06-18]
-o Add multicore futures, which are futures that are resolved
- asynchronously in a separate process. These are only
- supported on Unix-like systems, but not on Windows.
+NEW FEATURES:
+ o Add multicore futures, which are futures that are resolved
+ asynchronously in a separate process. These are only
+ supported on Unix-like systems, but not on Windows.
+
+
Version: 0.5.1 [2015-06-18]
-o Eager and lazy futures now records the result internally
- such that the expression is only evaluated once, even if
- their errored values are requested multiple times.
-o Eager futures are always created regardless of error or not.
-o All Future objects are environments themselves that record
- the expression, the call environment and optional variables.
+NEW FEATURES:
+
+ o Eager and lazy futures now records the result internally
+ such that the expression is only evaluated once, even if
+ their errored values are requested multiple times.
+
+ o Eager futures are always created regardless of error or not.
+ o All Future objects are environments themselves that record
+ the expression, the call environment and optional variables.
+
+
Version: 0.5.0 [2015-06-16]
-o lazy() "freezes" global variables at the time when
- the future is created. This way the result of a lazy
- future is more likely to be the same as an eager future.
- This is also how globals are likely to be handled by
- asynchronous futures.
+GLOBALS:
+ o lazy() "freezes" global variables at the time when
+ the future is created. This way the result of a lazy
+ future is more likely to be the same as an eager future.
+ This is also how globals are likely to be handled by
+ asynchronous futures.
+
+
Version: 0.4.2 [2015-06-15]
-o plan() records the call.
-o Added demo("mandelbrot", package="future"), which can be
- re-used by other future packages.
+NEW FEATURES:
+ o plan() records the call.
+
+DOCUMENTATION:
+
+ o Added demo("mandelbrot", package="future"), which can be
+ re-used by other future packages.
+
+
Version: 0.4.1 [2015-06-14]
-o Added plan().
-o Added eager future - useful for troubleshooting.
+NEW FEATURES:
+ o Added plan().
+
+ o Added eager future - useful for troubleshooting.
+
+
Version: 0.4.0 [2015-06-07]
-o Distilled Future API from (in-house) 'async' package.
+
+ o Distilled Future API from (in-house) 'async' package.
diff --git a/OVERVIEW.md b/OVERVIEW.md
deleted file mode 100644
index 784f03a..0000000
--- a/OVERVIEW.md
+++ /dev/null
@@ -1,14 +0,0 @@
-<%
-## Reuse the future vignette
-md <- R.rsp::rstring(file="vignettes/future-1-overview.md.rsp", postprocess=FALSE)
-
-## Drop the header
-md <- unlist(strsplit(md, split="\n", fixed=TRUE))
-md <- md[-seq_len(grep("^## ", md)[1]-1)]
-
-## Drop the footer
-md <- md[seq_len(grep("^---", md)[1]-1)]
-
-## Output
-cat(md, sep="\n")
-%>
diff --git a/R/ClusterFuture-class.R b/R/ClusterFuture-class.R
index 420641a..c133bad 100644
--- a/R/ClusterFuture-class.R
+++ b/R/ClusterFuture-class.R
@@ -1,27 +1,38 @@
-#' A cluster future is a future whose value will be resolved asynchroneously in a parallel process
+#' A cluster future is a future whose value will be resolved asynchronously in a parallel process
#'
#' @inheritParams MultiprocessFuture-class
+#'
#' @param globals (optional) a logical, a character vector,
#' or a named list for controlling how globals are handled.
#' For details, see section 'Globals used by future expressions'
#' in the help for \code{\link{future}()}.
+#'
#' @param persistent If FALSE, the evaluation environment is cleared
#' from objects prior to the evaluation of the future.
+#'
#' @param workers A \code{\link[parallel:makeCluster]{cluster}}.
#' Alternatively, a character vector of host names or a numeric scalar,
#' for creating a cluster via \code{\link[parallel]{makeCluster}(workers)}.
+#'
#' @param revtunnel If TRUE, reverse SSH tunneling is used for the
#' PSOCK cluster nodes to connect back to the master R process. This
#' avoids the hassle of firewalls, port forwarding and having to know
#' the internal / public IP address of the master R session.
+#'
#' @param user (optional) The user name to be used when communicating
#' with another host.
+#'
#' @param master (optional) The hostname or IP address of the master
#' machine running this node.
+#'
#' @param homogeneous If TRUE, all cluster nodes is assumed to use the
#' same path to \file{Rscript} as the main R session. If FALSE, the
#' it is assumed to be on the PATH for each node.
#'
+#' @param sessioninfo If TRUE, session information is collected for each
+#' cluster node, otherwise not. This also servers as testing that each
+#' node is working properly.
+#'
#' @return An object of class \code{ClusterFuture}.
#'
#' @seealso
@@ -34,22 +45,18 @@
#' @importFrom digest digest
#' @name ClusterFuture-class
#' @keywords internal
-ClusterFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, local=!persistent, globals=TRUE, gc=FALSE, persistent=FALSE, workers=NULL, user=NULL, master=NULL, revtunnel=TRUE, homogeneous=TRUE, ...) {
- defaultCluster <- importParallel("defaultCluster")
-
- ## BACKWARD COMPATIBILITY
- args <- list(...)
- if ("cluster" %in% names(args)) {
- workers <- args$cluster
- .Deprecated(msg="Argument 'cluster' has been renamed to 'workers'. Please update your script/code that uses the future package.")
+ClusterFuture <- function(expr = NULL, envir = parent.frame(), substitute = FALSE, globals = TRUE, packages = NULL, local = !persistent, gc = FALSE, persistent = FALSE, workers = NULL, user = NULL, master = NULL, revtunnel = TRUE, homogeneous = TRUE, ...) {
+ if ("cluster" %in% names(list(...))) {
+ .Defunct(msg = "Argument 'cluster' has been renamed to 'workers'. Please update your script/code that uses the future package.")
}
if (substitute) expr <- substitute(expr)
if (is.null(workers)) {
+ defaultCluster <- importParallel("defaultCluster")
workers <- defaultCluster()
} else if (is.character(workers) || is.numeric(workers)) {
- workers <- ClusterRegistry("start", workers=workers, user=user, master=master, revtunnel=revtunnel, homogeneous=homogeneous)
+ workers <- ClusterRegistry("start", workers = workers, user = user, master = master, revtunnel = revtunnel, homogeneous = homogeneous)
} else {
workers <- as.cluster(workers)
}
@@ -59,7 +66,14 @@ ClusterFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, loc
stopifnot(length(workers) > 0)
## Attaching UUID for each cluster connection, unless already done.
- workers <- addClusterUUIDs(workers)
+ workers <- add_cluster_uuid(workers)
+
+ ## Attached workers' session information, unless already done.
+ ## FIXME: We cannot do this here, because it introduces a race condition
+ ## where multiple similar requests may appear at the same time bringing
+ ## the send/receive data to be out of sync and therefore corrupt the
+ ## futures' values.
+ ## workers <- add_cluster_session_info(workers)
## Attach name to cluster?
name <- attr(workers, "name")
@@ -70,14 +84,14 @@ ClusterFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, loc
}
## Global objects
- gp <- getGlobalsAndPackages(expr, envir=envir, persistent=persistent, globals=globals)
+ gp <- getGlobalsAndPackages(expr, envir = envir, persistent = persistent, globals = globals)
globals <- gp$globals
- packages <- gp$packages
+ packages <- unique(c(packages, gp$packages))
expr <- gp$expr
gp <- NULL
- f <- MultiprocessFuture(expr=expr, envir=envir, substitute=FALSE, local=local, gc=gc, persistent=persistent, globals=globals, packages=packages, workers=workers, node=NA_integer_, ...)
- structure(f, class=c("ClusterFuture", class(f)))
+ f <- MultiprocessFuture(expr = expr, envir = envir, substitute = FALSE, globals = globals, packages = packages, local = local, gc = gc, persistent = persistent, workers = workers, node = NA_integer_, ...)
+ structure(f, class = c("ClusterFuture", class(f)))
}
@@ -85,8 +99,12 @@ ClusterFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, loc
#' @importFrom parallel clusterCall clusterExport
#' @export
run.ClusterFuture <- function(future, ...) {
+ debug <- getOption("future.debug", FALSE)
+
if (future$state != 'created') {
- stop("A future can only be launched once.")
+ label <- future$label
+ if (is.null(label)) label <- "<none>"
+ stop(sprintf("A future ('%s') can only be launched once.", label))
}
## Assert that the process that created the future is
@@ -103,15 +121,13 @@ run.ClusterFuture <- function(future, ...) {
## Next available cluster node
- node <- requestNode(await=function() {
- mdebug("Waiting for free cluster node ...")
- FutureRegistry(reg, action="collect-first")
- mdebug("Waiting for free cluster node ... DONE")
- }, workers=workers)
- future$node <- node
+ node_idx <- requestNode(await = function() {
+ FutureRegistry(reg, action = "collect-first")
+ }, workers = workers)
+ future$node <- node_idx
## Cluster node to use
- cl <- workers[node]
+ cl <- workers[node_idx]
## WORKAROUND: When running covr::package_coverage(), the
@@ -119,10 +135,10 @@ run.ClusterFuture <- function(future, ...) {
## library path used by covr. We here add that path iff
## covr is being used. /HB 2016-01-15
if (is.element("covr", loadedNamespaces())) {
- mdebug("covr::package_coverage() workaround ...")
+ if (debug) mdebug("covr::package_coverage() workaround ...")
libPath <- .libPaths()[1]
- clusterCall(cl, fun=function() .libPaths(c(libPath, .libPaths())))
- mdebug("covr::package_coverage() workaround ... DONE")
+ clusterCall(cl, fun = function() .libPaths(c(libPath, .libPaths())))
+ if (debug) mdebug("covr::package_coverage() workaround ... DONE")
}
@@ -131,56 +147,71 @@ run.ClusterFuture <- function(future, ...) {
## may happen even if the future is evaluated inside a
## local, e.g. local({ a <<- 1 }).
if (!persistent) {
- clusterCall(cl, fun=grmall)
+ clusterCall(cl, fun = grmall)
}
## (ii) Attach packages that needs to be attached
- packages <- future$packages
- if (length(packages) > 0) {
- mdebug("Attaching %d packages (%s) on cluster node #%d ...",
- length(packages), hpaste(sQuote(packages)), node)
-
- clusterCall(cl, fun=requirePackages, packages)
-
- mdebug("Attaching %d packages (%s) on cluster node #%d ... DONE",
- length(packages), hpaste(sQuote(packages)), node)
+ ## NOTE: Already take care of by getExpression() of the Future class.
+ ## However, if we need to get an early error about missing packages,
+ ## we can get the error here before launching the future.
+ packages <- packages(future)
+ if (future$earlySignal && length(packages) > 0) {
+ if (debug) mdebug("Attaching %d packages (%s) on cluster node #%d ...",
+ length(packages), hpaste(sQuote(packages)), node_idx)
+
+ clusterCall(cl, fun = requirePackages, packages)
+
+ if (debug) mdebug("Attaching %d packages (%s) on cluster node #%d ... DONE",
+ length(packages), hpaste(sQuote(packages)), node_idx)
}
-
+
## (iii) Export globals
- globals <- future$globals
+ globals <- globals(future)
if (length(globals) > 0) {
+ if (debug) {
+ total_size <- asIEC(objectSize(globals))
+ mdebug("Exporting %d global objects (%s) to cluster node #%d ...", length(globals), total_size, node_idx)
+ }
for (name in names(globals)) {
## For instance sendData.SOCKnode(...) may generate warnings
## on packages not being available after serialization, e.g.
## In serialize(data, node$con) :
## package:future' may not be available when loading
## Here we'll suppress any such warnings.
- mdebug("Exported %s to cluster node #%d ...", sQuote(name), node)
+ value <- globals[[name]]
+ if (debug) {
+ size <- asIEC(objectSize(value))
+ mdebug("Exporting %s (%s) to cluster node #%d ...", sQuote(name), size, node_idx)
+ }
suppressWarnings({
- clusterCall(cl, fun=gassign, name, globals[[name]])
+ clusterCall(cl, fun = gassign, name, value)
})
- mdebug("Exported %s to cluster node #%d ... DONE", sQuote(name), node)
+ if (debug) mdebug("Exporting %s (%s) to cluster node #%d ... DONE", sQuote(name), size, node_idx)
+ value <- NULL
}
+ if (debug) mdebug("Exporting %d global objects (%s) to cluster node #%d ... DONE", length(globals), total_size, node_idx)
}
## Not needed anymore
globals <- NULL
## Add to registry
- FutureRegistry(reg, action="add", future=future, earlySignal=FALSE)
+ FutureRegistry(reg, action = "add", future = future, earlySignal = FALSE)
## (iv) Launch future
- sendCall(cl[[1L]], fun=geval, args=list(expr))
+ sendCall(cl[[1L]], fun = geval, args = list(expr))
future$state <- 'running'
+ if (debug) mdebug("%s started", class(future)[1])
+
invisible(future)
}
#' @export
-resolved.ClusterFuture <- function(x, timeout=0.2, ...) {
+resolved.ClusterFuture <- function(x, timeout = 0.2, ...) {
## Is future even launched?
if (x$state == 'created') return(FALSE)
@@ -192,19 +223,29 @@ resolved.ClusterFuture <- function(x, timeout=0.2, ...) {
assertOwner(x)
workers <- x$workers
- node <- x$node
- cl <- workers[node]
-
- ## WORKAROUND: Non-integer timeouts (at least < 2.0 seconds) may
- ## result in infinite waiting, cf.
- ## https://stat.ethz.ch/pipermail/r-devel/2016-October/073218.html
- if (.Platform$OS.type != "windows") {
- timeout <- round(timeout, digits = 0L)
- }
-
+ node_idx <- x$node
+ cl <- workers[node_idx]
+
## Check if workers socket connection is available for reading
- con <- cl[[1]]$con
- res <- socketSelect(list(con), write=FALSE, timeout=timeout)
+ node <- cl[[1]]
+
+ ## FIXME: This assumes that the worker has a connection, which
+ ## is _not_ the case for MPI clusters. /HB 2017-03-06
+ con <- node$con
+ if (!is.null(con)) {
+ ## WORKAROUND: Non-integer timeouts (at least < 2.0 seconds) may
+ ## result in infinite waiting, cf.
+ ## https://stat.ethz.ch/pipermail/r-devel/2016-October/073218.html
+ if (.Platform$OS.type != "windows") {
+ timeout <- round(timeout, digits = 0L)
+ }
+ res <- socketSelect(list(con), write = FALSE, timeout = timeout)
+ } else {
+ ## stop("Not yet implemented: ", paste(sQuote(class(node)), collapse = ", "))
+ warning(sprintf("resolved() is not yet implemented for workers of class %s. Will use value() instead and return TRUE", sQuote(class(node)[1])))
+ value(x, signal = FALSE)
+ res <- TRUE
+ }
## Signal conditions early? (happens only iff requested)
if (res) signalEarly(x, ...)
@@ -230,8 +271,8 @@ value.ClusterFuture <- function(future, ...) {
recvResult <- importParallel("recvResult")
workers <- future$workers
- node <- future$node
- cl <- workers[node]
+ node_idx <- future$node
+ cl <- workers[node_idx]
## If not, wait for process to finish, and
## then collect and record the value
@@ -242,20 +283,27 @@ value.ClusterFuture <- function(future, ...) {
if (inherits(ack, "simpleError")) {
- ## If the connection has changed, report on that!
- msg <- check_connection_uuid(cl[[1]], future = future)
- if (!is.null(msg)) {
- on_failure <- getOption("future.cluster.invalidConnection", "error")
- if (on_failure == "error") {
- stop(FutureError(msg, future=future))
+ ## If the worker uses a connection and that has changed, report on that!
+ node <- cl[[1]]
+ if (inherits(node$con, "connection")) {
+ msg <- check_connection_uuid(node, future = future)
+ if (!is.null(msg)) {
+ on_failure <- getOption("future.cluster.invalidConnection", "error")
+ if (on_failure == "error") {
+ stop(FutureError(msg, future = future))
+ }
+ warning(msg)
+ return(sprintf("EXCEPTIONAL ERROR: %s", msg))
}
- warning(msg)
- return(sprintf("EXCEPTIONAL ERROR: %s", msg))
}
-
- msg <- sprintf("Failed to retrieve the value of %s from cluster node #%d on %s. ", class(future)[1], node, sQuote(cl[[1]]$host))
+
+ ## AD HOC: This assumes that the worker has a hostname, which is not
+ ## the case for MPI workers. /HB 2017-03-07
+ info <- node$host
+ info <- if (is.null(info)) NA_character_ else sprintf("on %s", sQuote(info))
+ msg <- sprintf("Failed to retrieve the value of %s from cluster node #%d (%s). ", class(future)[1], node_idx, info)
msg <- sprintf("%s The reason reported was %s", msg, sQuote(ack$message))
- ex <- FutureError(msg, call=ack$call, future=future)
+ ex <- FutureError(msg, call = ack$call, future = future)
stop(ex)
}
stopifnot(isTRUE(ack))
@@ -282,80 +330,90 @@ value.ClusterFuture <- function(future, ...) {
reg <- sprintf("workers-%s", attr(workers, "name"))
## Remove from registry
- FutureRegistry(reg, action="remove", future=future, earlySignal=FALSE)
+ FutureRegistry(reg, action = "remove", future = future, earlySignal = FALSE)
## Garbage collect cluster worker?
if (future$gc) {
## Cleanup global environment while at it
- if (!future$persistent) clusterCall(cl[1], fun=grmall)
+ if (!future$persistent) clusterCall(cl[1], fun = grmall)
## WORKAROUND: Need to clear cluster worker before garbage collection,
## cf. https://github.com/HenrikBengtsson/Wishlist-for-R/issues/27
## UPDATE: This has been fixed in R (>= 3.3.2) /HB 2016-10-13
clusterCall(cl[1], function() NULL)
- clusterCall(cl[1], gc, verbose=FALSE, reset=FALSE)
+ clusterCall(cl[1], gc, verbose = FALSE, reset = FALSE)
}
NextMethod("value")
}
-requestNode <- function(await, workers, times=getOption("future.wait.times", 600L), delta=getOption("future.wait.interval", 0.001), alpha=getOption("future.wait.alpha", 1.01)) {
- stopifnot(is.function(await))
+requestNode <- function(await, workers, timeout = getOption("future.wait.timeout", 30 * 24 * 60 * 60), delta = getOption("future.wait.interval", 0.2), alpha = getOption("future.wait.alpha", 1.01)) {
+ debug <- getOption("future.debug", FALSE)
+
stopifnot(inherits(workers, "cluster"))
- times <- as.integer(times)
- stopifnot(is.finite(times), times > 0)
+ stopifnot(is.function(await))
+ stopifnot(is.finite(timeout), timeout >= 0)
stopifnot(is.finite(alpha), alpha > 0)
## Maximum number of nodes available
total <- length(workers)
## FutureRegistry to use
- reg <- sprintf("workers-%s", attr(workers, "name"))
-
+ name <- attr(workers, "name")
+ stopifnot(is.character(name), length(name) == 1L)
+ reg <- sprintf("workers-%s", name)
+
usedNodes <- function() {
## Number of unresolved cluster futures
- length(FutureRegistry(reg, action="list", earlySignal=FALSE))
+ length(FutureRegistry(reg, action = "list", earlySignal = FALSE))
}
+ t0 <- Sys.time()
+ dt <- 0
iter <- 1L
interval <- delta
finished <- FALSE
- while (iter <= times) {
- finished <- (usedNodes() < total)
+ while (dt <= timeout) {
+ ## Check for available nodes
+ used <- usedNodes()
+ finished <- (used < total)
if (finished) break
+ if (debug) mdebug("Poll #%d (%s): usedNodes() = %d, workers = %d", iter, format(round(dt, digits = 2L)), used, total)
+
## Wait
Sys.sleep(interval)
-
+ interval <- alpha * interval
+
## Finish/close workers, iff possible
await()
- interval <- alpha*interval
iter <- iter + 1L
+ dt <- difftime(Sys.time(), t0)
}
if (!finished) {
- msg <- sprintf("TIMEOUT: All %d workers are still occupied", total)
- mdebug(msg)
+ msg <- sprintf("TIMEOUT: All %d cluster nodes are still occupied after %s (polled %d times)", total, format(round(dt, digits = 2L)), iter)
+ if (debug) mdebug(msg)
stop(msg)
}
## Find which node is available
- avail <- rep(TRUE, times=length(workers))
- futures <- FutureRegistry(reg, action="list", earlySignal=FALSE)
- nodes <- unlist(lapply(futures, FUN=function(f) f$node), use.names=FALSE)
+ avail <- rep(TRUE, times = length(workers))
+ futures <- FutureRegistry(reg, action = "list", earlySignal = FALSE)
+ nodes <- unlist(lapply(futures, FUN = function(f) f$node), use.names = FALSE)
avail[nodes] <- FALSE
## Sanity check
stopifnot(any(avail))
- node <- which(avail)[1L]
- stopifnot(is.numeric(node), is.finite(node), node >= 1)
+ node_idx <- which(avail)[1L]
+ stopifnot(is.numeric(node_idx), is.finite(node_idx), node_idx >= 1)
- node
+ node_idx
}
@@ -365,6 +423,9 @@ requestNode <- function(await, workers, times=getOption("future.wait.times", 600
## (https://stat.ethz.ch/pipermail/r-devel/2016-October/073331.html)
check_connection_uuid <- function(worker, future, on_failure = "error") {
con <- worker$con
+ ## Not a worker with a connection
+ if (!inherits(con, "connection")) return(NULL)
+
uuid <- attr(con, "uuid")
uuid_now <- uuid_of_connection(con, keep_source = TRUE, must_work = FALSE)
if (uuid_now == uuid) return(NULL)
diff --git a/R/ClusterRegistry.R b/R/ClusterRegistry.R
index 9da090d..e3fd5a0 100644
--- a/R/ClusterRegistry.R
+++ b/R/ClusterRegistry.R
@@ -3,7 +3,7 @@ ClusterRegistry <- local({
last <- NULL
cluster <- NULL
- function(action=c("get", "start", "stop"), workers=NULL, ...) {
+ function(action = c("get", "start", "stop"), workers = NULL, ...) {
action <- match.arg(action)
if (is.null(workers)) {
@@ -27,12 +27,12 @@ ClusterRegistry <- local({
} else if (action == "start") {
## Already setup?
if (!identical(workers, last)) {
- ClusterRegistry(action="stop")
+ ClusterRegistry(action = "stop")
cluster <<- .makeCluster(workers, ...)
last <<- workers
}
} else if (action == "stop") {
- if (length(cluster) > 0L) try(stopCluster(cluster), silent=TRUE)
+ if (length(cluster) > 0L) try(stopCluster(cluster), silent = TRUE)
cluster <<- NULL
last <<- NULL
}
diff --git a/R/ConstantFuture-class.R b/R/ConstantFuture-class.R
index 63a3a65..efbc257 100644
--- a/R/ConstantFuture-class.R
+++ b/R/ConstantFuture-class.R
@@ -4,6 +4,7 @@
#' and therefore by definition already resolved upon creation.
#'
#' @inheritParams Future-class
+#'
#' @param \dots Not used.
#'
#' @return An object of class \code{ConstantFuture}.
@@ -11,11 +12,11 @@
#' @export
#' @name ConstantFuture-class
#' @keywords internal
-ConstantFuture <- function(expr=NULL, envir=emptyenv(), substitute=FALSE, local=FALSE, ...) {
+ConstantFuture <- function(expr = NULL, envir = emptyenv(), substitute = FALSE, globals = NULL, packages = NULL, local = FALSE, ...) {
expr <- force(expr)
- f <- Future(expr=expr, envir=emptyenv(), substitute=FALSE, local=FALSE, ...)
+ f <- Future(expr = expr, envir = emptyenv(), substitute = FALSE, globals = NULL, packages = NULL, local = FALSE, ...)
f$value <- expr
f$state <- "finished"
- structure(f, class=c("ConstantFuture", class(f)))
+ structure(f, class = c("ConstantFuture", class(f)))
f
}
diff --git a/R/DEPRECATED.R b/R/DEPRECATED.R
new file mode 100644
index 0000000..af222ca
--- /dev/null
+++ b/R/DEPRECATED.R
@@ -0,0 +1,32 @@
+#' @rdname sequential
+#' @export
+eager <- function(expr, envir = parent.frame(), substitute = TRUE, lazy = FALSE, seed = NULL, globals = TRUE, local = TRUE, earlySignal = FALSE, label = NULL, ...) {
+ .Defunct(msg = "Future strategy 'eager' is defunct. Instead, use 'sequential', which works identically.")
+}
+class(eager) <- c("eager", "uniprocess", "future", "function")
+
+#' @rdname sequential
+#' @export
+lazy <- function(expr, envir = parent.frame(), substitute = TRUE, lazy = TRUE, seed = NULL, globals = TRUE, local = TRUE, earlySignal = FALSE, label = NULL, ...) {
+ .Defunct(msg = "Future strategy 'lazy' is defunct. Instead, use f <- future(..., lazy = TRUE) or v %<-% { ... } %lazy% TRUE.")
+}
+class(lazy) <- c("lazy", "uniprocess", "future", "function")
+
+## WORKAROUND:
+## Avoid lazyeval::print.lazy() being called with print(lazy())
+## https://github.com/HenrikBengtsson/future/issues/52
+class(lazy) <- c("function", class(lazy))
+
+
+#' @rdname UniprocessFuture-class
+#' @export
+EagerFuture <- function(expr = NULL, envir = parent.frame(), substitute = FALSE, lazy = FALSE, globals = TRUE, local = TRUE, ...) {
+ .Defunct(msg = "EagerFuture is defunct. Instead, use SequentialFuture")
+}
+
+
+#' @rdname UniprocessFuture-class
+#' @export
+LazyFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, lazy=TRUE, globals=TRUE, local=TRUE, ...) {
+ .Defunct(msg = "LazyFuture is defunct. Instead, use f <- SequentialFuture(..., lazy = TRUE)")
+}
diff --git a/R/Future-class.R b/R/Future-class.R
index ba4250c..b3fbf9c 100644
--- a/R/Future-class.R
+++ b/R/Future-class.R
@@ -8,20 +8,39 @@
#' is available via \code{\link[future]{value}()}.
#'
#' @param expr An R \link[base]{expression}.
+#'
#' @param envir The \link{environment} in which the evaluation
#' is done (or inherits from if \code{local} is TRUE).
+#'
#' @param substitute If TRUE, argument \code{expr} is
#' \code{\link[base]{substitute}()}:ed, otherwise not.
+#'
+#' @param globals (optional) a named list of global objects needed in order
+#' for the future to be resolved correctly.
+#'
+#' @param packages (optional) a character vector specifying packages
+#' to be attached in the R environment evaluating the future.
+#'
+#' @param seed (optional) A L'Ecuyer-CMRG RNG seed.
+#'
+#' @param lazy If \code{FALSE} (default), the future is resolved
+#' eagerly (starting immediately), otherwise not.
+#'
#' @param local If TRUE, the expression is evaluated such that
#' all assignments are done to local temporary environment, otherwise
-#' the assignments are done in the calling environment.
+#' the assignments are done to the global environment of the \R process
+#' evaluating the future.
+#'
#' @param gc If TRUE, the garbage collector run (in the process that
#' evaluated the future) after the value of the future is collected.
#' \emph{Some types of futures ignore this argument.}
+#'
#' @param earlySignal Specified whether conditions should be signaled
#' as soon as possible or not.
+#'
#' @param label An optional character string label attached to the
#' future.
+#'
#' @param \dots Additional named elements of the future.
#'
#' @return An object of class \code{Future}.
@@ -37,16 +56,46 @@
#' right-hand-side (RHS) R expression and assigns its future value
#' to a variable as a \emph{\link[base]{promise}}.
#'
+#' @importFrom utils capture.output
#' @export
#' @name Future-class
-Future <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, local=TRUE, gc=FALSE, earlySignal=FALSE, label=NULL, ...) {
+Future <- function(expr = NULL, envir = parent.frame(), substitute = FALSE, globals = NULL, packages = NULL, seed = NULL, lazy = FALSE, local = TRUE, gc = FALSE, earlySignal = FALSE, label = NULL, ...) {
if (substitute) expr <- substitute(expr)
- args <- list(...)
+
+ if (!is.null(seed)) {
+ ## For RNGkind("L'Ecuyer-CMRG") we should have (see help('RNGkind')):
+ ## .Random.seed <- c(rng.kind, n)
+ ## where rng.kind == 407L and length(n) == 6L
+ if (!is.integer(seed) || length(seed) != 7 || !all(is.finite(seed)) || seed[1] != 407L) {
+ msg <- "Argument 'seed' must be L'Ecuyer-CMRG RNG seed as returned by parallel::nextRNGStream()"
+ mdebug(msg)
+ mdebug(capture.output(print(seed)))
+ stop(msg)
+ }
+ }
- core <- new.env(parent=emptyenv())
+ if (!is.null(globals)) {
+ stopifnot(is.list(globals),
+ length(globals) == 0 || inherits(globals, "Globals"))
+ }
+
+ if (!is.null(packages)) {
+ stopifnot(is.character(packages))
+ packages <- unique(packages)
+ stopifnot(!anyNA(packages), all(nzchar(packages)))
+ }
+
+ args <- list(...)
+
+ core <- new.env(parent = emptyenv())
core$expr <- expr
core$envir <- envir
+ core$globals <- globals
+ core$packages <- packages
core$owner <- session_uuid(attributes = TRUE)
+ core$lazy <- lazy
+ core$asynchronous <- TRUE ## Reserved for future version (Issue #109)
+ core$seed <- seed
core$local <- local
core$gc <- gc
core$earlySignal <- earlySignal
@@ -59,7 +108,7 @@ Future <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, local=TRUE
## Additional named arguments
for (key in names(args)) core[[key]] <- args[[key]]
- structure(core, class=c("Future", class(core)))
+ structure(core, class = c("Future", class(core)))
}
@@ -70,28 +119,47 @@ print.Future <- function(x, ...) {
cat(sprintf("%s:\n", class[1]))
label <- x$label
if (is.null(label)) label <- "<none>"
- cat("Label: ", sQuote(label), "\n", sep="")
+ cat("Label: ", sQuote(label), "\n", sep = "")
cat("Expression:\n")
print(x$expr)
+ cat(sprintf("Lazy evaluation: %s\n", x$lazy))
+ cat(sprintf("Asynchronous evaluation: %s\n", x$asynchronous))
cat(sprintf("Local evaluation: %s\n", x$local))
cat(sprintf("Environment: %s\n", capture.output(x$envir)))
- g <- x$globals
+
+ ## FIXME: Add method globals_of() for Future such that it's possible
+ ## also for SequentialFuture to return something here. /HB 2017-05-17
+ g <- globals(x)
ng <- length(g)
if (ng > 0) {
- gSizes <- sapply(g, FUN=object.size)
+ gSizes <- sapply(g, FUN = objectSize)
gTotalSize <- sum(gSizes)
- g <- head(g, n=5L)
- gSizes <- head(gSizes, n=5L)
- g <- sprintf("%s %s of %s", sapply(g, FUN=function(x) class(x)[1]), sQuote(names(g)), sapply(gSizes, FUN=asIEC))
+ g <- head(g, n = 5L)
+ gSizes <- head(gSizes, n = 5L)
+ g <- sprintf("%s %s of %s", sapply(g, FUN = function(x) class(x)[1]), sQuote(names(g)), sapply(gSizes, FUN = asIEC))
if (ng > 5L) g <- sprintf("%s ...", g)
cat(sprintf("Globals: %d objects totaling %s (%s)\n", ng, asIEC(gTotalSize), g))
} else {
cat("Globals: <none>\n")
}
+
+ p <- packages(x)
+ np <- length(p)
+ if (np > 0) {
+ cat(sprintf("Packages: %d packages (%s)\n", np, paste(sQuote(p), collapse = ", ")))
+ } else {
+ cat("Packages: <none>\n")
+ }
+
+ if (is.null(x$seed)) {
+ cat("L'Ecuyer-CMRG RNG seed: <none>\n")
+ } else {
+ cat(sprintf("L'Ecuyer-CMRG RNG seed: c(%s)\n", paste(x$seed, collapse = ", ")))
+ }
- hasValue <- exists("value", envir=x, inherits=FALSE)
+ hasValue <- exists("value", envir = x, inherits = FALSE)
- if (exists("value", envir=x, inherits=FALSE)) {
+ if (exists("value", envir = x, inherits = FALSE)) {
cat("Resolved: TRUE\n")
} else if (inherits(x, "UniprocessFuture") && x$lazy) {
## FIXME: Special case; will there every be other cases
@@ -109,13 +177,13 @@ print.Future <- function(x, ...) {
}
if (hasValue) {
- cat(sprintf("Value: %s of class %s\n", asIEC(object.size(x$value)), sQuote(class(x$value)[1])))
+ cat(sprintf("Value: %s of class %s\n", asIEC(objectSize(x$value)), sQuote(class(x$value)[1])))
} else {
cat("Value: <not collected>\n")
}
cat(sprintf("Early signalling: %s\n", isTRUE(x$earlySignal)))
cat(sprintf("Owner process: %s\n", x$owner))
- cat(sprintf("Class: %s\n", paste(sQuote(class), collapse=", ")))
+ cat(sprintf("Class: %s\n", paste(sQuote(class), collapse = ", ")))
} ## print()
@@ -127,7 +195,7 @@ assertOwner <- function(future, ...) {
}
if (!identical(future$owner, session_uuid(attributes = TRUE))) {
- stop(FutureError(sprintf("Invalid usage of futures: A future whose value has not yet been collected can only be queried by the R process (%s) that created it, not by any other R processes (%s): %s", hpid(future$owner), hpid(session_uuid()), hexpr(future$expr)), future=future))
+ stop(FutureError(sprintf("Invalid usage of futures: A future whose value has not yet been collected can only be queried by the R process (%s) that created it, not by any other R processes (%s): %s", hpid(future$owner), hpid(session_uuid()), hexpr(future$expr)), future = future))
}
invisible(future)
@@ -151,9 +219,12 @@ assertOwner <- function(future, ...) {
#' @rdname run
#' @export
#' @export run
+#' @keywords internal
run.Future <- function(future, ...) {
if (future$state != 'created') {
- stop("A future can only be launched once.")
+ label <- future$label
+ if (is.null(label)) label <- "<none>"
+ stop(sprintf("A future ('%s') can only be launched once.", label))
}
future
@@ -183,19 +254,19 @@ run <- function(...) UseMethod("run")
#' @rdname value
#' @export
#' @export value
-value.Future <- function(future, signal=TRUE, ...) {
- if (future$state == 'created') {
+value.Future <- function(future, signal = TRUE, ...) {
+ if (future$state == "created") {
future <- run(future)
}
- if (!future$state %in% c('finished', 'failed', 'interrupted')) {
+ if (!future$state %in% c("finished", "failed", "interrupted")) {
msg <- sprintf("Internal error: value() called on a non-finished future: %s", class(future)[1])
mdebug(msg)
- stop(FutureError(msg, future=future))
+ stop(FutureError(msg, future = future))
}
value <- future$value
- if (signal && future$state == 'failed') {
+ if (signal && future$state == "failed") {
mdebug("Future state: %s", sQuote(future$state))
mdebug("Future value: %s", sQuote(value))
stop(FutureError(future))
@@ -210,12 +281,12 @@ value <- function(...) UseMethod("value")
#' @export
resolved.Future <- function(x, ...) {
## Is future even launched?
- if (x$state == 'created') return(FALSE)
+ if (x$state == "created") return(FALSE)
## Signal conditions early, iff specified for the given future
signalEarly(x, ...)
- x$state %in% c('finished', 'failed', 'interrupted')
+ x$state %in% c("finished", "failed", "interrupted")
}
@@ -229,13 +300,12 @@ resolved.Future <- function(x, ...) {
#'
#' @details
#' If no next future strategy is specified, the default is to
-#' use \link{eager} futures. This conservative approach protects
+#' use \link{sequential} futures. This conservative approach protects
#' against spawning off recursive futures by mistake, especially
#' \link{multicore} and \link{multisession} ones.
-#' The default will also set \code{options(mc.cores=0L)}, which
-#' means that no \emph{additional} R processes may be spawned off
-#' by functions such as \code{\link[parallel:mclapply]{mclapply}()}
-#' and friends (*).
+#' The default will also set \code{options(mc.cores = 1L)} (*) so that
+#' no parallel R processes are spawned off by functions such as
+#' \code{\link[parallel:mclapply]{mclapply}()} and friends.
#'
#' Currently it is not possible to specify what type of nested
#' futures to be used, meaning the above default will always be
@@ -243,8 +313,8 @@ resolved.Future <- function(x, ...) {
#' See \href{https://github.com/HenrikBengtsson/future/issues/37}{Issue #37}
#' for plans on adding support for custom nested future types.
#'
-#' (*) Note that using \code{mc.cores=0} will unfortunately cause
-#' \code{mclapply()} and friends to generate an error saying
+#' (*) Ideally we would set \code{mc.cores = 0} but that will unfortunately
+#' cause \code{mclapply()} and friends to generate an error saying
#' "'mc.cores' must be >= 1". Ideally those functions should
#' fall back to using the non-multicore alternative in this
#' case, e.g. \code{mclapply(...)} => \code{lapply(...)}.
@@ -257,91 +327,121 @@ resolved.Future <- function(x, ...) {
getExpression <- function(future, ...) UseMethod("getExpression")
#' @export
-getExpression.Future <- function(future, mc.cores=NULL, ...) {
-## mdebug("getExpression() ...")
+getExpression.Future <- function(future, mc.cores = NULL, ...) {
+ debug <- getOption("future.debug", FALSE)
+ ## mdebug("getExpression() ...")
## Should 'mc.cores' be set?
if (!is.null(mc.cores)) {
-## mdebug("getExpression(): setting mc.cores=%d inside future", mc.cores)
+## mdebug("getExpression(): setting mc.cores = %d inside future", mc.cores)
## FIXME: How can we guarantee that '...future.mc.cores.old'
## is not overwritten? /HB 2016-03-14
enter <- bquote({
## covr: skip=2
...future.mc.cores.old <- getOption("mc.cores")
- options(mc.cores=.(mc.cores))
+ options(mc.cores = .(mc.cores))
})
exit <- bquote({
## covr: skip=1
- options(mc.cores=...future.mc.cores.old)
+ options(mc.cores = ...future.mc.cores.old)
})
} else {
enter <- exit <- NULL
}
+ ## Seed RNG seed?
+ if (!is.null(future$seed)) {
+ enter <- bquote({
+ ## covr: skip=2
+ .(enter)
+ ## NOTE: It is not needed to call eRNGkind("L'Ecuyer-CMRG") here
+ ## because the type of RNG is defined by .Random.seed, especially
+ ## .Random.seed[1]. See help("RNGkind"). /HB 2017-01-12
+ assign(".Random.seed", .(future$seed), envir = globalenv(), inherits = FALSE)
+ })
+ }
+
## Reset future strategies upon exit of future
strategies <- plan("list")
stopifnot(length(strategies) >= 1L)
exit <- bquote({
## covr: skip=2
.(exit)
- future::plan(.(strategies))
+ future::plan(.(strategies), .cleanup = FALSE, .init = FALSE)
})
## Pass down the default or the remain set of future strategies?
strategiesR <- strategies[-1]
-## mdebug("Number of remaining strategies: %d\n", length(strategiesR))
+ ## mdebug("Number of remaining strategies: %d\n", length(strategiesR))
+
+ ## Identify packages needed by the futures
+ pkgs <- NULL
+ if (length(strategiesR) > 0L) {
+ ## Identify package namespaces needed for strategies
+ pkgs <- lapply(strategiesR, FUN = environment)
+ pkgs <- lapply(pkgs, FUN = environmentName)
+ pkgs <- unique(unlist(pkgs, use.names = FALSE))
+ ## CLEANUP: Only keep those that are loaded in the current session
+ pkgs <- intersect(pkgs, loadedNamespaces())
+ if (debug) mdebug("Packages needed by future strategies (n = %d): %s", length(pkgs), paste(sQuote(pkgs), collapse = ", "))
+ } else {
+ if (debug) mdebug("Packages needed by future strategies (n = 0): <none>")
+ }
+
+ pkgsF <- packages(future)
+ if (length(pkgsF) > 0) {
+ if (debug) mdebug("Packages needed by the future expression (n = %d): %s", length(pkgsF), paste(sQuote(pkgsF), collapse = ", "))
+ pkgs <- unique(c(pkgs, pkgsF))
+ } else {
+ if (debug) mdebug("Packages needed by the future expression (n = 0): <none>")
+ }
+
+ ## Make sure to load and attach all package needed
+ if (length(pkgs) > 0L) {
+ ## Sanity check by verifying packages can be loaded already here
+ ## If there is somethings wrong in 'pkgs', we get the error
+ ## already before launching the future.
+ for (pkg in pkgs) loadNamespace(pkg)
+
+ enter <- bquote({
+ ## covr: skip=3
+ .(enter)
+ ## TROUBLESHOOTING: If the package fails to load, then library()
+ ## suppress that error and generates a generic much less
+ ## informative error message. Because of this, we load the
+ ## namespace first (to get a better error message) and then
+ ## calls library(), which attaches the package. /HB 2016-06-16
+ ## NOTE: We use local() here such that 'pkg' is not assigned
+ ## to the future environment. /HB 2016-07-03
+ local({
+ for (pkg in .(pkgs)) {
+ loadNamespace(pkg)
+ library(pkg, character.only = TRUE)
+ }
+ })
+ })
+ }
+
+ ## Make sure to set all nested future strategies needed
if (length(strategiesR) == 0L) {
## mdebug("Set plan('default') inside future")
## Use default future strategy
enter <- bquote({
## covr: skip=2
.(enter)
- future::plan("default")
+ future::plan("default", .cleanup = FALSE, .init = FALSE)
})
- } else {
-## mdebug("Set plan(<remaining strategies>) inside future")
- ## Identify package namespaces for strategies
- pkgs <- lapply(strategiesR, FUN=environment)
- pkgs <- lapply(pkgs, FUN=environmentName)
- pkgs <- unique(unlist(pkgs, use.names=FALSE))
- pkgs <- intersect(pkgs, loadedNamespaces())
-## mdebug("Packages to be loaded by expression (n=%d): %s", length(pkgs), paste(sQuote(pkgs), collapse=", "))
-
- if (length(pkgs) > 0L) {
- ## Sanity check by verifying packages can be loaded already here
- ## If there is somethings wrong in 'pkgs', we get the error
- ## already before launching the future.
- for (pkg in pkgs) loadNamespace(pkg)
-
- enter <- bquote({
- ## covr: skip=3
- .(enter)
- ## TROUBLESHOOTING: If the package fails to load, then library()
- ## suppress that error and generates a generic much less
- ## informative error message. Because of this, we load the
- ## namespace first (to get a better error message) and then
- ## calls library(), which attaches the package. /HB 2016-06-16
- ## NOTE: We use local() here such that 'pkg' is not assigned
- ## to the future environment. /HB 2016-07-03
- local({
- for (pkg in .(pkgs)) {
- loadNamespace(pkg)
- library(pkg, character.only=TRUE)
- }
- })
- })
- }
-
+ } else {
## Pass down future strategies
enter <- bquote({
## covr: skip=2
.(enter)
- future::plan(.(strategiesR))
+ future::plan(.(strategiesR), .cleanup = FALSE, .init = FALSE)
})
} ## if (length(strategiesR) > 0L)
- expr <- makeExpression(expr=future$expr, local=future$local, enter=enter, exit=exit)
+ expr <- makeExpression(expr = future$expr, local = future$local, enter = enter, exit = exit)
if (getOption("future.debug", FALSE)) {
print(expr)
}
@@ -352,11 +452,11 @@ getExpression.Future <- function(future, mc.cores=NULL, ...) {
} ## getExpression()
-makeExpression <- function(expr, local=TRUE, globals.onMissing=getOption("future.globals.onMissing", "error"), enter=NULL, exit=NULL) {
+makeExpression <- function(expr, local = TRUE, globals.onMissing = getOption("future.globals.onMissing", "error"), enter = NULL, exit = NULL) {
## Evaluate expression in a local() environment?
if (local) {
- a <- NULL; rm(list="a") ## To please R CMD check
- expr <- substitute(local(a), list(a=expr))
+ a <- NULL; rm(list = "a") ## To please R CMD check
+ expr <- substitute(local(a), list(a = expr))
}
## Set and reset certain future.* options
@@ -364,17 +464,17 @@ makeExpression <- function(expr, local=TRUE, globals.onMissing=getOption("future
## covr: skip=7
...future.oldOptions <- options(
## Prevent .future.R from being source():d when future is attached
- future.startup.loadScript=FALSE,
+ future.startup.loadScript = FALSE,
## Assert globals when future is created (or at run time)?
- future.globals.onMissing=globals.onMissing
+ future.globals.onMissing = globals.onMissing
)
enter
- }, env=list(globals.onMissing=globals.onMissing, enter=enter))
+ }, env = list(globals.onMissing = globals.onMissing, enter = enter))
exit <- substitute({
exit
options(...future.oldOptions)
- }, env=list(exit=exit))
+ }, env = list(exit = exit))
## NOTE: We don't want to use local(body) w/ on.exit() because
@@ -389,7 +489,20 @@ makeExpression <- function(expr, local=TRUE, globals.onMissing=getOption("future
}, finally = {
exit
})
- }, env=list(enter=enter, body=expr, exit=exit))
+ }, env = list(enter = enter, body = expr, exit = exit))
expr
} ## makeExpression()
+
+
+globals <- function(future, ...) UseMethod("globals")
+
+globals.Future <- function(future, ...) {
+ future[["globals"]]
+}
+
+packages <- function(future, ...) UseMethod("packages")
+
+packages.Future <- function(future, ...) {
+ future[["packages"]]
+}
diff --git a/R/FutureError-class.R b/R/FutureError-class.R
index 900b3eb..d8cff68 100644
--- a/R/FutureError-class.R
+++ b/R/FutureError-class.R
@@ -10,7 +10,7 @@
#' @aliases getOutput
#' @export
#' @keywords internal
-FutureError <- function(message, call=NULL, future=NULL, output=NULL) {
+FutureError <- function(message, call = NULL, future = NULL, output = NULL) {
## Support different types of input
## NOTE: We could turn this into an S3 method. /HB 2016-07-01
if (inherits(message, "Future")) {
@@ -25,7 +25,7 @@ FutureError <- function(message, call=NULL, future=NULL, output=NULL) {
}
## Create a basic error object
- cond <- simpleError(message, call=call)
+ cond <- simpleError(message, call = call)
## Record Future object and optional output messages
attr(cond, "future") <- future
@@ -64,7 +64,7 @@ print.FutureError <- function(x, ...) {
if (!is.null(output)) {
cat("Captured output:\n")
- cat(getOutput(x, tail=30L, collapse="\n"))
+ cat(getOutput(x, tail = 30L, collapse = "\n"))
cat("\n\n")
}
@@ -76,7 +76,7 @@ print.FutureError <- function(x, ...) {
#' @export
-getOutput.FutureError <- function(x, collapse=NULL, head=NULL, tail=NULL, ...) {
+getOutput.FutureError <- function(x, collapse = NULL, head = NULL, tail = NULL, ...) {
output <- attr(x, "output")
## Return "as is"?
@@ -85,17 +85,17 @@ getOutput.FutureError <- function(x, collapse=NULL, head=NULL, tail=NULL, ...) {
## Truncate?
if (!is.null(head) && !is.null(tail)) {
idxs <- seq_along(output)
- idxs <- sort(unique(c(head(idxs, n=head), tail(idxs, n=tail))))
+ idxs <- sort(unique(c(head(idxs, n = head), tail(idxs, n = tail))))
output <- output[idxs]
idxs
} else if (!is.null(head)) {
- output <- head(output, n=head)
+ output <- head(output, n = head)
} else if (!is.null(tail)) {
- output <- tail(output, n=tail)
+ output <- tail(output, n = tail)
}
## Collapse? (add line endings)
- if (!is.null(collapse)) output <- paste(output, collapse=collapse)
+ if (!is.null(collapse)) output <- paste(output, collapse = collapse)
output
} ## getOutput()
diff --git a/R/FutureGlobals-class.R b/R/FutureGlobals-class.R
new file mode 100644
index 0000000..43641f3
--- /dev/null
+++ b/R/FutureGlobals-class.R
@@ -0,0 +1,139 @@
+#' A representation of a set of globals used with futures
+#'
+#' @param object A named list.
+#'
+#' @param resolved A logical indicating whether these globals
+#' have been scanned for and resolved futures or not.
+#'
+#' @param total_size The total size of all globals, if known.
+#'
+#' @param \dots Not used.
+#'
+#' @return An object of class \code{FutureGlobals}.
+#'
+#' @details
+#' This class extends the \link[globals]{Globals} class by add
+#' attribute \code{resolved}.
+#'
+#' @aliases as.FutureGlobals as.FutureGlobals.FutureGlobals
+#' as.FutureGlobals.Globals as.FutureGlobals.list [.FutureGlobals
+#' c.FutureGlobals unique.FutureGlobals
+#'
+#' @importFrom globals Globals
+#' @export
+FutureGlobals <- function(object = list(), resolved = FALSE, total_size = NA_real_, ...) {
+ if (!is.list(object)) {
+ stop("Argument 'object' is not a list: ", class(object)[1])
+ }
+
+ if (!inherits(object, "Globals")) {
+ object <- Globals(object, ...)
+ attr(object, "resolved") <- resolved
+ attr(object, "total_size") <- total_size
+ } else if (!inherits(object, "FutureGlobals")) {
+ attr(object, "resolved") <- resolved
+ attr(object, "total_size") <- total_size
+ }
+
+ structure(object, class = c("FutureGlobals", class(object)))
+}
+
+#' @export
+resolved.FutureGlobals <- function(x, ...) attr(x, "resolved")
+
+#' @export
+as.FutureGlobals <- function(x, ...) UseMethod("as.FutureGlobals")
+
+#' @export
+as.FutureGlobals.FutureGlobals <- function(x, ...) x
+
+#' @export
+as.FutureGlobals.Globals <- function(x, ...) {
+ class(x) <- c("FutureGlobals", class(x))
+ attr(x, "resolved") <- FALSE
+ attr(x, "total_size") <- NA_real_
+ x
+}
+
+#' @export
+as.FutureGlobals.list <- function(x, ...) {
+ as.FutureGlobals(as.Globals(x, ...))
+}
+
+#' @export
+`[.FutureGlobals` <- function(x, i) {
+ resolved <- attr(x, "resolved")
+ size <- attr(x, "total_size")
+ x <- NextMethod("[")
+ attr(x, "resolved") <- resolved
+ attr(x, "total_size") <- size
+ x
+}
+
+#' @export
+c.FutureGlobals <- function(x, ...) {
+ args <- list(...)
+ if (length(args) == 0) return(x)
+
+ ## Are all imputs resolved?
+ resolved <- attr(x, "resolved")
+ resolved_args <- lapply(args, FUN = function(x) isTRUE(attr(x, "resolved")))
+ resolved_args <- unlist(resolved_args, use.names = FALSE)
+ resolved <- resolved && all(resolved_args)
+
+ ## Total size?
+ size <- attr(x, "total_size")
+ if (!is.na(size)) {
+ size_args <- lapply(args, FUN = function(z) {
+ size <- attr(z, "total_size")
+ if (is.null(size)) NA_real_ else size
+ })
+ size_args <- unlist(size_args, use.names = FALSE)
+ size <- size + sum(size_args, na.rm = FALSE)
+ }
+
+ x <- NextMethod("c")
+
+ attr(x, "resolved") <- resolved
+ attr(x, "total_size") <- size
+
+ x
+}
+
+#' @export
+unique.FutureGlobals <- function(x, ...) {
+ nx <- length(x)
+ if (nx == 0) return(x)
+
+ resolved <- attr(x, "resolved")
+ size <- attr(x, "total_size")
+ x <- NextMethod("unique")
+ attr(x, "resolved") <- resolved
+
+ ## Were any elements dropped?
+ if (length(x) != nx) size <- NA_real_
+ attr(x, "total_size") <- size
+
+ x
+}
+
+
+#' @export
+resolve.FutureGlobals <- function(x, ...) {
+ ## Nothing to do?
+ if (length(x) == 0) return(x)
+
+ ## Already resolved?
+ if (isTRUE(attr(x, "resolved"))) return(x)
+
+ x <- NextMethod("resolve")
+
+ ## At this point we consider these future globals resolved (regardless of 'recursive')
+ attr(x, "resolved") <- TRUE
+
+ ## Since we don't know whether the above turned any futures into their
+ ## values, we cannot make any assumption about the total size.
+ attr(x, "total_size") <- NA_real_
+
+ x
+}
diff --git a/R/FutureRegistry.R b/R/FutureRegistry.R
index d598109..6d4e3dd 100644
--- a/R/FutureRegistry.R
+++ b/R/FutureRegistry.R
@@ -8,7 +8,7 @@ FutureRegistry <- local({
NA_integer_
}
- collectValues <- function(where, futures, firstOnly=TRUE) {
+ collectValues <- function(where, futures, firstOnly = TRUE) {
for (ii in seq_along(futures)) {
future <- futures[[ii]]
@@ -21,13 +21,13 @@ FutureRegistry <- local({
if (resolved(future)) {
## (a) Let future cleanup after itself, iff needed.
## This, this may result in a call to
- ## FutureRegistry(..., action="remove").
- value(future, signal=FALSE)
+ ## FutureRegistry(..., action = "remove").
+ value(future, signal = FALSE)
## (b) Make sure future is removed from registry, unless
## already done via above value() call.
futuresDB <- db[[where]]
- idx <- indexOf(futuresDB, future=future)
+ idx <- indexOf(futuresDB, future = future)
if (!is.na(idx)) {
futuresDB[[idx]] <- NULL
db[[where]] <<- futuresDB
@@ -42,7 +42,7 @@ FutureRegistry <- local({
} ## collectValues()
- function(where, action=c("add", "remove", "list", "collect-first", "reset"), future=NULL, earlySignal=TRUE, ...) {
+ function(where, action = c("add", "remove", "list", "collect-first", "reset"), future = NULL, earlySignal = TRUE, ...) {
stopifnot(length(where) == 1, nzchar(where))
futures <- db[[where]]
@@ -53,7 +53,7 @@ FutureRegistry <- local({
}
if (action == "add") {
- idx <- indexOf(futures, future=future)
+ idx <- indexOf(futures, future = future)
if (!is.na(idx)) {
msg <- sprintf("Cannot add to %s registry. %s is already registered.", sQuote(where), class(future)[1])
mdebug("ERROR: %s", msg)
@@ -62,7 +62,7 @@ FutureRegistry <- local({
futures[[length(futures)+1L]] <- future
db[[where]] <<- futures
} else if (action == "remove") {
- idx <- indexOf(futures, future=future)
+ idx <- indexOf(futures, future = future)
if (is.na(idx)) {
msg <- sprintf("Cannot remove from %s registry. %s not registered.", sQuote(where), class(future)[1])
mdebug("ERROR: %s", msg)
@@ -71,7 +71,7 @@ FutureRegistry <- local({
futures[[idx]] <- NULL
db[[where]] <<- futures
} else if (action == "collect-first") {
- collectValues(where, futures=futures, firstOnly=TRUE)
+ collectValues(where, futures = futures, firstOnly = TRUE)
} else if (action == "reset") {
db[[where]] <<- list()
} else if (action == "list") {
@@ -84,14 +84,14 @@ FutureRegistry <- local({
## Early signaling of conditions?
if (earlySignal && length(futures) > 0L) {
## Which futures have early signaling enabled?
- idxs <- lapply(futures, FUN=function(f) f$earlySignal)
- idxs <- which(unlist(idxs, use.names=FALSE))
+ idxs <- lapply(futures, FUN = function(f) f$earlySignal)
+ idxs <- which(unlist(idxs, use.names = FALSE))
## Any futures to be scanned for early signaling?
if (length(idxs) > 0) {
## Collect values, which will trigger signaling during
## calls to resolved().
- collectValues(where, futures=futures[idxs], firstOnly=FALSE)
+ collectValues(where, futures = futures[idxs], firstOnly = FALSE)
}
}
diff --git a/R/MulticoreFuture-class.R b/R/MulticoreFuture-class.R
index d1ff1a8..2e33574 100644
--- a/R/MulticoreFuture-class.R
+++ b/R/MulticoreFuture-class.R
@@ -1,4 +1,4 @@
-#' An multicore future is a future whose value will be resolved asynchroneously in a parallel process
+#' An multicore future is a future whose value will be resolved asynchronously in a parallel process
#'
#' @inheritParams MultiprocessFuture-class
#'
@@ -11,16 +11,20 @@
#' @export
#' @name MulticoreFuture-class
#' @keywords internal
-MulticoreFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, globals=TRUE, ...) {
+MulticoreFuture <- function(expr = NULL, envir = parent.frame(), substitute = FALSE, globals = TRUE, ...) {
if (substitute) expr <- substitute(expr)
+ args <- list(...)
+ lazy <- args$lazy
+ if (is.null(lazy)) lazy <- FALSE
+
## Global objects
assignToTarget <- (is.list(globals) || inherits(globals, "Globals"))
- gp <- getGlobalsAndPackages(expr, envir=envir, tweak=tweakExpression, globals=globals, resolve=TRUE)
+ gp <- getGlobalsAndPackages(expr, envir = envir, tweak = tweakExpression, globals = globals)
## Assign?
- if (assignToTarget && length(gp) > 0) {
- target <- new.env(parent=envir)
+ if (length(gp) > 0 && (lazy || assignToTarget)) {
+ target <- new.env(parent = envir)
globalsT <- gp$globals
for (name in names(globalsT)) {
target[[name]] <- globalsT[[name]]
@@ -30,15 +34,19 @@ MulticoreFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, g
}
gp <- NULL
- f <- MultiprocessFuture(expr=expr, envir=envir, substitute=FALSE, job=NULL, ...)
- structure(f, class=c("MulticoreFuture", class(f)))
+ f <- MultiprocessFuture(expr = expr, envir = envir, substitute = FALSE, job = NULL, ...)
+ structure(f, class = c("MulticoreFuture", class(f)))
}
#' @export
run.MulticoreFuture <- function(future, ...) {
+ debug <- getOption("future.debug", FALSE)
+
if (future$state != 'created') {
- stop("A future can only be launched once.")
+ label <- future$label
+ if (is.null(label)) label <- "<none>"
+ stop(sprintf("A future ('%s') can only be launched once.", label))
}
## Assert that the process that created the future is
@@ -51,24 +59,26 @@ run.MulticoreFuture <- function(future, ...) {
envir <- future$envir
requestCore(
- await=function() FutureRegistry("multicore", action="collect-first"),
- workers=future$workers
+ await = function() FutureRegistry("multicore", action = "collect-first"),
+ workers = future$workers
)
## Add to registry
- FutureRegistry("multicore", action="add", future=future)
+ FutureRegistry("multicore", action = "add", future = future)
future.args <- list(expr)
- job <- do.call(parallel::mcparallel, args=future.args, envir=envir)
+ job <- do.call(parallel::mcparallel, args = future.args, envir = envir)
future$job <- job
future$state <- 'running'
+ if (debug) mdebug("%s started", class(future)[1])
+
invisible(future)
}
#' @export
-resolved.MulticoreFuture <- function(x, timeout=0.2, ...) {
+resolved.MulticoreFuture <- function(x, timeout = 0.2, ...) {
## Is future even launched?
if (x$state == 'created') return(FALSE)
@@ -83,11 +93,11 @@ resolved.MulticoreFuture <- function(x, timeout=0.2, ...) {
job <- x$job
stopifnot(inherits(job, "parallelJob"))
- ## NOTE: We cannot use mcollect(job, wait=FALSE, timeout=0.2),
+ ## NOTE: We cannot use mcollect(job, wait = FALSE, timeout = 0.2),
## because that will return NULL if there's a timeout, which is
## an ambigous value because the future expression may return NULL.
## WORKAROUND: Adopted from parallel::mccollect().
- pid <- selectChildren(job, timeout=timeout)
+ pid <- selectChildren(job, timeout = timeout)
res <- (is.integer(pid) || is.null(pid))
## Signal conditions early? (happens only iff requested)
@@ -97,7 +107,7 @@ resolved.MulticoreFuture <- function(x, timeout=0.2, ...) {
}
#' @export
-value.MulticoreFuture <- function(future, signal=TRUE, ...) {
+value.MulticoreFuture <- function(future, signal = TRUE, ...) {
## Has the value already been collected?
if (future$state %in% c('finished', 'failed', 'interrupted')) {
return(NextMethod("value"))
@@ -116,14 +126,14 @@ value.MulticoreFuture <- function(future, signal=TRUE, ...) {
mccollect <- importParallel("mccollect")
job <- future$job
stopifnot(inherits(job, "parallelJob"))
- res <- mccollect(job, wait=TRUE)[[1L]]
+ res <- mccollect(job, wait = TRUE)[[1L]]
## SPECIAL: Check for fallback 'fatal error in wrapper code'
## try-error from parallel:::mcparallel(). If detected, then
## turn into an error with a more informative error message, cf.
## https://github.com/HenrikBengtsson/future/issues/35
- if (identical(res, structure("fatal error in wrapper code", class="try-error"))) {
- stop(FutureError(sprintf("Detected an error ('%s') by the 'parallel' package while trying to retrieve the value of a %s (%s). This could be because the forked R process that evalutes the future was terminated before it was completed.", res, class(future)[1], sQuote(hexpr(future$expr))), future=future))
+ if (identical(res, structure("fatal error in wrapper code", class = "try-error"))) {
+ stop(FutureError(sprintf("Detected an error ('%s') by the 'parallel' package while trying to retrieve the value of a %s (%s). This could be because the forked R process that evaluates the future was terminated before it was completed.", res, class(future)[1], sQuote(hexpr(future$expr))), future = future))
}
## Update value and state
@@ -138,13 +148,13 @@ value.MulticoreFuture <- function(future, signal=TRUE, ...) {
res <- NULL ## Not needed anymore
## Remove from registry
- FutureRegistry("multicore", action="remove", future=future)
+ FutureRegistry("multicore", action = "remove", future = future)
NextMethod("value")
}
#' @export
-getExpression.MulticoreFuture <- function(future, mc.cores=0L, ...) {
- NextMethod("getExpression", mc.cores=mc.cores)
+getExpression.MulticoreFuture <- function(future, mc.cores = 1L, ...) {
+ NextMethod("getExpression", mc.cores = mc.cores)
}
diff --git a/R/MultiprocessFuture-class.R b/R/MultiprocessFuture-class.R
index e3be606..c79799e 100644
--- a/R/MultiprocessFuture-class.R
+++ b/R/MultiprocessFuture-class.R
@@ -1,10 +1,7 @@
-#' An multiprocess future is a future whose value will be resolved asynchroneously in a parallel process
+#' An multiprocess future is a future whose value will be resolved asynchronously in a parallel process
#'
#' @inheritParams Future-class
-#' @param local If TRUE, the expression is evaluated such that
-#' all assignments are done to local temporary environment, otherwise
-#' the assignments are done to the global environment of the \R process
-#' evaluating the future.
+#'
#' @param \dots Additional named elements passed to \code{\link{Future}()}.
#'
#' @return An object of class \code{MultiprocessFuture}.
@@ -16,9 +13,9 @@
#' @export
#' @name MultiprocessFuture-class
#' @keywords internal
-MultiprocessFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, ...) {
+MultiprocessFuture <- function(expr = NULL, envir = parent.frame(), substitute = FALSE, ...) {
if (substitute) expr <- substitute(expr)
- f <- Future(expr=expr, envir=envir, substitute=FALSE, ...)
- structure(f, class=c("MultiprocessFuture", class(f)))
+ f <- Future(expr = expr, envir = envir, substitute = FALSE, ...)
+ structure(f, class = c("MultiprocessFuture", class(f)))
}
diff --git a/R/MultisessionFuture-class.R b/R/MultisessionFuture-class.R
index 6becac6..48a6d16 100644
--- a/R/MultisessionFuture-class.R
+++ b/R/MultisessionFuture-class.R
@@ -2,14 +2,14 @@
#'
## Currently aliased in ClusterFuture
#' @export
-MultisessionFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, local=!persistent, globals=TRUE, persistent=FALSE, workers=NULL, ...) {
+MultisessionFuture <- function(expr = NULL, envir = parent.frame(), substitute = FALSE, globals = TRUE, persistent = FALSE, local = !persistent, workers = NULL, ...) {
if (substitute) expr <- substitute(expr)
- f <- ClusterFuture(expr=expr, envir=envir, substitute=FALSE, local=local, globals=globals, persistent=persistent, workers=workers, ...)
- structure(f, class=c("MultisessionFuture", class(f)))
+ f <- ClusterFuture(expr = expr, envir = envir, substitute = FALSE, globals = globals, persistent = persistent, local = local, workers = workers, ...)
+ structure(f, class = c("MultisessionFuture", class(f)))
}
#' @export
-getExpression.MultisessionFuture <- function(future, mc.cores=0L, ...) {
- NextMethod("getExpression", mc.cores=mc.cores)
+getExpression.MultisessionFuture <- function(future, mc.cores = 1L, ...) {
+ NextMethod("getExpression", mc.cores = mc.cores)
}
diff --git a/R/UniprocessFuture-class.R b/R/UniprocessFuture-class.R
index 181b180..6d66196 100644
--- a/R/UniprocessFuture-class.R
+++ b/R/UniprocessFuture-class.R
@@ -1,8 +1,10 @@
-#' An uniprocess future is a future whose value will be resolved synchroneously in the current process
+#' An uniprocess future is a future whose value will be resolved synchronously in the current process
#'
#' @inheritParams Future-class
+#'
#' @param lazy If \code{FALSE} (default), then the setup and validation of
-#' global variables are done for eager evaluation, otherwise not.
+#' global variables are done for eager evaluation, otherwise not.
+#'
#' @param \dots Additional named elements passed to \code{\link{Future}()}.
#'
#' @return An object of class \code{UniprocessFuture}.
@@ -14,27 +16,28 @@
#' @export
#' @name UniprocessFuture-class
#' @keywords internal
-UniprocessFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, globals=TRUE, local=TRUE, lazy=FALSE, ...) {
+UniprocessFuture <- function(expr = NULL, envir = parent.frame(), substitute = FALSE, globals = TRUE, packages = NULL, lazy = FALSE, local = TRUE, ...) {
if (substitute) expr <- substitute(expr)
if (lazy) {
## Evaluate in a local environment?
if (local) {
- envir <- new.env(parent=envir)
+ envir <- new.env(parent = envir)
} else {
if (!is.logical(globals) || globals) {
- stop("Non-supported use of lazy uniprocess futures: Whenever argument 'local' is FALSE, then argument 'globals' must also be FALSE. Lazy uniprocess future evaluation in the calling environment (local=FALSE) can only be done if global objects are resolved at the same time.")
+ stop("Non-supported use of lazy uniprocess futures: Whenever argument 'local' is FALSE, then argument 'globals' must also be FALSE. Lazy uniprocess future evaluation in the calling environment (local = FALSE) can only be done if global objects are resolved at the same time.")
}
}
}
## Global objects
assignToTarget <- (is.list(globals) || inherits(globals, "Globals"))
- gp <- getGlobalsAndPackages(expr, envir=envir, tweak=tweakExpression, globals=globals, resolve=TRUE)
+ gp <- getGlobalsAndPackages(expr, envir = envir, tweak = tweakExpression, globals = globals)
- ## Assign?
- if (length(gp) > 0 && (lazy || assignToTarget)) {
- target <- new.env(parent=envir)
+ ## Assign globals to "target" environment?
+ if (length(gp$globals) > 0 && (lazy || assignToTarget)) {
+ target <- new.env(parent = envir)
+ target[["...future_has_globals"]] <- TRUE
globalsT <- gp$globals
for (name in names(globalsT)) {
target[[name]] <- globalsT[[name]]
@@ -42,17 +45,27 @@ UniprocessFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE,
globalsT <- NULL
envir <- target
}
+
+ ## Record packages?
+ if (length(packages) > 0 || (length(gp$packages) > 0 && lazy)) {
+ packages <- unique(c(gp$packages, packages))
+ }
+
gp <- NULL
- f <- Future(expr=expr, envir=envir, substitute=FALSE, local=local, lazy=lazy, ...)
- structure(f, class=c("UniprocessFuture", class(f)))
+ f <- Future(expr = expr, envir = envir, substitute = FALSE, lazy = lazy, asynchronous = FALSE, local = local, globals = NULL, packages = packages, ...)
+ structure(f, class = c("UniprocessFuture", class(f)))
}
#' @export
run.UniprocessFuture <- function(future, ...) {
+ debug <- getOption("future.debug", FALSE)
+
if (future$state != 'created') {
- stop("A future can only be launched once.")
+ label <- future$label
+ if (is.null(label)) label <- "<none>"
+ stop(sprintf("A future ('%s') can only be launched once.", label))
}
## Assert that the process that created the future is
@@ -73,7 +86,7 @@ run.UniprocessFuture <- function(future, ...) {
current <- sys.nframe()
tryCatch({
withCallingHandlers({
- future$value <- eval(expr, envir=envir)
+ future$value <- eval(expr, envir = envir)
future$state <- 'finished'
}, error = function(ex) {
calls <- sys.calls()
@@ -81,7 +94,7 @@ run.UniprocessFuture <- function(future, ...) {
calls <- calls[seq_len(length(calls)-2L)]
## Drop fluff added by outer tryCatch()
calls <- calls[-seq_len(current+7L)]
- ## Drop fluff added by outer local=TRUE
+ ## Drop fluff added by outer local = TRUE
if (future$local) calls <- calls[-seq_len(6L)]
ex$traceback <- calls
future$value <- ex
@@ -89,9 +102,11 @@ run.UniprocessFuture <- function(future, ...) {
})
}, error = function(ex) {})
+ if (debug) mdebug("%s started (and completed)", class(future)[1])
+
## Signal conditions early, iff specified for the given future
- signalEarly(future, collect=FALSE)
-
+ signalEarly(future, collect = FALSE)
+
invisible(future)
}
@@ -104,25 +119,24 @@ resolved.UniprocessFuture <- function(x, ...) {
## such that the future gets resolved. The reason for this
## is so that polling is always possible, e.g.
## while(!resolved(f)) Sys.sleep(5);
- value(x, signal=FALSE)
+ value(x, signal = FALSE)
}
NextMethod("resolved")
}
-
-#' @rdname UniprocessFuture-class
-#' @export
-EagerFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, globals=TRUE, local=TRUE, lazy=FALSE, ...) {
- if (substitute) expr <- substitute(expr)
- f <- UniprocessFuture(expr=expr, envir=envir, substitute=FALSE, globals=globals, local=local, lazy=FALSE, ...)
- structure(f, class=c("EagerFuture", class(f)))
+globals.UniprocessFuture <- function(future, ...) {
+ envir <- future$envir
+ globals <- names(envir)
+ if (!"...future_has_globals" %in% globals) return(NULL)
+ globals <- setdiff(globals, "...future_has_globals")
+ mget(globals, envir = envir, inherits = FALSE)
}
#' @rdname UniprocessFuture-class
#' @export
-LazyFuture <- function(expr=NULL, envir=parent.frame(), substitute=FALSE, globals=TRUE, local=TRUE, lazy=FALSE, ...) {
+SequentialFuture <- function(expr = NULL, envir = parent.frame(), substitute = FALSE, lazy = FALSE, globals = TRUE, local = TRUE, ...) {
if (substitute) expr <- substitute(expr)
- f <- UniprocessFuture(expr=expr, envir=envir, substitute=FALSE, globals=globals, local=local, lazy=TRUE, ...)
- structure(f, class=c("LazyFuture", class(f)))
+ f <- UniprocessFuture(expr = expr, envir = envir, substitute = FALSE, lazy = lazy, globals = globals, local = local, ...)
+ structure(f, class = c("SequentialFuture", class(f)))
}
diff --git a/R/availableCores.R b/R/availableCores.R
index e942742..c9ebb63 100644
--- a/R/availableCores.R
+++ b/R/availableCores.R
@@ -6,21 +6,25 @@
#' @param constraints An optional character specifying under what
#' constraints ("purposes") we are requesting the values.
#' For instance, on systems where multicore processing is not supported
-#' (i.e. Windows), using \code{constrains="multicore"} will force a
+#' (i.e. Windows), using \code{constrains = "multicore"} will force a
#' single core to be reported.
+#'
#' @param methods A character vector specifying how to infer the number
#' of available cores.
+#'
#' @param na.rm If TRUE, only non-missing settings are considered/returned.
+#'
#' @param default The default number of cores to return if no non-missing
#' settings are available.
+#'
#' @param which A character specifying which settings to return.
#' If \code{"min"}, the minimum value is returned.
#' If \code{"max"}, the maximum value is returned (be careful!)
#' If \code{"all"}, all values are returned.
#'
-#' @return Return a positive (>=1) integer.
-#' If \code{which="all"}, then more than one value may be returned.
-#' Together with \code{na.rm=FALSE} missing values may also be returned.
+#' @return Return a positive (>= 1) integer.
+#' If \code{which = "all"}, then more than one value may be returned.
+#' Together with \code{na.rm = FALSE} missing values may also be returned.
#'
#' @details
#' The following settings ("methods") for inferring the number of cores
@@ -28,12 +32,12 @@
#' \itemize{
#' \item \code{"system"} -
#' Query \code{\link[parallel]{detectCores}()}.
-#' \item \code{"mc.cores+1"} -
+#' \item \code{"mc.cores"} -
#' If available, returns the value of option
-#' \code{\link[base:options]{mc.cores}} + 1.
+#' \code{\link[base:options]{mc.cores}}.
#' Note that \option{mc.cores} is defined as the number of
#' \emph{additional} R processes that can be used in addition to the
-#' main R process. This means that with \code{mc.cores=0} all
+#' main R process. This means that with \code{mc.cores = 0} all
#' calculations should be done in the main R process, i.e. we have
#' exactly one core available for our calculations.
#' The \option{mc.cores} option defaults to environment variable
@@ -41,16 +45,17 @@
#' package is loaded). The \option{mc.cores} option is used by for
#' instance \code{\link[parallel]{mclapply}()}.
#' \item \code{"PBS"} -
-#' Query Torque/PBS environment variable \env{PBS_NUM_PPN}.
-#' Depending on PBS system configuration, this \emph{resource} parameter
-#' may or may not default to one. It can be specified when submitting
-#' a job as in, for instance, \code{qsub -l nodes=4:ppn=2}, which
-#' requests four nodes each with two cores.
+#' Query TORQUE/PBS environment variable \env{PBS_NUM_PPN}.
+#' Depending on PBS system configuration, this \emph{resource}
+#' parameter may or may not default to one.
+#' An example of a job submission that results in this is
+#' \code{qsub -l nodes=1:ppn=2}, which requests one node with two cores.
#' \item \code{"SGE"} -
#' Query Sun/Oracle Grid Engine (SGE) environment variable
#' \env{NSLOTS}.
-#' It can be specified when submitting a job as in, for instance,
-#' \code{qsub -pe by_node 2}, which two cores on a single machine.
+#' An example of a job submission that results in this is
+#' \code{qsub -pe smp 2} (or \code{qsub -pe by_node 2}), which
+#' requests two cores on a single machine.
#' \item \code{"Slurm"} -
#' Query Simple Linux Utility for Resource Management (Slurm)
#' environment variable \env{SLURM_CPUS_PER_TASK}.
@@ -64,19 +69,21 @@
#'
#' @section Advanced usage:
#' It is possible to override the maximum number of cores on the machine
-#' as reported by \code{availableCores(methods="system")}. This can be
+#' as reported by \code{availableCores(methods = "system")}. This can be
#' done by first specifying
-#' \code{options(future.availableCores.methods="mc.cores+1")} and
-#' then the number of cores to use (in addition to the main R process),
-#' e.g. \code{options(mc.cores=8)} will cause the value of
-#' \code{availableCores()} to be 9 (=8+1).
+#' \code{options(future.availableCores.methods = "mc.cores")} and
+#' then the number of cores to use, e.g. \code{options(mc.cores = 8)}.
#' Having said this, it is almost always better to do this by explicitly
#' setting the number of workers when specifying the future strategy,
-#' e.g. \code{plan(multiprocess, workers=9)}.
+#' e.g. \code{plan(multiprocess, workers = 8)}.
+#'
+#' @seealso
+#' To get the number of available workers regardless of machine,
+#' see \code{\link{availableWorkers}()}.
#'
#' @export
#' @keywords internal
-availableCores <- function(constraints=NULL, methods=getOption("future.availableCores.methods", c("system", "mc.cores+1", "_R_CHECK_LIMIT_CORES_", "PBS", "SGE", "Slurm")), na.rm=TRUE, default=c(current=1L), which=c("min", "max", "all")) {
+availableCores <- function(constraints = NULL, methods = getOption("future.availableCores.methods", c("system", "mc.cores", "_R_CHECK_LIMIT_CORES_", "PBS", "SGE", "Slurm", "fallback")), na.rm = TRUE, default = c(current = 1L), which = c("min", "max", "all")) {
## Local functions
getenv <- function(name) {
as.integer(trim(Sys.getenv(name, NA_character_)))
@@ -89,7 +96,7 @@ availableCores <- function(constraints=NULL, methods=getOption("future.available
which <- match.arg(which)
stopifnot(length(default) == 1, is.finite(default), default >= 1L)
- ncores <- rep(NA_integer_, times=length(methods))
+ ncores <- rep(NA_integer_, times = length(methods))
names(ncores) <- methods
for (kk in seq_along(methods)) {
method <- methods[kk]
@@ -97,14 +104,15 @@ availableCores <- function(constraints=NULL, methods=getOption("future.available
## Number of cores assigned by Slurm
n <- getenv("SLURM_CPUS_PER_TASK")
} else if (method == "PBS") {
- ## Number of cores assigned by Torque/PBS
+ ## Number of cores assigned by TORQUE/PBS
n <- getenv("PBS_NUM_PPN")
} else if (method == "SGE") {
## Number of cores assigned by Sun/Oracle Grid Engine (SGE)
n <- getenv("NSLOTS")
} else if (method == "mc.cores") {
- n <- getopt("mc.cores") + 1L
- .Deprecated(msg="Method 'mc.cores' for future::availableCores() is deprecated; use 'mc.cores+1' instead.")
+ ## Number of cores by option defined by 'parallel' package
+ n <- getopt("mc.cores")
+ if (!is.na(n) && n == 0) n <- 1L ## Because options(mc.cores = 0) may be set
} else if (method == "mc.cores+1") {
## Number of cores by option defined by 'parallel' package
n <- getopt("mc.cores") + 1L
@@ -113,14 +121,18 @@ availableCores <- function(constraints=NULL, methods=getOption("future.available
## cores allowed to be use in package tests. Here we
## acknowledge this and sets number of cores to the
## maximum two allowed. This way we don't have to explicitly
- ## use options(mc.cores=2L) in example code, which may be
+ ## use options(mc.cores = 2L) in example code, which may be
## misleading to the reader.
chk <- tolower(Sys.getenv("_R_CHECK_LIMIT_CORES_", ""))
chk <- (nzchar(chk) && (chk != "false"))
- if (chk) n <- 3L ## = 2+1
+ n <- if (chk) 2L else NA_integer_
} else if (method == "system") {
## Number of cores available according to parallel::detectCores()
n <- detectCores()
+ } else if (method == "fallback") {
+ ## Number of cores available according to future.availableCores.fallback
+ n <- getOption("future.availableCores.fallback", NA_integer_)
+ n <- as.integer(n)
} else {
## covr: skip=3
## Fall back to querying option and system environment variable
@@ -136,7 +148,7 @@ availableCores <- function(constraints=NULL, methods=getOption("future.available
ncoresT <- ncoresT[ncoresT <= 0]
if (length(ncoresT) > 0) {
msg <- sprintf("Detected invalid (zero or less) core settings: %s",
- paste(paste0(sQuote(names(ncoresT)), "=", ncoresT), collapse=", "))
+ paste(paste0(sQuote(names(ncoresT)), " = ", ncoresT), collapse = ", "))
mdebug(msg)
stop(msg)
}
@@ -149,17 +161,33 @@ availableCores <- function(constraints=NULL, methods=getOption("future.available
## Fall back to the default?
if (length(ncores) == 0) ncores <- default
- if (which == "min") {
- ## which.min() to preserve name
- ncores <- ncores[which.min(ncores)]
- } else if (which == "max") {
- ## which.max() to preserve name
- ncores <- ncores[which.max(ncores)]
+ ## Keep only one
+ if (length(ncores) >= 2 && (which %in% c("min", "max"))) {
+ ## SPECIAL: The 'fallback' should only be used as a fallback if no other
+ ## options are explicitly set / available.
+ idx_fallback <- which(names(ncores) == "fallback")
+ if (length(idx_fallback) == 1) {
+ ## If 'system' and 'fallback' are the only options, then use 'fallback' ...
+ if (length(ncores) == 2 && "system" %in% names(ncores)) {
+ ncores <- ncores[idx_fallback]
+ } else {
+ ## ... otherwise, ignore 'fallback'.
+ ncores <- ncores[-idx_fallback]
+ }
+ }
+
+ if (which == "min") {
+ ## which.min() to preserve name
+ ncores <- ncores[which.min(ncores)]
+ } else if (which == "max") {
+ ## which.max() to preserve name
+ ncores <- ncores[which.max(ncores)]
+ }
}
if (!is.null(constraints)) {
if (constraints == "multicore") {
- ## SPECIAL: On some supports such as Windows, multicore processing
+ ## SPECIAL: On some OSes such as Windows, multicore processing
## is not supported. If so, we should override all values to
## to reflect that only a single core is available
if (!supportsMulticore()) ncores[] <- 1L
@@ -167,7 +195,7 @@ availableCores <- function(constraints=NULL, methods=getOption("future.available
}
## Sanity check
- stopifnot(all(ncores >= 1L, na.rm=TRUE))
+ stopifnot(all(ncores >= 1L, na.rm = TRUE))
ncores
} # availableCores()
@@ -190,9 +218,8 @@ supportsMulticore <- local({
function() {
if (is.na(supported)) {
ns <- getNamespace("parallel")
- supported <<- exists("mcparallel", mode="function", envir=ns, inherits=FALSE)
+ supported <<- exists("mcparallel", mode = "function", envir = ns, inherits = FALSE)
}
supported
}
})
-
diff --git a/R/availableWorkers.R b/R/availableWorkers.R
new file mode 100644
index 0000000..c59fe1a
--- /dev/null
+++ b/R/availableWorkers.R
@@ -0,0 +1,287 @@
+#' Get set of available workers
+#'
+#' @param methods A character vector specifying how to infer the number
+#' of available cores.
+#'
+#' @param na.rm If TRUE, only non-missing settings are considered/returned.
+#'
+#' @param default The default set of workers.
+#'
+#' @param which A character specifying which set / sets to return.
+#' If \code{"auto"}, the first non-empty set found.
+#' If \code{"min"}, the minimum value is returned.
+#' If \code{"max"}, the maximum value is returned (be careful!)
+#' If \code{"all"}, all values are returned.
+#'
+#' @return Return a character vector of workers, which typically consists
+#' of names of machines / compute nodes, but may also be IP numbers.
+#'
+#' @details
+#' The default set of workers for each method is
+#' \code{rep("localhost", times = availableCores(method))}, which means
+#' that each will at least use as many parallel workers on the current
+#' machine that \code{\link{availableCores}()} allows for that method.
+#'
+#' In addition, the following settings ("methods") are also acknowledged:
+#' \itemize{
+#' \item \code{"PBS"} -
+#' Query TORQUE/PBS environment variable \env{PBS_NODEFILE}.
+#' If this is set and specifies an existing file, then the set
+#' of workers is read from that file, where one worker (node)
+#' is given per line.
+#' An example of a job submission that results in this is
+#' \code{qsub -l nodes = 4:ppn = 2}, which requests four nodes each
+#' with two cores.
+#' \item \code{"SGE"} -
+#' Query Sun/Oracle Grid Engine (SGE) environment variable
+#' \env{PE_HOSTFILE}.
+#' An example of a job submission that results in this is
+#' \code{qsub -pe mpi 8} (or \code{qsub -pe ompi 8}), which
+#' requests eight cores on a any number of machines.
+#' }
+#'
+#' @seealso
+#' To get the number of available workers on the current machine,
+#' see \code{\link{availableCores}()}.
+#'
+#' @importFrom utils file_test
+#' @export
+#' @keywords internal
+availableWorkers <- function(methods = getOption("future.availableWorkers.methods", c("mc.cores", "_R_CHECK_LIMIT_CORES_", "PBS", "SGE", "Slurm", "system", "fallback")), na.rm = TRUE, default = "localhost", which = c("auto", "min", "max", "all")) {
+ ## Local functions
+ getenv <- function(name) {
+ as.character(trim(Sys.getenv(name, NA_character_)))
+ }
+
+ getopt <- function(name) {
+ as.character(getOption(name, NA_character_))
+ }
+
+ split <- function(s) {
+ x <- unlist(strsplit(s, split = "[,]", fixed = FALSE), use.names = FALSE)
+ x <- trim(x)
+ x <- x[nzchar(x)]
+ x
+ }
+
+ which <- match.arg(which)
+ stopifnot(is.character(default), length(default) >= 1, !anyNA(default))
+
+
+ ## Default is to use the current machine
+ ncores <- availableCores(methods = methods, na.rm = FALSE, which = "all")
+
+ workers <- lapply(ncores, FUN = function(n) {
+ if (length(n) == 0 || is.na(n)) n <- 0L
+ rep("localhost", times = n)
+ })
+
+ ## Acknowledge known HPC settings (skip others)
+ methods_localhost <- c("_R_CHECK_LIMIT_CORES_", "mc.cores", "mc.cores+1", "system")
+ methodsT <- setdiff(methods, methods_localhost)
+ for (method in methodsT) {
+ if (method == "PBS") {
+ pathname <- getenv("PBS_NODEFILE")
+ if (is.na(pathname)) next
+ if (!file_test("-f", pathname)) {
+ warning(sprintf("Environment variable %s was set but no such file %s exists", sQuote("PBS_NODEFILE"), sQuote(pathname)))
+ next
+ }
+ data <- read_pbs_nodefile(pathname)
+ w <- data$node
+
+ ## Sanity checks
+ pbs_np <- as.integer(getenv("PBS_NP"))
+ if (!identical(pbs_np, length(w))) {
+ warning(sprintf("Identified %d workers from the %s file (%s), which does not match environment variable %s = %d", length(w), sQuote("PBS_NODEFILE"), sQuote(pathname), sQuote("PBS_NP"), pbs_np))
+ }
+
+ pbs_nodes <- as.integer(getenv("PBS_NUM_NODES"))
+ pbs_ppn <- as.integer(getenv("PBS_NUM_PPN"))
+ pbs_np <- pbs_nodes * pbs_ppn
+ if (!identical(pbs_np, length(w))) {
+ warning(sprintf("Identified %d workers from the %s file (%s), which does not match environment variables %s * %s = %d * %d = %d", length(w), sQuote("PBS_NODEFILE"), sQuote(pathname), sQuote("PBS_NUM_NODES"), sQuote("PBS_NUM_PPN"), pbs_nodes, pbs_ppn, pbs_np))
+ }
+
+ ## TO DO: Add validation of 'w' (from PBS_HOSTFILE) toward
+ ## counts in PBS_NP and / or PBS_NUM_NODES * PBS_NUM_PPN.
+ } else if (method == "SGE") {
+ pathname <- getenv("PE_HOSTFILE")
+ if (is.na(pathname)) next
+ if (!file_test("-f", pathname)) {
+ warning(sprintf("Environment variable %s was set but no such file %s exists", sQuote("PE_HOSTFILE"), sQuote(pathname)))
+ next
+ }
+ data <- read_pe_hostfile(pathname)
+ w <- expand_nodes(data)
+
+ ## Sanity checks
+ nslots <- as.integer(getenv("NSLOTS"))
+ if (!identical(nslots, length(w))) {
+ warning(sprintf("Identified %d workers from the %s file (%s), which does not match environment variable %s = %d", length(w), sQuote("PE_HOSTFILE"), sQuote(pathname), sQuote("NSLOTS"), nslots))
+ }
+ } else {
+ ## Fall back to querying option and system environment variable
+ ## with the given name
+ w <- getopt(method)
+ if (is.na(w)) w <- getenv(method)
+ if (is.na(w)) next
+ w <- split(w)
+ }
+
+ ## Drop missing or empty values?
+ if (na.rm) w <- w[!is.na(w)]
+
+ workers[[method]] <- w
+ }
+
+
+ nnodes <- unlist(lapply(workers, FUN = length), use.names = TRUE)
+
+ if (which == "auto") {
+ ## For default localhost sets, use the minimum allowed number of
+ ## workers **according to availableCores()**.
+ methodsT <- intersect(names(workers), methods_localhost)
+ methodsT <- methodsT[is.finite(ncores[methodsT]) & ncores[methodsT] > 0]
+ if (length(methodsT) > 1L) {
+ min <- min(ncores[methodsT], na.rm = TRUE)
+ if (is.finite(min)) {
+ nnodes[methodsT] <- min
+ workers[methodsT] <- list(rep("localhost", times = min))
+ }
+ }
+
+ workers <- apply_fallback(workers)
+ nnodes <- unlist(lapply(workers, FUN = length), use.names = TRUE)
+
+ ## Now, pick the first positive and finite value
+ idx <- which(nnodes > 0L, useNames = FALSE)[1]
+ workers <- if (is.na(idx)) character(0L) else workers[[idx]]
+ } else if (which == "min") {
+ workers <- apply_fallback(workers)
+ nnodes <- unlist(lapply(workers, FUN = length), use.names = TRUE)
+ idx <- which.min(nnodes)
+ workers <- workers[[idx]]
+ } else if (which == "max") {
+ workers <- apply_fallback(workers)
+ nnodes <- unlist(lapply(workers, FUN = length), use.names = TRUE)
+ idx <- which.max(nnodes)
+ workers <- workers[[idx]]
+ }
+
+ ## Fall back to default?
+ if (is.character(workers) && length(workers) == 0) workers <- default
+
+ ## Sanity checks
+ min_count <- as.integer(na.rm)
+ if (is.list(workers)) {
+ lapply(workers, FUN = function(w) {
+ stopifnot(is.character(w), length(w) >= 0L, all(nchar(w) > 0))
+ })
+ } else {
+ stopifnot(is.character(workers), length(workers) >= min_count, all(nchar(workers) > 0))
+ }
+
+ workers
+} # availableWorkers()
+
+
+
+#' @importFrom utils read.table
+read_pbs_nodefile <- function(pathname, sort = TRUE) {
+ ## One (node) per line
+ lines <- readLines(pathname, warn = FALSE)
+ lines <- trim(lines)
+
+ ## Sanity checks
+ stopifnot(
+ all(nzchar(lines)),
+ !anyNA(lines),
+ !any(grepl("[[:space:]]", lines))
+ )
+
+ data <- data.frame(node = lines, stringsAsFactors = FALSE)
+
+ if (sort) {
+ data <- data[order(data$node), , drop = FALSE]
+ }
+
+ data
+}
+
+
+#' @importFrom utils read.table
+read_pe_hostfile <- function(pathname, sort = TRUE) {
+ ## One (node, ncores, queue, comment) per line, e.g.
+ ## opt88 3 short.q at opt88 UNDEFINED
+ ## iq242 2 short.q at iq242 UNDEFINED
+ ## opt116 1 short.q at opt116 UNDEFINED
+ data <- read.table(pathname, header = FALSE, sep = " ", stringsAsFactors = FALSE)
+
+ ## Sanity checks
+ stopifnot(ncol(data) >= 2)
+
+ colnames(data)[1:2] <- c("node", "count")
+ if (ncol(data) >= 3) colnames(data)[3] <- "via"
+ if (ncol(data) >= 4) colnames(data)[4] <- "notes"
+
+ stopifnot(
+ is.character(data$node),
+ !anyNA(data$node),
+ !any(grepl("[[:space:]]", data$node)),
+ is.integer(data$count),
+ !anyNA(data$count),
+ all(is.finite(data$count)),
+ all(data$count > 0)
+ )
+
+ if (sort) {
+ data <- data[order(data$node, data$count), , drop = FALSE]
+ }
+
+ data
+}
+
+## Used after read_pe_hostfile()
+expand_nodes <- function(data) {
+ nodes <- mapply(data$node, data$count, FUN = function(node, count) {
+ rep(node, times = count)
+ }, SIMPLIFY = FALSE, USE.NAMES = FALSE)
+ unlist(nodes, recursive = FALSE, use.names = FALSE)
+}
+
+## Used by availableWorkers()
+apply_fallback <- function(workers) {
+ ## No 'fallback'?
+ idx_fallback <- which(names(workers) == "fallback")
+ if (length(idx_fallback) == 0) return(workers)
+
+ ## Number of workers per set
+ nnodes <- unlist(lapply(workers, FUN = length), use.names = TRUE)
+
+ ## No 'fallback' workers?
+ if (nnodes[idx_fallback] == 0) return(workers)
+
+ ## Only consider non-empty sets
+ nonempty <- which(nnodes > 0)
+ workers_nonempty <- workers[nonempty]
+
+ ## Nothing to do?
+ n_nonempty <- length(workers_nonempty)
+ if (n_nonempty <= 1) return(workers)
+
+ ## Drop 'fallback'?
+ if (n_nonempty > 2) {
+ workers <- workers[-idx_fallback]
+ return(workers)
+ }
+
+ ## No 'system' to override?
+ idx_system <- which(names(workers) == "system")
+ if (length(idx_system) == 0) return(workers)
+
+ ## Drop 'system' in favor or 'fallback'
+ workers <- workers[-idx_system]
+
+ workers
+} ## apply_fallback()
diff --git a/R/backtrace.R b/R/backtrace.R
index b8feec9..0dbfc7f 100644
--- a/R/backtrace.R
+++ b/R/backtrace.R
@@ -7,13 +7,13 @@
#' @return A list of calls.
#'
#' @export
-backtrace <- function(future, envir=parent.frame(), ...) {
+backtrace <- function(future, envir = parent.frame(), ...) {
## Argument 'expr':
expr <- substitute(future)
if (!is.null(expr)) {
- target <- parse_env_subset(expr, envir=envir, substitute=FALSE)
- future <- get_future(target, mustExist=TRUE)
+ target <- parse_env_subset(expr, envir = envir, substitute = FALSE)
+ future <- get_future(target, mustExist = TRUE)
}
if (!resolved(future)) {
@@ -32,4 +32,3 @@ backtrace <- function(future, envir=parent.frame(), ...) {
calls
} ## backtrace()
-
diff --git a/R/cluster.R b/R/cluster.R
index eff61bf..b594ad8 100644
--- a/R/cluster.R
+++ b/R/cluster.R
@@ -1,4 +1,4 @@
-#' Create a cluster future whose value will be resolved asynchroneously in a parallel process
+#' Create a cluster future whose value will be resolved asynchronously in a parallel process
#'
#' A cluster future is a future that uses cluster evaluation,
#' which means that its \emph{value is computed and resolved in
@@ -36,17 +36,16 @@
#' and \code{\link{\%<-\%}} will create \emph{cluster futures}.
#'
#' @export
-cluster <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, persistent=FALSE, workers=NULL, user=NULL, revtunnel=TRUE, homogeneous=TRUE, gc=FALSE, earlySignal=FALSE, label=NULL, ...) {
- ## BACKWARD COMPATIBILITY
- args <- list(...)
- if ("cluster" %in% names(args)) {
- workers <- args$cluster
- .Deprecated(msg="Argument 'cluster' has been renamed to 'workers'. Please update your script/code that uses the future package.")
+cluster <- function(expr, envir = parent.frame(), substitute = TRUE, lazy = FALSE, seed = NULL, globals = TRUE, persistent = FALSE, workers = availableWorkers(), user = NULL, revtunnel = TRUE, homogeneous = TRUE, gc = FALSE, earlySignal = FALSE, label = NULL, ...) {
+ if ("cluster" %in% names(list(...))) {
+ .Defunct(msg = "Argument 'cluster' has been renamed to 'workers'. Please update your script/code that uses the future package.")
}
if (substitute) expr <- substitute(expr)
- future <- ClusterFuture(expr=expr, envir=envir, substitute=FALSE, globals=globals, persistent=persistent, workers=workers, user=user, revtunnel=revtunnel, homogeneous=homogeneous, gc=gc, earlySignal=earlySignal, label=label, ...)
- run(future)
+ future <- ClusterFuture(expr = expr, envir = envir, substitute = FALSE, lazy = lazy, seed = seed, globals = globals, persistent = persistent, workers = workers, user = user, revtunnel = revtunnel, homogeneous = homogeneous, gc = gc, earlySignal = earlySignal, label = label, ...)
+ if (!future$lazy) future <- run(future)
+ invisible(future)
}
class(cluster) <- c("cluster", "multiprocess", "future", "function")
+attr(cluster, "init") <- TRUE
diff --git a/R/constant.R b/R/constant.R
index 53d611f..951cc65 100644
--- a/R/constant.R
+++ b/R/constant.R
@@ -1,5 +1,5 @@
## Used only internally
constant <- function(value, ...) {
- ConstantFuture(value=value, ...)
+ ConstantFuture(value = value, ...)
}
class(constant) <- c("constant", "uniprocess", "future", "function")
diff --git a/R/flapply.R b/R/flapply.R
deleted file mode 100644
index 2e73e40..0000000
--- a/R/flapply.R
+++ /dev/null
@@ -1,36 +0,0 @@
-
-#' Apply a Function over a List or Vector via Futures
-#'
-#' @param x A vector-like object to iterate over.
-#' @param FUN A function taking at least one argument.
-#' @param ... (optional) Additional arguments pass to \code{FUN()}.
-#'
-#' @return A list.
-#'
-#' @keywords internal
-flapply <- function(x, FUN, ...) {
- stopifnot(is.function(FUN))
-
- ## 1. Collect all globals (once)
- envir <- environment()
- x_ii <- NULL
- globals <- c("FUN", "x_ii", names(list(...)), "...")
- globals <- globalsByName(globals, envir=envir, mustExist=FALSE)
-
- ## 2. Apply function with fixed set of globals
- fs <- vector("list", length = length(x))
- names(fs) <- names(x)
-
- for (ii in seq_along(x)) {
- ## Subsetting outside future is more efficient
- globals$x_ii <- x[[ii]]
-
- fs[[ii]] <- future(FUN(x_ii, ...), envir=envir, globals=globals)
- }
-
- ## Not needed anymore
- rm(list=c("x_ii", "globals", "ii", "envir"))
-
- ## Resolve and return as values
- values(fs)
-} # flapply()
diff --git a/R/future.R b/R/future.R
index 0b70538..2bd2f27 100644
--- a/R/future.R
+++ b/R/future.R
@@ -6,21 +6,32 @@
#' using \code{\link{plan}()} such that it is evaluated in parallel on,
#' for instance, the current machine, on a remote machine, or via a
#' job queue on a compute cluster.
-#' Importantly, \R code using futures remains the same regardless
-#' on these settings.
+#' Importantly, any \R code using futures remains the same regardless
+#' on these settings and there is no need to modify the code when
+#' switching from, say, sequential to parallel processing.
+#'
+#' @param expr,value An \R \link[base]{expression} to be evaluated.
#'
-#' @param expr An \R \link[base]{expression}.
#' @param envir The \link{environment} from where global
-#' objects should be identified. Depending on "evaluator",
-#' it may also be the environment in which the expression
-#' is evaluated.
+#' objects should be identified. Depending on the future
+#' strategy (the \code{evaluator}), it may also be the environment
+#' in which the expression is evaluated.
+#'
#' @param substitute If TRUE, argument \code{expr} is
#' \code{\link[base]{substitute}()}:ed, otherwise not.
+#'
#' @param globals A logical, a character vector,
#' or a named list for controlling how globals are handled.
#' For details, see below section.
-#' This argument can be specified via the \dots arguments
-#' for \code{future()} and \code{futureCall()}.
+#'
+#' @param packages (optional) a character vector specifying packages
+#' to be attached in the R environment evaluating the future.
+#'
+#' @param lazy Specifies whether a future should be resolved
+#' lazily or eagerly (default).
+#'
+#' @param seed (optional) A L'Ecuyer-CMRG RNG seed.
+#'
#' @param evaluator The actual function that evaluates
#' the future expression and returns a \link{Future}.
#' The evaluator function should accept all of the same
@@ -28,22 +39,12 @@
#' (except \code{evaluator}, \code{FUN} and \code{args}).
#' The default evaluator function is the one that the user
#' has specified via \code{\link{plan}()}.
+#'
#' @param ... Additional arguments passed to the "evaluator".
#'
#' @return
#' \code{f <- future(expr)} creates a \link{Future} \code{f} that evaluates expression \code{expr}, the value of the future is retrieved using \code{v <- value(f)}.
#'
-#' \code{f <- futureCall(FUN, args)} creates a \link{Future} \code{f} that calls function \code{FUN} with arguments \code{args}, where the value of the future is retrieved using \code{v <- value(f)}.
-#'
-#' \code{futureAssign("v", expr)} and \code{v \%<-\% expr} (a future assignment) create a \link{Future} that evaluates expression \code{expr} and binds its value (as a \link[base]{promise}) to a variable \code{v}. The value of the future is automatically retrieved when the assigned variable (promise) is queried.
-#' The future itself is returned invisibly, e.g.
-#' \code{f <- futureAssign("v", expr)} and \code{f <- (v \%<-\% expr)}.
-#' Alternatively, the future of a future variable \code{v} can be retrieved
-#' without blocking using \code{f <- \link{futureOf}(v)}.
-#' Both the future and the variable (promise) are assigned to environment
-#' \code{assign.env} where the name of the future is \code{.future_<name>}.
-#'
-#'
#' @details
#' The state of a future is either unresolved or resolved.
#' The value of a future can be retrieved using \code{v <- \link{value}(f)}.
@@ -52,22 +53,38 @@
#' It is possible to check whether a future is resolved or not
#' without blocking by using \code{\link{resolved}(f)}.
#'
-#' For a future created via a future assignment, the value is bound to
-#' a promise, which when queried will internally call \code{\link{value}()}
-#' on the future and which will then be resolved into a regular variable
-#' bound to that value. For example, with future assignment
-#' \code{v \%<-\% expr}, the first time variable \code{v} is queried
-#' the call blocks if (and only if) the future is not yet resolved. As soon
-#' as it is resolved, and any succeeding queries, querying \code{v} will
-#' immediately give the value.
+#' For a future created via a future assignment
+#' (\code{x \%<-\% value} or \code{futureAssign("x", value)}), the value
+#' is bound to a promise, which when queried will internally call
+#' \code{\link{value}()} on the future and which will then be resolved
+#' into a regular variable bound to that value. For example, with future
+#' assignment \code{x \%<-\% value}, the first time variable \code{x} is
+#' queried the call blocks if (and only if) the future is not yet resolved.
+#' As soon as it is resolved, and any succeeding queries, querying \code{x}
+#' will immediately give the value.
+#'
+#' The future assignment construct \code{x \%<-\% value} is not a formal
+#' assignment per se, but a binary infix operator on objects \code{x}
+#' and expression \code{value}. However, by using non-standard evaluation,
+#' this constructs can emulate an assignment operator similar to
+#' \code{x <- value}. Due to \R's precedence rules of operators,
+#' future expressions often needs to be explicitly bracketed, e.g.
+#' \code{x \%<-\% { a + b }}.
+#'
+#'
+#' @section Eager or lazy evaluation:
+#' By default, a future is resolved using \emph{eager} evaluation
+#' (\code{lazy = FALSE}). This means that the expression starts to
+#' be evaluated as soon as the future is created.
+#'
+#' As an alternative, the future can be resolved using \emph{lazy}
+#' evaluation (\code{lazy = TRUE}). This means that the expression
+#' will only be evaluated when the value of the future is requested.
+#' \emph{Note that this means that the expression may not be evaluated
+#' at all - it is guaranteed to be evaluated if the value is requested}.
#'
-#' The future assignment construct \code{v \%<-\% expr} is not a formal
-#' assignment per se, but a binary infix operator on objects \code{v}
-#' and \code{expr}. However, by using non-standard evaluation, this
-#' constructs can emulate an assignment operator similar to
-#' \code{v <- expr}. Due to \R's precedence rules of operators,
-#' future expressions that contain multiple statements need to be
-#' explicitly bracketed, e.g. \code{v \%<-\% { a <- 2; a^2 }}.
+#' For future assignments, lazy evaluation can be controlled via the
+#' \code{\%lazy\%} operator, e.g. \code{x \%<-\% { expr } \%lazy\% TRUE}.
#'
#'
#' @section Globals used by future expressions:
@@ -90,7 +107,7 @@
#' More precisely, globals are identified via code inspection of the
#' future expression \code{expr} and their values are retrieved with
#' environment \code{envir} as the starting point (basically via
-#' \code{get(global, envir=envir, inherits=TRUE)}).
+#' \code{get(global, envir = envir, inherits = TRUE)}).
#' \emph{In most cases, such automatic collection of globals is sufficient
#' and less tedious and error prone than if they are manually specified}.
#'
@@ -116,7 +133,7 @@
#'
#' Specifying globals explicitly avoids the overhead added from
#' automatically identifying the globals and gathering their values.
-#' Futhermore, if we know that the future expression does not make use
+#' Furthermore, if we know that the future expression does not make use
#' of any global variables, we can disable the automatic search for
#' globals by using
#' \preformatted{
@@ -174,24 +191,26 @@
#'
#'
#' @seealso
-#' How, when and where futures are resolved is given by the future strategy,
-#' which can be set by the \code{\link{plan}()} function.
+#' How, when and where futures are resolved is given by the
+#' \emph{future strategy}, which can be set by the end user using the
+#' \code{\link{plan}()} function. The future strategy must not be
+#' set by the developer, e.g. it must not be called within a package.
#'
#' @aliases futureCall
#' @export
#' @name future
-future <- function(expr, envir=parent.frame(), substitute=TRUE, evaluator=plan("next"), ...) {
+future <- function(expr, envir = parent.frame(), substitute = TRUE, globals = TRUE, packages = NULL, lazy = FALSE, seed = NULL, evaluator = plan("next"), ...) {
if (substitute) expr <- substitute(expr)
if (!is.function(evaluator)) {
stop("Argument 'evaluator' must be a function: ", typeof(evaluator))
}
- future <- evaluator(expr, envir=envir, substitute=FALSE, ...)
+ future <- evaluator(expr, envir = envir, substitute = FALSE, lazy = lazy, seed = seed, globals = globals, packages = packages, ...)
## Assert that a future was returned
if (!inherits(future, "Future")) {
- stop("Argument 'evaluator' specifies a function that does not return a Future object: ", paste(sQuote(class(future)), collapse=", "))
+ stop("Argument 'evaluator' specifies a function that does not return a Future object: ", paste(sQuote(class(future)), collapse = ", "))
}
future
diff --git a/R/futureAssign.R b/R/futureAssign.R
index ddc3abc..abdaa51 100644
--- a/R/futureAssign.R
+++ b/R/futureAssign.R
@@ -1,22 +1,49 @@
-#' @param x the name of a future variable.
-#' @param value the \R \link[base]{expression} to be evaluated in
-#' the future and whose value will be assigned to the variable.
+#' @param x the name of a future variable, which will hold the value
+#' of the future expression (as a promise).
+#'
#' @param assign.env The \link[base]{environment} to which the variable
#' should be assigned.
#'
+#' @return
+#' \code{x \%<-\% value} (a future assignment) and
+#' \code{futureAssign("x", value)} create a \link{Future} that evaluates
+#' expression \code{expr} and binds its value (as a \link[base]{promise}) to
+#' a variable \code{x}. The value of the future is automatically retrieved
+#' when the assigned variable (promise) is queried.
+#' The future itself is returned invisibly, e.g.
+#' \code{f <- futureAssign("x", expr)} and \code{f <- (x \%<-\% expr)}.
+#' Alternatively, the future of a future variable \code{x} can be retrieved
+#' without blocking using \code{f <- \link{futureOf}(x)}.
+#' Both the future and the variable (promise) are assigned to environment
+#' \code{assign.env} where the name of the future is \code{.future_<name>}.
+#'
#' @rdname future
#' @export
-futureAssign <- function(x, value, envir=parent.frame(), assign.env=envir, substitute=TRUE) {
+futureAssign <- function(x, value, envir = parent.frame(), substitute = TRUE, lazy = FALSE, seed = NULL, globals = TRUE, ..., assign.env = envir) {
stopifnot(is.character(x), !is.na(x), nzchar(x))
if (substitute) value <- substitute(value)
+
+ ## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ ## (1) Arguments passed to future()
+ ## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ future.args <- list(value, envir = envir, lazy = lazy, seed = seed, globals = globals, ...)
+
+ ## Any arguments set via disposible option?
+ args <- getOption("future.disposable", NULL)
+ if (!is.null(args)) {
+ for (name in names(args)) future.args[name] <- args[name]
+ on.exit(options(future.disposable = NULL))
+ }
+
+
## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- ## (1) Create future
+ ## (2) Create future
## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
## Name of "future" saved in parallel with the "promise"
future_name <- sprintf(".future_%s", x)
- if (exists(future_name, envir=envir)) {
- msg <- sprintf("A future with name %s already exists in environment %s: %s", sQuote(future_name), sQuote(environmentName(envir)), hpaste(ls(envir=envir, all.names=TRUE)))
+ if (exists(future_name, envir = envir)) {
+ msg <- sprintf("A future with name %s already exists in environment %s: %s", sQuote(future_name), sQuote(environmentName(envir)), hpaste(ls(envir = envir, all.names = TRUE)))
## warning(msg)
}
@@ -24,13 +51,12 @@ futureAssign <- function(x, value, envir=parent.frame(), assign.env=envir, subst
## a variable as a "promise".
## NOTE: We make sure to pass 'envir' in order for globals to
## be located properly.
- future.args <- list(value, envir=envir)
- future <- do.call(future::future, args=future.args, envir=assign.env)
+ future <- do.call(future::future, args = future.args, envir = assign.env)
## Assign future to assignment environment
future_without_gc <- future
future_without_gc$.gcenv <- NULL
- assign(future_name, future_without_gc, envir=assign.env)
+ assign(future_name, future_without_gc, envir = assign.env)
## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -44,9 +70,9 @@ futureAssign <- function(x, value, envir=parent.frame(), assign.env=envir, subst
delayedAssign(x, local({
value <- value(future)
## Remove internal future variable
- rm(list=future_name, envir=assign.env)
+ rm(list = future_name, envir = assign.env)
value
- }), eval.env=env, assign.env=assign.env)
+ }), eval.env = env, assign.env = assign.env)
invisible(future)
}
diff --git a/R/futureAssign_OP.R b/R/futureAssign_OP.R
index 49355f3..0a8ba17 100644
--- a/R/futureAssign_OP.R
+++ b/R/futureAssign_OP.R
@@ -1,61 +1,42 @@
#' @usage x \%<-\% value
#'
#' @rdname future
-#' @aliases %<-% %->% %<=% %=>%
-#' @export %<-% %->% %<=% %=>%
+#' @aliases %<-% %->%
+#' @export %<-% %->%
`%<-%` <- function(x, value) {
target <- substitute(x)
expr <- substitute(value)
envir <- parent.frame(1)
- futureAssignInternal(target, expr, envir=envir, substitute=FALSE)
+ futureAssignInternal(target, expr, envir = envir, substitute = FALSE)
}
`%->%` <- function(value, x) {
target <- substitute(x)
expr <- substitute(value)
envir <- parent.frame(1)
- futureAssignInternal(target, expr, envir=envir, substitute=FALSE)
-}
-
-## Deprecated
-`%<=%` <- function(x, value) {
- target <- substitute(x)
- expr <- substitute(value)
- envir <- parent.frame(1)
- .Deprecated("%<-%")
- futureAssignInternal(target, expr, envir=envir, substitute=FALSE)
-}
-
-`%=>%` <- function(value, x) {
- target <- substitute(x)
- expr <- substitute(value)
- envir <- parent.frame(1)
- .Deprecated("%->%")
- futureAssignInternal(target, expr, envir=envir, substitute=FALSE)
+ futureAssignInternal(target, expr, envir = envir, substitute = FALSE)
}
#' @importFrom listenv get_variable parse_env_subset
-futureAssignInternal <- function(target, expr, envir=parent.frame(), substitute=FALSE) {
- target <- parse_env_subset(target, envir=envir, substitute=substitute)
+futureAssignInternal <- function(target, expr, envir = parent.frame(), substitute = FALSE) {
+ target <- parse_env_subset(target, envir = envir, substitute = substitute)
assign.env <- target$envir
name <- target$name
if (inherits(target$envir, "listenv")) {
if (target$exists) {
- name <- get_variable(target$envir, target$idx, mustExist=TRUE, create=FALSE)
+ name <- get_variable(target$envir, target$idx, mustExist = TRUE, create = FALSE)
} else {
if (!is.na(name) && nzchar(name)) {
- name <- get_variable(target$envir, name, mustExist=FALSE, create=TRUE)
+ name <- get_variable(target$envir, name, mustExist = FALSE, create = TRUE)
} else if (all(is.finite(target$idx))) {
- name <- get_variable(target$envir, target$idx, mustExist=FALSE, create=TRUE)
+ name <- get_variable(target$envir, target$idx, mustExist = FALSE, create = TRUE)
} else {
- msg <- "INTERNAL ERROR: Zero length variable name and unknown index."
- mdebug(msg)
- stop(msg)
+ stop("INTERNAL ERROR: Zero length variable name and unknown index.")
}
}
}
- futureAssign(name, expr, envir=envir, assign.env=assign.env, substitute=FALSE)
+ futureAssign(name, expr, envir = envir, assign.env = assign.env, substitute = FALSE)
} # futureAssignInternal()
diff --git a/R/futureCall.R b/R/futureCall.R
index 8e05dc9..4042eaa 100644
--- a/R/futureCall.R
+++ b/R/futureCall.R
@@ -1,16 +1,19 @@
-#' @param FUN A \link[base]{function} object.
+#' @param FUN A \link[base]{function} to be evaluated.
+#'
#' @param args A \link[base]{list} of arguments passed to function \code{FUN}.
#'
-#' @rdname future
+#' @return
+#' \code{f <- futureCall(FUN, args)} creates a \link{Future} \code{f} that calls function \code{FUN} with arguments \code{args}, where the value of the future is retrieved using \code{x <- value(f)}.
#'
+#' @rdname future
#' @export
-futureCall <- function(FUN, args=NULL, envir=parent.frame(), globals=TRUE, evaluator=plan("next"), ...) {
+futureCall <- function(FUN, args = NULL, envir = parent.frame(), lazy = FALSE, seed = NULL, globals = TRUE, evaluator = plan("next"), ...) {
stopifnot(is.function(FUN))
stopifnot(is.list(args))
- envirT <- new.env(parent=envir)
+ envirT <- new.env(parent = envir)
envirT$FUN <- FUN
envirT$args <- args
- future(do.call(what=FUN, args=args), substitute=TRUE, envir=envirT, globals=globals, evaluator=evaluator)
-} # futureCall()
+ future(do.call(what = FUN, args = args), substitute = TRUE, envir = envirT, lazy = lazy, seed = seed, globals = globals, evaluator = evaluator)
+}
diff --git a/R/futureOf.R b/R/futureOf.R
index 35cb07a..3752d25 100644
--- a/R/futureOf.R
+++ b/R/futureOf.R
@@ -19,15 +19,15 @@
#'
#' @export
#' @importFrom listenv map parse_env_subset
-futureOf <- function(var=NULL, envir=parent.frame(), mustExist=TRUE, default=NA, drop=FALSE) {
+futureOf <- function(var = NULL, envir = parent.frame(), mustExist = TRUE, default = NA, drop = FALSE) {
## Argument 'expr':
expr <- substitute(var)
## Inspect by expression?
if (!is.null(expr)) {
- target <- parse_env_subset(expr, envir=envir, substitute=FALSE)
- future <- get_future(target, mustExist=mustExist)
+ target <- parse_env_subset(expr, envir = envir, substitute = FALSE)
+ future <- get_future(target, mustExist = mustExist)
return(future)
}
@@ -40,26 +40,26 @@ futureOf <- function(var=NULL, envir=parent.frame(), mustExist=TRUE, default=NA,
names(res) <- names(map)
for (idx in seq_along(res)) {
- target <- parse_env_subset(idx, envir=envir, substitute=FALSE)
- future <- get_future(target, mustExist=FALSE, default=default)
+ target <- parse_env_subset(idx, envir = envir, substitute = FALSE)
+ future <- get_future(target, mustExist = FALSE, default = default)
if (!is.null(future) || !is.atomic(future) || !is.na(future)) {
res[[idx]] <- future
}
}
} else {
## names(x) is only supported in R (>= 3.2.0)
- vars <- ls(envir=envir, all.names=TRUE)
- vars <- grep("^.future_", vars, invert=TRUE, value=TRUE)
- res <- lapply(vars, FUN=function(var) {
- target <- parse_env_subset(var, envir=envir, substitute=FALSE)
- get_future(target, mustExist=FALSE, default=default)
+ vars <- ls(envir = envir, all.names = TRUE)
+ vars <- grep("^.future_", vars, invert = TRUE, value = TRUE)
+ res <- lapply(vars, FUN = function(var) {
+ target <- parse_env_subset(var, envir = envir, substitute = FALSE)
+ get_future(target, mustExist = FALSE, default = default)
})
names(res) <- vars
}
## Keep only futures?
if (drop && length(res) > 0) {
- keep <- sapply(res, FUN=inherits, "Future")
+ keep <- sapply(res, FUN = inherits, "Future")
res <- res[keep]
} else {
## Preserve dimensions
@@ -74,14 +74,14 @@ futureOf <- function(var=NULL, envir=parent.frame(), mustExist=TRUE, default=NA,
}
-get_future <- function(target, mustExist=TRUE, default=NA) {
+get_future <- function(target, mustExist = TRUE, default = NA) {
res <- default
if (!target$exists) {
msg <- sprintf("No such future variable: %s", target$code)
if (mustExist) {
mdebug("ERROR: %s", msg)
- stop(msg, call.=FALSE)
+ stop(msg, call. = FALSE)
}
attr(res, "reason") <- msg
return(res)
@@ -99,13 +99,13 @@ get_future <- function(target, mustExist=TRUE, default=NA) {
name <- target$name
}
future_name <- sprintf(".future_%s", name)
- if (exists(future_name, envir=envir, inherits=FALSE)) {
- return(get(future_name, envir=envir, inherits=FALSE))
+ if (exists(future_name, envir = envir, inherits = FALSE)) {
+ return(get(future_name, envir = envir, inherits = FALSE))
}
## (b) Check if element itself is a future object
- if (exists(name, envir=envir, inherits=FALSE)) {
- future <- get(name, envir=envir, inherits=FALSE)
+ if (exists(name, envir = envir, inherits = FALSE)) {
+ future <- get(name, envir = envir, inherits = FALSE)
if (inherits(future, "Future")) return(future)
}
@@ -113,7 +113,7 @@ get_future <- function(target, mustExist=TRUE, default=NA) {
msg <- sprintf("Future (%s) not found in %s %s: %s", sQuote(name), class(envir)[1], sQuote(envirName), sQuote(target$code))
if (mustExist) {
mdebug("ERROR: %s", msg)
- stop(msg, call.=FALSE)
+ stop(msg, call. = FALSE)
}
attr(res, "reason") <- msg
diff --git a/R/future_lapply.R b/R/future_lapply.R
new file mode 100644
index 0000000..8eda9c9
--- /dev/null
+++ b/R/future_lapply.R
@@ -0,0 +1,398 @@
+#' Apply a Function over a List or Vector via Futures
+#'
+#' @param x A vector-like object to iterate over.
+#'
+#' @param FUN A function taking at least one argument.
+#'
+#' @param ... (optional) Additional arguments pass to \code{FUN()}.
+#'
+#' @param future.globals A logical, a character vector, or a named list for
+#' controlling how globals are handled. For details, see below section.
+#'
+#' @param future.packages (optional) a character vector specifying packages
+#' to be attached in the R environment evaluating the future.
+#'
+#' @param future.seed A logical or an integer (of length one or seven),
+#' or a list of \code{length(x)} with pre-generated random seeds.
+#' For details, see below section.
+#'
+#' @param future.lazy Specifies whether the futures should be resolved
+#' lazily or eagerly (default).
+#'
+#' @param future.scheduling Average number of futures ("chunks") per worker.
+#' If \code{0.0}, then a single future is used to process all elements
+#' of \code{x}.
+#' If \code{1.0} or \code{TRUE}, then one future per worker is used.
+#' If \code{2.0}, then each worker will process two futures
+#' (if there are enough elements in \code{x}).
+#' If \code{Inf} or \code{FALSE}, then one future per element of
+#' \code{x} is used.
+#'
+#' @return A list with same length and names as \code{x}.
+#'
+#' @section Global variables:
+#' Argument \code{future.globals} may be used to control how globals
+#' should be handled similarly how the \code{globals} argument is used with
+#' \code{\link{future}()}.
+#' Since all function calls use the same set of globals, this function can do
+#' any gathering of globals upfront (once), which is more efficient than if
+#' it would be done for each future independently.
+#' If \code{TRUE}, \code{NULL} or not is specified (default), then globals
+#' are automatically identified and gathered.
+#' If a character vector of names is specified, then those globals are gathered.
+#' If a named list, then those globals are used as is.
+#' In all cases, \code{FUN} and any \code{...} arguments are automatically
+#' passed as globals to each future created as they are always needed.
+#'
+#' @section Reproducible random number generation (RNG):
+#' Unless \code{future.seed = FALSE}, this function guarantees to generate
+#' the exact same sequence of random numbers \emph{given the same initial
+#' seed / RNG state} - this regardless of type of futures and scheduling
+#' ("chunking") strategy.
+#'
+#' RNG reproducibility is achieved by pregenerating the random seeds for all
+#' iterations (over \code{x}) by using L'Ecuyer-CMRG RNG streams. In each
+#' iteration, these seeds are set before calling \code{FUN(x[[ii]], ...)}.
+#' \emph{Note, for large \code{length(x)} this may introduce a large overhead.}
+#' As input (\code{future.seed}), a fixed seed (integer) may be given, either
+#' as a full L'Ecuyer-CMRG RNG seed (vector of 1+6 integers) or as a seed
+#' generating such a full L'Ecuyer-CMRG seed.
+#' If \code{future.seed = TRUE}, then \code{\link[base:Random]{.Random.seed}}
+#' is returned if it holds a L'Ecuyer-CMRG RNG seed, otherwise one is created
+#' randomly.
+#' If \code{future.seed = NA}, a L'Ecuyer-CMRG RNG seed is randomly created.
+#' If none of the function calls \code{FUN(x[[i]], ...)} uses random number
+#' generation, then \code{future.seed = FALSE} may be used.
+#'
+#' In addition to the above, it is possible to specify a pre-generated
+#' sequence of RNG seeds as a list such that
+#' \code{length(future.seed) == length(x)} and where each element is an
+#' integer seed that can be assigned to \code{\link[base:Random]{.Random.seed}}.
+#' Use this alternative with caution.
+#' \emph{Note that as.list(seq_along(x)) is \emph{not} a valid set of such
+#' \code{.Random.seed} values.}
+#'
+#' In all cases but \code{future.seed = FALSE}, the RNG state of the calling
+#' R processes after this function returns is guaranteed to be
+#' "forwarded one step" from the RNG state that was before the call and
+#' in the same way regardless of \code{future.seed}, \code{future.scheduling}
+#' and future strategy used. This is done in order to guarantee that an \R
+#' script calling \code{future_lapply()} multiple times should be numerically
+#' reproducible given the same initial seed.
+#'
+#' @example incl/future_lapply.R
+#'
+#' @importFrom globals globalsByName cleanup
+#' @importFrom parallel nextRNGStream nextRNGSubStream splitIndices
+#' @importFrom utils capture.output str
+#' @export
+#' @keywords internal
+future_lapply <- function(x, FUN, ..., future.globals = TRUE, future.packages = NULL, future.seed = FALSE, future.lazy = FALSE, future.scheduling = 1.0) {
+ stopifnot(is.function(FUN))
+
+ stopifnot(is.logical(future.lazy))
+
+ stopifnot(!is.null(future.seed))
+
+ stopifnot(length(future.scheduling) == 1, !is.na(future.scheduling),
+ is.numeric(future.scheduling) || is.logical(future.scheduling))
+
+ ## Nothing to do?
+ nx <- length(x)
+ if (nx == 0) return(list())
+
+ debug <- getOption("future.debug", FALSE)
+
+ if (debug) mdebug("future_lapply() ...")
+
+ ## NOTE TO SELF: We'd ideally have a 'future.envir' argument also for
+ ## future_lapply(), cf. future(). However, it's not yet clear to me how
+ ## to do this, because we need to have globalsOf() to search for globals
+ ## from the current environment in order to identify the globals of
+ ## arguments 'FUN' and '...'. /HB 2017-03-10
+ future.envir <- environment() ## Not used; just to clarify the above.
+
+ envir <- future.envir
+
+ ## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ ## 1. Global variables
+ ## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ ## The default is to gather globals
+ if (is.null(future.globals)) future.globals <- TRUE
+
+ packages <- NULL
+ globals <- future.globals
+ if (is.logical(globals)) {
+ ## Gather all globals?
+ if (globals) {
+ if (debug) mdebug("Finding globals ...")
+
+ expr <- do.call(call, args = c(list("FUN"), list(...)))
+ gp <- getGlobalsAndPackages(expr, envir = envir, tweak = tweakExpression, globals = TRUE)
+ globals <- gp$globals
+ packages <- gp$packages
+ gp <- NULL
+
+ if (debug) {
+ mdebug(" - globals found: [%d] %s", length(globals), hpaste(sQuote(names(globals))))
+ mdebug(" - needed namespaces: [%d] %s", length(packages), hpaste(sQuote(packages)))
+ mdebug("Finding globals ... DONE")
+ }
+ } else {
+ ## globals = FALSE
+ globals <- c("FUN", names(list(...)), "...")
+ globals <- globalsByName(globals, envir = envir, mustExist = FALSE)
+ }
+ } else if (is.character(globals)) {
+ globals <- unique(c(globals, "FUN", names(list(...)), "..."))
+ globals <- globalsByName(globals, envir = envir, mustExist = FALSE)
+ } else if (is.list(globals)) {
+ names <- names(globals)
+ if (length(globals) > 0 && is.null(names)) {
+ stop("Invalid argument 'future.globals'. All globals must be named.")
+ }
+ } else {
+ stop("Invalid argument 'future.globals': ", mode(globals))
+ }
+ globals <- as.FutureGlobals(globals)
+ stopifnot(inherits(globals, "FutureGlobals"))
+
+ names <- names(globals)
+ if (!is.element("FUN", names)) {
+ globals <- c(globals, FUN = FUN)
+ }
+
+ if (!is.element("...", names)) {
+ if (debug) mdebug("Getting '...' globals ...")
+ dotdotdot <- globalsByName("...", envir = envir, mustExist = TRUE)
+ dotdotdot <- as.FutureGlobals(dotdotdot)
+ dotdotdot <- resolve(dotdotdot)
+ attr(dotdotdot, "total_size") <- objectSize(dotdotdot)
+ if (debug) mdebug("Getting '...' globals ... DONE")
+ globals <- c(globals, dotdotdot)
+ }
+
+ ## Assert there are no reserved variables names among globals
+ reserved <- intersect(c("...future.FUN", "...future.x_ii",
+ "...future.seeds_ii"), names)
+ if (length(reserved) > 0) {
+ stop("Detected globals using reserved variables names: ",
+ paste(sQuote(reserved), collapse = ", "))
+ }
+
+ ## Avoid FUN() clash with lapply(..., FUN) below.
+ names <- names(globals)
+ names[names == "FUN"] <- "...future.FUN"
+ names(globals) <- names
+
+ if (debug) {
+ mdebug("Globals to be used in all futures:")
+ mdebug(paste(capture.output(str(globals)), collapse = "\n"))
+ }
+
+
+ ## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ ## 2. Packages
+ ## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ if (!is.null(future.packages)) {
+ stopifnot(is.character(future.packages))
+ future.packages <- unique(future.packages)
+ stopifnot(!anyNA(future.packages), all(nzchar(future.packages)))
+ packages <- unique(c(packages, future.packages))
+ }
+
+ if (debug) {
+ mdebug("Packages to be attached in all futures:")
+ mdebug(paste(capture.output(str(packages)), collapse = "\n"))
+ }
+
+
+ ## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ ## 3. Reproducible RNG (for sequential and parallel processing)
+ ## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ seed <- future.seed
+
+ ## Placeholder for all RNG stream seeds.
+ seeds <- NULL
+
+ ## Don't use RNGs? (seed = FALSE)
+ if (is.logical(seed) && !is.na(seed) && !seed) seed <- NULL
+
+ # Use RNGs?
+ if (!is.null(seed)) {
+ if (debug) mdebug("Generating random seeds ...")
+
+ ## future_lapply() should return with the same RNG state regardless of
+ ## future strategy used. This is be done such that RNG kind is preserved
+ ## and the seed is "forwarded" one step from what it was when this
+ ## function was called. The forwarding is done by generating one random
+ ## number. Note that this approach is also independent on length(x) and
+ ## the diffent FUN() calls.
+ oseed <- next_random_seed()
+ on.exit(set_random_seed(oseed))
+
+ ## A pregenerated sequence of random seeds?
+ if (is.list(seed)) {
+ if (debug) mdebug("Using a pre-define stream of random seeds ...", nx)
+
+ nseed <- length(seed)
+ if (nseed != nx) {
+ stop("Argument 'seed' is a list, which specifies the sequence of seeds to be used for each element in 'x', but length(seed) != length(x): ", nseed, " != ", nx)
+ }
+
+ ## Assert same type of RNG seeds?
+ ns <- unique(unlist(lapply(seed, FUN = length), use.names = FALSE))
+ if (length(ns) != 1) {
+ stop("The elements of the list specified in argument 'seed' are not all of the same lengths (did you really pass RNG seeds?): ", hpaste(ns))
+ }
+
+ ## Did use specify scalar integers as meant for set.seed()?
+ if (ns == 1L) {
+ stop("Argument 'seed' is invalid. Pre-generated random seeds must be valid .Random.seed seeds, which means they should be all integers and consists of two or more elements, not just one.")
+ }
+
+ types <- unlist(lapply(seed, FUN = typeof), use.names = FALSE)
+ if (!all(types == "integer")) {
+ stop("The elements of the list specified in argument 'seed' are not all integers (did you really pass RNG seeds?): ", hpaste(unique(types)))
+ }
+
+ ## Check if valid random seeds are specified.
+ ## For efficiency, only look at the first one.
+ if (!is_valid_random_seed(seed[[1]])) {
+ stop("The list in argument 'seed' does not seem to hold elements that are valid .Random.seed values: ", capture.output(str(seeds[[1]])))
+ }
+
+ seeds <- seed
+
+ if (debug) mdebug("Using a pre-define stream of random seeds ... DONE", nx)
+ } else {
+ if (debug) mdebug("Generating random seed streams for %d elements ...", nx)
+
+ ## Generate sequence of _all_ RNG seeds starting with an initial seed
+ ## '.seed' that is based on argument 'seed'.
+ .seed <- as_lecyer_cmrg_seed(seed)
+
+ seeds <- vector("list", length = nx)
+ for (ii in seq_len(nx)) {
+ ## RNG substream seed used in call FUN(x[[ii]], ...):
+ ## This way each future can in turn generate further seeds, also
+ ## recursively, with minimal risk of generating the same seeds as
+ ## another future. This should make it safe to recursively call
+ ## future_lapply(). /HB 2017-01-11
+ seeds[[ii]] <- nextRNGSubStream(.seed)
+
+ ## Main random seed for next iteration (= ii + 1)
+ .seed <- nextRNGStream(.seed)
+ }
+
+ if (debug) mdebug("Generating random seed streams for %d elements ... DONE", nx)
+ }
+
+ if (debug) mdebug("Generating random seeds ... DONE")
+ } ## if (!is.null(seed))
+
+
+ ## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ ## 4. Load balancing ("chunking")
+ ## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ if (is.logical(future.scheduling)) {
+ if (future.scheduling) {
+ nbr_of_futures <- nbrOfWorkers()
+ if (nbr_of_futures > nx) nbr_of_futures <- nx
+ } else {
+ nbr_of_futures <- nx
+ }
+ } else {
+ ## Treat 'future.scheduling' as the number of futures per worker.
+ stopifnot(future.scheduling >= 0)
+ nbr_of_workers <- nbrOfWorkers()
+ if (nbr_of_workers > nx) nbr_of_workers <- nx
+ nbr_of_futures <- future.scheduling * nbr_of_workers
+ if (nbr_of_futures < 1) {
+ nbr_of_futures <- 1L
+ } else if (nbr_of_futures > nx) {
+ nbr_of_futures <- nx
+ }
+ }
+
+ chunks <- splitIndices(nx, ncl = nbr_of_futures)
+ if (debug) mdebug("Number of chunks: %d", length(chunks))
+
+
+ ## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ ## 5. Create futures
+ ## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ ## Add argument placeholders
+ globals_extra <- as.FutureGlobals(list(...future.x_ii = NULL, ...future.seeds_ii = NULL))
+ attr(globals_extra, "resolved") <- TRUE
+ attr(globals_extra, "total_size") <- 0
+ globals <- c(globals, globals_extra)
+
+ ## At this point a globals should be resolved and we should know their total size
+## stopifnot(attr(globals, "resolved"), !is.na(attr(globals, "total_size")))
+
+ ## To please R CMD check
+ ...future.FUN <- ...future.x_ii <- ...future.seeds_ii <- NULL
+
+ nchunks <- length(chunks)
+ fs <- vector("list", length = nchunks)
+ if (debug) mdebug("Number of futures (= number of chunks): %d", nchunks)
+
+ if (debug) mdebug("Launching %d futures (chunks) ...", nchunks)
+ for (ii in seq_along(chunks)) {
+ chunk <- chunks[[ii]]
+ if (debug) mdebug("Chunk #%d of %d ...", ii, length(chunks))
+
+ ## Subsetting outside future is more efficient
+ globals_ii <- globals
+ globals_ii[["...future.x_ii"]] <- x[chunk]
+## stopifnot(attr(globals_ii, "resolved"))
+
+ ## Using RNG seeds or not?
+ if (is.null(seeds)) {
+ if (debug) mdebug(" - seeds: <none>")
+ fs[[ii]] <- future({
+ lapply(seq_along(...future.x_ii), FUN = function(jj) {
+ ...future.x_jj <- ...future.x_ii[[jj]]
+ ...future.FUN(...future.x_jj, ...)
+ })
+ }, envir = envir, lazy = future.lazy, globals = globals_ii, packages = packages)
+ } else {
+ if (debug) mdebug(" - seeds: [%d] <seeds>", length(chunk))
+ globals_ii[["...future.seeds_ii"]] <- seeds[chunk]
+ fs[[ii]] <- future({
+ lapply(seq_along(...future.x_ii), FUN = function(jj) {
+ ...future.x_jj <- ...future.x_ii[[jj]]
+ assign(".Random.seed", ...future.seeds_ii[[jj]], envir = globalenv(), inherits = FALSE)
+ ...future.FUN(...future.x_jj, ...)
+ })
+ }, envir = envir, lazy = future.lazy, globals = globals_ii, packages = packages)
+ }
+
+ ## Not needed anymore
+ rm(list = c("chunk", "globals_ii"))
+
+ if (debug) mdebug("Chunk #%d of %d ... DONE", ii, nchunks)
+ } ## for (ii ...)
+ if (debug) mdebug("Launching %d futures (chunks) ... DONE", nchunks)
+
+ ## Not needed anymore
+ rm(list = c("chunks", "globals", "envir"))
+
+ ## 4. Resolving futures
+ if (debug) mdebug("Resolving %d futures (chunks) ...", nchunks)
+ values <- values(fs)
+ if (debug) mdebug("Resolving %d futures (chunks) ... DONE", nchunks)
+
+ ## Not needed anymore
+ rm(list = "fs")
+
+ if (debug) mdebug("Reducing values from %d chunks ...", nchunks)
+ values <- Reduce(c, values)
+ names(values) <- names(x)
+ if (debug) mdebug("Reducing values from %d chunks ... DONE", nchunks)
+
+ if (debug) mdebug("future_lapply() ... DONE")
+
+ values
+}
diff --git a/R/futures.R b/R/futures.R
index 2dea29b..7aa3173 100644
--- a/R/futures.R
+++ b/R/futures.R
@@ -26,7 +26,7 @@ futures.list <- function(x, ...) {
#' @export
futures.environment <- function(x, ...) {
- fs <- futureOf(envir=x, mustExist=FALSE, drop=FALSE)
+ fs <- futureOf(envir = x, mustExist = FALSE, drop = FALSE)
## Create object of same class as 'x'
res <- new.env()
@@ -45,7 +45,7 @@ futures.environment <- function(x, ...) {
#' @export
#' @importFrom listenv listenv
futures.listenv <- function(x, ...) {
- fs <- futureOf(envir=x, mustExist=FALSE, drop=FALSE)
+ fs <- futureOf(envir = x, mustExist = FALSE, drop = FALSE)
## Create object of same class as 'x'
res <- listenv()
diff --git a/R/globals.R b/R/globals.R
index 9adabe0..2d420d9 100644
--- a/R/globals.R
+++ b/R/globals.R
@@ -13,15 +13,18 @@
#' @seealso Internally, \code{\link[globals]{globalsOf}()} is used to identify globals and associated packages from the expression.
#'
#' @importFrom globals globalsOf globalsByName as.Globals packagesOf cleanup
-#' @importFrom utils object.size
+#' @export
#'
#' @keywords internal
-getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpression, globals=TRUE, resolve=getOption("future.globals.resolve", FALSE), persistent=FALSE, ...) {
+getGlobalsAndPackages <- function(expr, envir = parent.frame(), tweak = tweakExpression, globals = TRUE, resolve = getOption("future.globals.resolve", FALSE), persistent = FALSE, ...) {
## Nothing to do?
if (is.logical(globals) && !globals) {
- return(list(expr=expr, globals=list(), packages=character(0)))
+ return(list(expr = expr, globals = list(), packages = character(0)))
}
-
+
+ debug <- getOption("future.debug", FALSE)
+ if (debug) mdebug("getGlobalsAndPackages() ...")
+
## Assert that all identified globals exists when future is created?
if (persistent) {
## If future relies on persistent storage, then the globals may
@@ -32,7 +35,7 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
## Note: It's possible to switch between 'ignore' and 'error'
## at any time. Tests handles both cases. /HB 2016-06-18
globals.onMissing <- getOption("future.globals.onMissing", "ignore")
- globals.onMissing <- match.arg(globals.onMissing, choices=c("error", "ignore"))
+ globals.onMissing <- match.arg(globals.onMissing, choices = c("error", "ignore"))
mustExist <- is.element(globals.onMissing, "error")
}
@@ -40,42 +43,66 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
## Alt 1. Identify globals based on expr, envir and tweak
if (is.logical(globals)) {
stopifnot(length(globals) == 1, !is.na(globals))
+ if (debug) mdebug("Searching for globals ...")
## Algorithm for identifying globals
globals.method <- getOption("future.globals.method", "ordered")
globals <- globalsOf(
## Passed to globals::findGlobals()
- expr, envir=envir, substitute=FALSE, tweak=tweak,
+ expr, envir = envir, substitute = FALSE, tweak = tweak,
## Passed to globals::findGlobals() via '...'
- dotdotdot="return",
- method=globals.method,
- unlist=TRUE,
+ dotdotdot = "return",
+ method = globals.method,
+ unlist = TRUE,
## Passed to globals::globalsByName()
- mustExist=mustExist
+ mustExist = mustExist,
+ recursive = TRUE
)
+ if (debug) mdebug("- globals found: [%d] %s", length(globals), hpaste(sQuote(names(globals))))
+ if (debug) mdebug("Searching for globals ... DONE")
} else if (is.character(globals)) {
- globals <- globalsByName(globals, envir=envir, mustExist=mustExist)
+ if (debug) mdebug("Retrieving globals ...")
+ globals <- globalsByName(globals, envir = envir, mustExist = mustExist)
+ if (debug) mdebug("- globals retrieved: [%d] %s", length(globals), hpaste(sQuote(names(globals))))
+ if (debug) mdebug("Retrieving globals ... DONE")
} else if (inherits(globals, "Globals")) {
- ## Keep as is
+ if (debug) mdebug("- globals passed as-is: [%d] %s", length(globals), hpaste(sQuote(names(globals))))
} else if (is.list(globals)) {
- globals <- as.Globals(globals)
+ if (debug) mdebug("- globals passed as-list: [%d] %s", length(globals), hpaste(sQuote(names(globals))))
} else {
stop("Argument 'globals' must be either a logical scalar or a character vector: ", mode(globals))
}
- stopifnot(inherits(globals, "Globals"))
+ ## Make sure to preserve 'resolved' attribute
+ globals <- as.FutureGlobals(globals)
+ stopifnot(inherits(globals, "FutureGlobals"))
## Nothing more to do?
if (length(globals) == 0) {
- return(list(expr=expr, globals=list(), packages=character(0)))
+ if (debug) {
+ mdebug("- globals: [0] <none>")
+ mdebug("getGlobalsAndPackages() ... DONE")
+ }
+ return(list(expr = expr, globals = list(), packages = character(0)))
}
-
+
+ ## Are globals already resolved?
+ t <- attr(globals, "resolved")
+ if (isTRUE(t)) {
+ resolve <- FALSE
+ if (debug) mdebug("Resolving globals: %s (because already done)", resolve)
+ } else {
+ if (debug) mdebug("Resolving globals: %s", resolve)
+ }
+ stopifnot(is.logical(resolve), length(resolve) == 1L, !is.na(resolve))
+
exprOrg <- expr
## Tweak expression to be called with global ... arguments?
- if (length(globals) > 0 && inherits(globals$`...`, "DotDotDotList")) {
+ if (length(globals) > 0 && inherits(globals[["..."]], "DotDotDotList")) {
+ if (debug) mdebug("Tweak future expression to call with '...' arguments ...")
## Missing global '...'?
- if (!is.list(globals$`...`)) {
+ if (!is.list(globals[["..."]])) {
msg <- sprintf("Did you mean to create the future within a function? Invalid future expression tries to use global '...' variables that do not exist: %s", hexpr(exprOrg))
- mdebug(msg)
+ if (debug) mdebug(msg)
stop(msg)
}
@@ -86,11 +113,12 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
## To please R CMD check
a <- `future.call.arguments` <- NULL
- rm(list=c("a", "future.call.arguments"))
+ rm(list = c("a", "future.call.arguments"))
expr <- substitute({
## covr: skip=1
- do.call(function(...) a, args=`future.call.arguments`)
- }, list(a=expr))
+ do.call(function(...) a, args = `future.call.arguments`)
+ }, list(a = expr))
+ if (debug) mdebug("Tweak future expression to call with '...' arguments ... DONE")
}
## Resolve futures and turn into already-resolved "constant" futures
@@ -98,17 +126,26 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
## recursively try to resolve everything in every global which may
## or may not point to packages (include base R package)
if (resolve && length(globals) > 0L) {
- mdebug("Resolving globals that are futures ...")
- idxs <- which(unlist(lapply(globals, FUN=inherits, "Future"), use.names=FALSE))
- mdebug("Number of global futures: %d", length(idxs))
- if (length(idxs) > 0) {
- mdebug("Global futures: %s", hpaste(sQuote(names(globals[idxs]))))
- valuesF <- values(globals[idxs])
- globals[idxs] <- lapply(valuesF, FUN=ConstantFuture)
- valuesF <- NULL ## Not needed anymore
+ if (debug) mdebug("Resolving any globals that are futures ...")
+ globals <- as.FutureGlobals(globals)
+
+ ## Unless already resolved, perform a shallow resolve
+ if (attr(globals, "resolved")) {
+ idxs <- which(unlist(lapply(globals, FUN = inherits, "Future"), use.names = FALSE))
+ if (debug) mdebug("Number of global futures: %d", length(idxs))
+
+ ## Nothing to do?
+ if (length(idxs) > 0) {
+ if (debug) mdebug("Global futures (not constant): %s", hpaste(sQuote(names(globals[idxs]))))
+ valuesF <- values(globals[idxs])
+ globals[idxs] <- lapply(valuesF, FUN = ConstantFuture)
+ }
+ }
+
+ if (debug) {
+ mdebug("- globals: [%d] %s", length(globals), hpaste(sQuote(names(globals))))
+ mdebug("Resolving any globals that are futures ... DONE")
}
- idxs <- NULL ## Not needed anymore
- mdebug("Resolving globals that are futures ... DONE")
}
@@ -129,28 +166,24 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
where <- attr(globals, "where")
names <- names(globals)
- keep <- rep(TRUE, times=length(globals))
+ keep <- rep(TRUE, times = length(globals))
names(keep) <- names
for (name in names) {
pkg <- environmentName(where[[name]])
pkg <- gsub("^package:", "", pkg)
if (pkg %in% pkgs) {
## Only drop exported objects
- if (exists(name, envir=asPkgEnvironment(pkg)))
+ if (exists(name, envir = asPkgEnvironment(pkg)))
keep[name] <- FALSE
}
}
+
if (!all(keep)) globals <- globals[keep]
## Now drop globals that are primitive functions or
## that are part of the base packages, which now are
## part of 'pkgs' if needed.
globals <- cleanup(globals)
-
- ## Nothing more to do?
- if (length(globals) == 0) {
- return(list(expr=expr, globals=list(), packages=pkgs))
- }
}
@@ -161,44 +194,53 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
## only dive into such environments if they have a certain flag
## set. /HB 2016-02-04
if (resolve && length(globals) > 0L) {
- mdebug("Resolving futures part of globals (recursively) ...")
- globals <- resolve(globals, value=TRUE, recursive=TRUE)
- mdebug("Resolving futures part of globals (recursively) ... DONE")
+ if (debug) mdebug("Resolving futures part of globals (recursively) ...")
+ globals <- resolve(globals, value = TRUE, recursive = TRUE)
+ if (debug) {
+ mdebug("- globals: [%d] %s", length(globals), hpaste(sQuote(names(globals))))
+ mdebug("Resolving futures part of globals (recursively) ... DONE")
+ }
}
## Protect against user error exporting too large objects?
- ## Maximum size of globals (to prevent too large exports) = 500 MiB
- maxSizeOfGlobals <- getOption("future.globals.maxSize", 500*1024^2)
- maxSizeOfGlobals <- as.numeric(maxSizeOfGlobals)
- stopifnot(!is.na(maxSizeOfGlobals), maxSizeOfGlobals > 0)
- if (length(globals) > 0L && is.finite(maxSizeOfGlobals)) {
- sizes <- lapply(globals, FUN=object.size)
- sizes <- unlist(sizes, use.names=TRUE)
- totalExportSize <- sum(sizes, na.rm=TRUE)
- if (totalExportSize > maxSizeOfGlobals) {
- n <- length(sizes)
- o <- order(sizes, decreasing=TRUE)[1:3]
- o <- o[is.finite(o)]
- sizes <- sizes[o]
- classes <- lapply(globals[o], FUN=mode)
- classes <- unlist(classes, use.names=FALSE)
- largest <- sprintf("%s (%s of class %s)", sQuote(names(sizes)), asIEC(sizes), sQuote(classes))
- msg <- sprintf("The total size of all global objects that need to be exported for the future expression (%s) is %s. This exceeds the maximum allowed size of %s (option 'future.global.maxSize').", sQuote(hexpr(exprOrg)), asIEC(totalExportSize), asIEC(maxSizeOfGlobals))
- if (n == 1) {
- fmt <- "%s There is one global: %s."
- } else if (n == 2) {
- fmt <- "%s There are two globals: %s."
- } else if (n == 3) {
- fmt <- "%s There are three globals: %s."
- } else {
- fmt <- "%s The three largest globals are %s."
- }
- msg <- sprintf(fmt, msg, hpaste(largest, lastCollapse=" and "))
- mdebug(msg)
- stop(msg)
+ if (length(globals) > 0L) {
+ ## Maximum size of globals (to prevent too large exports) = 500 MiB
+ maxSizeOfGlobals <- getOption("future.globals.maxSize", 500 * 1024 ^ 2)
+ maxSizeOfGlobals <- as.numeric(maxSizeOfGlobals)
+ stopifnot(!is.na(maxSizeOfGlobals), maxSizeOfGlobals > 0)
+
+ if (is.finite(maxSizeOfGlobals)) {
+ sizes <- lapply(globals, FUN = objectSize)
+ sizes <- unlist(sizes, use.names = TRUE)
+ total_size <- sum(sizes, na.rm = TRUE)
+ attr(globals, "total_size") <- total_size
+ if (debug) mdebug("The total size of the %d globals is %s (%s bytes)", length(globals), asIEC(total_size), total_size)
+
+ if (total_size > maxSizeOfGlobals) {
+ n <- length(sizes)
+ o <- order(sizes, decreasing = TRUE)[1:3]
+ o <- o[is.finite(o)]
+ sizes <- sizes[o]
+ classes <- lapply(globals[o], FUN = mode)
+ classes <- unlist(classes, use.names = FALSE)
+ largest <- sprintf("%s (%s of class %s)", sQuote(names(sizes)), asIEC(sizes), sQuote(classes))
+ msg <- sprintf("The total size of the %d globals that need to be exported for the future expression (%s) is %s. This exceeds the maximum allowed size of %s (option 'future.globals.maxSize').", length(globals), sQuote(hexpr(exprOrg)), asIEC(total_size), asIEC(maxSizeOfGlobals))
+ if (n == 1) {
+ fmt <- "%s There is one global: %s."
+ } else if (n == 2) {
+ fmt <- "%s There are two globals: %s."
+ } else if (n == 3) {
+ fmt <- "%s There are three globals: %s."
+ } else {
+ fmt <- "%s The three largest globals are %s."
+ }
+ msg <- sprintf(fmt, msg, hpaste(largest, lastCollapse = " and "))
+ if (debug) mdebug(msg)
+ stop(msg)
+ } ## if (total_size > ...)
}
- }
+ } ## if (length(globals) > 0)
## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -211,7 +253,7 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
## Local functions
attachedPackages <- function() {
pkgs <- search()
- pkgs <- grep("^package:", pkgs, value=TRUE)
+ pkgs <- grep("^package:", pkgs, value = TRUE)
pkgs <- gsub("^package:", "", pkgs)
pkgs
}
@@ -223,5 +265,13 @@ getGlobalsAndPackages <- function(expr, envir=parent.frame(), tweak=tweakExpress
pkgs <- pkgs[isAttached]
}
- list(expr=expr, globals=globals, packages=pkgs)
+ if (debug) {
+ mdebug("- globals: [%d] %s", length(globals), hpaste(sQuote(names(globals))))
+ mdebug("- packages: [%d] %s", length(pkgs), hpaste(sQuote(pkgs)))
+ mdebug("getGlobalsAndPackages() ... DONE")
+ }
+
+ stopifnot(inherits(globals, "FutureGlobals"))
+
+ list(expr = expr, globals = globals, packages = pkgs)
} ## getGlobalsAndPackages()
diff --git a/R/globals_OP.R b/R/globals_OP.R
index ed71159..ce035d0 100644
--- a/R/globals_OP.R
+++ b/R/globals_OP.R
@@ -6,13 +6,15 @@
#' \code{x \%<-\% \{ expr \}}.
#' @inheritParams multiprocess
#'
-#' @details
-#' \code{a \%globals\% b} is short for \code{a \%tweak\% list(globals = b)}.
-#'
#' @export
`%globals%` <- function(fassignment, globals) {
fassignment <- substitute(fassignment)
envir <- parent.frame(1)
- args <- list(fassignment, list(globals=globals))
- do.call(`%tweak%`, args=args, envir=envir)
+
+ ## Temporarily set 'globals' argument
+ args <- getOption("future.disposable", list())
+ args["globals"] <- list(globals)
+ options(future.disposable = args)
+
+ eval(fassignment, envir = envir)
}
diff --git a/R/label_OP.R b/R/label_OP.R
index 528918a..9cb79f1 100644
--- a/R/label_OP.R
+++ b/R/label_OP.R
@@ -6,13 +6,15 @@
#' \code{x \%<-\% \{ expr \}}.
#' @inheritParams multiprocess
#'
-#' @details
-#' \code{a \%label\% b} is short for \code{a \%tweak\% list(label = b)}.
-#'
#' @export
`%label%` <- function(fassignment, label) {
fassignment <- substitute(fassignment)
envir <- parent.frame(1)
- args <- list(fassignment, list(label=label))
- do.call(`%tweak%`, args=args, envir=envir)
+
+ ## Temporarily set 'label' argument
+ args <- getOption("future.disposable", list())
+ args["label"] <- list(label)
+ options(future.disposable = args)
+
+ eval(fassignment, envir = envir)
}
diff --git a/R/lazy_OP.R b/R/lazy_OP.R
new file mode 100644
index 0000000..b827fb1
--- /dev/null
+++ b/R/lazy_OP.R
@@ -0,0 +1,20 @@
+#' Control lazy / eager evaluation for a future assignment
+#'
+#' @usage fassignment \%lazy\% lazy
+#'
+#' @param fassignment The future assignment, e.g.
+#' \code{x \%<-\% \{ expr \}}.
+#' @inheritParams multiprocess
+#'
+#' @export
+`%lazy%` <- function(fassignment, lazy) {
+ fassignment <- substitute(fassignment)
+ envir <- parent.frame(1)
+
+ ## Temporarily set 'lazy' argument
+ args <- getOption("future.disposable", list())
+ args["lazy"] <- list(lazy)
+ options(future.disposable = args)
+
+ eval(fassignment, envir = envir)
+}
diff --git a/R/makeClusterPSOCK.R b/R/makeClusterPSOCK.R
index 4429584..0627772 100644
--- a/R/makeClusterPSOCK.R
+++ b/R/makeClusterPSOCK.R
@@ -1,18 +1,29 @@
-#' Create a Parallel Socket Cluster
+#' Create a cluster of \R workers for parallel processing
+#'
+#' The \code{makeClusterPSOCK()} function creates a cluster of \R workers
+#' for parallel processing. These \R workers may be background \R sessions
+#' on the current machine, \R sessions on external machines (local or remote),
+#' or a mix of such. For external workers, the default is to use SSH to connect
+#' to those external machines. This function works similarly to
+#' \code{\link[parallel:makePSOCKcluster]{makePSOCKcluster}} of the
+#' \pkg{parallel} package, but provides additional and more flexibility options
+#' for controlling the setup of the system calls that launch the background
+#' \R workers, and how to connect to external machines.
#'
-#' @param workers The host names of workers (as a character vector) or
-#' the number of localhost workers (as a positive integer).
-#' @param makeNode A function that creates a \code{"SOCKnode"}
-#' or \code{"SOCK0node"} object, which represents a connection
-#' to a worker.
-#' @param ... Optional arguments passed to \code{makeNode(workers[i], ..., rank=i)} where \code{i = seq_along{workers}}.
+#' @param workers The hostnames of workers (as a character vector) or the number
+#' of localhost workers (as a positive integer).
+#'
+#' @param makeNode A function that creates a \code{"SOCKnode"} or
+#' \code{"SOCK0node"} object, which represents a connection to a worker.
+#'
+#' @param ... Optional arguments passed to
+#' \code{makeNode(workers[i], ..., rank = i)} where
+#' \code{i = seq_along(workers)}.
+#'
#' @param verbose If TRUE, informative messages are outputted.
#'
#' @return An object of class \code{c("SOCKcluster", "cluster")} consisting
-#' of a list of \code{"SOCKnode"} or \code{"SOCK0node"} workers.
-#'
-#' @details
-#' The \code{makeClusterPSOCK()} function is similar to \code{\link[parallel:makePSOCKcluster]{makePSOCKcluster}} of the \pkg{parallel} package, but provides more flexibility in controlling the setup of the system calls that launch the background R workers and how to connect to external machines.
+#' of a list of \code{"SOCKnode"} or \code{"SOCK0node"} workers.
#'
#' @example incl/makeClusterPSOCK.R
#'
@@ -28,6 +39,10 @@ makeClusterPSOCK <- function(workers, makeNode = makeNodePSOCK, port = c("auto",
}
workers <- rep("localhost", times = workers)
}
+ if (verbose) {
+ message(sprintf("Workers: [n = %d] %s",
+ length(workers), hpaste(sQuote(workers))))
+ }
if (is.character(port)) {
port <- match.arg(port)
@@ -48,75 +63,210 @@ makeClusterPSOCK <- function(workers, makeNode = makeNodePSOCK, port = c("auto",
if (is.na(port) || port < 0L || port > 65535L) {
stop("Invalid port: ", port)
}
+ if (verbose) message(sprintf("Base port: %d", port))
- cl <- vector("list", length = length(workers))
- for (ii in seq_along(cl)) {
- cl[[ii]] <- makeNode(workers[[ii]], port = port, ..., rank = ii, verbose = verbose)
- }
+ n <- length(workers)
+ cl <- vector("list", length = n)
class(cl) <- c("SOCKcluster", "cluster")
+ for (ii in seq_along(cl)) {
+ if (verbose) message(sprintf("Creating node %d of %d ...", ii, n))
+ if (verbose) message("- setting up node")
+ cl[[ii]] <- makeNode(workers[[ii]], port = port, ..., rank = ii,
+ verbose = verbose)
+
+ ## Attaching UUID for each cluster connection. This is done because
+ ## https://stat.ethz.ch/pipermail/r-devel/2016-October/073331.html
+ if (verbose) message("- assigning connection UUID")
+ cl[ii] <- add_cluster_uuid(cl[ii])
- ## Attaching UUID for each cluster connection. This is done because
- ## https://stat.ethz.ch/pipermail/r-devel/2016-October/073331.html
- cl <- addClusterUUIDs(cl)
-
+ ## Attaching session information for each worker. This is done to assert
+ ## that we have a working cluster already here. It will also collect
+ ## useful information otherwise not available, e.g. the PID.
+ if (verbose) message("- collecting session information")
+ cl[ii] <- add_cluster_session_info(cl[ii])
+
+ if (verbose) message(sprintf("Creating node %d of %d ... done", ii, n))
+ }
+
cl
} ## makeClusterPSOCK()
-#' @param worker The host name or IP number of the machine where the worker should run.
-#' @param master The host name or IP number of the master / calling machine, as known to the workers. If NULL (default), then the default is \code{Sys.info()[["nodename"]]} unless \code{worker} is the localhost (\code{"localhost"} or \code{"127.0.0.1"}) or \code{revtunnel = TRUE} in case it is \code{"localhost"}.
-#' @param port The port number of the master used to for communicating with all the workers (via socket connections). If an integer vector of ports, then a random one among those is chosen. If \code{"random"}, then a random port in \code{11000:11999} is chosen. If \code{"auto"} (default), then the default is taken from environment variable \env{R_PARALLEL_PORT}, otherwise \code{"random"} is used.
-#' @param connectTimeout The maximum time (in seconds) allowed for each sockect connection between the master and a worker to be established (defaults to 2 minutes). \emph{See note below on current lack of support on Linux and macOS systems.}
-#' @param timeout The maximum time (in seconds) allowed to pass without the master and a worker communicate with each other (defaults to 30 days).
-#' @param rscript,homogeneous The system command for launching Rscript on the worker. If \code{NULL} (default), the default is \code{"Rscript"} unless \code{homogenenous} is TRUE, which in case it is \code{file.path(R.home("bin"), "Rscript")}. Argument \code{homogenenous} defaults to FALSE, unless \code{master} is the localhost (\code{"localhost"} or \code{"127.0.0.1"}).
-#' @param rscript_args Additional arguments to \code{Rscript} (as a character vector).
+#' @param worker The hostname or IP number of the machine where the worker
+#' should run.
+#'
+#' @param master The hostname or IP number of the master / calling machine, as
+#' known to the workers. If NULL (default), then the default is
+#' \code{Sys.info()[["nodename"]]} unless \code{worker} is \emph{localhost} or
+#' \code{revtunnel = TRUE} in case it is \code{"localhost"}.
+#'
+#' @param port The port number of the master used to for communicating with all
+#' the workers (via socket connections). If an integer vector of ports, then a
+#' random one among those is chosen. If \code{"random"}, then a random port in
+#' \code{11000:11999} is chosen. If \code{"auto"} (default), then the default
+#' is taken from environment variable \env{R_PARALLEL_PORT}, otherwise
+#' \code{"random"} is used.
+#'
+#' @param connectTimeout The maximum time (in seconds) allowed for each socket
+#' connection between the master and a worker to be established (defaults to
+#' 2 minutes). \emph{See note below on current lack of support on Linux and
+#' macOS systems.}
+#'
+#' @param timeout The maximum time (in seconds) allowed to pass without the
+#' master and a worker communicate with each other (defaults to 30 days).
+#'
+#' @param rscript,homogeneous The system command for launching \command{Rscript}
+#' on the worker and whether it is installed in the same path as the calling
+#' machine or not. For more details, see below.
+#'
+#' @param rscript_args Additional arguments to \command{Rscript} (as a character
+#' vector).
+#'
#' @param methods If TRUE, then the \pkg{methods} package is also loaded.
-#' @param useXDR If TRUE, the communication between master and workers, which is binary, will be use big-endian (XDR).
-#' @param outfile Where to direct the \link[base:stdout]{stdout} and \link[base:stderr]{stderr} connection output from the workers.
-#' @param renice A numerical 'niceness' (priority) to set for the worker processes.
+#'
+#' @param useXDR If TRUE, the communication between master and workers, which is
+#' binary, will use big-endian (XDR).
+#'
+#' @param outfile Where to direct the \link[base:stdout]{stdout} and
+#' \link[base:stderr]{stderr} connection output from the workers.
+#'
+#' @param renice A numerical 'niceness' (priority) to set for the worker
+#' processes.
+#'
#' @param rank A unique one-based index for each worker (automatically set).
-#' @param rshcmd The command to be run on the master to launch a process on another host. Only applicable if \code{machine} is not localhost.
-#' @param user (optional) The user name to be used when communicating with another host.
-#' @param revtunnel If TRUE, a reverse SSH tunneling is set up for each worker such that the worker R process sets up a socket connect to its local port \code{(port - rank + 1)} which then reaches the master on port \code{port}. If FALSE, then the worker will try to connect directly to port \code{port} on \code{master}.
-#' @param rshopts Additional arguments to \code{rshcmd} (as a character vector).
-#' @param manual If TRUE the workers will need to be run manually.
-#' @param dryrun If TRUE, nothing is set up, but a message suggesting how to launch the worker from the terminal is outputted. This is useful for troubleshooting.
+#'
+#' @param rshcmd,rshopts The command (character vector) to be run on the master
+#' to launch a process on another host and any additional arguments (character
+#' vector). These arguments are only applied if \code{machine} is not
+#' \emph{localhost}. For more details, see below.
+#'
+#' @param user (optional) The user name to be used when communicating with
+#' another host.
+#'
+#' @param revtunnel If TRUE, a reverse SSH tunnel is set up for each worker such
+#' that the worker R process sets up a socket connection to its local port
+#' \code{(port - rank + 1)} which then reaches the master on port \code{port}.
+#' If FALSE, then the worker will try to connect directly to port \code{port} on
+#' \code{master}. For more details, see below.
+#'
+#' @param manual If TRUE the workers will need to be run manually. The command
+#' to run will be displayed.
+#'
+#' @param dryrun If TRUE, nothing is set up, but a message suggesting how to
+#' launch the worker from the terminal is outputted. This is useful for
+#' troubleshooting.
#'
-#' @return \code{makeNodePSOCK()} returns a
-#' \code{"SOCKnode"} or \code{"SOCK0node"} object
-#' representing an established connection to a worker.
+#' @return \code{makeNodePSOCK()} returns a \code{"SOCKnode"} or
+#' \code{"SOCK0node"} object representing an established connection to a worker.
#'
-#' @details
-#' The default is to use reverse SSH tunnelling for workers
-#' running on other machines. This avoids the complication of
-#' otherwise having to configure port forwarding in firewalls,
-#' which often requires static IP address but which also most
-#' users don't have priviligies to do themselves.
-#' It also has the advantage of not having to know the internal
-#' and / or the public IP address / host name of the master.
+#' @section Definition of \emph{localhost}:
+#' A hostname is considered to be \emph{localhost} if it equals:
+#' \itemize{
+#' \item \code{"localhost"},
+#' \item \code{"127.0.0.1"}, or
+#' \item \code{Sys.info()[["nodename"]]}.
+#' }
+#' It is also considered \emph{localhost} if it appears on the same line
+#' as the value of \code{Sys.info()[["nodename"]]} in file \file{/etc/hosts}.
+#'
+#' @section Default values of arguments \code{rshcmd} and \code{rshopts}:
+#' Arguments \code{rshcmd} and \code{rshopts} are only used when connecting
+#' to an external host.
+#'
+#' The default method for connecting to an external host is via SSH and the
+#' system executable for this is given by argument \code{rshcmd}. The default
+#' is given by option \code{future.makeNodePSOCK.rshcmd} and if that is not
+#' set the default is either of \command{ssh} and \command{plink -ssh}.
+#' Most Unix-like systems, including macOS, have \command{ssh} preinstalled
+#' on the \code{PATH}. It is less common to find this command on Windows
+#' system, which are more likely to have the \command{PuTTY} software and
+#' its SSH client \command{plink} installed.
+#' If neither \command{ssh} nor \command{plink} is found, an informative error
+#' message is produced.
+#' It is also possible to specify the absolute path to the SSH client. To do
+#' this for PuTTY, specify the absolute path in the first element and option
+#' \command{-ssh} in the second as in
+#' \code{rshcmd = c("C:/Path/PuTTY/plink.exe", "-ssh")}.
+#' This is because all elements of \code{rshcmd} are individually "shell"
+#' quoted and element \code{rshcmd[1]} must be on the system \code{PATH}.
#'
-#' If there is no communication between the master and a
-#' worker within the \code{timeout} limit, then the corresponding
-#' socket connection will be closed automatically. This will
-#' eventually result in an error in code trying to access the
-#' connection.
+#' Additional SSH options may be specified via argument \code{rshopts}, which
+#' defaults to option \code{future.makeNodePSOCK.rshopts}. For instance, a
+#' private SSH key can be provided as
+#' \code{rshopts = c("-i", "~/.ssh/my_private_key")}. PuTTY users should
+#' specify a PuTTY PPK file, e.g.
+#' \code{rshopts = c("-i", "C:/Users/joe/.ssh/my_keys.ppk")}.
+#' Contrary to \code{rshcmd}, elements of \code{rshopts} are not quoted.
+#'
+#' @section Reverse SSH tunneling:
+#' The default is to use reverse SSH tunneling (\code{revtunnel = TRUE}) for
+#' workers running on other machines. This avoids the complication of
+#' otherwise having to configure port forwarding in firewalls, which often
+#' requires static IP address as well as privilieges to edit the firewall,
+#' something most users don't have.
+#' It also has the advantage of not having to know the internal and / or the
+#' public IP address / hostname of the master.
+#' Yet another advantage is that there will be no need for a DNS lookup by the
+#' worker machines to the master, which may not be configured or is disabled
+#' on some systems, e.g. compute clusters.
#'
+#' @section Default value of argument \code{rscript}:
+#' If \code{homogenenous} is FALSE, the \code{rscript} defaults to
+#' \code{"Rscript"}, i.e. it is assumed that the \command{Rscript} executable
+#' is available on the \code{PATH} of the worker.
+#' If \code{homogenenous} is TRUE, the \code{rscript} defaults to
+#' \code{file.path(R.home("bin"), "Rscript")}, i.e. it is basically assumed
+#' that the worker and the caller share the same file system and R installation.
+#'
+#' @section Default value of argument \code{homogeneous}:
+#' The default value of \code{homogenenous} is TRUE if and only if either
+#' of the following is fullfilled:
+#' \itemize{
+#' \item \code{worker} is \emph{localhost}
+#' \item \code{revtunnel} is FALSE and \code{master} is \emph{localhost}
+#' \item \code{worker} is neither an IP number nor a fully qualified domain
+#' name (FQDN). A hostname is considered to be a FQDN if it contains
+#' one or more periods
+#' }
+#'
#' @section Connection time out:
-#' Argument \code{connectTimeout} does \emph{not} work properly
-#' on Unix and macOS due to limitation in \R itself. For more details
-#' on this, please R devel thread 'BUG?: On Linux setTimeLimit() fails
-#' to propagate timeout error when it occurs (works on Windows)' on
-#' 2016-10-26 (\url{https://stat.ethz.ch/pipermail/r-devel/2016-October/073309.html}). When used, the timeout will eventually trigger an error, but
-#' it won't happen until the socket connection timeout \code{timeout}
-#' itself happens.
+#' Argument \code{connectTimeout} does \emph{not} work properly on Unix and
+#' macOS due to limitation in \R itself. For more details on this, please see
+#' R-devel thread 'BUG?: On Linux setTimeLimit() fails to propagate timeout
+#' error when it occurs (works on Windows)' on 2016-10-26
+#' (\url{https://stat.ethz.ch/pipermail/r-devel/2016-October/073309.html}).
+#' When used, the timeout will eventually trigger an error, but it won't happen
+#' until the socket connection timeout \code{timeout} itself happens.
+#'
+#' @section Communication time out:
+#' If there is no communication between the master and a worker within the
+#' \code{timeout} limit, then the corresponding socket connection will be
+#' closed automatically. This will eventually result in an error in code
+#' trying to access the connection.
#'
#' @rdname makeClusterPSOCK
#' @export
-makeNodePSOCK <- function(worker = "localhost", master = NULL, port, connectTimeout = 2*60, timeout = 30*24*60*60, rscript = NULL, homogeneous = NULL, rscript_args = NULL, methods = TRUE, useXDR = TRUE, outfile = "/dev/null", renice = NA_integer_, rshcmd = "ssh", user = NULL, revtunnel = TRUE, rshopts = NULL, rank = 1L, manual = FALSE, dryrun = FALSE, verbose = FALSE) {
+makeNodePSOCK <- function(worker = "localhost", master = NULL, port, connectTimeout = getOption("future.makeNodePSOCK.connectTimeout", 2 * 60), timeout = getOption("future.makeNodePSOCK.timeout", 30 * 24 * 60 * 60), rscript = NULL, homogeneous = NULL, rscript_args = NULL, methods = TRUE, useXDR = TRUE, outfile = "/dev/null", renice = NA_integer_, rshcmd = getOption("future.makeNodePSOCK.rshcmd", NULL), user = NULL, revtunnel = TRUE, rshopts = getOption("future.makeNodePSOCK.rshopts", NUL [...]
localMachine <- is.element(worker, c("localhost", "127.0.0.1"))
- rshcmd <- as.character(rshcmd)
- stopifnot(length(rshcmd) >= 1L)
+ ## Could it be that the worker specifies the name of the localhost?
+ ## Note, this approach preserves worker == "127.0.0.1" if that is given.
+ if (!localMachine) {
+ localMachine <- is_localhost(worker)
+ if (localMachine) worker <- "localhost"
+ }
+
+ manual <- as.logical(manual)
+ stopifnot(length(manual) == 1L, !is.na(manual))
+
+ dryrun <- as.logical(dryrun)
+ stopifnot(length(dryrun) == 1L, !is.na(dryrun))
+
+ ## Locate a default SSH client?
+ if (!is.null(rshcmd)) {
+ rshcmd <- as.character(rshcmd)
+ stopifnot(length(rshcmd) >= 1L)
+ }
rshopts <- as.character(rshopts)
@@ -145,9 +295,13 @@ makeNodePSOCK <- function(worker = "localhost", master = NULL, port, connectTime
methods <- as.logical(methods)
stopifnot(length(methods) == 1L, !is.na(methods))
-
+
if (is.null(homogeneous)) {
- homogeneous <- is.element(master, c("localhost", "127.0.0.1"))
+ homogeneous <- {
+ localMachine ||
+ (!revtunnel && is_localhost(master)) ||
+ (!is_ip_number(worker) && !is_fqdn(worker))
+ }
}
homogeneous <- as.logical(homogeneous)
stopifnot(length(homogeneous) == 1L, !is.na(homogeneous))
@@ -170,12 +324,6 @@ makeNodePSOCK <- function(worker = "localhost", master = NULL, port, connectTime
rank <- as.integer(rank)
stopifnot(length(rank) == 1L, !is.na(rank))
- manual <- as.logical(manual)
- stopifnot(length(manual) == 1L, !is.na(manual))
-
- dryrun <- as.logical(dryrun)
- stopifnot(length(dryrun) == 1L, !is.na(dryrun))
-
verbose <- as.logical(verbose)
stopifnot(length(verbose) == 1L, !is.na(verbose))
@@ -207,6 +355,11 @@ makeNodePSOCK <- function(worker = "localhost", master = NULL, port, connectTime
}
if (!localMachine) {
+ ## Find default SSH client
+ if (is.null(rshcmd)) {
+ rshcmd <- find_rshcmd(!localMachine && !manual && !dryrun)
+ }
+
## Local commands
rshcmd <- paste(shQuote(rshcmd), collapse = " ")
if (length(user) == 1L) rshopts <- c("-l", user, rshopts)
@@ -267,13 +420,17 @@ makeNodePSOCK <- function(worker = "localhost", master = NULL, port, connectTime
## actually work with the same connection. See R-devel thread
## 'closeAllConnections() can really mess things up' on 2016-10-30
## (https://stat.ethz.ch/pipermail/r-devel/2016-October/073331.html)
-addClusterUUIDs <- function(cl) {
+add_cluster_uuid <- function(cl) {
stopifnot(inherits(cl, "cluster"))
for (ii in seq_along(cl)) {
node <- cl[[ii]]
if (is.null(node)) next ## Happens with dryrun = TRUE
+
+ ## Worker does not use connections? Then nothing to do.
con <- node$con
+ if (is.null(con)) next
+
uuid <- attr(con, "uuid")
if (is.null(uuid)) {
attr(con, "uuid") <- uuid_of_connection(con, keep_source = TRUE)
@@ -283,4 +440,132 @@ addClusterUUIDs <- function(cl) {
}
cl
-} ## addClusterUUIDs()
+} ## add_cluster_uuid()
+
+
+## Checks if a given worker is the same as the localhost. It is, iff:
+##
+## * worker == "localhost"
+## * worker == "127.0.0.1"
+## * worker == hostname
+## * worker and hostname appears on the same line in /etc/hosts
+##
+## This should cover cases such as:
+## * Calling is_localhost("n3") from machine n3
+## * Calling is_localhost("n3.myserver.org") from machine n3[.myserver.org]
+##
+## References:
+## * https://en.wikipedia.org/wiki/Hostname
+is_localhost <- local({
+ localhosts <- c("localhost", "127.0.0.1")
+ non_localhosts <- character(0L)
+
+ function(worker, hostname = Sys.info()[["nodename"]], pathnames = "/etc/hosts") {
+ ## INTERNAL: Clear list of known local hosts?
+ if (is.null(worker) && is.null(hostname)) {
+ localhosts <<- c("localhost", "127.0.0.1")
+ non_localhosts <<- character(0L)
+ return(NA)
+ }
+
+ stopifnot(length(worker) == 1, length(hostname) == 1)
+
+ ## Already known to a localhost or not to one?
+ if (worker %in% localhosts) return(TRUE)
+ if (worker %in% non_localhosts) return(FALSE)
+
+ if (worker == hostname) {
+ ## Add worker to the list of known local hosts.
+ localhosts <<- unique(c(localhosts, worker))
+ return(TRUE)
+ }
+
+ ## Scan known "hosts" files
+ pathnames <- pathnames[file_test("-f", pathnames)]
+ if (length(pathnames) == 0L) return(FALSE)
+
+ ## Search for (hostname, worker) and (worker, hostname)
+ ## occuring on the same line and are separates by one or
+ ## more whitespace symbols (but nothing else).
+ pattern <- sprintf("^((|.*[[:space:]])%s[[:space:]]+%s([[:space:]]+|)|(|.*[[:space:]])%s[[:space:]]+%s([[:space:]]+|))$", hostname, worker, worker, hostname)
+
+ for (pathname in pathnames) {
+ bfr <- readLines(pathname, warn = FALSE)
+ if (any(grepl(pattern, bfr, ignore.case = TRUE))) {
+ ## Add worker to the list of known local hosts.
+ localhosts <<- unique(c(localhosts, worker))
+ return(TRUE)
+ }
+ }
+
+ ## Add worker to the list of known non-local hosts.
+ non_localhosts <<- unique(c(non_localhosts, worker))
+
+ FALSE
+ }
+}) ## is_localhost()
+
+
+## Checks if a worker is specified by its IP number.
+is_ip_number <- function(worker) {
+ ip <- strsplit(worker, split = ".", fixed = TRUE)[[1]]
+ if (length(ip) != 4) return(FALSE)
+ ip <- as.integer(ip)
+ if (anyNA(ip)) return(FALSE)
+ all(0 <= ip & ip <= 255)
+}
+
+## Checks if a worker is specified as a fully qualified domain name (FQDN)
+is_fqdn <- function(worker) {
+ grepl(".", worker, fixed = TRUE)
+}
+
+
+## Locate an SSH client
+find_rshcmd <- function(must_work = TRUE) {
+ cmds <- list("ssh", c("plink", "-ssh"))
+ for (cmd in cmds) {
+ if (nzchar(Sys.which(cmd[1]))) return(cmd)
+ }
+
+ cmds_checked <- unlist(lapply(cmds, FUN = function(x) x[1]))
+ msg <- sprintf("Failed to locate a default SSH client (checked: %s). Please specify one via argument 'rshcmd'.", paste(sQuote(cmds_checked), collapse = ", ")) #nolint
+ if (must_work) stop(msg)
+
+ cmd <- cmds[[1]]
+ msg <- sprintf("%s Will use %s.", msg, sQuote(paste(cmd, collapse = " ")))
+ warning(msg)
+ cmd
+}
+
+
+session_info <- function() {
+ list(
+ r = c(R.version, os.type = .Platform$OS.type),
+ system = as.list(Sys.info()),
+ process = list(pid = Sys.getpid())
+ )
+}
+
+
+add_cluster_session_info <- function(cl) {
+ stopifnot(inherits(cl, "cluster"))
+
+ for (ii in seq_along(cl)) {
+ node <- cl[[ii]]
+ if (is.null(node)) next ## Happens with dryrun = TRUE
+
+ ## Session information already collected?
+ if (!is.null(node$session_info)) next
+
+ pid <- capture.output(print(node))
+ pid <- as.integer(gsub(".* ", "", pid))
+
+ info <- clusterCall(cl[ii], fun = session_info)[[1]]
+ stopifnot(info$process$pid == pid)
+ node$session_info <- info
+ cl[[ii]] <- node
+ }
+
+ cl
+} ## add_cluster_session_info()
diff --git a/R/mandelbrot.R b/R/mandelbrot.R
index 0d423b2..08dab80 100644
--- a/R/mandelbrot.R
+++ b/R/mandelbrot.R
@@ -17,58 +17,56 @@
#' non-negative counts.
#'
#' @examples
-#' counts <- mandelbrot(xmid=-0.75, ymid=0, side=3)
+#' counts <- mandelbrot(xmid = -0.75, ymid = 0, side = 3)
#' str(counts)
#' \dontrun{
#' plot(counts)
#' }
#'
#' \dontrun{
-#' demo("mandelbrot", package="future", ask=FALSE)
+#' demo("mandelbrot", package = "future", ask = FALSE)
#' }
#'
#' @author The internal Mandelbrot algorithm was inspired by and
#' adopted from similar GPL code of Martin Maechler (available
#' from ftp://stat.ethz.ch/U/maechler/R/ on 2005-02-18 [sic!]).
#'
-#' @aliases as.raster.Mandelbrot plot.Mandelbrot mandelbrotTiles
+#' @aliases as.raster.Mandelbrot plot.Mandelbrot mandelbrot_tiles
#' @export
#'
#' @keywords internal
mandelbrot <- function(...) UseMethod("mandelbrot")
#' @export
-mandelbrot.matrix <- function(Z, maxIter=200L, tau=2.0, ...) {
+mandelbrot.matrix <- function(Z, maxIter = 200L, tau = 2.0, ...) {
stopifnot(is.matrix(Z), mode(Z) == "complex")
-
+
## By default, assume none of the elements will converge
- counts <- matrix(maxIter, nrow=nrow(Z), ncol=ncol(Z))
+ counts <- matrix(maxIter, nrow = nrow(Z), ncol = ncol(Z))
- ## But as a start, flag the to all be non-diverged
- nonDiverged <- rep(TRUE, times=length(Z))
- idxOfNonDiverged <- seq_along(nonDiverged)
+ ## But as a start, mark all be non-diverged
+ idx_of_non_diverged <- seq_along(Z)
## SPEEDUP: The Mandelbrot sequence will only be calculated on the
## "remaining set" of complex numbers that yet hasn't diverged.
sZ <- Z ## The Mandelbrot sequence of the "remaining" set
Zr <- Z ## The original complex number of the "remaining" set
- for (ii in seq_len(maxIter-1L)) {
- sZ <- sZ*sZ + Zr
+ for (ii in seq_len(maxIter - 1L)) {
+ sZ <- sZ * sZ + Zr
## Did any of the "remaining" points diverge?
diverged <- (Mod(sZ) > tau)
if (any(diverged)) {
## Record at what iteration divergence occurred
- counts[idxOfNonDiverged[diverged]] <- ii
+ counts[idx_of_non_diverged[diverged]] <- ii
## Early stopping?
keep <- which(!diverged)
if (length(keep) == 0) break
## Drop from remain calculations
- idxOfNonDiverged <- idxOfNonDiverged[keep]
- nonDiverged[nonDiverged] <- !diverged
+ idx_of_non_diverged <- idx_of_non_diverged[keep]
## Update the "remaining" set of complex numbers
sZ <- sZ[keep]
@@ -76,16 +74,18 @@ mandelbrot.matrix <- function(Z, maxIter=200L, tau=2.0, ...) {
}
}
- attr(counts, "params") <- list(Z=Z, maxIter=maxIter, tau=tau)
+ attr(counts, "params") <- list(Z = Z, maxIter = maxIter, tau = tau)
class(counts) <- c("Mandelbrot", class(counts))
counts
-} ## mandelbrot() for matrix
+}
#' @export
-mandelbrot.numeric <- function(xmid=-0.75, ymid=0, side=3, resolution=400L, maxIter=200L, tau=2, ...) {
+mandelbrot.numeric <- function(xmid = -0.75, ymid = 0.0, side = 3.0,
+ resolution = 400L, maxIter = 200L,
+ tau = 2.0, ...) {
## Validate arguments
stopifnot(side > 0)
resolution <- as.integer(resolution)
@@ -97,17 +97,17 @@ mandelbrot.numeric <- function(xmid=-0.75, ymid=0, side=3, resolution=400L, maxI
## The nx-by-ny bins
nx <- ny <- resolution
- ## Setup (x,y) bins
- xrange <- xmid + c(-1,1)*side/2
- yrange <- ymid + c(-1,1)*side/2
- x <- seq(from=xrange[1], to=xrange[2], length.out=nx)
- y <- seq(from=yrange[1], to=yrange[2], length.out=ny)
+ ## Setup (x, y) bins
+ xrange <- xmid + c(-1, 1) * side / 2
+ yrange <- ymid + c(-1, 1) * side / 2
+ x <- seq(from = xrange[1], to = xrange[2], length.out = nx)
+ y <- seq(from = yrange[1], to = yrange[2], length.out = ny)
## Set of complex numbers to be investigated
- Z <- outer(y, x, FUN=function(y,x) complex(real=x, imaginary=y))
+ Z <- outer(y, x, FUN = function(y, x) complex(real = x, imaginary = y))
- mandelbrot(Z, maxIter=maxIter, tau=tau)
-} ## mandelbrot() for numeric
+ mandelbrot(Z, maxIter = maxIter, tau = tau)
+}
#' @export
@@ -115,22 +115,22 @@ mandelbrot.numeric <- function(xmid=-0.75, ymid=0, side=3, resolution=400L, maxI
#' @keywords internal
as.raster.Mandelbrot <- function(x, ...) {
maxIter <- attr(x, "params")$maxIter
- img <- hsv(h=x/maxIter, s=1, v=1)
+ img <- hsv(h = x / maxIter, s = 1, v = 1)
img[x == maxIter] <- "#000000"
dim(img) <- dim(x)
img <- t(img)
- img <- structure(img, class="raster")
+ img <- structure(img, class = "raster")
img
-} ## as.raster()
+}
#' @export
#' @importFrom grDevices as.raster
#' @importFrom graphics par plot
#' @keywords internal
-plot.Mandelbrot <- function(x, y, ..., mar=c(0,0,0,0)) {
+plot.Mandelbrot <- function(x, y, ..., mar = c(0, 0, 0, 0)) {
if (!is.null(mar)) {
- opar <- par(mar=c(0,0,0,0))
+ opar <- par(mar = c(0, 0, 0, 0))
on.exit(par(opar))
}
plot(as.raster(x), ...)
@@ -138,9 +138,11 @@ plot.Mandelbrot <- function(x, y, ..., mar=c(0,0,0,0)) {
#' @export
-mandelbrotTiles <- function(xmid=-0.75, ymid=0.0, side=3.0, nrow=2L, ncol=nrow, resolution=400L, truncate=TRUE) {
+mandelbrot_tiles <- function(xmid = -0.75, ymid = 0.0, side = 3.0,
+ nrow = 2L, ncol = nrow,
+ resolution = 400L, truncate = TRUE) {
## Validate arguments
- stopifnot(side > 0)
+ stopifnot(side > 0)
resolution <- as.integer(resolution)
stopifnot(resolution > 0)
@@ -155,14 +157,14 @@ mandelbrotTiles <- function(xmid=-0.75, ymid=0.0, side=3.0, nrow=2L, ncol=nrow,
## Truncate so all tiles have identical dimensions?
if (truncate) {
nx <- ncol * dx
- ny <- nrow * dy
+ ny <- nrow * dy
}
-
- ## Setup (x,y) bins
- xrange <- xmid + c(-1,1)*side/2
- yrange <- ymid + c(-1,1)*side/2
- x <- seq(from=xrange[1], to=xrange[2], length.out=nx)
- y <- seq(from=yrange[1], to=yrange[2], length.out=ny)
+
+ ## Setup (x, y) bins
+ xrange <- xmid + c(-1, 1) * side / 2
+ yrange <- ymid + c(-1, 1) * side / 2
+ x <- seq(from = xrange[1], to = xrange[2], length.out = nx)
+ y <- seq(from = yrange[1], to = yrange[2], length.out = ny)
## Generate tiles row by row
@@ -170,19 +172,21 @@ mandelbrotTiles <- function(xmid=-0.75, ymid=0.0, side=3.0, nrow=2L, ncol=nrow,
for (rr in seq_len(nrow)) {
yrr <- if (rr < nrow) y[1:dy] else y
y <- y[-(1:dy)]
-
+
xrr <- x
for (cc in seq_len(ncol)) {
xcc <- if (cc < ncol) xrr[1:dx] else xrr
xrr <- xrr[-(1:dx)]
-
- Ccc <- outer(yrr, xcc, FUN=function(y,x) complex(real=x, imaginary=y))
- attr(Ccc, "region") <- list(xrange=range(xcc), yrange=range(yrr))
+
+ Ccc <- outer(yrr, xcc, FUN = function(y, x) {
+ complex(real = x, imaginary = y)
+ })
+ attr(Ccc, "region") <- list(xrange = range(xcc), yrange = range(yrr))
attr(Ccc, "tile") <- c(rr, cc)
res <- c(res, list(Ccc))
}
}
dim(res) <- c(nrow, ncol)
-
+
res
-} ## mandelbrotTiles()
+}
diff --git a/R/multicore.R b/R/multicore.R
index 7bcd040..68ab448 100644
--- a/R/multicore.R
+++ b/R/multicore.R
@@ -1,4 +1,4 @@
-#' Create a multicore future whose value will be resolved asynchroneously in a forked parallel process
+#' Create a multicore future whose value will be resolved asynchronously in a forked parallel process
#'
#' A multicore future is a future that uses multicore evaluation,
#' which means that its \emph{value is computed and resolved in
@@ -12,7 +12,7 @@
#' @return A \link{MulticoreFuture}
#' If \code{workers == 1}, then all processing using done in the
#' current/main R session and we therefore fall back to using
-#' an eager future. This is also the case whenever multicore
+#' an sequential future. This is also the case whenever multicore
#' processing is not supported, e.g. on Windows.
#'
#' @example incl/multicore.R
@@ -27,7 +27,7 @@
#' Not all systems support multicore futures. For instance,
#' it is not supported on Microsoft Windows. Trying to create
#' multicore futures on non-supported systems will silently
-#' fall back to using \link{eager} futures, which effectively
+#' fall back to using \link{sequential} futures, which effectively
#' corresponds to a multicore future that can handle one parallel
#' process (the current one) before blocking.
#'
@@ -50,31 +50,29 @@
#' system.
#'
#' @export
-multicore <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, workers=availableCores(constraints="multicore"), earlySignal=FALSE, label=NULL, ...) {
- ## BACKWARD COMPATIBILITY
- args <- list(...)
- if ("maxCores" %in% names(args)) {
- workers <- args$maxCores
- .Deprecated(msg="Argument 'maxCores' has been renamed to 'workers'. Please update you script/code that uses the future package.")
+multicore <- function(expr, envir = parent.frame(), substitute = TRUE, lazy = FALSE, seed = NULL, globals = TRUE, workers = availableCores(constraints = "multicore"), earlySignal = FALSE, label = NULL, ...) {
+ if ("maxCores" %in% names(list(...))) {
+ .Defunct(msg = "Argument 'maxCores' has been renamed to 'workers'. Please update you script/code that uses the future package.")
}
if (substitute) expr <- substitute(expr)
workers <- as.integer(workers)
stopifnot(is.finite(workers), workers >= 1L)
- ## Fall back to eager uniprocess futures if only a single additional R process
+ ## Fall back to sequential futures if only a single additional R process
## can be spawned off, i.e. then use the current main R process.
- ## Uniprocess futures best reflect how multicore futures handle globals.
+ ## Sequential futures best reflect how multicore futures handle globals.
if (workers == 1L || !supportsMulticore()) {
## covr: skip=1
- return(uniprocess(expr, envir=envir, substitute=FALSE, globals=globals, local=TRUE, label=label, lazy=FALSE))
+ return(sequential(expr, envir = envir, substitute = FALSE, lazy = lazy, seed = seed, globals = globals, local = TRUE, label = label))
}
- oopts <- options(mc.cores=workers)
+ oopts <- options(mc.cores = workers)
on.exit(options(oopts))
- future <- MulticoreFuture(expr=expr, envir=envir, substitute=FALSE, globals=globals, workers=workers, earlySignal=earlySignal, label=label)
- run(future)
+ future <- MulticoreFuture(expr = expr, envir = envir, substitute = FALSE, lazy = lazy, seed = seed, globals = globals, workers = workers, earlySignal = earlySignal, label = label, ...)
+ if (!future$lazy) future <- run(future)
+ invisible(future)
}
class(multicore) <- c("multicore", "multiprocess", "future", "function")
@@ -82,25 +80,25 @@ class(multicore) <- c("multicore", "multiprocess", "future", "function")
#' Get number of cores currently used
#'
-#' Get number of children plus one (for the current process)
+#' Get number of children (and don't count the current process)
#' used by the current R session. The number of children
#' is the total number of subprocesses launched by this
#' process that are still running and whose values have yet
#' not been collected.
#'
-#' @return A positive integer equal or greater than one.
+#' @return A non-negative integer.
#'
#' @keywords internal
usedCores <- function() {
## Number of unresolved multicore futures
- futures <- FutureRegistry("multicore", action="list")
+ futures <- FutureRegistry("multicore", action = "list")
nfutures <- length(futures)
ncores <- nfutures
## Total number of multicore processes
## To please R CMD check
ns <- getNamespace("parallel")
- children <- get("children", envir=ns, mode="function")
+ children <- get("children", envir = ns, mode = "function")
nchildren <- length(children())
## Any multicore processes that are not futures?
@@ -116,12 +114,12 @@ usedCores <- function() {
## However, ...
if (nfutures == 0L) {
- warning(sprintf("Hmm... %d active multicore processes were detected, but without any active multicore futures (it is not clear by what mechanism they were created). Because of this, the 'future' package do not know how to resolve/collect them and will therefore threat them as they do not exist.", nchildren))
+ warning(sprintf("Hmm... %d active multicore processes were detected, but without any active multicore futures (it is not clear by what mechanism they were created). Because of this, the 'future' package do not know how to resolve/collect them and will therefore treat them as they do not exist.", nchildren))
ncores <- 0L
}
}
- return(ncores + 1L)
+ ncores
}
@@ -133,11 +131,15 @@ usedCores <- function() {
#'
#' @param await A function used to try to "collect"
#' finished multicore subprocesses.
+#'
#' @param workers Total number of workers available.
-#' @param times Then maximum number of times subprocesses
-#' should be collected before timeout.
+#'
+#' @param timeout Maximum waiting time (in seconds) allowed
+#' before a timeout error is generated.
+#'
#' @param delta Then base interval (in seconds) to wait
#' between each try.
+#'
#' @param alpha A multiplicative factor used to increase
#' the wait interval after each try.
#'
@@ -145,45 +147,50 @@ usedCores <- function() {
#' extensive waiting, then a timeout error is thrown.
#'
#' @keywords internal
-requestCore <- function(await, workers=availableCores(), times=getOption("future.wait.times", 1000), delta=getOption("future.wait.interval", 1.0), alpha=getOption("future.wait.alpha", 1.01)) {
+requestCore <- function(await, workers = availableCores(), timeout = getOption("future.wait.timeout", 30 * 24 * 60 * 60), delta = getOption("future.wait.interval", 0.2), alpha = getOption("future.wait.alpha", 1.01)) {
stopifnot(length(workers) == 1L, is.numeric(workers), is.finite(workers), workers >= 1)
stopifnot(is.function(await))
- times <- as.integer(times)
- stopifnot(is.finite(times), times > 0)
+ stopifnot(is.finite(timeout), timeout >= 0)
stopifnot(is.finite(alpha), alpha > 0)
- mdebug(sprintf("requestCore(): workers = %d", workers))
+ debug <- getOption("future.debug", FALSE)
+ if (debug) mdebug("requestCore(): workers = %d", workers)
## No additional cores available?
- if (workers == 1L) {
- stop("INTERNAL ERROR: requestCore() was asked to find a free core, but there is only one core available, which is already occupied by the main R process.")
+ if (workers == 0L) {
+ stop("INTERNAL ERROR: requestCore() was asked to find a free core, but no cores are available (workers = 0).")
}
+
+ t0 <- Sys.time()
+ dt <- 0
iter <- 1L
interval <- delta
finished <- FALSE
- while (iter <= times) {
+ while (dt <= timeout) {
+ ## Check for available cores
used <- usedCores()
finished <- (used < workers)
if (finished) break
- mdebug(sprintf("usedCores() = %d, workers = %d", used, workers))
+ if (debug) mdebug("Poll #%d (%s): usedCores() = %d, workers = %d", iter, format(round(dt, digits = 2L)), used, workers)
## Wait
Sys.sleep(interval)
+ interval <- alpha * interval
## Finish/close cores, iff possible
await()
- interval <- alpha*interval
iter <- iter + 1L
+ dt <- difftime(Sys.time(), t0)
}
if (!finished) {
- msg <- sprintf("TIMEOUT: All %d CPU cores are still occupied", workers)
- mdebug(msg)
+ msg <- sprintf("TIMEOUT: All %d cores are still occupied after %s (polled %d times)", workers, format(round(dt, digits = 2L)), iter)
+ if (debug) mdebug(msg)
stop(msg)
}
- invisible(finished)
+ invisible(unname(finished))
}
diff --git a/R/multiprocess.R b/R/multiprocess.R
index 50e4f45..1a3d2c6 100644
--- a/R/multiprocess.R
+++ b/R/multiprocess.R
@@ -1,4 +1,4 @@
-#' Create a multiprocess future whose value will be resolved asynchroneously using multicore or a multisession evaluation
+#' Create a multiprocess future whose value will be resolved asynchronously using multicore or a multisession evaluation
#'
#' A multiprocess future is a future that uses \link{multicore} evaluation
#' if supported, otherwise it uses \link{multisession} evaluation.
@@ -6,6 +6,8 @@
#' parallel in another process}.
#'
#' @inheritParams future
+#' @param lazy If \code{FALSE} (default), the future is resolved eagerly
+#' (immediately), otherwise not.
#' @param globals (optional) a logical, a character vector,
#' or a named list for controlling how globals are handled.
#' For details, see section 'Globals used by future expressions'
@@ -29,9 +31,9 @@
#' are used.
#'
#' @export
-multiprocess <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, workers=availableCores(), gc=FALSE, earlySignal=FALSE, label=NULL, ...) {
+multiprocess <- function(expr, envir = parent.frame(), substitute = TRUE, lazy = FALSE, seed = NULL, globals = TRUE, workers = availableCores(), gc = FALSE, earlySignal = FALSE, label = NULL, ...) {
if (substitute) expr <- substitute(expr)
fun <- if (supportsMulticore()) multicore else multisession
- fun(expr=expr, envir=envir, substitute=FALSE, globals=globals, workers=workers, gc=gc, earlySignal=earlySignal, label=label, ...)
+ fun(expr = expr, envir = envir, substitute = FALSE, lazy = lazy, seed = seed, globals = globals, workers = workers, gc = gc, earlySignal = earlySignal, label = label, ...)
}
class(multiprocess) <- c("multiprocess", "future", "function")
diff --git a/R/multisession.R b/R/multisession.R
index 30a6a51..43477f1 100644
--- a/R/multisession.R
+++ b/R/multisession.R
@@ -1,4 +1,4 @@
-#' Create a multisession future whose value will be resolved asynchroneously in a parallel R session
+#' Create a multisession future whose value will be resolved asynchronously in a parallel R session
#'
#' A multisession future is a future that uses multisession evaluation,
#' which means that its \emph{value is computed and resolved in
@@ -54,31 +54,27 @@
#' cores that are available for the current R session.
#'
#' @export
-multisession <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, persistent=FALSE, workers=availableCores(), gc=FALSE, earlySignal=FALSE, label=NULL, ...) {
- ## BACKWARD COMPATIBILITY
- args <- list(...)
- if ("maxCores" %in% names(args)) {
- workers <- args$maxCores
- .Deprecated(msg="Argument 'maxCores' has been renamed to 'workers'. Please update you script/code that uses the future package.")
+multisession <- function(expr, envir = parent.frame(), substitute = TRUE, lazy = FALSE, seed = NULL, globals = TRUE, persistent = FALSE, workers = availableCores(), gc = FALSE, earlySignal = FALSE, label = NULL, ...) {
+ if ("maxCores" %in% names(list(...))) {
+ .Defunct(msg = "Argument 'maxCores' has been renamed to 'workers'. Please update you script/code that uses the future package.")
}
if (substitute) expr <- substitute(expr)
workers <- as.integer(workers)
stopifnot(length(workers) == 1, is.finite(workers), workers >= 1)
- ## Fall back to lazy uniprocess futures if only a single R session can be used,
+ ## Fall back to lazy sequential futures if only a single R session can be used,
## i.e. the use the current main R process.
if (workers == 1L) {
## FIXME: How to handle argument 'persistent'? /HB 2016-03-19
- return(uniprocess(expr, envir=envir, substitute=FALSE, globals=globals, local=TRUE, label=label, lazy=TRUE))
+ return(sequential(expr, envir = envir, substitute = FALSE, lazy = TRUE, seed = seed, globals = globals, local = TRUE, label = label))
}
- ## IMPORTANT: When we setup a multisession cluster, we need to
- ## account for the main R process as well, i.e. we should setup
- ## a cluster with one less process.
- workers <- ClusterRegistry("start", workers=workers-1L)
+ workers <- ClusterRegistry("start", workers = workers)
- future <- MultisessionFuture(expr=expr, envir=envir, substitute=FALSE, globals=globals, persistent=persistent, workers=workers, gc=gc, earlySignal=earlySignal, label=label, ...)
- run(future)
+ future <- MultisessionFuture(expr = expr, envir = envir, substitute = FALSE, lazy = lazy, seed = seed, globals = globals, persistent = persistent, workers = workers, gc = gc, earlySignal = earlySignal, label = label, ...)
+ if (!future$lazy) future <- run(future)
+ invisible(future)
}
class(multisession) <- c("multisession", "cluster", "multiprocess", "future", "function")
+attr(multisession, "init") <- TRUE
diff --git a/R/nbrOfWorkers.R b/R/nbrOfWorkers.R
index 7946101..cb0b606 100644
--- a/R/nbrOfWorkers.R
+++ b/R/nbrOfWorkers.R
@@ -4,17 +4,31 @@
#' If NULL (default), the current evaluator as returned
#' by \code{\link{plan}()} is used.
#'
-#' @return A number in [1,Inf].
+#' @return A number in [1, Inf].
#'
#' @example incl/nbrOfWorkers.R
#'
#' @export
-nbrOfWorkers <- function(evaluator=NULL) {
+nbrOfWorkers <- function(evaluator = NULL) {
UseMethod("nbrOfWorkers")
}
#' @export
+nbrOfWorkers.cluster <- function(evaluator) {
+ expr <- formals(evaluator)$workers
+ workers <- eval(expr)
+ stopifnot(is.character(workers) || is.numeric(workers) || inherits(workers, "cluster"), !anyNA(workers))
+ if (is.character(workers)) {
+ workers <- length(workers)
+ } else if (inherits(workers, "cluster")) {
+ workers <- length(workers)
+ }
+ stopifnot(length(workers) == 1, is.finite(workers), workers >= 1)
+ workers
+}
+
+#' @export
nbrOfWorkers.uniprocess <- function(evaluator) 1L
#' @export
diff --git a/R/options.R b/R/options.R
index 89f2ca0..5874683 100644
--- a/R/options.R
+++ b/R/options.R
@@ -6,15 +6,15 @@
#'
#' @section Options for controlling futures:
#' \describe{
-#' \item{\option{future.plan}:}{Default future strategy plan used unless otherwise specified via \code{\link{plan}()}. This will also be the future plan set when calling \code{plan("default")}. If not specified, this option may be set when the \pkg{future} package is \emph{loaded} if command-line option \code{--parallel=ncores} (short \code{-p ncores}) is specified; if \code{ncores > 1}, then option \option{future.plan} is set to \code{multiprocess} otherwise \code{eager} (in addition [...]
+#' \item{\option{future.plan}:}{Default future strategy plan used unless otherwise specified via \code{\link{plan}()}. This will also be the future plan set when calling \code{plan("default")}. If not specified, this option may be set when the \pkg{future} package is \emph{loaded} if command-line option \code{--parallel=ncores} (short \code{-p ncores}) is specified; if \code{ncores > 1}, then option \option{future.plan} is set to \code{multiprocess} otherwise \code{sequential} (in addi [...]
#' \item{\option{future.globals.onMissing}:}{Action to take when non-existing global variables ("globals" or "unknowns") are identified when the future is created. If \code{"error"}, an error is generated immediately. If \code{"ignore"}, no action is taken and an attempt to evaluate the future expression will be made. The latter is useful when there is a risk for false-positive globals being identified, e.g. when future expression contains non-standard evaluation (NSE). (Default: \c [...]
#' \item{\option{future.globals.method}:}{Method used to identify globals. For details, see \code{\link[globals]{globalsOf}()}. (Default: \code{"ordered"})}
-#' \item{\option{future.globals.maxSize}:}{Maximum allowed total size (in bytes) of global variables identified. Used to prevent too large exports. (Default: \code{500*1024^2} = 500 MiB)}
+#' \item{\option{future.globals.maxSize}:}{Maximum allowed total size (in bytes) of global variables identified. Used to prevent too large exports. (Default: \code{500 * 1024 ^ 2} = 500 MiB)}
#' \item{\option{future.globals.resolve}:}{If \code{TRUE}, globals that are \code{\link{Future}} objects (typically created as \emph{explicit} futures) will be resolved and have their values (using \code{value()}) collected. Because searching for unresolved futures among globals (including their content) can be expensive, the default is not to do it and instead leave it to the run-time checks that assert proper ownership when resolving futures and collecting their values. (Default: \co [...]
#' \item{\option{future.resolve.recursive}:}{An integer specifying the maximum recursive depth to which futures should be resolved. If negative, nothing is resolved. If \code{0}, only the future itself is resolved. If \code{1}, the future and any of its elements that are futures are resolved, and so on. If \code{+Inf}, infinite search depth is used. (Default: \code{0})}
-#' \item{\option{future.wait.times}:}{Maximum number of times a future is polled waiting for it to be resolved.}
-#' \item{\option{future.wait.interval}:}{Initial interval (in seconds) between polls.}
-#' \item{\option{future.wait.alpha}:}{Positive scale factor used to increase the interval after each poll. (Default: \code{0.01})}
+#' \item{\option{future.wait.timeout}:}{Maximum waiting time (in seconds) for a free worker before a timeout error is generated. (Default: \code{30 * 24 * 60 * 60} (= 30 days))}
+#' \item{\option{future.wait.interval}:}{Initial interval (in seconds) between polls. (Default: \code{0.2} (0.2 seconds))}
+#' \item{\option{future.wait.alpha}:}{Positive scale factor used to increase the interval after each poll. (Default: \code{1.01})}
#' }
#'
#' @section Options for debugging futures:
@@ -29,10 +29,15 @@
#' \item{\option{future.cmdargs}:}{Overrides \code{\link[base]{commandArgs}()} when the \pkg{future} package is \emph{loaded}.}
#' }
#'
-#' @section Options for configurating low-level system behaviors:
+#' @section Options for configuring low-level system behaviors:
#' \describe{
-#' \item{\option{future.availableCores.methods}:}{Default lookup methods for \code{\link{availableCores}()}. (Default: \code{c("system", "mc.cores+1", "_R_CHECK_LIMIT_CORES_", "Slurm", "PBS", "SGE")})}
-#' \item{\option{future.availableCores.system}:}{Number of "system" cores used instead of what is reported by \code{\link{availableCores}(which="system")}. If not specified, this option is set according to system environment variable \env{R_FUTURE_AVAILABLECORES_SYSTEM} when the \pkg{future} package is \emph{loaded}.}
+#' \item{\option{future.availableCores.methods}:}{Default lookup methods for \code{\link{availableCores}()}. (Default: \code{c("system", "mc.cores", "_R_CHECK_LIMIT_CORES_", "PBS", "SGE", "Slurm", "fallback")})}
+#'
+#' \item{\option{future.availableWorkers.methods}:}{Default lookup methods for \code{\link{availableWorkers}()}. (Default: \code{c("mc.cores", "_R_CHECK_LIMIT_CORES_", "PBS", "SGE", "Slurm", "system", "fallback")})}
+#'
+#' \item{\option{future.availableCores.fallback}:}{Number of cores to use when no core-specifying settings are detected other than \code{"system"}. If not specified, this option is set according to system environment variable \env{R_FUTURE_AVAILABLECORES_FALLBACK} when the \pkg{future} package is \emph{loaded}. This options makes it possible to set the default number of cores returned by \code{availableCores()} / \code{availableWorkers()} yet allow users and schedulers to override it. I [...]
+#'
+#' \item{\option{future.availableCores.system}:}{Number of "system" cores used instead of what is reported by \code{\link{availableCores}(which = "system")}. If not specified, this option is set according to system environment variable \env{R_FUTURE_AVAILABLECORES_SYSTEM} when the \pkg{future} package is \emph{loaded}. This option allows you to effectively override what \code{parallel::detectCores()} reports the system has.}
#' }
#'
#' @section Options for demos:
@@ -42,7 +47,7 @@
#' }
#'
#' @seealso
-#' To set \R options when \R starts (even before the \pkg{future} package is loaded), see the \link[base]{Startup} help page.
+#' To set \R options when \R starts (even before the \pkg{future} package is loaded), see the \link[base]{Startup} help page. The \href{https://cran.r-project.org/package=startup}{\pkg{startup}} package provides a friendly mechanism for configurating \R's startup process.
#'
#' @aliases future.availableCores.methods future.cmdargs future.cores future.debug future.globals.maxSize future.globals.method future.globals.onMissing future.globals.resolve future.plan future.progress future.resolve.recursive future.startup.script future.wait.alpha future.wait.interval future.wait.times R_FUTURE_PLAN R_FUTURE_CORES future.demo.mandelbrot.region future.demo.mandelbrot.nrow
#' @keywords internal
diff --git a/R/plan_OP.R b/R/plan_OP.R
index ab65a3b..cda1fa4 100644
--- a/R/plan_OP.R
+++ b/R/plan_OP.R
@@ -18,8 +18,8 @@
## Temporarily use a different plan
oplan <- plan("list")
- on.exit(plan(oplan, substitute=FALSE, .call=NULL))
- plan(strategy, substitute=FALSE, .call=NULL)
+ on.exit(plan(oplan, substitute = FALSE, .call = NULL, .cleanup = FALSE, .init = FALSE))
+ plan(strategy, substitute = FALSE, .call = NULL, .cleanup = FALSE, .init = FALSE)
- eval(fassignment, envir=envir)
+ eval(fassignment, envir = envir)
}
diff --git a/R/remote.R b/R/remote.R
index 2641114..8a9fb47 100644
--- a/R/remote.R
+++ b/R/remote.R
@@ -1,4 +1,4 @@
-#' Create a remote future whose value will be resolved asynchroneously in a remote process
+#' Create a remote future whose value will be resolved asynchronously in a remote process
#'
#' A remote future is a future that uses remote cluster evaluation,
#' which means that its \emph{value is computed and resolved
@@ -15,13 +15,13 @@
#' @example incl/remote.R
#'
#' @details
-#' Note that remote futures use \code{persistent=TRUE} by default.
+#' Note that remote futures use \code{persistent = TRUE} by default.
#'
#' @export
-remote <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, persistent=TRUE, workers=NULL, user=NULL, revtunnel=TRUE, gc=FALSE, earlySignal=FALSE, myip=NULL, label=NULL, ...) {
+remote <- function(expr, envir = parent.frame(), substitute = TRUE, lazy = FALSE, seed = NULL, globals = TRUE, persistent = TRUE, workers = NULL, user = NULL, revtunnel = TRUE, gc = FALSE, earlySignal = FALSE, myip = NULL, label = NULL, ...) {
if (substitute) expr <- substitute(expr)
- stopifnot(length(workers) >= 1L, is.character(workers), !anyNA(workers))
+ stopifnot(length(workers) >= 1L, !anyNA(workers))
if (is.character(workers)) {
homogeneous <- FALSE ## Calls plain 'Rscript'
@@ -40,7 +40,7 @@ remote <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, pe
## for 'localhost' anyways, but just in case)
revtunnel <- FALSE
- ## (c) We use myip='127.0.0.1' because it's slightly more
+ ## (c) We use myip = '127.0.0.1' because it's slightly more
## generic than 'localhost'.
myip <- "127.0.0.1"
} else {
@@ -64,7 +64,9 @@ remote <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, pe
stop("Argument 'workers' is not of class 'cluster': ", class(workers)[1])
}
- future <- ClusterFuture(expr=expr, envir=envir, substitute=FALSE, globals=globals, persistent=persistent, workers=workers, user=user, master=myip, revtunnel=revtunnel, homogeneous=homogeneous, gc=gc, earlySignal=earlySignal, label=label, ...)
- run(future)
+ future <- ClusterFuture(expr = expr, envir = envir, substitute = FALSE, lazy = lazy, seed = seed, globals = globals, persistent = persistent, workers = workers, user = user, master = myip, revtunnel = revtunnel, homogeneous = homogeneous, gc = gc, earlySignal = earlySignal, label = label, ...)
+ if (!future$lazy) future <- run(future)
+ invisible(future)
}
-class(remote) <- c("remote", "multiprocess", "future", "function")
+class(remote) <- c("remote", "cluster", "multiprocess", "future", "function")
+attr(remote, "init") <- TRUE
diff --git a/R/resolve.R b/R/resolve.R
index c9ea99b..6e665f4 100644
--- a/R/resolve.R
+++ b/R/resolve.R
@@ -1,37 +1,48 @@
-#' Wait until all existing futures in an environment are resolved
+#' Resolve one or more futures synchronously
#'
-#' The environment is first scanned for futures and then the futures
-#' are polled until all are resolved. When a resolved future is
-#' detected its value is retrieved (optionally).
-#' This provides an efficient mechanism for waiting for a set of
-#' futures to be resolved and in the meanwhile retrieving values
-#' of already resolved futures.
-#'
-#' @param x an environment holding futures.
-#' @param idxs subset of elements to check.
+#' This function provides an efficient mechanism for waiting for multiple
+#' futures in a container (e.g. list or environment) to be resolved while in
+#' the meanwhile retrieving values of already resolved futures.
+#'
+#' @param x a list, an environment, or a list environment holding futures
+#' that should be resolved. May also be a single \link{Future}.
+#'
+#' @param idxs (optional) integer or logical index specifying the subset of
+#' elements to check.
+#'
#' @param value If TRUE, the values are retrieved, otherwise not.
-#' @param recursive A non-negative number specifying how deep of
-#' a recursion should be done. If TRUE, an infintive recursion
-#' is used. If FALSE or zero, no recursion is performed.
-#' @param sleep Number of seconds to wait before checking
-#' if futures have been resolved since last time.
-#' @param progress If TRUE textual progress summary is outputted.
-#' If a function, the it is called as \code{progress(done, total)}
-#' every time a future is resolved.
+#'
+#' @param recursive A non-negative number specifying how deep of a recursion
+#' should be done. If TRUE, an infinite recursion is used. If FALSE or zero,
+#' no recursion is performed.
+#'
+#' @param sleep Number of seconds to wait before checking if futures have been
+#' resolved since last time.
+#'
+#' @param progress If TRUE textual progress summary is outputted. If a
+#' function, the it is called as \code{progress(done, total)} every time a
+#' future is resolved.
+#'
#' @param \dots Not used
#'
#' @return Returns \code{x} (regardless of subsetting or not).
#'
-#' @seealso futureOf
+#' @details
+#' This function is resolves synchronously, i.e. it blocks until \code{x} and
+#' any containing futures are resolved.
+#'
+#' @seealso To resolve a future \emph{variable}, first retrieve its
+#' \link{Future} object using \code{\link{futureOf}()}, e.g.
+#' \code{resolve(futureOf(x))}.
#'
#' @export
-resolve <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=1.0, progress=getOption("future.progress", FALSE), ...) UseMethod("resolve")
+resolve <- function(x, idxs = NULL, value = FALSE, recursive = 0, sleep = 1.0, progress = getOption("future.progress", FALSE), ...) UseMethod("resolve")
#' @export
resolve.default <- function(x, ...) x
#' @export
-resolve.Future <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, progress=getOption("future.progress", FALSE), ...) {
+resolve.Future <- function(x, idxs = NULL, value = FALSE, recursive = 0, sleep = 0.1, progress = getOption("future.progress", FALSE), ...) {
if (is.logical(recursive)) {
if (recursive) recursive <- getOption("future.resolve.recursive", 99)
}
@@ -40,7 +51,10 @@ resolve.Future <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, pr
## Nothing to do?
if (recursive < 0) return(x)
- ## Poll for Future to finish
+ ## Lazy future that is not yet launched?
+ if (x$state == 'created') x <- run(x)
+
+ ## Poll for the Future to finish
while (!resolved(x)) {
Sys.sleep(sleep)
}
@@ -58,7 +72,7 @@ resolve.Future <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, pr
## Recursively resolve the value?
if (!is.atomic(v)) {
- v <- resolve(v, value=TRUE, recursive=recursive-1, sleep=sleep, progress=FALSE, ...)
+ v <- resolve(v, value = TRUE, recursive = recursive-1, sleep = sleep, progress = FALSE, ...)
msg <- sprintf("%s (and resolved itself)", msg)
}
@@ -80,7 +94,7 @@ resolve.Future <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, pr
#' @export
-resolve.list <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, progress=getOption("future.progress", FALSE), ...) {
+resolve.list <- function(x, idxs = NULL, value = FALSE, recursive = 0, sleep = 0.1, progress = getOption("future.progress", FALSE), ...) {
if (is.logical(recursive)) {
if (recursive) recursive <- getOption("future.resolve.recursive", 99)
}
@@ -88,9 +102,11 @@ resolve.list <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, prog
## Nothing to do?
if (recursive < 0) return(x)
+
+ nx <- .length(x)
## Nothing to do?
- if (length(x) == 0) return(x)
+ if (nx == 0) return(x)
x0 <- x
@@ -99,10 +115,10 @@ resolve.list <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, prog
## Setup default progress function?
if (hasProgress && !is.function(progress)) {
progress <- function(done, total) {
- msg <- sprintf("Progress: %.0f%% (%d/%d)", 100*done/total, done, total)
+ msg <- sprintf("Progress: %.0f%% (%d/%d)", 100 * done / total, done, total)
if (done < total) {
- bs <- paste(rep("\b", times=nchar(msg)), collapse="")
- message(paste(msg, bs, sep=""), appendLF=FALSE)
+ bs <- paste(rep("\b", times = nchar(msg)), collapse = "")
+ message(paste(msg, bs, sep = ""), appendLF = FALSE)
} else {
message(msg)
}
@@ -117,14 +133,14 @@ resolve.list <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, prog
## Multi-dimensional indices?
if (is.matrix(idxs)) {
- idxs <- whichIndex(idxs, dim=dim(x), dimnames=dimnames(x))
+ idxs <- whichIndex(idxs, dim = dim(x), dimnames = dimnames(x))
}
idxs <- unique(idxs)
if (is.numeric(idxs)) {
idxs <- as.numeric(idxs)
- if (any(idxs < 1 | idxs > length(x))) {
- stop(sprintf("Indices out of range [1,%d]: %s", length(x), hpaste(idxs)))
+ if (any(idxs < 1 | idxs > nx)) {
+ stop(sprintf("Indices out of range [1,%d]: %s", nx, hpaste(idxs)))
}
} else {
names <- names(x)
@@ -140,13 +156,16 @@ resolve.list <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, prog
}
x <- x[idxs]
+ nx <- .length(x)
}
- mdebug("resolve() on list ...")
- mdebug(" recursive: %s", recursive)
+ debug <- getOption("future.debug", FALSE)
+ if (debug) {
+ mdebug("resolve() on list ...")
+ mdebug(" recursive: %s", recursive)
+ }
- ## Everything is considered non-resolved by default
- nx <- length(x)
+ ## NOTE: Everything is considered non-resolved by default
## Total number of values to resolve
total <- nx
@@ -157,8 +176,10 @@ resolve.list <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, prog
progress(done, total)
}
- mdebug(" length: %d", nx)
- mdebug(" elements: %s", hpaste(sQuote(names(x))))
+ if (debug) {
+ mdebug(" length: %d", nx)
+ mdebug(" elements: %s", hpaste(sQuote(names(x))))
+ }
## Resolve all elements
while (length(remaining) > 0) {
@@ -167,16 +188,20 @@ resolve.list <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, prog
if (!is.atomic(obj)) {
## If an unresolved future, move on to the next object
- ## so that future can be resolved in the asynchroneously
- if (inherits(obj, "Future") && !resolved(obj)) next
+ ## so that future can be resolved in the asynchronously
+ if (inherits(obj, "Future")) {
+ ## Lazy future that is not yet launched?
+ if (obj$state == 'created') obj <- run(obj)
+ if (!resolved(obj)) next
+ }
## In all other cases, try to resolve
- resolve(obj, value=value, recursive=recursive-1, sleep=sleep, progress=FALSE, ...)
+ resolve(obj, value = value, recursive = recursive - 1, sleep = sleep, progress = FALSE, ...)
}
## Assume resolved at this point
remaining <- setdiff(remaining, ii)
- mdebug(" length: %d (resolved future %s)", length(remaining), ii)
+ if (debug) mdebug(" length: %d (resolved future %s)", length(remaining), ii)
stopifnot(!anyNA(remaining))
if (hasProgress) {
@@ -191,14 +216,14 @@ resolve.list <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, prog
if (hasProgress && done != done0) progress(done, total)
- mdebug("resolve() on list ... DONE")
+ if (debug) mdebug("resolve() on list ... DONE")
x0
} ## resolve() for list
#' @export
-resolve.environment <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, progress=FALSE, ...) {
+resolve.environment <- function(x, idxs = NULL, value = FALSE, recursive = 0, sleep = 0.1, progress = FALSE, ...) {
if (is.logical(recursive)) {
if (recursive) recursive <- getOption("future.resolve.recursive", 99)
}
@@ -207,21 +232,23 @@ resolve.environment <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.
## Nothing to do?
if (recursive < 0) return(x)
+ nx <- .length(x)
+
## Nothing to do?
- if (length(x) == 0) return(x)
+ if (nx == 0) return(x)
## Subset?
if (is.null(idxs)) {
## names(x) is only supported in R (>= 3.2.0)
- idxs <- ls(envir=x, all.names=TRUE)
+ idxs <- ls(envir = x, all.names = TRUE)
} else {
## Nothing to do?
if (length(idxs) == 0) return(x)
## names(x) is only supported in R (>= 3.2.0)
- names <- ls(envir=x, all.names=TRUE)
+ names <- ls(envir = x, all.names = TRUE)
- ## Sanity check (because length(x) == 0 returns early above)
+ ## Sanity check (because nx == 0 returns early above)
stopifnot(length(names) > 0)
idxs <- unique(idxs)
@@ -238,20 +265,23 @@ resolve.environment <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.
nx <- length(idxs)
if (nx == 0) return(x)
- mdebug("resolve() on environment ...")
- mdebug(" recursive: %s", recursive)
+ debug <- getOption("future.debug", FALSE)
+ if (debug) {
+ mdebug("resolve() on environment ...")
+ mdebug(" recursive: %s", recursive)
+ }
## Coerce future promises into Future objects
x0 <- x
x <- futures(x)
- nx <- length(x)
- idxs <- ls(envir=x, all.names=TRUE)
+ nx <- .length(x)
+ idxs <- ls(envir = x, all.names = TRUE)
stopifnot(length(idxs) == nx)
## Everything is considered non-resolved by default
remaining <- idxs
- mdebug(" elements: [%d] %s", nx, hpaste(sQuote(idxs)))
+ if (debug) mdebug(" elements: [%d] %s", nx, hpaste(sQuote(idxs)))
## Resolve all elements
while (length(remaining) > 0) {
@@ -260,16 +290,20 @@ resolve.environment <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.
if (!is.atomic(obj)) {
## If an unresolved future, move on to the next object
- ## so that future can be resolved in the asynchroneously
- if (inherits(obj, "Future") && !resolved(obj)) next
+ ## so that future can be resolved in the asynchronously
+ if (inherits(obj, "Future")) {
+ ## Lazy future that is not yet launched?
+ if (obj$state == 'created') obj <- run(obj)
+ if (!resolved(obj)) next
+ }
## In all other cases, try to resolve
- resolve(obj, value=value, recursive=recursive-1, sleep=sleep, progress=FALSE, ...)
+ resolve(obj, value = value, recursive = recursive-1, sleep = sleep, progress = FALSE, ...)
}
## Assume resolved at this point
remaining <- setdiff(remaining, ii)
- mdebug(" length: %d (resolved future %s)", length(remaining), ii)
+ if (debug) mdebug(" length: %d (resolved future %s)", length(remaining), ii)
stopifnot(!anyNA(remaining))
} # for (ii ...)
@@ -277,14 +311,14 @@ resolve.environment <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.
if (length(remaining) > 0) Sys.sleep(sleep)
} # while (...)
- mdebug("resolve() on environment ... DONE")
+ if (debug) mdebug("resolve() on environment ... DONE")
x0
} ## resolve() for environment
#' @export
-resolve.listenv <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, progress=FALSE, ...) {
+resolve.listenv <- function(x, idxs = NULL, value = FALSE, recursive = 0, sleep = 0.1, progress = FALSE, ...) {
if (is.logical(recursive)) {
if (recursive) recursive <- getOption("future.resolve.recursive", 99)
}
@@ -293,8 +327,12 @@ resolve.listenv <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, p
## Nothing to do?
if (recursive < 0) return(x)
+ ## NOTE: Contrary to other implementations that use .length(x), we here
+ ## do need to use generic length() that dispatches on class.
+ nx <- length(x)
+
## Nothing to do?
- if (length(x) == 0) return(x)
+ if (nx == 0) return(x)
## Subset?
if (is.null(idxs)) {
@@ -305,18 +343,18 @@ resolve.listenv <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, p
## Multi-dimensional indices?
if (is.matrix(idxs)) {
- idxs <- whichIndex(idxs, dim=dim(x), dimnames=dimnames(x))
+ idxs <- whichIndex(idxs, dim = dim(x), dimnames = dimnames(x))
}
idxs <- unique(idxs)
if (is.numeric(idxs)) {
- if (any(idxs < 1 | idxs > length(x))) {
- stop(sprintf("Indices out of range [1,%d]: %s", length(x), hpaste(idxs)))
+ if (any(idxs < 1 | idxs > nx)) {
+ stop(sprintf("Indices out of range [1,%d]: %s", nx, hpaste(idxs)))
}
} else {
names <- names(x)
- ## Sanity check (because length(x) == 0 returns early above)
+ ## Sanity check (because nx == 0 returns early above)
stopifnot(length(names) > 0)
idxs <- as.character(idxs)
@@ -332,8 +370,11 @@ resolve.listenv <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, p
if (nx == 0) return(x)
- mdebug("resolve() on list environment ...")
- mdebug(" recursive: %s", recursive)
+ debug <- getOption("future.debug", FALSE)
+ if (debug) {
+ mdebug("resolve() on list environment ...")
+ mdebug(" recursive: %s", recursive)
+ }
## Coerce future promises into Future objects
x0 <- x
@@ -343,8 +384,10 @@ resolve.listenv <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, p
## Everything is considered non-resolved by default
remaining <- seq_len(nx)
- mdebug(" length: %d", nx)
- mdebug(" elements: %s", hpaste(sQuote(names(x))))
+ if (debug) {
+ mdebug(" length: %d", nx)
+ mdebug(" elements: %s", hpaste(sQuote(names(x))))
+ }
## Resolve all elements
while (length(remaining) > 0) {
@@ -353,16 +396,20 @@ resolve.listenv <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, p
if (!is.atomic(obj)) {
## If an unresolved future, move on to the next object
- ## so that future can be resolved in the asynchroneously
- if (inherits(obj, "Future") && !resolved(obj)) next
+ ## so that future can be resolved in the asynchronously
+ if (inherits(obj, "Future")) {
+ ## Lazy future that is not yet launched?
+ if (obj$state == 'created') obj <- run(obj)
+ if (!resolved(obj)) next
+ }
## In all other cases, try to resolve
- resolve(obj, value=value, recursive=recursive-1, sleep=sleep, progress=FALSE, ...)
+ resolve(obj, value = value, recursive = recursive-1, sleep = sleep, progress = FALSE, ...)
}
## Assume resolved at this point
remaining <- setdiff(remaining, ii)
- mdebug(" length: %d (resolved future %s)", length(remaining), ii)
+ if (debug) mdebug(" length: %d (resolved future %s)", length(remaining), ii)
stopifnot(!anyNA(remaining))
} # for (ii ...)
@@ -370,7 +417,7 @@ resolve.listenv <- function(x, idxs=NULL, value=FALSE, recursive=0, sleep=0.1, p
if (length(remaining) > 0) Sys.sleep(sleep)
} # while (...)
- mdebug("resolve() on list environment ... DONE")
+ if (debug) mdebug("resolve() on list environment ... DONE")
x0
} ## resolve() for list environment
diff --git a/R/resolved.R b/R/resolved.R
index 4dbf8a7..dc8324f 100644
--- a/R/resolved.R
+++ b/R/resolved.R
@@ -13,7 +13,7 @@
#' the Future API. The implementation must never throw an error,
#' but only return either TRUE or FALSE.
#' It should also be possible to use the method for polling the
-#' future until it is resolved (without having to wait infinitly long),
+#' future until it is resolved (without having to wait infinitely long),
#' e.g. \code{while (!resolved(future)) Sys.sleep(5)}.
#'
#' @export
@@ -33,7 +33,7 @@ resolved.list <- function(x, ...) {
## Allocate results. Assume everything
## is resolved unless not.
- res <- rep(TRUE, times=length(fs))
+ res <- rep(TRUE, times = length(fs))
for (ii in seq_along(fs)) {
f <- fs[[ii]]
if (inherits(f, "Future")) res[[ii]] <- resolved(f)
diff --git a/R/seed_OP.R b/R/seed_OP.R
new file mode 100644
index 0000000..c17608c
--- /dev/null
+++ b/R/seed_OP.R
@@ -0,0 +1,20 @@
+#' Set random seed for future assignment
+#'
+#' @usage fassignment \%seed\% seed
+#'
+#' @param fassignment The future assignment, e.g.
+#' \code{x \%<-\% \{ expr \}}.
+#' @inheritParams multiprocess
+#'
+#' @export
+`%seed%` <- function(fassignment, seed) {
+ fassignment <- substitute(fassignment)
+ envir <- parent.frame(1)
+
+ ## Temporarily set 'seed' argument
+ args <- getOption("future.disposable", list())
+ args["seed"] <- list(seed)
+ options(future.disposable = args)
+
+ eval(fassignment, envir = envir)
+}
diff --git a/R/sequential.R b/R/sequential.R
new file mode 100644
index 0000000..274729f
--- /dev/null
+++ b/R/sequential.R
@@ -0,0 +1,52 @@
+#' Create a sequential future whose value will be in the current R session
+#'
+#' A sequential future is a future that is evaluated sequentially in the
+#' current R session similarly to how \R expressions are evaluated in R.
+#' The only difference to R itself is that globals are validated
+#' by default just as for all other types of futures in this package.
+#'
+#'
+#' @inheritParams future
+#' @inheritParams multiprocess
+#' @param local If TRUE, the expression is evaluated such that
+#' all assignments are done to local temporary environment, otherwise
+#' the assignments are done in the calling environment.
+#'
+#' @return A \link{SequentialFuture}.
+#'
+#' @example incl/sequential.R
+#'
+#' @details
+#' The preferred way to create a sequential future is not to call these functions
+#' directly, but to register them via \code{\link{plan}(sequential)} such that
+#' it becomes the default mechanism for all futures. After this
+#' \code{\link{future}()} and \code{\link{\%<-\%}} will create
+#' \emph{sequential futures}.
+#'
+#' @section transparent futures:
+#' Transparent futures are sequential futures configured to emulate how R
+#' evaluates expressions as far as possible. For instance, errors and
+#' warnings are signaled immediately and assignments are done to the
+#' calling environment (without \code{local()} as default for all other
+#' types of futures). This makes transparent futures ideal for
+#' troubleshooting, especially when there are errors.
+#'
+#' @aliases uniprocess
+#' @export
+sequential <- function(expr, envir = parent.frame(), substitute = TRUE, lazy = FALSE, seed = NULL, globals = TRUE, local = TRUE, earlySignal = FALSE, label = NULL, ...) {
+ if (substitute) expr <- substitute(expr)
+ local <- as.logical(local)
+
+ future <- SequentialFuture(expr = expr, envir = envir, substitute = FALSE, lazy = lazy, seed = seed, globals = globals, local = local, earlySignal = earlySignal, label = label, ...)
+ if (!future$lazy) future <- run(future)
+ invisible(future)
+}
+class(sequential) <- c("sequential", "uniprocess", "future", "function")
+
+#' @rdname sequential
+#' @export
+transparent <- function(expr, envir = parent.frame(), substitute = TRUE, lazy = FALSE, seed = NULL, globals = FALSE, local = FALSE, earlySignal = TRUE, label = NULL, ...) {
+ if (substitute) expr <- substitute(expr)
+ sequential(expr, envir = envir, substitute = FALSE, lazy = lazy, seed = seed, globals = globals, local = local, earlySignal = earlySignal, label = label, ...)
+}
+class(transparent) <- c("transparent", "sequential", "uniprocess", "future", "function")
diff --git a/R/sessionDetails.R b/R/sessionDetails.R
index 3244614..ba52c80 100644
--- a/R/sessionDetails.R
+++ b/R/sessionDetails.R
@@ -5,7 +5,7 @@
#' @return Invisibly a list of all details.
#'
#' @details
-#' Note that remote futures use \code{persistent=TRUE} by default.
+#' Note that remote futures use \code{persistent = TRUE} by default.
#'
#' @importFrom utils sessionInfo
#' @export
diff --git a/R/signalEarly.R b/R/signalEarly.R
index 0ed329b..5600a64 100644
--- a/R/signalEarly.R
+++ b/R/signalEarly.R
@@ -1,4 +1,4 @@
-signalEarly <- function(future, collect=TRUE, ...) {
+signalEarly <- function(future, collect = TRUE, ...) {
## Future is not yet launched
if (future$state == "created") return(future)
@@ -7,24 +7,27 @@ signalEarly <- function(future, collect=TRUE, ...) {
## Don't signal early?
if (!earlySignal) return(future)
- mdebug("signalEarly(): Retrieving value ...")
+ debug <- getOption("future.debug", FALSE)
+ if (debug) mdebug("signalEarly(): Retrieving value ...")
## Collect value?
if (collect) {
- mdebug("signalEarly(): v <- value(f, signal=FALSE)")
- value <- value(future, signal=FALSE)
+ if (debug) mdebug("signalEarly(): v <- value(f, signal = FALSE)")
+ value <- value(future, signal = FALSE)
} else {
- mdebug("signalEarly(): v <- f$value")
+ if (debug) mdebug("signalEarly(): v <- f$value")
value <- future$value
}
- mdebug("signalEarly(): class(v) = c(%s)", paste(sQuote(class(value)), collapse=", "))
- mdebug("signalEarly(): Retrieving value ... DONE")
+ if (debug) {
+ mdebug("signalEarly(): class(v) = c(%s)", paste(sQuote(class(value)), collapse = ", "))
+ mdebug("signalEarly(): Retrieving value ... DONE")
+ }
## Was a condition caught?
if (!inherits(value, "condition")) return(future)
- mdebug("signalEarly(): signalCondition(v)")
+ if (debug) mdebug("signalEarly(): signalCondition(v)")
## Signal detected condition
if (inherits(value, "error")) {
@@ -38,7 +41,7 @@ signalEarly <- function(future, collect=TRUE, ...) {
signalCondition(value)
}
- mdebug("signalEarly() ... DONE")
+ if (debug) mdebug("signalEarly() ... DONE")
invisible(future)
}
diff --git a/R/tweak.R b/R/tweak.R
index 91af0e9..e175efd 100644
--- a/R/tweak.R
+++ b/R/tweak.R
@@ -13,21 +13,21 @@
#' new default strategy.
#'
#' @export
-tweak <- function(strategy, ..., penvir=parent.frame()) UseMethod("tweak")
+tweak <- function(strategy, ..., penvir = parent.frame()) UseMethod("tweak")
#' @export
-tweak.character <- function(strategy, ..., penvir=parent.frame()) {
+tweak.character <- function(strategy, ..., penvir = parent.frame()) {
## Search attached packages and the 'future' package
## for a future function with this name
- envirs <- list(penvir, future=getNamespace("future"), NULL)
+ envirs <- list(penvir, future = getNamespace("future"), NULL)
for (envir in envirs) {
## Reached the end? Nothing found.
if (is.null(envir)) {
stop("No such strategy for futures: ", sQuote(strategy))
}
- if (exists(strategy, mode="function", envir=envir, inherits=TRUE)) {
- strategy <- get(strategy, mode="function", envir=envir, inherits=TRUE)
+ if (exists(strategy, mode = "function", envir = envir, inherits = TRUE)) {
+ strategy <- get(strategy, mode = "function", envir = envir, inherits = TRUE)
break
}
}
@@ -35,11 +35,11 @@ tweak.character <- function(strategy, ..., penvir=parent.frame()) {
## Sanity check
stopifnot(is.function(strategy))
- tweak(strategy, ..., penvir=penvir)
+ tweak(strategy, ..., penvir = penvir)
}
#' @export
-tweak.future <- function(strategy, ..., penvir=parent.frame()) {
+tweak.future <- function(strategy, ..., penvir = parent.frame()) {
args <- list(...)
## Nothing to tweak?
@@ -53,6 +53,10 @@ tweak.future <- function(strategy, ..., penvir=parent.frame()) {
## Arguments that must not be tweaked
if (is.element("lazy", names)) {
stop("Future argument 'lazy' must not be tweaked / set via plan()")
+ } else if (is.element("asynchronous", names)) {
+ stop("Future argument 'asynchronous' must not be tweaked / set via plan()")
+ } else if (is.element("seed", names)) {
+ stop("Future argument 'seed' must not be tweaked / set via plan()")
}
## formals()<- drops any attributes including class
@@ -62,13 +66,11 @@ tweak.future <- function(strategy, ..., penvir=parent.frame()) {
## Tweak arguments
formals <- names(formals(strategy))
- ## BACKWARD COMPATIBILITY
- for (name in c("maxCores", "cluster")) {
- if (name %in% names) {
- if ("workers" %in% formals) {
- names <- gsub(name, "workers", names, fixed=TRUE)
- names(args) <- names
- .Deprecated(msg=sprintf("Argument '%s' has been renamed to 'workers'. Please update you script/code that uses the future package.", name))
+ ## DEFUNCT
+ if ("workers" %in% formals) {
+ for (name in c("maxCores", "cluster")) {
+ if (name %in% names) {
+ .Defunct(msg = sprintf("Argument '%s' has been renamed to 'workers'. Please update you script/code that uses the future package.", name))
}
}
}
@@ -83,7 +85,7 @@ tweak.future <- function(strategy, ..., penvir=parent.frame()) {
}
}
if (length(unknown) > 0L) {
- warning(sprintf("Ignored %d unknown arguments: %s", length(unknown), paste(sQuote(unknown), collapse=", ")))
+ warning(sprintf("Ignored %d unknown arguments: %s", length(unknown), paste(sQuote(unknown), collapse = ", ")))
}
## Restore attributes including class
@@ -94,3 +96,41 @@ tweak.future <- function(strategy, ..., penvir=parent.frame()) {
strategy
} ## tweak()
+
+
+#' @export
+tweak.function <- function(strategy, ...) {
+ strategy_name <- NULL
+
+ ## Try to find the name of the function
+ env <- environment(strategy)
+ env_name <- environmentName(env)
+ if (nchar(env_name) == 0) env_name <- "<unknown>"
+
+ names <- ls(envir = env, all.names = TRUE)
+ if (length(names) > 0) {
+ is_fcn <- sapply(names, FUN = exists, mode = "function",
+ envir = env, inherits = FALSE)
+ names <- names[is_fcn]
+ if (length(names) > 0) {
+ for (name in names) {
+ fcn <- get(name, mode = "function", envir = env, inherits = FALSE)
+ if (identical(fcn, strategy)) {
+ strategy_name <- sprintf("%s::%s", env_name, name)
+ break
+ }
+ }
+ }
+ }
+
+ msg <- "Trying to use non-future function"
+ if (!is.null(strategy_name)) {
+ msg <- sprintf("%s %s", msg, sQuote(strategy_name))
+ } else if (nzchar(env_name)) {
+ msg <- sprintf("%s from environment / package %s", msg, sQuote(env_name))
+ }
+
+ args <- deparse(args(strategy), width.cutoff = 500L)[1L]
+ msg <- sprintf("%s: %s { ... }", msg, args)
+ stop(msg)
+}
diff --git a/R/tweakExpression.R b/R/tweakExpression.R
index 541fa0a..83ff7e7 100644
--- a/R/tweakExpression.R
+++ b/R/tweakExpression.R
@@ -24,8 +24,10 @@ tweakFormulaCall <- function(expr) {
rhs <- expr[[3]]
}
- ## covr: skip=1
- substitute({ lhs; rhs; e }, list(lhs=lhs, rhs=rhs, e=expr))
+ substitute(
+ { lhs; rhs; e },
+ list(lhs = lhs, rhs = rhs, e = expr)
+ )
} ## tweakFormulaCall()
@@ -52,9 +54,9 @@ tweakFormulaCall <- function(expr) {
## AST: (<- ([ x a) value)
## tweaked expression: x; x["a"] <- value
##
-## expression: x[1,2,3] <- value
+## expression: x[1, 2, 3] <- value
## AST: (<- ([ x 1 2 3) value)
-## tweaked expression: x; x[1,2,3] <- value
+## tweaked expression: x; x[1, 2, 3] <- value
tweakSubassignmentCall <- function(expr) {
if (!is.call(expr)) return(expr)
op <- expr[[1]]
@@ -77,8 +79,10 @@ tweakSubassignmentCall <- function(expr) {
target <- expr2[[2]]
- ## covr: skip=1
- substitute({ target; e }, list(target=target, e=expr))
+ substitute(
+ { target; e },
+ list(target = target, e = expr)
+ )
} ## tweakSubassignmentCall()
@@ -95,28 +99,40 @@ tweakFutureAssignmentCall <- function(expr) {
if (n != 3) return(expr)
op <- as.character(op)
- if (op %in% c("<<-", "%<-%", "%<=%")) {
+ if (op %in% c("<<-", "%<-%")) {
lhs <- expr[[2]]
rhs <- expr[[3]]
- ## covr: skip=1
- expr <- substitute({a <- b; e}, list(a=lhs, b=rhs, e=expr))
- } else if (op %in% c("->>", "%->%", "%=>%")) {
+ expr <- substitute(
+ {a <- b; e},
+ list(a = lhs, b = rhs, e = expr)
+ )
+ } else if (op %in% c("->>", "%->%")) {
lhs <- expr[[3]]
rhs <- expr[[2]]
- ## covr: skip=1
- expr <- substitute({a <- b; e}, list(a=lhs, b=rhs, e=expr))
+ expr <- substitute(
+ {a <- b; e},
+ list(a = lhs, b = rhs, e = expr)
+ )
}
expr
} ## tweakFutureAssignmentCall()
+# Tweaks a future expression prior to searching for globals
+#
+# @param expr An \R expression
+#
+# @return An \R expression
+#
+# @keywords internal
+#
#' @importFrom globals walkAST
tweakExpression <- function(expr) {
if (!is.language(expr)) return(expr)
-
- expr <- walkAST(expr, call=tweakFutureAssignmentCall)
- expr <- walkAST(expr, call=tweakFormulaCall)
- expr <- walkAST(expr, call=tweakSubassignmentCall)
-
+
+ expr <- walkAST(expr, call = tweakFutureAssignmentCall)
+ expr <- walkAST(expr, call = tweakFormulaCall)
+ expr <- walkAST(expr, call = tweakSubassignmentCall)
+
expr
-} # tweakExpression()
+}
diff --git a/R/tweak_OP.R b/R/tweak_OP.R
index 74711b8..e4fbf52 100644
--- a/R/tweak_OP.R
+++ b/R/tweak_OP.R
@@ -17,13 +17,13 @@
## Temporarily use a different plan
oplan <- plan("list")
- on.exit(plan(oplan, substitute=FALSE, .call=NULL))
+ on.exit(plan(oplan, substitute = FALSE, .call = NULL, .cleanup = FALSE, .init = FALSE))
## Tweak current strategy and apply
plans <- oplan
- args <- c(list(plans[[1]], penvir=envir), tweaks)
- plans[[1]] <- do.call(tweak, args=args)
- plan(plans, substitute=FALSE, .call=NULL)
+ args <- c(list(plans[[1]], penvir = envir), tweaks)
+ plans[[1]] <- do.call(tweak, args = args)
+ plan(plans, substitute = FALSE, .call = NULL, .cleanup = FALSE, .init = FALSE)
- eval(fassignment, envir=envir)
+ eval(fassignment, envir = envir)
}
diff --git a/R/uniprocess.R b/R/uniprocess.R
deleted file mode 100644
index 8b6107e..0000000
--- a/R/uniprocess.R
+++ /dev/null
@@ -1,81 +0,0 @@
-#' Create a uniprocess future whose value will be in the current R session
-#'
-#' A uniprocess future is a future that is evaluated sequentially in the
-#' current R session. The default is to resolve it eagerly, which means
-#' that its \emph{value is computed and resolved immediately}, which is
-#' how regular expressions are evaluated in R.
-#' The only difference to R itself is that globals are validated
-#' by default just as for all other types of futures in this package.
-#'
-#'
-#' @inheritParams future
-#' @inheritParams multiprocess
-#' @param local If TRUE, the expression is evaluated such that
-#' all assignments are done to local temporary environment, otherwise
-#' the assignments are done in the calling environment.
-#' @param lazy If \code{FALSE} (default), the future is resolved eagerly
-#' (immediately), otherwise not.
-#'
-#' @return A \link{UniprocessFuture}.
-#'
-#' @example incl/uniprocess.R
-#'
-#' @details
-#' The preferred way to create a uniprocess future is not to call these functions
-#' directly, but to register them via \code{\link{plan}(eager)} such that it
-#' becomes the default mechanism for all futures. After this
-#' \code{\link{future}()} and \code{\link{\%<-\%}} will create
-#' \emph{eager uniprocess futures}.
-#'
-#' @section transparent futures:
-#' Transparent futures are eager uniprocess futures configured to emulate how R
-#' evaluates expressions as far as possible. For instance, errors and
-#' warnings are signaled immediately and assignments are done to the
-#' calling environment (without \code{local()} as default for all other
-#' types of futures). This makes transparent futures ideal for
-#' troubleshooting, especially when there are errors.
-#'
-#' @aliases uniprocess
-#' @export
-eager <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, local=TRUE, earlySignal=FALSE, label=NULL, lazy=FALSE, ...) {
- if (substitute) expr <- substitute(expr)
- local <- as.logical(local)
-
- future <- EagerFuture(expr=expr, envir=envir, substitute=FALSE, globals=globals, local=local, earlySignal=earlySignal, label=label, lazy=lazy, ...)
- run(future)
-}
-class(eager) <- c("eager", "uniprocess", "future", "function")
-
-#' @rdname eager
-#' @export
-transparent <- function(expr, envir=parent.frame(), substitute=TRUE, globals=FALSE, local=FALSE, earlySignal=TRUE, label=NULL, lazy=FALSE, ...) {
- if (substitute) expr <- substitute(expr)
- uniprocess(expr, envir=envir, substitute=FALSE, globals=globals, local=local, earlySignal=earlySignal, label=label, lazy=lazy, ...)
-}
-class(transparent) <- c("transparent", "uniprocess", "future", "function")
-
-#' @rdname eager
-#' @export
-lazy <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, local=TRUE, earlySignal=FALSE, label=NULL, lazy=TRUE, ...) {
- if (substitute) expr <- substitute(expr)
- local <- as.logical(local)
-
- LazyFuture(expr=expr, envir=envir, local=local, globals=globals, earlySignal=earlySignal, label=label, lazy=lazy, ...)
-}
-class(lazy) <- c("lazy", "uniprocess", "future", "function")
-
-## WORKAROUND:
-## Avoid lazyeval::print.lazy() being called with print(lazy())
-## https://github.com/HenrikBengtsson/future/issues/52
-class(lazy) <- c("function", class(lazy))
-
-
-## Keep private for now until name has been decided, cf.
-## https://github.com/HenrikBengtsson/future/issues/109
-uniprocess <- function(expr, envir=parent.frame(), substitute=TRUE, globals=TRUE, local=TRUE, earlySignal=FALSE, label=NULL, lazy=FALSE, ...) {
- if (substitute) expr <- substitute(expr)
- future <- UniprocessFuture(expr=expr, envir=envir, substitute=FALSE, globals=globals, local=local, earlySignal=earlySignal, label=label, lazy=lazy, ...)
- if (!lazy) future <- run(future)
- invisible(future)
-}
-class(uniprocess) <- c("uniprocess", "future", "function")
diff --git a/R/utils.R b/R/utils.R
index a1b9abf..5711f7b 100644
--- a/R/utils.R
+++ b/R/utils.R
@@ -1,9 +1,9 @@
## From R.utils 2.0.2 (2015-05-23)
-hpaste <- function(..., sep="", collapse=", ", lastCollapse=NULL, maxHead=if (missing(lastCollapse)) 3 else Inf, maxTail=if (is.finite(maxHead)) 1 else Inf, abbreviate="...") {
+hpaste <- function(..., sep = "", collapse = ", ", lastCollapse = NULL, maxHead = if (missing(lastCollapse)) 3 else Inf, maxTail = if (is.finite(maxHead)) 1 else Inf, abbreviate = "...") {
if (is.null(lastCollapse)) lastCollapse <- collapse
# Build vector 'x'
- x <- paste(..., sep=sep)
+ x <- paste(..., sep = sep)
n <- length(x)
# Nothing todo?
@@ -20,10 +20,10 @@ hpaste <- function(..., sep="", collapse=", ", lastCollapse=NULL, maxHead=if (mi
if (!is.null(collapse) && n > 1) {
if (lastCollapse == collapse) {
- x <- paste(x, collapse=collapse)
+ x <- paste(x, collapse = collapse)
} else {
- xT <- paste(x[1:(n-1)], collapse=collapse)
- x <- paste(xT, x[n], sep=lastCollapse)
+ xT <- paste(x[1:(n-1)], collapse = collapse)
+ x <- paste(xT, x[n], sep = lastCollapse)
}
}
@@ -36,16 +36,16 @@ trim <- function(s) {
} # trim()
-hexpr <- function(expr, trim=TRUE, collapse="; ", maxHead=6L, maxTail=3L, ...) {
+hexpr <- function(expr, trim = TRUE, collapse = "; ", maxHead = 6L, maxTail = 3L, ...) {
code <- deparse(expr)
if (trim) code <- trim(code)
- hpaste(code, collapse=collapse, maxHead=maxHead, maxTail=maxTail, ...)
+ hpaste(code, collapse = collapse, maxHead = maxHead, maxTail = maxTail, ...)
} # hexpr()
## From R.filesets
-asIEC <- function(size, digits=2L) {
- if (length(size) > 1L) return(sapply(size, FUN=asIEC, digits=digits))
+asIEC <- function(size, digits = 2L) {
+ if (length(size) > 1L) return(sapply(size, FUN = asIEC, digits = digits))
units <- c("bytes", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB")
for (unit in units) {
if (size < 1000) break;
@@ -71,8 +71,8 @@ mdebug <- function(...) {
#' @importFrom digest digest
uuid <- function(source, keep_source = FALSE) {
uuid <- digest(source)
- uuid <- strsplit(uuid, split="")[[1]]
- uuid <- paste(c(uuid[1:8], "-", uuid[9:12], "-", uuid[13:16], "-", uuid[17:20], "-", uuid[21:32]), collapse="")
+ uuid <- strsplit(uuid, split = "")[[1]]
+ uuid <- paste(c(uuid[1:8], "-", uuid[9:12], "-", uuid[13:16], "-", uuid[17:20], "-", uuid[21:32]), collapse = "")
if (keep_source) attr(uuid, "source") <- source
uuid
} ## uuid()
@@ -110,11 +110,11 @@ session_uuid <- local({
host <- Sys.getenv(c("HOST", "HOSTNAME", "COMPUTERNAME"))
host <- host[nzchar(host)][1]
info <- list(
- host=host,
- info=info,
- pid=Sys.getpid(),
- time=Sys.time(),
- random=sample.int(.Machine$integer.max, size=1L)
+ host = host,
+ info = info,
+ pid = Sys.getpid(),
+ time = Sys.time(),
+ random = sample.int(.Machine$integer.max, size = 1L)
)
uuid <- uuid(info, keep_source = TRUE)
value <<- uuid
@@ -133,41 +133,41 @@ session_uuid <- local({
## and all of the package content would be exported.
## Removes all variables in the global environment.
-grmall <- local(function(envir=.GlobalEnv) {
- vars <- ls(envir=envir, all.names=TRUE)
- rm(list=vars, envir=envir, inherits=FALSE)
+grmall <- local(function(envir = .GlobalEnv) {
+ vars <- ls(envir = envir, all.names = TRUE)
+ rm(list = vars, envir = envir, inherits = FALSE)
})
## Assigns a value to the global environment.
-gassign <- local(function(name, value, envir=.GlobalEnv) {
- assign(name, value=value, envir=envir)
+gassign <- local(function(name, value, envir = .GlobalEnv) {
+ assign(name, value = value, envir = envir)
NULL
})
## Evaluates an expression in global environment.
-geval <- local(function(expr, substitute=FALSE, envir=.GlobalEnv, ...) {
+geval <- local(function(expr, substitute = FALSE, envir = .GlobalEnv, ...) {
if (substitute) expr <- substitute(expr)
- eval(expr, envir=envir)
+ eval(expr, envir = envir)
})
## Vectorized version of require() with bells and whistles
requirePackages <- local(function(pkgs) {
requirePackage <- function(pkg) {
- if (require(pkg, character.only=TRUE)) return()
+ if (require(pkg, character.only = TRUE)) return()
## Failed to attach package
msg <- sprintf("Failed to attach package %s in %s", sQuote(pkg), R.version$version.string)
data <- utils::installed.packages()
## Installed, but fails to load/attach?
- if (is.element(pkg, data[,"Package"])) {
- keep <- (data[,"Package"] == pkg)
- data <- data[keep,,drop=FALSE]
- pkgs <- sprintf("%s %s (in %s)", data[,"Package"], data[, "Version"], sQuote(data[,"LibPath"]))
- msg <- sprintf("%s, although the package is installed: %s", msg, paste(pkgs, collapse=", "))
+ if (is.element(pkg, data[, "Package"])) {
+ keep <- (data[, "Package"] == pkg)
+ data <- data[keep, ,drop = FALSE]
+ pkgs <- sprintf("%s %s (in %s)", data[, "Package"], data[, "Version"], sQuote(data[, "LibPath"]))
+ msg <- sprintf("%s, although the package is installed: %s", msg, paste(pkgs, collapse = ", "))
} else {
paths <- .libPaths()
- msg <- sprintf("%s, because the package is not installed in any of the libraries (%s), which contain %d installed packages.", msg, paste(sQuote(paths), collapse=", "), nrow(data))
+ msg <- sprintf("%s, because the package is not installed in any of the libraries (%s), which contain %d installed packages.", msg, paste(sQuote(paths), collapse = ", "), nrow(data))
}
stop(msg)
@@ -175,7 +175,7 @@ requirePackages <- local(function(pkgs) {
## require() all packages
pkgs <- unique(pkgs)
- lapply(pkgs, FUN=requirePackage)
+ lapply(pkgs, FUN = requirePackage)
}) ## requirePackages()
@@ -185,8 +185,8 @@ requirePackages <- local(function(pkgs) {
## is used.
getOption <- local({
go <- base::getOption
- function(x, default=NULL) {
- if (missing(default) || match(x, table=names(.Options), nomatch=0L) > 0L) go(x) else default
+ function(x, default = NULL) {
+ if (missing(default) || match(x, table = names(.Options), nomatch = 0L) > 0L) go(x) else default
}
}) ## getOption()
@@ -222,23 +222,36 @@ detectCores <- local({
## We are currently importing the following non-exported functions:
-## * parallel:::defaultCluster()
-## * parallel:::recvResult()
-## * parallel:::selectChildren()
-## * parallel:::sendCall()
+## * cluster futures:
+## - parallel:::defaultCluster() ## non-critical / not really needed
+## - parallel:::sendCall() ## run()
+## - parallel:::recvResult() ## value()
+## * multicore futures:
+## - parallel:::selectChildren() ## resolved()
## As well as the following ones (because they are not exported on Windows):
-## * parallel:::mccollect()
-## * parallel:::mcparallel()
-importParallel <- function(name=NULL) {
- ns <- getNamespace("parallel")
- if (!exists(name, mode="function", envir=ns, inherits=FALSE)) {
- ## covr: skip=3
- msg <- sprintf("This type of future processing is not supported on this system (%s), because parallel function %s() is not available", sQuote(.Platform$OS.type), name)
- mdebug(msg)
- stop(msg, call.=FALSE)
+## * multicore futures:
+## - parallel::mcparallel() ## run()
+## - parallel::mccollect() ## value()
+importParallel <- local({
+ ns <- NULL
+ cache <- list()
+
+ function(name = NULL) {
+ res <- cache[[name]]
+ if (is.null(res)) {
+ ns <<- getNamespace("parallel")
+ if (!exists(name, mode = "function", envir = ns, inherits = FALSE)) {
+ ## covr: skip=3
+ msg <- sprintf("This type of future processing is not supported on this system (%s), because parallel function %s() is not available", sQuote(.Platform$OS.type), name)
+ mdebug(msg)
+ stop(msg, call. = FALSE)
+ }
+ res <- get(name, mode = "function", envir = ns, inherits = FALSE)
+ cache[[name]] <<- res
+ }
+ res
}
- get(name, mode="function", envir=ns, inherits=FALSE)
-}
+})
parseCmdArgs <- function() {
@@ -260,13 +273,13 @@ parseCmdArgs <- function() {
value <- as.integer(gsub("--parallel=", "", cmdarg))
}
- max <- availableCores(methods="system")
+ max <- availableCores(methods = "system")
if (is.na(value) || value <= 0L) {
msg <- sprintf("future: Ignoring invalid number of processes specified in command-line option: %s", cmdarg)
- warning(msg, call.=FALSE, immediate.=TRUE)
+ warning(msg, call. = FALSE, immediate. = TRUE)
} else if (value > max) {
- msg <- sprintf("future: Ignoring requested number of processes, because it is greater than the number of cores/child processes available (=%d) to this R process: %s", max, cmdarg)
- warning(msg, call.=FALSE, immediate.=TRUE)
+ msg <- sprintf("future: Ignoring requested number of processes, because it is greater than the number of cores/child processes available (= %d) to this R process: %s", max, cmdarg)
+ warning(msg, call. = FALSE, immediate. = TRUE)
} else {
args$p <- value
}
@@ -278,16 +291,18 @@ parseCmdArgs <- function() {
myExternalIP <- local({
ip <- NULL
- function(force=FALSE, mustWork=TRUE) {
+ function(force = FALSE, mustWork = TRUE) {
if (!force && !is.null(ip)) return(ip)
## FIXME: The identification of the external IP number relies on a
## single third-party server. This could be improved by falling back
## to additional servers, cf. https://github.com/phoemur/ipgetter
urls <- c(
+ "https://httpbin.org/ip",
"https://myexternalip.com/raw",
"https://diagnostic.opendns.com/myip",
"https://api.ipify.org/",
+ "http://httpbin.org/ip",
"http://myexternalip.com/raw",
"http://diagnostic.opendns.com/myip",
"http://api.ipify.org/"
@@ -295,24 +310,36 @@ myExternalIP <- local({
value <- NULL
for (url in urls) {
value <- tryCatch(readLines(url), error = function(ex) NULL)
- if (!is.null(value)) break
- }
+
+ ## Nothing found?
+ if (is.null(value)) next
+
+ ## Keep only lines that look like they contain IP v4 numbers
+ ip4_pattern <- ".*[^[:digit:]]+([[:digit:]]+[.][[:digit:]]+[.][[:digit:]]+[.][[:digit:]]+).*"
+ value <- grep(ip4_pattern, value, value = TRUE)
+
+ ## Extract the IP numbers
+ value <- gsub(ip4_pattern, "\\1", value)
+
+ ## Trim and drop empty results (just in case)
+ value <- trim(value)
+ value <- value[nzchar(value)]
+
+ ## Nothing found?
+ if (length(value) == 0) next
+
+ ## Match?
+ if (length(value) == 1 && nzchar(value)) break
+ } ## for (url ...)
## Nothing found?
if (is.null(value)) {
if (mustWork) {
- stop(sprintf("Failed to identify external IP from any of the %d external services: %s", length(urls), paste(sQuote(urls), collapse=", ")))
+ stop(sprintf("Failed to identify external IP from any of the %d external services: %s", length(urls), paste(sQuote(urls), collapse = ", ")))
}
return(NA_character_)
}
- ## Trim and drop empty results (just in case)
- value <- trim(value)
- value <- value[nzchar(value)]
-
- ## Nothing found?
- if (length(value) == 0 && !mustWork) return(NA_character_)
-
## Sanity check
stopifnot(length(value) == 1, is.character(value), !is.na(value), nzchar(value))
@@ -333,9 +360,9 @@ myInternalIP <- local({
## (3) 192.168.0.0 - 192.168.255.255
## https://en.wikipedia.org/wiki/Private_network#Private_IPv4_address_spaces
isPrivateIP <- function(ips) {
- ips <- strsplit(ips, split=".", fixed=TRUE)
- ips <- lapply(ips, FUN=as.integer)
- res <- logical(length=length(ips))
+ ips <- strsplit(ips, split = ".", fixed = TRUE)
+ ips <- lapply(ips, FUN = as.integer)
+ res <- logical(length = length(ips))
for (kk in seq_along(ips)) {
ip <- ips[[kk]]
if (ip[1] == 10) {
@@ -349,7 +376,7 @@ myInternalIP <- local({
res
} ## isPrivateIP()
- function(force=FALSE, which=c("first", "last", "all"), mustWork=TRUE) {
+ function(force = FALSE, which = c("first", "last", "all"), mustWork = TRUE) {
if (!force && !is.null(ip)) return(ip)
which <- match.arg(which)
@@ -359,46 +386,46 @@ myInternalIP <- local({
if (grepl("^linux", os)) {
## (i) Try command 'hostname -I'
res <- tryCatch({
- system2("hostname", args="-I", stdout=TRUE)
+ system2("hostname", args = "-I", stdout = TRUE)
}, error = identity)
## (ii) Try commands 'ifconfig'
if (inherits(res, "simpleError")) {
res <- tryCatch({
- system2("ifconfig", stdout=TRUE)
+ system2("ifconfig", stdout = TRUE)
}, error = identity)
}
## (ii) Try command '/sbin/ifconfig'
if (inherits(res, "simpleError")) {
res <- tryCatch({
- system2("/sbin/ifconfig", stdout=TRUE)
+ system2("/sbin/ifconfig", stdout = TRUE)
}, error = identity)
}
## Failed?
if (inherits(res, "simpleError")) res <- NA_character_
- res <- grep(pattern, res, value=TRUE)
- res <- unlist(strsplit(res, split="[ ]+", fixed=FALSE), use.names=FALSE)
- res <- grep(pattern, res, value=TRUE)
- res <- unlist(strsplit(res, split=":", fixed=FALSE), use.names=FALSE)
- res <- grep(pattern, res, value=TRUE)
+ res <- grep(pattern, res, value = TRUE)
+ res <- unlist(strsplit(res, split = "[ ]+", fixed = FALSE), use.names = FALSE)
+ res <- grep(pattern, res, value = TRUE)
+ res <- unlist(strsplit(res, split = ":", fixed = FALSE), use.names = FALSE)
+ res <- grep(pattern, res, value = TRUE)
res <- unique(trim(res))
## Keep private network IPs only (just in case)
value <- res[isPrivateIP(res)]
} else if (grepl("^mingw", os)) {
- res <- system2("ipconfig", stdout=TRUE)
- res <- grep("IPv4", res, value=TRUE)
- res <- grep(pattern, res, value=TRUE)
- res <- unlist(strsplit(res, split="[ ]+", fixed=FALSE), use.names=FALSE)
- res <- grep(pattern, res, value=TRUE)
+ res <- system2("ipconfig", stdout = TRUE)
+ res <- grep("IPv4", res, value = TRUE)
+ res <- grep(pattern, res, value = TRUE)
+ res <- unlist(strsplit(res, split = "[ ]+", fixed = FALSE), use.names = FALSE)
+ res <- grep(pattern, res, value = TRUE)
res <- unique(trim(res))
## Keep private network IPs only (just in case)
value <- res[isPrivateIP(res)]
} else {
if (mustWork) {
- stop(sprintf("remote(..., myip='<internal>') is yet not implemented for this operating system (%s). Please specify the 'myip' IP number manually.", os))
+ stop(sprintf("remote(..., myip = '<internal>') is yet not implemented for this operating system (%s). Please specify the 'myip' IP number manually.", os))
}
return(NA_character_)
}
@@ -428,3 +455,249 @@ myInternalIP <- local({
ip
}
}) ## myInternalIP()
+
+
+
+
+## A *rough* estimate of size of an object + its environment.
+#' @importFrom utils object.size
+objectSize <- function(x, depth = 3L) {
+ # Nothing to do?
+ if (isNamespace(x)) return(0)
+ if (depth <= 0) return(0)
+
+ if (!is.list(x) && !is.environment(x)) {
+ size <- unclass(object.size(x))
+ x <- environment(x)
+ } else {
+ size <- 0
+ }
+
+ ## Nothing more to do?
+ if (depth == 1) return(size)
+
+ .scannedEnvs <- new.env()
+ scanned <- function(e) {
+ for (name in names(.scannedEnvs)) if (identical(e, .scannedEnvs[[name]])) return(TRUE)
+ FALSE
+ }
+
+ objectSize.FutureGlobals <- function(x, ...) {
+ size <- attr(x, "total_size")
+ if (!is.na(size)) return(size)
+ objectSize.list(x, ...)
+ }
+
+ objectSize.list <- function(x, depth) {
+ # Nothing to do?
+ if (depth <= 0) return(0)
+ depth <- depth - 1L
+ size <- 0
+
+ ## Use the true length that corresponds to what .subset2() uses
+ nx <- .length(x)
+
+ for (kk in seq_len(nx)) {
+ ## NOTE: Use non-class dispatching subsetting to avoid infinite loop,
+ ## e.g. x <- packageVersion("future") gives x[[1]] == x.
+ x_kk <- .subset2(x, kk)
+ if (is.list(x_kk)) {
+ size <- size + objectSize.list(x_kk, depth = depth)
+ } else if (is.environment(x_kk)) {
+ if (!scanned(x_kk)) size <- size + objectSize.env(x_kk, depth = depth)
+ } else {
+ size <- size + unclass(object.size(x_kk))
+ }
+ }
+ size
+ } ## objectSize.list()
+
+ objectSize.env <- function(x, depth) {
+ # Nothing to do?
+ if (depth <= 0) return(0)
+ depth <- depth - 1L
+ if (isNamespace(x)) return(0)
+## if (inherits(x, "Future")) return(0)
+
+ size <- 0
+
+ ## Get all objects in the environment
+ elements <- ls(envir = x, all.names = TRUE)
+ if (length(elements) == 0) return(0)
+
+ ## Skip variables that are future promises in order
+ ## to avoid inspecting promises that are already
+ ## under investigation.
+ skip <- grep("^.future_", elements, value = TRUE)
+ if (length(skip) > 0) {
+ skip <- gsub("^.future_", "", elements)
+ elements <- setdiff(elements, skip)
+ if (length(elements) == 0) return(0)
+ }
+
+ ## Avoid scanning the current environment again
+ name <- sprintf("env_%d", length(.scannedEnvs))
+ .scannedEnvs[[name]] <- x
+
+ for (element in elements) {
+ ## FIXME: Some elements may not exist, although ls() returns them
+ ## and exists() say they do exist, cf. Issue #161 /HB 2017-08-24
+ ## NOTE: Hmm... is it possible to test for the existence or are
+ ## we doomed to have to use of tryCatch() here?
+ res <- tryCatch({
+ x_kk <- .subset2(x, element)
+ NULL ## So that 'x_kk' is not returned, which may be missing()
+ }, error = identity)
+
+ ## A promise that cannot be resolved? This could be a false positive,
+ ## e.g. an expression not to be resolved, cf. Issue #161 /HB 2017-08-24
+ if (inherits(res, "error")) next
+
+ ## Nothing to do?
+ if (missing(x_kk)) next
+
+ if (is.list(x_kk)) {
+ size <- size + objectSize.list(x_kk, depth = depth)
+ } else if (is.environment(x_kk)) {
+## if (!inherits(x_kk, "Future") && !scanned(x_kk)) {
+ if (!scanned(x_kk)) {
+ size <- size + objectSize.env(x_kk, depth = depth)
+ }
+ } else {
+ size <- size + unclass(object.size(x_kk))
+ }
+ }
+
+ size
+ } ## objectSize.env()
+
+ ## Suppress "Warning message:
+ ## In doTryCatch(return(expr), name, parentenv, handler) :
+ ## restarting interrupted promise evaluation
+ suppressWarnings({
+ if (is.list(x)) {
+ size <- size + objectSize.list(x, depth = depth - 1L)
+ } else if (is.environment(x)) {
+ size <- size + objectSize.env(x, depth = depth - 1L)
+ }
+ })
+
+ size
+}
+
+
+get_random_seed <- function() {
+ env <- globalenv()
+ env$.Random.seed
+}
+
+set_random_seed <- function(seed) {
+ env <- globalenv()
+ if (is.null(seed)) {
+ rm(list = ".Random.seed", envir = env, inherits = FALSE)
+ } else {
+ env$.Random.seed <- seed
+ }
+}
+
+next_random_seed <- function(seed = get_random_seed()) {
+ sample.int(n = 1L, size = 1L, replace = FALSE)
+ seed_next <- get_random_seed()
+ stopifnot(!any(seed_next != seed))
+ invisible(seed_next)
+}
+
+is_valid_random_seed <- function(seed) {
+ oseed <- get_random_seed()
+ on.exit(set_random_seed(oseed))
+ env <- globalenv()
+ env$.Random.seed <- seed
+ res <- tryCatch({
+ sample.int(n = 1L, size = 1L, replace = FALSE)
+ }, FUN = function(w) w)
+ !inherits(res, "warning")
+}
+
+is_lecyer_cmrg_seed <- function(seed) {
+ is.numeric(seed) && length(seed) == 7L &&
+ all(is.finite(seed)) && seed[1] == 407L
+}
+
+# @importFrom utils capture.output
+as_lecyer_cmrg_seed <- function(seed) {
+ ## Generate a L'Ecuyer-CMRG seed (existing or random)?
+ if (is.logical(seed)) {
+ stopifnot(length(seed) == 1L)
+ if (!is.na(seed) && !seed) {
+ stop("Argument 'seed' must be TRUE if logical: ", seed)
+ }
+
+ oseed <- get_random_seed()
+
+ ## Already a L'Ecuyer-CMRG seed? Then use that as is.
+ if (!is.na(seed) && seed) {
+ if (is_lecyer_cmrg_seed(oseed)) return(oseed)
+ }
+
+ ## Otherwise, generate a random one.
+ on.exit(set_random_seed(oseed), add = TRUE)
+ RNGkind("L'Ecuyer-CMRG")
+ return(get_random_seed())
+ }
+
+ stopifnot(is.numeric(seed), all(is.finite(seed)))
+ seed <- as.integer(seed)
+
+ ## Already a L'Ecuyer-CMRG seed?
+ if (length(seed) == 7L) {
+ if (seed[1] != 407L) {
+ stop("Argument 'seed' must be L'Ecuyer-CMRG RNG seed as returned by parallel::nextRNGStream() or an single integer: ", capture.output(str(seed)))
+ }
+ return(seed)
+ }
+
+ ## Generate a new L'Ecuyer-CMRG seed?
+ if (length(seed) == 1L) {
+ oseed <- get_random_seed()
+ on.exit(set_random_seed(oseed), add = TRUE)
+ RNGkind("L'Ecuyer-CMRG")
+ set.seed(seed)
+ return(get_random_seed())
+ }
+
+ stop("Argument 'seed' must be of length 1 or 7 (= 1+6):", capture.output(str(seed)))
+}
+
+
+#' Gets the length of an object without dispatching
+#'
+#' @param x Any R object.
+#'
+#' @return A non-negative integer.
+#'
+#' @details
+#' This function returns \code{length(unclass(x))}, but tries to avoid
+#' calling \code{unclass(x)} unless necessary.
+#'
+#' @seealso \code{\link{.subset}()} and \code{\link{.subset2}()}.
+#'
+#' @keywords internal
+#' @rdname private_length
+.length <- function(x) {
+ nx <- length(x)
+
+ ## Can we trust base::length(x), i.e. is there a risk that there is
+ ## a method that overrides with another definition?
+ classes <- class(x)
+ if (length(classes) == 1L && classes == "list") return(nx)
+
+ ## Identify all length() methods for this object
+ mthds <- sprintf("length.%s", classes)
+ keep <- lapply(mthds, FUN = exists, mode = "function", inherits = TRUE)
+ keep <- unlist(keep, use.names = FALSE)
+
+ ## If found, don't trust them
+ if (any(keep)) nx <- length(unclass(x))
+
+ nx
+} ## .length()
diff --git a/R/values.R b/R/values.R
index 0404eaf..c9cc37c 100644
--- a/R/values.R
+++ b/R/values.R
@@ -39,7 +39,7 @@ values.list <- function(x, ...) {
values.environment <- function(x, ...) {
y <- futures(x)
y <- resolve(y)
- names <- ls(envir=y, all.names=TRUE)
+ names <- ls(envir = y, all.names = TRUE)
for (key in names) {
tmp <- y[[key]]
if (inherits(tmp, "Future")) y[[key]] <- value(tmp, ...)
diff --git a/R/whichIndex.R b/R/whichIndex.R
index 4836ff1..2c3e10d 100644
--- a/R/whichIndex.R
+++ b/R/whichIndex.R
@@ -1,4 +1,4 @@
-whichIndex <- function(I, dim, dimnames=NULL) {
+whichIndex <- function(I, dim, dimnames = NULL) {
ndim <- length(dim)
stopifnot((is.matrix(I) || is.data.frame(I)), ncol(I) == ndim)
if (!is.null(dimnames)) stopifnot(length(dimnames) == ndim)
@@ -6,7 +6,7 @@ whichIndex <- function(I, dim, dimnames=NULL) {
if (is.data.frame(I)) {
## Convert each column to indices
- I2 <- array(NA_integer_, dim=dim(I))
+ I2 <- array(NA_integer_, dim = dim(I))
for (kk in 1:ndim) {
idxs <- I[[kk]]
if (is.numeric(idxs)) {
@@ -17,45 +17,45 @@ whichIndex <- function(I, dim, dimnames=NULL) {
idxs <- as.character(idxs)
idxs <- match(idxs, dimnames[[kk]])
if (anyNA(idxs)) {
- unknown <- I[is.na(idxs),kk]
+ unknown <- I[is.na(idxs), kk]
stop("Unknown indices: ", hpaste(sQuote(unknown)))
}
}
- I2[,kk] <- idxs
+ I2[, kk] <- idxs
}
I <- I2
I2 <- NULL
} else if (is.numeric(I)) {
for (kk in 1:ndim) {
- idxs <- I[,kk]
+ idxs <- I[, kk]
if (any(idxs < 1 | idxs > dim[kk])) {
stop("Index out of range.")
}
}
} else {
## Convert dimnames to dimindices
- I2 <- array(NA_integer_, dim=dim(I))
+ I2 <- array(NA_integer_, dim = dim(I))
for (kk in 1:ndim) {
## Could be, say, factor
- idxs <- I[,kk]
+ idxs <- I[, kk]
idxs <- as.character(idxs)
idxs <- match(idxs, dimnames[[kk]])
if (anyNA(idxs)) {
- unknown <- I[is.na(idxs),kk]
+ unknown <- I[is.na(idxs), kk]
stop("Unknown indices: ", hpaste(sQuote(unknown)))
}
- I2[,kk] <- idxs
+ I2[, kk] <- idxs
}
I <- I2
I2 <- NULL
}
## Nothing more to do?
- if (ndim == 1) return(I[,1L])
+ if (ndim == 1) return(I[, 1L])
base <- cumprod(dim[-ndim])
for (kk in 2:ndim) {
- I[,kk] <- (I[,kk]-1) * base[kk-1L]
+ I[, kk] <- (I[, kk] - 1) * base[kk - 1L]
}
rowSums(I)
}
diff --git a/R/zzz.R b/R/zzz.R
index b8daaea..79b6f68 100644
--- a/R/zzz.R
+++ b/R/zzz.R
@@ -1,24 +1,47 @@
## covr: skip=all
.onLoad <- function(libname, pkgname) {
- ## Unless already set, set option 'future.availableCores.system' according
- ## to system environment variable 'R_FUTURE_AVAILABLECORES_SYSTEM'.
+ debug <- getOption("future.debug", FALSE)
+
+ ## Unless already set, set option 'future.availableCores.fallback'
+ ## according to environment variable 'R_FUTURE_AVAILABLECORES_FALLBACK'.
+ ncores <- getOption("future.availableCores.fallback")
+ if (is.null(ncores)) {
+ ncores <- trim(Sys.getenv("R_FUTURE_AVAILABLECORES_FALLBACK"))
+ if (nzchar(ncores)) {
+ if (debug) mdebug("R_FUTURE_AVAILABLECORES_FALLBACK=%s", sQuote(ncores))
+ if (is.element(ncores, c("NA_integer_", "NA"))) {
+ ncores <- NA_integer_
+ } else {
+ ncores <- as.integer(ncores)
+ }
+ if (debug) mdebug(" => options(future.availableCores.fallback = %d)", ncores)
+ options(future.availableCores.fallback = ncores)
+ }
+ ncores <- getOption("future.availableCores.fallback")
+ }
+ if (!is.null(ncores)) {
+ if (debug) mdebug("Option 'future.availableCores.fallback = %d", ncores)
+ }
+
+ ## Unless already set, set option 'future.availableCores.system'
+ ## according to environment variable 'R_FUTURE_AVAILABLECORES_SYSTEM'.
ncores <- getOption("future.availableCores.system")
if (is.null(ncores)) {
ncores <- trim(Sys.getenv("R_FUTURE_AVAILABLECORES_SYSTEM"))
if (nzchar(ncores)) {
- mdebug("R_FUTURE_AVAILABLECORES_SYSTEM=%s", sQuote(ncores))
+ if (debug) mdebug("R_FUTURE_AVAILABLECORES_SYSTEM=%s", sQuote(ncores))
if (is.element(ncores, c("NA_integer_", "NA"))) {
ncores <- NA_integer_
} else {
ncores <- as.integer(ncores)
}
- mdebug("=> options(future.availableCores.system=%d)", ncores)
- options(future.availableCores.system=ncores)
+ if (debug) mdebug(" => options(future.availableCores.system = %d)", ncores)
+ options(future.availableCores.system = ncores)
}
ncores <- getOption("future.availableCores.system")
}
if (!is.null(ncores)) {
- mdebug("Option 'future.availableCores.system=%d", ncores)
+ if (debug) mdebug("Option 'future.availableCores.system = %d", ncores)
}
## Unless already set, set option 'future.plan' according to
@@ -27,47 +50,55 @@
if (is.null(strategy)) {
strategy <- trim(Sys.getenv("R_FUTURE_PLAN"))
if (nzchar(strategy)) {
- mdebug("R_FUTURE_PLAN=%s", sQuote(strategy))
- mdebug("=> options(future.plan='%s')", strategy)
- options(future.plan=strategy)
+ if (debug) {
+ mdebug("R_FUTURE_PLAN=%s", sQuote(strategy))
+ mdebug(" => options(future.plan = '%s')", strategy)
+ }
+ options(future.plan = strategy)
}
strategy <- getOption("future.plan")
}
if (!is.null(strategy)) {
- if (is.character(strategy)) {
- mdebug("Option 'future.plan'=%s", sQuote(strategy))
- } else {
- mdebug("Option 'future.plan' of type %s", sQuote(mode(strategy)))
+ if (debug) {
+ if (is.character(strategy)) {
+ mdebug("Option 'future.plan' = %s", sQuote(strategy))
+ } else {
+ mdebug("Option 'future.plan' of type %s", sQuote(mode(strategy)))
+ }
}
}
args <- parseCmdArgs()
p <- args$p
if (!is.null(p)) {
- mdebug("R command-line argument: -p %s", p)
+ if (debug) mdebug("R command-line argument: -p %s", p)
## Apply
- options(mc.cores=p-1L)
- ## options(Ncpus=p-1L) ## FIXME: Does it make sense? /HB 2016-04-02
+ options(mc.cores = p)
+ ## options(Ncpus = p) ## FIXME: Does it make sense? /HB 2016-04-02
## Set 'future.plan' option?
if (!is.null(strategy)) {
- mdebug("=> 'future.plan' already set.")
+ if (debug) mdebug(" => 'future.plan' already set.")
} else if (p == 1L) {
- mdebug("=> options(future.plan=eager)")
- options(future.plan=eager)
+ if (debug) mdebug(" => options(future.plan = sequential)")
+ options(future.plan = sequential)
} else {
- mdebug("=> options(future.plan=tweak(multiprocess, workers=%s))", p)
- options(future.plan=tweak(multiprocess, workers=p))
+ if (debug) mdebug(" => options(future.plan = tweak(multiprocess, workers = %s))", p)
+ options(future.plan = tweak(multiprocess, workers = p))
}
}
## Create UUID for this process
id <- session_uuid(attributes = TRUE)
- mdebug("R process uuid: %s", id)
- mdebug("Setting plan('default')")
- plan("default")
+ if (debug) {
+ mdebug("R process uuid: %s", id)
+ mdebug("Setting plan('default')")
+ }
+
+ ## NOTE: Don't initiate during startup - it might hang / give an error
+ plan("default", .init = FALSE)
} ## .onLoad()
@@ -77,21 +108,25 @@
## Load .future.R script?
loadDotFuture <- getOption("future.startup.loadScript", TRUE)
if (isTRUE(loadDotFuture)) {
+ debug <- getOption("future.debug", FALSE)
+
pathnames <- c(".future.R", "~/.future.R")
pathnames <- pathnames[file_test("-f", pathnames)]
if (length(pathnames) == 0) {
- mdebug("Future startup scripts identified: <none>")
+ if (debug) mdebug("Future startup scripts identified: <none>")
return()
}
- mdebug("Future startup scripts identified: %s", paste(sQuote(pathnames), collapse=", "))
pathname <- pathnames[1]
- mdebug("Future startup script to load: %s", sQuote(pathname))
+ if (debug) {
+ mdebug("Future startup scripts identified: %s", paste(sQuote(pathnames), collapse = ", "))
+ mdebug("Future startup script to load: %s", sQuote(pathname))
+ }
tryCatch({
- source(pathname, chdir=FALSE, echo=FALSE, local=FALSE)
+ source(pathname, chdir = FALSE, echo = FALSE, local = FALSE)
}, error = function(ex) {
msg <- sprintf("Failed to source %s file while attaching the future package. Will ignore this error, but please investigate. The error message was: %s", sQuote(pathname), sQuote(ex$message))
- mdebug(msg)
+ if (debug) mdebug(msg)
warning(msg)
})
}
diff --git a/R/zzz.plan.R b/R/zzz.plan.R
index 5d38c25..6334ddf 100644
--- a/R/zzz.plan.R
+++ b/R/zzz.plan.R
@@ -6,36 +6,41 @@
#'
#' @param strategy The evaluation function (or name of it) to use
#' for resolving a future. If NULL, then the current strategy is returned.
+#'
#' @param \dots Additional arguments overriding the default arguments
-#' of the evaluation function.
+#' of the evaluation function. Which additional arguments are supported
+#' depends on what evaluation function is used, e.g. several support
+#' argument \code{workers} but not all. For details, see the individual
+#' functions of which some are linked to below.
+#"
#' @param substitute If TRUE, the \code{strategy} expression is
#' \code{substitute()}:d, otherwise not.
+#'
#' @param .call (internal) Used for recording the call to this function.
#'
+#' @param .cleanup (internal) Used to stop implicitly started clusters.
+#'
+#' @param .init (internal) Used to initiate workers.
+#'
#' @return If a new strategy is chosen, then the previous one is returned
#' (invisible), otherwise the current one is returned (visibly).
#'
#' @example incl/plan.R
#'
#' @details
-#' The default strategy is \code{\link{eager}}, but the default can be
+#' The default strategy is \code{\link{sequential}}, but the default can be
#' configured by option \option{future.plan} and, if that is not set,
#' system environment variable \env{R_FUTURE_PLAN}.
#' To reset the strategy back to the default, use \code{plan("default")}.
#'
#' @section Implemented evaluation strategies:
#' \itemize{
-#' \item{\code{\link{eager}}:}{
+#' \item{\code{\link{sequential}}:}{
#' Resolves futures sequentially in the current R process.
#' }
-#' \item{\code{\link{lazy}}:}{
-#' Resolves futures synchronously (sequentially) in the current
-#' R process, but only if their values are requested. Futures for
-#' which the values are never requested will not be evaluated.
-#' }
#' \item{\code{\link{transparent}}:}{
-#' Resolves futures synchronously (sequentially) in the current
-#' R process and assignments will be done to the calling environment.
+#' Resolves futures sequentially in the current R process and
+#' assignments will be done to the calling environment.
#' Early stopping is enabled by default.
#' }
#' \item{\code{\link{multisession}}:}{
@@ -49,7 +54,7 @@
#' }
#' \item{\code{\link{multiprocess}}:}{
#' If multicore evaluation is supported, that will be used,
-# otherwise multisession evaluation will be used.
+#' otherwise multisession evaluation will be used.
#' }
#' \item{\code{\link{cluster}}:}{
#' Resolves futures asynchronously (in parallel) in separate
@@ -71,14 +76,52 @@
#'
#' @export
plan <- local({
- defaultStrategy <- structure(eager, call=substitute(plan(eager)))
- defaultStack <- structure(list(defaultStrategy), class = c("FutureStrategyList", "list"))
-
+ defaultStrategy <- structure(sequential, call = substitute(plan(sequential)))
+
+ defaultStack <- structure(list(defaultStrategy),
+ class = c("FutureStrategyList", "list"))
+
## Stack of type of futures to use
stack <- defaultStack
+ plan_cleanup <- function() {
+ ClusterRegistry(action = "stop")
+ }
+
+ plan_init <- function() {
+ evaluator <- stack[[1L]]
+ init <- attr(evaluator, "init")
+ if (identical(init, TRUE)) {
+ debug <- getOption("future.debug", FALSE)
+ if (debug) {
+ mdebug("plan(): plan_init() of %s ...",
+ paste(sQuote(class(evaluator)), collapse = ", "))
+ mdebug(paste(capture.output(print(evaluator)), collapse = "\n"))
+ }
+
+ ## IMPORANT: Initiate only once. This avoids an infinite
+ ## recursive loop caused by other plan() calls.
+ attr(evaluator, "init") <- "done"
+ stack[[1L]] <<- evaluator
+
+ ## Create dummy future to trigger setup (minimum overhead)
+ f <- evaluator(NA, globals = FALSE, lazy = FALSE)
+
+ ## Cleanup, but resolving it
+ ## (otherwise the garbage collector would have to do it)
+ v <- value(f)
+
+ if (debug) {
+ mdebug("plan(): plan_init() of %s ... DONE",
+ paste(sQuote(class(evaluator)), collapse = ", "))
+ }
+ }
+ }
+
+
## Main function
- function(strategy=NULL, ..., substitute=TRUE, .call=TRUE) {
+ function(strategy = NULL, ..., substitute = TRUE, .call = TRUE,
+ .cleanup = TRUE, .init = TRUE) {
if (substitute) strategy <- substitute(strategy)
if (is.logical(.call)) stopifnot(length(.call) == 1L, !is.na(.call))
@@ -89,16 +132,19 @@ plan <- local({
class(strategy) <- c("FutureStrategy", class(strategy))
return(strategy)
} else if (identical(strategy, "default")) {
- strategy <- getOption("future.plan", eager)
+ strategy <- getOption("future.plan", sequential)
} else if (identical(strategy, "list")) {
## List stack of future strategies?
return(stack)
} else if (identical(strategy, "reset")) {
- ## Rest stack of future strategies?
+ ## Reset stack of future strategies?
stack <<- defaultStack
+ ## Stop any (implicitly started) clusters?
+ if (.cleanup) plan_cleanup()
return(stack)
} else if (identical(strategy, "pop")) {
- ## Pop strategy stack and return old stack (so it can be pushed back later)
+ ## Pop strategy stack and return old stack
+ ## (so it can be pushed back later)
oldStack <- stack
stack <<- stack[-1L]
if (length(stack) == 0L) stack <<- defaultStack
@@ -109,14 +155,29 @@ plan <- local({
oldStack <- stack
newStack <- NULL
+ ## Arguments to be tweaked
+ targs <- list(...)
+
## Set new stack?
if (is.list(strategy)) {
stopifnot(is.list(strategy), length(strategy) >= 1L)
+
+ ## Check for usage of defunct eager() and lazy()
for (ii in seq_along(strategy)) {
stopifnot(is.function(strategy[[ii]]))
+ if (inherits(strategy[[ii]], "lazy")) {
+ .Defunct(msg = "Future strategy 'lazy' is defunct. Lazy evaluation can no longer be set via plan(). Instead, use f <- future(..., lazy = TRUE) or v %<-% { ... } %lazy% TRUE.")
+ } else if (inherits(strategy[[ii]], "lazy")) {
+ .Defunct(msg = "Future strategy 'eager' is defunct. Please use 'sequential' instead, which works identical.")
+ }
}
+
class(strategy) <- unique(c("FutureStrategyList", class(strategy)))
stack <<- strategy
+ ## Stop any (implicitly started) clusters?
+ if (.cleanup) plan_cleanup()
+ ## Initiate future workers?
+ if (.init) plan_init()
return(invisible(oldStack[[1L]]))
}
@@ -124,24 +185,26 @@ plan <- local({
if (is.language(strategy)) {
first <- as.list(strategy)[[1]]
if (is.symbol(first)) {
- first <- eval(first, envir=parent.frame())
+ first <- eval(first, envir = parent.frame())
## A list object, e.g. plan(oplan)?
if (is.list(first)) {
strategies <- first
- res <- plan(strategies, substitute=FALSE)
+ res <- plan(strategies, substitute = FALSE,
+ .cleanup = .cleanup, .init = .init)
return(invisible(res))
}
- ## Example: plan(list(eager, lazy))
+ ## Example: plan(list(sequential, multicore))
if (is.function(first) && identical(first, list)) {
## Specified explicitly using plan(list(...))?
- strategies <- eval(strategy, envir=parent.frame())
+ strategies <- eval(strategy, envir = parent.frame())
stopifnot(is.list(strategies), length(strategies) >= 1L)
- ## Coerce strings to functions, e.g. plan(list("eager", lazy))
+ ## Coerce strings to functions, e.g.
+ ## plan(list("sequential", multicore))
for (kk in seq_along(strategies)) {
strategy_kk <- strategies[[kk]]
if (is.character(strategy_kk)) {
- strategy_kk <- tweak(strategy_kk, penvir=parent.frame())
+ strategy_kk <- tweak(strategy_kk, penvir = parent.frame())
strategies[[kk]] <- strategy_kk
}
}
@@ -152,30 +215,27 @@ plan <- local({
## (b) Otherwise, assume a single future strategy
if (is.null(newStack)) {
- ## Arguments to be tweaked
- targs <- list(...)
-
if (is.symbol(strategy)) {
- strategy <- eval(strategy, envir=parent.frame())
+ strategy <- eval(strategy, envir = parent.frame())
} else if (is.language(strategy)) {
strategyT <- as.list(strategy)
## tweak(...)?
if (strategyT[[1]] == as.symbol("tweak")) {
- strategy <- eval(strategy, envir=parent.frame())
+ strategy <- eval(strategy, envir = parent.frame())
} else {
- isSymbol <- sapply(strategyT, FUN=is.symbol)
+ isSymbol <- sapply(strategyT, FUN = is.symbol)
if (!all(isSymbol)) {
targs <- c(targs, strategyT[-1L])
strategy <- strategyT[[1L]]
}
- strategy <- eval(strategy, envir=parent.frame())
+ strategy <- eval(strategy, envir = parent.frame())
}
}
## Tweak future strategy accordingly
- args <- c(list(strategy), targs, penvir=parent.frame())
- tstrategy <- do.call(tweak, args=args)
+ args <- c(list(strategy), targs, penvir = parent.frame())
+ tstrategy <- do.call(tweak, args = args)
## Setup a new stack of future strategies (with a single one)
newStack <- list(tstrategy)
@@ -197,17 +257,37 @@ plan <- local({
}
}
+ using_lazy <- lapply(newStack, FUN = inherits, "lazy")
+ using_lazy <- any(unlist(using_lazy, use.names = FALSE))
+ if (using_lazy) {
+ .Defunct(msg = "Future strategy 'lazy' is defunct. Lazy evaluation can no longer be set via plan(). Instead, use f <- future(..., lazy = TRUE) or v %<-% { ... } %lazy% TRUE.")
+ }
+
+ using_eager <- lapply(newStack, FUN = inherits, "eager")
+ using_eager <- any(unlist(using_eager, use.names = FALSE))
+ if (using_eager) {
+ .Defunct(msg = "Future strategy 'eager' is defunct. Please use 'sequential' instead, which works identical.")
+ }
+
## Set new strategy for futures
class(newStack) <- c("FutureStrategyList", class(newStack))
stack <<- newStack
stopifnot(is.list(stack), length(stack) >= 1L)
+ ## Stop any (implicitly started) clusters?
+ if (.cleanup) plan_cleanup()
+
+ ## Initiate future workers?
+ if (.init) plan_init()
+
invisible(oldStack[[1L]])
} # function()
}) # plan()
-supportedStrategies <- function(strategies=c("lazy", "eager", "multicore", "multisession", "multiprocess")) {
+supportedStrategies <- function(strategies = c("sequential", "multicore",
+ "multisession", "multiprocess",
+ "cluster")) {
if (!supportsMulticore()) strategies <- setdiff(strategies, "multicore")
strategies
}
@@ -228,7 +308,7 @@ print.future <- function(x, ...) {
specs <- sprintf("- %s: %s", names(specs), unlist(specs))
s <- c(s, specs)
s <- paste(s, collapse = "\n")
- cat(s, "\n", sep="")
+ cat(s, "\n", sep = "")
invisible(x)
}
@@ -239,7 +319,7 @@ print.FutureStrategy <- print.future
#' @export
print.FutureStrategyList <- function(x, ...) {
s <- "List of future strategies:"
-
+
for (kk in seq_along(x)) {
x_kk <- x[[kk]]
class <- setdiff(class(x_kk), c("tweaked", "function"))
@@ -257,6 +337,6 @@ print.FutureStrategyList <- function(x, ...) {
}
s <- paste(s, collapse = "\n")
- cat(s, "\n", sep="")
+ cat(s, "\n", sep = "")
invisible(x)
}
diff --git a/appveyor.yml b/appveyor.yml
deleted file mode 100644
index bf6075d..0000000
--- a/appveyor.yml
+++ /dev/null
@@ -1,72 +0,0 @@
-#----------------------------------------------------------------
-# AppVeyor configuration for R packages
-#
-# REFERENCES:
-# * AppVeyor CI: https://ci.appveyor.com/
-# * r-appveyor: https://github.com/krlmlr/r-appveyor
-#
-# Validate your .appveyor.yml file at
-# https://ci.appveyor.com/tools/validate-yaml
-#----------------------------------------------------------------
-environment:
- _R_CHECK_FORCE_SUGGESTS_: false
- USE_RTOOLS: true
-
- matrix:
- - R_VERSION: devel
- R_ARCH: x64
-
- - R_VERSION: devel
- R_ARCH: i386
-
- - R_VERSION: release
- R_ARCH: x64
-
-
-# DO NOT CHANGE the "init" and "install" sections below
-
-# Download script file from GitHub
-init:
- ps: |
- $ErrorActionPreference = "Stop"
- Invoke-WebRequest http://raw.github.com/krlmlr/r-appveyor/master/scripts/appveyor-tool.ps1 -OutFile "..\appveyor-tool.ps1"
- Import-Module '..\appveyor-tool.ps1'
-install:
- ps: |
- Bootstrap
- InstallRtools
-
-# Adapt as necessary starting from here
-
-build_script:
- - echo Current directory=%CD%
- - travis-tool.sh install_r digest globals listenv markdown R.rsp
-
-test_script:
- - travis-tool.sh run_tests
-
-after_test:
- - 7z a all-Rout.zip *.Rcheck\**\*.Rout *.Rcheck\**\*.fail
-
-artifacts:
- - path: '*.Rcheck\**\*.log'
- name: Logs
-
- - path: '*.Rcheck\**\*.out'
- name: Logs
-
- - path: '*.Rcheck\**\*.fail'
- name: Logs
-
- - path: '\*_*.tar.gz'
- name: Bits
-
- - path: '\*_*.zip'
- name: Bits
-
- - path: all-Rout.zip
- name: AllRout
-
-on_failure:
- - 7z a failure.zip *.Rcheck\*
- - appveyor PushArtifact failure.zip
diff --git a/build/vignette.rds b/build/vignette.rds
new file mode 100644
index 0000000..bf89cf7
Binary files /dev/null and b/build/vignette.rds differ
diff --git a/cran-comments.md b/cran-comments.md
deleted file mode 100644
index 7bbfbcb..0000000
--- a/cran-comments.md
+++ /dev/null
@@ -1,49 +0,0 @@
-# CRAN submission future 1.2.0
-on 2015-11-12
-
-I've verified that this submission causes *no* issues for
-any of the 19 reverse (non-recursive) package dependencies
-available on CRAN and Bioconductor.
-
-Thanks in advance
-
-
-## Notes not sent to CRAN
-The package has been verified using `R CMD check --as-cran` on:
-
-* Platform x86_64-apple-darwin13.4.0 (64-bit) [Travis CI]:
- - R 3.2.4 Revised (2016-03-16)
- - R version 3.3.2 (2016-10-31)
-
-* Platform x86_64-unknown-linux-gnu (64-bit) [Travis CI]:
- - R version 3.2.5 (2016-04-14)
- - R version 3.3.1 (2016-06-21)
- - R Under development (unstable) (2016-11-11 r71647)
-
-* Platform x86_64-pc-linux-gnu (64-bit):
- - R version 3.1.2 (2014-10-31)
- - R version 3.3.0 (2016-05-03)
- - R version 3.3.2 (2016-10-31)
- - R version 3.3.2 Patched (2016-11-09 r71642)
-
-* Platform x86_64-pc-linux-gnu (64-bit) [r-hub]:
- - R version 3.3.1 (2016-06-21)
- - R Under development (unstable) (2016-10-30 r71610)
-
-* Platform i686-pc-linux-gnu (32-bit):
- - R version 3.2.3 (2015-12-10)
- - R version 3.3.2 (2016-10-31)
-
-* Platform i386-w64-mingw32 (32-bit) [Appveyor CI]:
- - R Under development (unstable) (2016-11-11 r71647)
-
-* Platform x86_64-w64-mingw32/x64 (64-bit) [Appveyor CI]:
- - R version 3.3.2 (2016-10-31)
- - R Under development (unstable) (2016-11-11 r71647)
-
-* Platform x86_64-w64-mingw32 (64-bit) [r-hub]:
- - R version 3.2.5 (2016-04-14)
-
-* Platform x86_64-w64-mingw32/x64 (64-bit) [win-builder]:
- - R version 3.3.2 (2016-10-31)
- - R Under development (unstable) (2016-11-09 r71641)
diff --git a/demo/fibonacci.R b/demo/fibonacci.R
index d6a5e69..23ab19d 100644
--- a/demo/fibonacci.R
+++ b/demo/fibonacci.R
@@ -5,13 +5,13 @@ library("listenv")
## (0, 1, 1, 2, 3, 5, 8, ...)
## but calculate only the ones need when
## a number is actually requested.
-oplan <- plan(lazy)
+oplan <- plan(sequential)
x <- listenv()
x[[1]] <- 0
x[[2]] <- 1
for (i in 3:100) {
- x[[i]] %<-% { x[[i-2]] + x[[i-1]] }
+ x[[i]] %<-% { x[[i - 2]] + x[[i - 1]] } %lazy% TRUE
}
## At this point nothing has been calculated,
diff --git a/demo/mandelbrot.R b/demo/mandelbrot.R
index 5e2c1a2..7d85d34 100644
--- a/demo/mandelbrot.R
+++ b/demo/mandelbrot.R
@@ -1,7 +1,7 @@
library("future")
library("graphics")
-plotWhatIsDone <- function(counts) {
+plot_what_is_done <- function(counts) {
for (kk in seq_along(counts)) {
f <- counts[[kk]]
@@ -10,49 +10,52 @@ plotWhatIsDone <- function(counts) {
## Not resolved?
if (!resolved(f)) next
-
+
cat(sprintf("Plotting tile #%d of %d ...\n", kk, n))
counts[[kk]] <- value(counts[[kk]])
screen(kk)
plot(counts[[kk]])
- } # for (kk ...)
+ }
counts
-} # plotWhatIsDone()
+}
## Options
region <- getOption("future.demo.mandelbrot.region", 1L)
if (!is.list(region)) {
if (region == 1L) {
- region <- list(xmid=-0.75, ymid=0.0, side=3.0)
+ region <- list(xmid = -0.75, ymid = 0.0, side = 3.0)
} else if (region == 2L) {
- region <- list(xmid=0.283, ymid=-0.0095, side=0.00026)
+ region <- list(xmid = 0.283, ymid = -0.0095, side = 0.00026)
} else if (region == 3L) {
- region <- list(xmid=0.282989, ymid=-0.01, side=3e-8)
+ region <- list(xmid = 0.282989, ymid = -0.01, side = 3e-8)
}
}
nrow <- getOption("future.demo.mandelbrot.nrow", 3L)
resolution <- getOption("future.demo.mandelbrot.resolution", 400L)
delay <- getOption("future.demo.mandelbrot.delay", interactive())
if (isTRUE(delay)) {
- delay <- function(counts) Sys.sleep(rexp(1, rate=2))
+ delay <- function(counts) Sys.sleep(rexp(1, rate = 2))
} else if (!is.function(delay)) {
delay <- function(counts) {}
}
## Generate Mandelbrot tiles to be computed
-Cs <- mandelbrotTiles(xmid=region$xmid, ymid=region$ymid,
- side=region$side, nrow=nrow, resolution=resolution)
-
+Cs <- mandelbrot_tiles(xmid = region$xmid, ymid = region$ymid,
+ side = region$side, nrow = nrow,
+ resolution = resolution)
if (interactive()) {
+ if (.Platform$GUI == "RStudio") {
+ if (!"RStudioGD" %in% names(dev.list())) dev.new()
+ }
dev.new()
plot.new()
split.screen(dim(Cs))
for (ii in seq_along(Cs)) {
screen(ii)
- par(mar=c(0,0,0,0))
- text(x=1/2, y=1/2, sprintf("Future #%d\nunresolved", ii), cex=2)
+ par(mar = c(0, 0, 0, 0))
+ text(x = 1 / 2, y = 1 / 2, sprintf("Future #%d\nunresolved", ii), cex = 2)
}
} else {
split.screen(dim(Cs))
@@ -64,27 +67,27 @@ n <- length(Cs)
for (ii in seq_len(n)) {
cat(sprintf("Mandelbrot tile #%d of %d ...\n", ii, n))
C <- Cs[[ii]]
-
+
counts[[ii]] <- future({
cat(sprintf("Calculating tile #%d of %d ...\n", ii, n))
fit <- mandelbrot(C)
-
+
## Emulate slowness
delay(fit)
-
+
cat(sprintf("Calculating tile #%d of %d ... done\n", ii, n))
fit
})
## Plot tiles that are already resolved
- counts <- plotWhatIsDone(counts)
+ counts <- plot_what_is_done(counts)
}
## Plot remaining tiles
repeat {
- counts <- plotWhatIsDone(counts)
- if (!any(sapply(counts, FUN=inherits, "Future"))) break
+ counts <- plot_what_is_done(counts)
+ if (!any(sapply(counts, FUN = inherits, "Future"))) break
}
diff --git a/incl/cluster.R b/incl/cluster.R
deleted file mode 100644
index 3850c7d..0000000
--- a/incl/cluster.R
+++ /dev/null
@@ -1,33 +0,0 @@
-## Cluster futures gives an error on R CMD check on
-## Windows (but not Linux or OS X) for unknown reasons.
-## The same code works in package tests.
-\donttest{
-
-## Use cluster futures
-cl <- parallel::makeCluster(2L)
-plan(cluster, workers=cl)
-
-## A global variable
-a <- 0
-
-## Create multicore future (explicitly)
-f <- future({
- b <- 3
- c <- 2
- a * b * c
-})
-
-## A cluster future is evaluated in a separate process.
-## Changing the value of a global variable will not
-## affect the result of the future.
-a <- 7
-print(a)
-
-v <- value(f)
-print(v)
-stopifnot(v == 0)
-
-## CLEANUP
-parallel::stopCluster(cl)
-
-}
diff --git a/incl/future.R b/incl/future.R
deleted file mode 100644
index 339e3b9..0000000
--- a/incl/future.R
+++ /dev/null
@@ -1,64 +0,0 @@
-## Cluster futures gives an error on R CMD check on
-## Windows (but not Linux or OS X) for unknown reasons.
-## The same code works in package tests.
-\donttest{
-## Evaluate futures in parallel
-plan(multiprocess)
-}
-
-## Data
-x <- rnorm(100)
-y <- 2*x + 0.2 + rnorm(100)
-w <- 1 + x^2
-
-
-## (1) Regular assignments (evaluated sequentially)
-fitA <- lm(y ~ x, weights = w) ## with offset
-fitB <- lm(y ~ x - 1, weights = w) ## without offset
-fitC <- {
- w <- 1 + abs(x) ## Different weights
- lm(y ~ x, weights = w)
-}
-print(fitA)
-print(fitB)
-print(fitC)
-
-
-## (2) Future assignments (evaluated in parallel)
-fitA %<-% lm(y ~ x, weights = w) ## with offset
-fitB %<-% lm(y ~ x - 1, weights = w) ## without offset
-fitC %<-% {
- w <- 1 + abs(x)
- lm(y ~ x, weights = w)
-}
-print(fitA)
-print(fitB)
-print(fitC)
-
-
-## (3) Explicitly create futures (evaluated in parallel)
-## and retrieve their values
-fA <- future( lm(y ~ x, weights = w) )
-fB <- future( lm(y ~ x - 1, weights = w) )
-fC <- future({
- w <- 1 + abs(x)
- lm(y ~ x, weights = w)
-})
-fitA <- value(fA)
-fitB <- value(fB)
-fitC <- value(fC)
-print(fitA)
-print(fitB)
-print(fitC)
-
-
-## (4) Explit future assignments (evaluated in parallel)
-futureAssign("fitA", lm(y ~ x, weights = w))
-futureAssign("fitB", lm(y ~ x - 1, weights = w))
-futureAssign("fitC", {
- w <- 1 + abs(x)
- lm(y ~ x, weights = w)
-})
-print(fitA)
-print(fitB)
-print(fitC)
diff --git a/incl/futureOf.R b/incl/futureOf.R
deleted file mode 100644
index 5f9cf4b..0000000
--- a/incl/futureOf.R
+++ /dev/null
@@ -1,33 +0,0 @@
-a %<-% { 1 }
-
-f <- futureOf(a)
-print(f)
-
-b %<-% { 2 }
-
-f <- futureOf(b)
-print(f)
-
-## All futures
-fs <- futureOf()
-print(fs)
-
-
-## Futures part of environment
-env <- new.env()
-env$c %<-% { 3 }
-
-f <- futureOf(env$c)
-print(f)
-
-f2 <- futureOf(c, envir=env)
-print(f2)
-
-f3 <- futureOf("c", envir=env)
-print(f3)
-
-fs <- futureOf(envir=env)
-print(fs)
-
-
-
diff --git a/incl/lazy.R b/incl/lazy.R
deleted file mode 100644
index ca262c9..0000000
--- a/incl/lazy.R
+++ /dev/null
@@ -1,60 +0,0 @@
-## Use lazy futures
-plan(lazy)
-
-## A global variable
-a <- 0
-
-## Create lazy future (explicitly)
-f <- future({
- b <- 3
- c <- 2
- a * b * c
-})
-
-## Although 'f' is a _lazy_ future and therefore
-## resolved/evaluates the future expression only
-## when the value is requested, any global variables
-## identified in the expression (here 'a') are
-## "frozen" at the time point when the future is
-## created. Because of this, the 'a' in the
-## the future expression preserved the zero value
-## although we reassign it in the global environment
-a <- 7
-print(a)
-
-v <- value(f)
-print(v)
-stopifnot(v == 0)
-
-
-## Another example illustrating that lazy futures go
-## hand-in-hand with lazy evaluation of arguments
-
-## A function that may or may not touch it's argument
-foo <- function(a, use=FALSE) {
- cat("foo() called\n")
- if (use) cat("a=", a, "\n", sep="")
-}
-
-## Create a future
-x %<-% { cat("Pow!\n"); 1 }
-
-## Lazy evaluation where argument is not used
-foo(x, use=FALSE)
-# Outputs:
-# foo() called
-
-## Lazy evaluation where argument is used
-## Hint: 'x' will be resolved
-foo(x, use=TRUE)
-# Outputs:
-# foo() called
-# Pow!
-# a=1
-
-## Lazy evaluation where argument is used (again)
-## Hint: 'x' is already resolved
-foo(x, use=TRUE)
-# Outputs:
-# foo() called
-# a=1
diff --git a/incl/makeClusterPSOCK.R b/incl/makeClusterPSOCK.R
deleted file mode 100644
index 3d4727e..0000000
--- a/incl/makeClusterPSOCK.R
+++ /dev/null
@@ -1,119 +0,0 @@
-## Setup of three R workers on two remote machines are set up
-workers <- c("n1.remote.org", "n2.remote.org", "n1.remote.org")
-cl <- makeClusterPSOCK(workers, dryrun = TRUE)
-
-## Same setup when the two machines are on the local network and
-## have identical software setups
-cl <- makeClusterPSOCK(
- workers,
- revtunnel = FALSE, homogeneous = TRUE,
- dryrun = TRUE
-)
-
-## Setup of remote worker with more detailed control on
-## authentication and reverse SSH tunnelling
-cl <- makeClusterPSOCK(
- "remote.server.org", user = "johnny",
- ## Manual configuration of reverse SSH tunnelling
- revtunnel = FALSE,
- rshopts = c("-v", "-R 11000:gateway:11942"),
- master = "gateway", port = 11942,
- ## Run Rscript nicely and skip any startup scripts
- rscript = c("nice", "/path/to/Rscript"),
- rscript_args = c("--vanilla"),
- dryrun = TRUE
-)
-
-## Setup of Docker worker running rocker/r-base
-## (requires installation of future package)
-cl <- makeClusterPSOCK(
- "localhost",
- ## Launch Rscript inside Docker container
- rscript = c(
- "docker", "run", "--net=host", "rocker/r-base",
- "Rscript"
- ),
- ## Install future package
- rscript_args = c(
- "-e", shQuote("install.packages('future')")
- ),
- dryrun = TRUE
-)
-
-
-## Setup of udocker.py worker running rocker/r-base
-## (requires installation of future package and extra quoting)
-cl <- makeClusterPSOCK(
- "localhost",
- ## Launch Rscript inside Docker container (using udocker)
- rscript = c(
- "udocker.py", "run", "rocker/r-base",
- "Rscript"
- ),
- ## Install future package and manually launch parallel workers
- ## (need double shQuote():s because udocker.py drops one level)
- rscript_args = c(
- "-e", shQuote(shQuote("install.packages('future')")),
- "-e", shQuote(shQuote("parallel:::.slaveRSOCK()"))
- ),
- dryrun = TRUE
-)
-
-
-## Launching worker on Amazon AWS EC2 running one of the
-## Amazon Machine Images (AMI) provided by RStudio
-## (http://www.louisaslett.com/RStudio_AMI/)
-public_ip <- "1.2.3.4"
-ssh_private_key_file <- "~/.ssh/my-private-aws-key.pem"
-cl <- makeClusterPSOCK(
- ## Public IP number of EC2 instance
- public_ip,
- ## User name (always 'ubuntu')
- user = "ubuntu",
- ## Use private SSH key registered with AWS
- rshopts = c(
- "-o", "StrictHostKeyChecking=no",
- "-o", "IdentitiesOnly=yes",
- "-i", ssh_private_key_file
- ),
- ## Set up .libPaths() for the 'ubuntu' user and
- ## install future package
- rscript_args = c(
- "-e", shQuote("local({
- p <- Sys.getenv('R_LIBS_USER')
- dir.create(p, recursive = TRUE, showWarnings = FALSE)
- .libPaths(p)
- })"),
- "-e", shQuote("install.packages('future')")
- ),
- dryrun = TRUE
-)
-
-
-## Launching worker on Google Cloud Engine (GCE) running a
-## container based VM (with a #cloud-config specification)
-public_ip <- "1.2.3.4"
-user <- "johnny"
-ssh_private_key_file <- "~/.ssh/google_compute_engine"
-cl <- makeClusterPSOCK(
- ## Public IP number of GCE instance
- public_ip,
- ## User name (== SSH key label (sic!))
- user = user,
- ## Use private SSH key registered with GCE
- rshopts = c(
- "-o", "StrictHostKeyChecking=no",
- "-o", "IdentitiesOnly=yes",
- "-i", ssh_private_key_file
- ),
- ## Launch Rscript inside Docker container
- rscript = c(
- "docker", "run", "--net=host", "rocker/r-base",
- "Rscript"
- ),
- ## Install future package
- rscript_args = c(
- "-e", shQuote("install.packages('future')")
- ),
- dryrun = TRUE
-)
diff --git a/incl/mandelbrot.R b/incl/mandelbrot.R
deleted file mode 100644
index d214b0d..0000000
--- a/incl/mandelbrot.R
+++ /dev/null
@@ -1,6 +0,0 @@
-counts <- mandelbrot(x=-0.75, y=0, side=3)
-plot(counts)
-
-\dontrun{
-demo("mandelbrot", package="future", ask=FALSE)
-}
diff --git a/incl/multicore.R b/incl/multicore.R
deleted file mode 100644
index 04bb746..0000000
--- a/incl/multicore.R
+++ /dev/null
@@ -1,22 +0,0 @@
-## Use multicore futures
-plan(multicore)
-
-## A global variable
-a <- 0
-
-## Create multicore future (explicitly)
-f <- future({
- b <- 3
- c <- 2
- a * b * c
-})
-
-## A multicore future is evaluated in a separate forked
-## process. Changing the value of a global variable
-## will not affect the result of the future.
-a <- 7
-print(a)
-
-v <- value(f)
-print(v)
-stopifnot(v == 0)
diff --git a/incl/multiprocess.R b/incl/multiprocess.R
deleted file mode 100644
index 4ffd7d0..0000000
--- a/incl/multiprocess.R
+++ /dev/null
@@ -1,29 +0,0 @@
-## Multiprocess futures gives an error on R CMD check on
-## Windows (but not Linux or OS X) for unknown reasons.
-## The same code works in package tests.
-\donttest{
-
-## Use multiprocess futures
-plan(multiprocess)
-
-## A global variable
-a <- 0
-
-## Create multicore future (explicitly)
-f <- future({
- b <- 3
- c <- 2
- a * b * c
-})
-
-## A multiprocess future is evaluated in a separate R process.
-## Changing the value of a global variable will not affect
-## the result of the future.
-a <- 7
-print(a)
-
-v <- value(f)
-print(v)
-stopifnot(v == 0)
-
-}
diff --git a/incl/multisession.R b/incl/multisession.R
deleted file mode 100644
index c43b3d7..0000000
--- a/incl/multisession.R
+++ /dev/null
@@ -1,29 +0,0 @@
-## Multisession futures gives an error on R CMD check on
-## Windows (but not Linux or OS X) for unknown reasons.
-## The same code works in package tests.
-\donttest{
-
-## Use multisession futures
-plan(multisession)
-
-## A global variable
-a <- 0
-
-## Create multicore future (explicitly)
-f <- future({
- b <- 3
- c <- 2
- a * b * c
-})
-
-## A multisession future is evaluated in a separate R session.
-## Changing the value of a global variable will not affect
-## the result of the future.
-a <- 7
-print(a)
-
-v <- value(f)
-print(v)
-stopifnot(v == 0)
-
-}
diff --git a/incl/nbrOfWorkers.R b/incl/nbrOfWorkers.R
deleted file mode 100644
index f262ae6..0000000
--- a/incl/nbrOfWorkers.R
+++ /dev/null
@@ -1,8 +0,0 @@
-plan(multiprocess)
-nbrOfWorkers() ## == availableCores()
-
-plan(multiprocess, workers=2)
-nbrOfWorkers() ## == 2
-
-plan(lazy)
-nbrOfWorkers() ## == 1
diff --git a/incl/plan.R b/incl/plan.R
deleted file mode 100644
index b0e23aa..0000000
--- a/incl/plan.R
+++ /dev/null
@@ -1,59 +0,0 @@
-a <- b <- c <- NA_real_
-
-# A lazy uniprocess future
-plan(lazy)
-f <- future({
- a <- 7
- b <- 3
- c <- 2
- a * b * c
-})
-y <- value(f)
-print(y)
-str(list(a=a, b=b, c=c)) ## All NAs
-
-
-# An eager uniprocess future
-plan(eager)
-f <- future({
- a <- 7
- b <- 3
- c <- 2
- a * b * c
-})
-y <- value(f)
-print(y)
-str(list(a=a, b=b, c=c)) ## All NAs
-
-
-# A multicore future
-plan(multicore)
-f <- future({
- a <- 7
- b <- 3
- c <- 2
- a * b * c
-})
-y <- value(f)
-print(y)
-str(list(a=a, b=b, c=c)) ## All NAs
-
-
-## Multisession futures gives an error on R CMD check on
-## Windows (but not Linux or OS X) for unknown reasons.
-## The same code works in package tests.
-\donttest{
-
-# A multisession future
-plan(multisession)
-f <- future({
- a <- 7
- b <- 3
- c <- 2
- a * b * c
-})
-y <- value(f)
-print(y)
-str(list(a=a, b=b, c=c)) ## All NAs
-
-}
diff --git a/incl/remote.R b/incl/remote.R
deleted file mode 100644
index 476339b..0000000
--- a/incl/remote.R
+++ /dev/null
@@ -1,11 +0,0 @@
-\dontrun{\donttest{
-
-## Use a remote machine
-plan(remote, workers="remote.server.org")
-
-## Evaluate expression remotely
-host %<-% { Sys.info()[["nodename"]] }
-host
-[1] "remote.server.org"
-
-}}
diff --git a/incl/uniprocess.R b/incl/uniprocess.R
deleted file mode 100644
index a4b78ea..0000000
--- a/incl/uniprocess.R
+++ /dev/null
@@ -1,23 +0,0 @@
-## Use an eager uniprocess futures
-plan(eager)
-
-## A global variable
-a <- 0
-
-## Create an eager uniprocess future (explicitly)
-f <- future({
- b <- 3
- c <- 2
- a * b * c
-})
-
-## Since 'a' is a global variable in future 'f' which
-## is eagerly resolved (default), this global has already
-## been resolved / incorporated, and any changes to 'a'
-## at this point will _not_ affect the value of 'f'.
-a <- 7
-print(a)
-
-v <- value(f)
-print(v)
-stopifnot(v == 0)
diff --git a/inst/doc/future-1-overview.html b/inst/doc/future-1-overview.html
new file mode 100644
index 0000000..9528192
--- /dev/null
+++ b/inst/doc/future-1-overview.html
@@ -0,0 +1,808 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+
+<title>A Future for R: A Comprehensive Overview</title>
+
+<script type="text/javascript">
+window.onload = function() {
+ var imgs = document.getElementsByTagName('img'), i, img;
+ for (i = 0; i < imgs.length; i++) {
+ img = imgs[i];
+ // center an image if it is the only element of its parent
+ if (img.parentElement.childElementCount === 1)
+ img.parentElement.style.textAlign = 'center';
+ }
+};
+</script>
+
+<!-- Styles for R syntax highlighter -->
+<style type="text/css">
+ pre .operator,
+ pre .paren {
+ color: rgb(104, 118, 135)
+ }
+
+ pre .literal {
+ color: #990073
+ }
+
+ pre .number {
+ color: #099;
+ }
+
+ pre .comment {
+ color: #998;
+ font-style: italic
+ }
+
+ pre .keyword {
+ color: #900;
+ font-weight: bold
+ }
+
+ pre .identifier {
+ color: rgb(0, 0, 0);
+ }
+
+ pre .string {
+ color: #d14;
+ }
+</style>
+
+<!-- R syntax highlighter -->
+<script type="text/javascript">
+var hljs=new function(){function m(p){return p.replace(/&/gm,"&").replace(/</gm,"<")}function f(r,q,p){return RegExp(q,"m"+(r.cI?"i":"")+(p?"g":""))}function b(r){for(var p=0;p<r.childNodes.length;p++){var q=r.childNodes[p];if(q.nodeName=="CODE"){return q}if(!(q.nodeType==3&&q.nodeValue.match(/\s+/))){break}}}function h(t,s){var p="";for(var r=0;r<t.childNodes.length;r++){if(t.childNodes[r].nodeType==3){var q=t.childNodes[r].nodeValue;if(s){q=q.replace(/\n/g,"")}p+=q}else{if(t.chi [...]
+hljs.initHighlightingOnLoad();
+</script>
+
+
+
+<style type="text/css">
+body, td {
+ font-family: sans-serif;
+ background-color: white;
+ font-size: 13px;
+}
+
+body {
+ max-width: 800px;
+ margin: auto;
+ padding: 1em;
+ line-height: 20px;
+}
+
+tt, code, pre {
+ font-family: 'DejaVu Sans Mono', 'Droid Sans Mono', 'Lucida Console', Consolas, Monaco, monospace;
+}
+
+h1 {
+ font-size:2.2em;
+}
+
+h2 {
+ font-size:1.8em;
+}
+
+h3 {
+ font-size:1.4em;
+}
+
+h4 {
+ font-size:1.0em;
+}
+
+h5 {
+ font-size:0.9em;
+}
+
+h6 {
+ font-size:0.8em;
+}
+
+a:visited {
+ color: rgb(50%, 0%, 50%);
+}
+
+pre, img {
+ max-width: 100%;
+}
+pre {
+ overflow-x: auto;
+}
+pre code {
+ display: block; padding: 0.5em;
+}
+
+code {
+ font-size: 92%;
+ border: 1px solid #ccc;
+}
+
+code[class] {
+ background-color: #F8F8F8;
+}
+
+table, td, th {
+ border: none;
+}
+
+blockquote {
+ color:#666666;
+ margin:0;
+ padding-left: 1em;
+ border-left: 0.5em #EEE solid;
+}
+
+hr {
+ height: 0px;
+ border-bottom: none;
+ border-top-width: thin;
+ border-top-style: dotted;
+ border-top-color: #999999;
+}
+
+ at media print {
+ * {
+ background: transparent !important;
+ color: black !important;
+ filter:none !important;
+ -ms-filter: none !important;
+ }
+
+ body {
+ font-size:12pt;
+ max-width:100%;
+ }
+
+ a, a:visited {
+ text-decoration: underline;
+ }
+
+ hr {
+ visibility: hidden;
+ page-break-before: always;
+ }
+
+ pre, blockquote {
+ padding-right: 1em;
+ page-break-inside: avoid;
+ }
+
+ tr, img {
+ page-break-inside: avoid;
+ }
+
+ img {
+ max-width: 100% !important;
+ }
+
+ @page :left {
+ margin: 15mm 20mm 15mm 10mm;
+ }
+
+ @page :right {
+ margin: 15mm 10mm 15mm 20mm;
+ }
+
+ p, h2, h3 {
+ orphans: 3; widows: 3;
+ }
+
+ h2, h3 {
+ page-break-after: avoid;
+ }
+}
+</style>
+
+<meta name="keywords" content="R, package, vignette, future, promise, lazy evaluation, synchronous, asynchronous, parallel, cluster">
+<meta name="author" content="Henrik Bengtsson">
+
+</head>
+
+<body>
+<h1>A Future for R: A Comprehensive Overview</h1>
+
+<h2>Introduction</h2>
+
+<p>The purpose of the <a href="https://cran.r-project.org/package=future">future</a> package is to provide a very simple and uniform way of evaluating R expressions asynchronously using various resources available to the user.</p>
+
+<p>In programming, a <em>future</em> is an abstraction for a <em>value</em> that may be available at some point in the future. The state of a future can either be <em>unresolved</em> or <em>resolved</em>. As soon as it is resolved, the value is available instantaneously. If the value is queried while the future is still unresolved, the current process is <em>blocked</em> until the future is resolved. It is possible to check whether a future is resolved or not without blocking. Exact [...]
+
+<p>Here is an example illustrating how the basics of futures work. First, consider the following code snippet that uses plain R code:</p>
+
+<pre><code class="r">> v <- {
++ cat("Resolving...\n")
++ 3.14
++ }
+Resolving...
+> v
+[1] 3.14
+</code></pre>
+
+<p>It works by assigning the value of an expression to variable <code>v</code> and we then print the value of <code>v</code>. Moreover, when the expression for <code>v</code> is evaluated we also print a message.</p>
+
+<p>Here is the same code snippet modified to use futures instead:</p>
+
+<pre><code class="r">> library("future")
+> v %<-% {
++ cat("Resolving...\n")
++ 3.14
++ }
+Resolving...
+> v
+[1] 3.14
+</code></pre>
+
+<p>The difference is in how <code>v</code> is constructed; with plain R we use <code><-</code> whereas with futures we use <code>%<-%</code>.</p>
+
+<p>So why are futures useful? Because we can choose to evaluate the future expression in a separate R process asynchronously by simply switching settings as:</p>
+
+<pre><code class="r">> library("future")
+> plan(multiprocess)
+> v %<-% {
++ cat("Resolving...\n")
++ 3.14
++ }
+> v
+[1] 3.14
+</code></pre>
+
+<p>With asynchronous futures, the current/main R process does <em>not</em> block, which means it is available for further processing while the futures are being resolved
+in separates processes running in the background. In other words, futures provide a simple but yet powerful construct for parallel and / or distributed processing in R.</p>
+
+<p>Now, if you cannot be bothered to read all the nitty-gritty details about futures, but just want to try them out, then skip to the end to play with the Mandelbrot demo using both parallel and non-parallel evaluation.</p>
+
+<h2>Implicit or Explicit Futures</h2>
+
+<p>Futures can be created either <em>implicitly</em> or <em>explicitly</em>. In the introductory example above we used <em>implicit futures</em> created via the <code>v %<-% { expr }</code> construct. An alternative is <em>explicit futures</em> using the <code>f <- future({ expr })</code> and <code>v <- value(f)</code> constructs. With these, our example could alternatively be written as:</p>
+
+<pre><code class="r">> library("future")
+> f <- future({
++ cat("Resolving...\n")
++ 3.14
++ })
+Resolving...
+> v <- value(f)
+> v
+[1] 3.14
+</code></pre>
+
+<p>Either style of future construct works equally(*) well. The implicit style is most similar to how regular R code is written. In principle, all you have to do is to replace <code><-</code> with a <code>%<-%</code> to turn the assignment into a future assignment. On the other hand, this simplicity can also be deceiving, particularly when asynchronous futures are being used. In contrast, the explicit style makes it much clearer that futures are being used, which lowers the risk [...]
+
+<p>(*) There are cases where <code>%<-%</code> cannot be used without some (small) modifications. We will return to this in Section 'Constraints when using Implicit Futures' near the end of this document.</p>
+
+<p>To summarize, for explicit futures, we use:</p>
+
+<ul>
+<li><code>f <- future({ expr })</code> - creates a future</li>
+<li><code>v <- value(f)</code> - gets the value of the future (blocks if not yet resolved)</li>
+</ul>
+
+<p>For implicit futures, we use:</p>
+
+<ul>
+<li><code>v %<-% { expr }</code> - creates a future and a promise to its value</li>
+</ul>
+
+<p>To keep it simple, we will use the implicit style in the rest of this document, but everything discussed will also apply to explicit futures.</p>
+
+<h2>Controlling How Futures are Resolved</h2>
+
+<p>The future package implements the following types of futures:</p>
+
+<table><thead>
+<tr>
+<th align="left">Name</th>
+<th align="left">OSes</th>
+<th align="left">Description</th>
+</tr>
+</thead><tbody>
+<tr>
+<td align="left"><em>synchronous:</em></td>
+<td align="left"></td>
+<td align="left"><em>non-parallel:</em></td>
+</tr>
+<tr>
+<td align="left"><code>sequential</code></td>
+<td align="left">all</td>
+<td align="left">sequentially and in the current R process</td>
+</tr>
+<tr>
+<td align="left"><code>transparent</code></td>
+<td align="left">all</td>
+<td align="left">as sequential w/ early signaling and w/out local (for debugging)</td>
+</tr>
+<tr>
+<td align="left"><em>asynchronous:</em></td>
+<td align="left"></td>
+<td align="left"><em>parallel</em>:</td>
+</tr>
+<tr>
+<td align="left"><code>multiprocess</code></td>
+<td align="left">all</td>
+<td align="left">multicore iff supported, otherwise multisession</td>
+</tr>
+<tr>
+<td align="left"><code>multisession</code></td>
+<td align="left">all</td>
+<td align="left">background R sessions (on current machine)</td>
+</tr>
+<tr>
+<td align="left"><code>multicore</code></td>
+<td align="left">not Windows</td>
+<td align="left">forked R processes (on current machine)</td>
+</tr>
+<tr>
+<td align="left"><code>cluster</code></td>
+<td align="left">all</td>
+<td align="left">external R sessions on current, local, and/or remote machines</td>
+</tr>
+<tr>
+<td align="left"><code>remote</code></td>
+<td align="left">all</td>
+<td align="left">Simple access to remote R sessions</td>
+</tr>
+</tbody></table>
+
+<p><em>Note</em>: Prior to future 1.3.0, <code>eager</code> and <code>lazy</code> were also options, but has since been deprecated and are both defunct as of 1.6.0. The reason for this is that whether a future should be resolved by lazy evaluation or not has to, in some cases, be in the control of the developer, and if the end user would be able change that, the code may not function as intended.</p>
+
+<p>The future package is designed such that support for additional strategies can be implemented as well. For instance, the <a href="https://cran.r-project.org/package=future.batchtools">future.batchtools</a> package provides futures for all types of <em>cluster functions</em> (“backends”) that the <a href="https://cran.r-project.org/package=batchtools">batchtools</a> package supports. Specifically, futures for evaluating R expressions via job schedulers such as Slurm, TORQ [...]
+
+<p>By default, future expressions are evaluated eagerly (= instantaneously) and synchronously (in the current R session). This evaluation strategy is referred to as “sequential”. In this section, we will go through each of these strategies and discuss what they have in common and how they differ.</p>
+
+<h3>Consistent Behavior Across Futures</h3>
+
+<p>Before going through each of the different future strategies, it is probably helpful to clarify the objectives the Future API (as defined by the future package). When programming with futures, it should not really matter what future strategy is used for executing code. This is because we cannot really know what computational resources the user has access to so the choice of evaluation strategy should be in the hands of the user and not the developer. In other words, the code should [...]
+
+<p>One of the designs of the Future API was to encapsulate any differences such that all types of futures will appear to work the same. This despite expressions may be evaluated locally in the current R session or across the world in remote R sessions. Another obvious advantage of having a consistent API and behavior among different types of futures is that it helps while prototyping. Typically one would use sequential evaluation while building up a script and, later, when the script [...]
+
+<p>Because of this, the defaults of the different strategies are such that the results and side effects of evaluating a future expression are as similar as possible. More specifically, the following is true for all futures:</p>
+
+<ul>
+<li><p>All <em>evaluation is done in a local environment</em> (i.e. <code>local({ expr })</code>) so that assignments do not affect the calling environment. This is natural when evaluating in an external R process, but is also enforced when evaluating in the current R session.</p></li>
+<li><p>When a future is constructed, <em>global variables are identified</em>. For asynchronous evaluation, globals are exported to the R process/session that will be evaluating the future expression. For sequential futures with lazy evaluation (<code>lazy = TRUE</code>), globals are “frozen” (cloned to a local environment of the future). Also, in order to protect against exporting too large objects by mistake, there is a built-in assertion that the total size of all globa [...]
+<li><p>Future <em>expressions are only evaluated once</em>. As soon as the value (or an error) has been collected it will be available for all succeeding requests.</p></li>
+</ul>
+
+<p>Here is an example illustrating that all assignments are done to a local environment:</p>
+
+<pre><code class="r">> plan(sequential)
+> a <- 1
+> x %<-% {
++ a <- 2
++ 2 * a
++ }
+> x
+[1] 4
+> a
+[1] 1
+</code></pre>
+
+<p>Now we are ready to explore the different future strategies.</p>
+
+<h3>Synchronous Futures</h3>
+
+<p>Synchronous futures are resolved one after another and most commonly by the R process that creates them. When a synchronous future is being resolved it blocks the main process until resolved. There are two types of synchronous futures in the future package, <em>sequential</em> and <em>transparent</em>. (In future 1.2.0 and before, there was also <em>lazy</em> futures, which has now been deprecated in favor of <code>f <- future(..., lazy = TRUE)</code> and <code>v %<-% { ... } [...]
+
+<h4>Sequential Futures</h4>
+
+<p>Sequential futures are the default unless otherwise specified. They were designed to behave as similar as possible to regular R evaluation while still fulfilling the Future API and its behaviors. Here is an example illustrating their properties:</p>
+
+<pre><code class="r">> plan(sequential)
+> pid <- Sys.getpid()
+> pid
+[1] 28518
+> a %<-% {
++ pid <- Sys.getpid()
++ cat("Resolving 'a' ...\n")
++ 3.14
++ }
+Resolving 'a' ...
+> b %<-% {
++ rm(pid)
++ cat("Resolving 'b' ...\n")
++ Sys.getpid()
++ }
+Resolving 'b' ...
+> c %<-% {
++ cat("Resolving 'c' ...\n")
++ 2 * a
++ }
+Resolving 'c' ...
+> b
+[1] 28518
+> c
+[1] 6.28
+> a
+[1] 3.14
+> pid
+[1] 28518
+</code></pre>
+
+<p>Since eager sequential evaluation is taking place, each of the three futures is resolved instantaneously in the moment it is created. Note also how <code>pid</code> in the calling environment, which was assigned the process ID of the current process, is neither overwritten nor removed. This is because futures are evaluated in a local environment. Since synchronous (uni-)processing is used, future <code>b</code> is resolved by the main R process (still in a local environment), which [...]
+
+<h4>Transparent Futures</h4>
+
+<p>For troubleshooting, <em>transparent</em> futures can be used by specifying <code>plan(transparent)</code>. A transparent future is technically a sequential future with instant signaling of conditions (including errors and warnings) and where evaluation, and therefore also assignments, take place in the calling environment. Transparent futures are particularly useful for troubleshooting errors that are otherwise hard to narrow down.</p>
+
+<h3>Asynchronous Futures</h3>
+
+<p>Next, we will turn to asynchronous futures, which are futures that are resolved in the background. By design, these futures are non-blocking, that is, after being created the calling process is available for other tasks including creating additional futures. It is only when the calling process tries to access the value of a future that is not yet resolved, or trying to create another asynchronous future when all available R processes are busy serving other futures, that it blocks.</p>
+
+<h4>Multisession Futures</h4>
+
+<p>We start with multisession futures because they are supported by all operating systems. A multisession future is evaluated in a background R session running on the same machine as the calling R process. Here is our example with multisession evaluation:</p>
+
+<pre><code class="r">> plan(multisession)
+> pid <- Sys.getpid()
+> pid
+[1] 28518
+> a %<-% {
++ pid <- Sys.getpid()
++ cat("Resolving 'a' ...\n")
++ 3.14
++ }
+> b %<-% {
++ rm(pid)
++ cat("Resolving 'b' ...\n")
++ Sys.getpid()
++ }
+> c %<-% {
++ cat("Resolving 'c' ...\n")
++ 2 * a
++ }
+> b
+[1] 28539
+> c
+[1] 6.28
+> a
+[1] 3.14
+> pid
+[1] 28518
+</code></pre>
+
+<p>The first thing we observe is that the values of <code>a</code>, <code>c</code> and <code>pid</code> are the same as previously. However, we notice that <code>b</code> is different from before. This is because future <code>b</code> is evaluated in a different R process and therefore it returns a different process ID. Another difference is that the messages, generated by <code>cat()</code>, are no longer displayed. This is because they are outputted to the background sessions and n [...]
+
+<p>When multisession evaluation is used, the package launches a set of R sessions in the background that will serve multisession futures by evaluating their expressions as they are created. If all background sessions are busy serving other futures, the creation of the next multisession future is <em>blocked</em> until a background session becomes available again. The total number of background processes launched is decided by the value of <code>availableCores()</code>, e.g.</p>
+
+<pre><code class="r">> availableCores()
+mc.cores
+ 2
+</code></pre>
+
+<p>This particular result tells us that the <code>mc.cores</code> option was set such that we are allowed to use in total 2 processes including the main process. In other words, with these settings, there will be 2 background processes serving the multisession futures. The <code>availableCores()</code> is also agile to different options and system environment variables. For instance, if compute cluster schedulers are used (e.g. TORQUE/PBS and Slurm), they set specific environment vari [...]
+
+<h4>Multicore Futures</h4>
+
+<p>On operating systems where R supports <em>forking</em> of processes, which is basically all operating system except Windows, an alternative to spawning R sessions in the background is to fork the existing R process. Forking an R process is considered faster than working with a separate R session running in the background. One reason is that the overhead of exporting large globals to the background session can be greater than when forking is used.
+To use multicore futures, we specify:</p>
+
+<pre><code class="r">plan(multicore)
+</code></pre>
+
+<p>The only real different between using multicore and multisession futures is that any output written (to standard output or standard error) by a multicore process is instantaneously outputted in calling process. Other than this, the behavior of using multicore evaluation is very similar to that of using multisession evaluation.</p>
+
+<p>Just like for multisession futures, the maximum number of parallel processes running will be decided by <code>availableCores()</code>, since in both cases the evaluation is done on the local machine.</p>
+
+<h4>Multiprocess Futures</h4>
+
+<p>Sometimes we do not know whether multicore futures are supported or not, but it might still be that we would like to write platform-independent scripts or instructions that work everywhere. In such cases we can specify that we want to use “multiprocess” futures as in:</p>
+
+<pre><code class="r">plan(multiprocess)
+</code></pre>
+
+<p>A multiprocess future is not a formal class of futures by itself, but rather a convenient alias for either of the two. When this is specified, multisession evaluation will be used unless multicore evaluation is supported.</p>
+
+<h4>Cluster Futures</h4>
+
+<p>Cluster futures evaluate expressions on an ad-hoc cluster (as implemented by the parallel package). For instance, assume you have access to three nodes <code>n1</code>, <code>n2</code> and <code>n3</code>, you can then use these for asynchronous evaluation as:</p>
+
+<pre><code class="r">> plan(cluster, workers = c("n1", "n2", "n3"))
+> pid <- Sys.getpid()
+> pid
+[1] 28518
+> a %<-% {
++ pid <- Sys.getpid()
++ cat("Resolving 'a' ...\n")
++ 3.14
++ }
+> b %<-% {
++ rm(pid)
++ cat("Resolving 'b' ...\n")
++ Sys.getpid()
++ }
+> c %<-% {
++ cat("Resolving 'c' ...\n")
++ 2 * a
++ }
+> b
+[1] 28561
+> c
+[1] 6.28
+> a
+[1] 3.14
+> pid
+[1] 28518
+</code></pre>
+
+<p>Just as for most other asynchronous evaluation strategies, the output from <code>cat()</code> is not displayed on the current/calling machine.</p>
+
+<p>Any types of clusters that <code>parallel::makeCluster()</code> creates can be used for cluster futures. For instance, the above cluster can be explicitly set up as:</p>
+
+<pre><code class="r">cl <- parallel::makeCluster(c("n1", "n2", "n3"))
+plan(cluster, workers = cl)
+</code></pre>
+
+<p>Also, it is considered good style to shut down cluster <code>cl</code> when it is no longer needed, that is, calling <code>parallel::stopCluster(cl)</code>. However, it will shut itself down if the main process is terminated. For more information on how to set up and manage such clusters, see <code>help("makeCluster", package = "parallel")</code>.
+Clusters created implicitly using <code>plan(cluster, workers = hosts)</code> where <code>hosts</code> is a character vector will also be shut down when the main R session terminates, or when the future strategy is changed, e.g. by calling <code>plan(sequential)</code>.</p>
+
+<p>Note that with automatic authentication setup (e.g. SSH key pairs), there is nothing preventing us from using the same approach for using a cluster of remote machines.</p>
+
+<h3>Nested Futures and Evaluation Topologies</h3>
+
+<p>This far we have discussed what can be referred to as “flat topology” of futures, that is, all futures are created in and assigned to the same environment. However, there is nothing stopping us from using a “nested topology” of futures, where one set of futures may, in turn, create another set of futures internally and so on.</p>
+
+<p>For instance, here is an example of two “top” futures (<code>a</code> and <code>b</code>) that uses multiprocess evaluation and where the second future (<code>b</code>) in turn uses two internal futures:</p>
+
+<pre><code class="r">> plan(multiprocess)
+> pid <- Sys.getpid()
+> a %<-% {
++ cat("Resolving 'a' ...\n")
++ Sys.getpid()
++ }
+> b %<-% {
++ cat("Resolving 'b' ...\n")
++ b1 %<-% {
++ cat("Resolving 'b1' ...\n")
++ Sys.getpid()
++ }
++ b2 %<-% {
++ cat("Resolving 'b2' ...\n")
++ Sys.getpid()
++ }
++ c(b.pid = Sys.getpid(), b1.pid = b1, b2.pid = b2)
++ }
+> pid
+[1] 28518
+> a
+[1] 28584
+> b
+ b.pid b1.pid b2.pid
+ 28585 28585 28585
+</code></pre>
+
+<p>By inspection the process IDs, we see that there are in total three different processes involved for resolving the futures. There is the main R process (pid 28518), and there are the two processes used by <code>a</code> (pid 28584) and <code>b</code> (pid 28585). However, the two futures (<code>b1</code> and <code>b2</code>) that is nested by <code>b</code> are evaluated by the same R process as <code>b</code>. This is because nested futures use sequential evaluation unless otherwi [...]
+
+<p>To specify a different type of <em>evaluation topology</em>, other than the first level of futures being resolved by multiprocess evaluation and the second level by sequential evaluation, we can provide a list of evaluation strategies to <code>plan()</code>. First, the same evaluation strategies as above can be explicitly specified as:</p>
+
+<pre><code class="r">plan(list(multiprocess, sequential))
+</code></pre>
+
+<p>We would actually get the same behavior if we try with multiple levels of multiprocess evaluations;</p>
+
+<pre><code class="r">> plan(list(multiprocess, multiprocess))
+[...]
+> pid
+[1] 28518
+> a
+[1] 28586
+> b
+ b.pid b1.pid b2.pid
+ 28587 28587 28587
+</code></pre>
+
+<p>The reason for this is, also here, to protect us from launching more processes than what the machine can support. Internally, this is done by setting <code>mc.cores = 1</code> such that functions like <code>parallel::mclapply()</code> will fall back to run sequentially. This is the case for both multisession and multicore evaluation.</p>
+
+<p>Continuing, if we start off by sequential evaluation and then use multiprocess evaluation for any nested futures, we get:</p>
+
+<pre><code class="r">> plan(list(sequential, multiprocess))
+[...]
+Resolving 'a' ...
+Resolving 'b' ...
+> pid
+[1] 28518
+> a
+[1] 28518
+> b
+ b.pid b1.pid b2.pid
+ 28518 28588 28589
+</code></pre>
+
+<p>which clearly show that <code>a</code> and <code>b</code> are resolved in the calling process (pid 28518) whereas the two nested futures (<code>b1</code> and <code>b2</code>) are resolved in two separate R processes (pids 28588 and 28589).</p>
+
+<p>Having said this, it is indeed possible to use nested multiprocess evaluation strategies, if we explicitly specify (read <em>force</em>) the number of cores available at each level. In order to do this we need to “tweak” the default settings, which can be done as follows:</p>
+
+<pre><code class="r">> plan(list(tweak(multiprocess, workers = 2L), tweak(multiprocess,
++ workers = 2L)))
+[...]
+> pid
+[1] 28518
+> a
+[1] 28590
+> b
+ b.pid b1.pid b2.pid
+ 28591 28592 28594
+</code></pre>
+
+<p>First, we see that both <code>a</code> and <code>b</code> are resolved in different processes (pids 28590 and 28591) than the calling process (pid 28518). Second, the two nested futures (<code>b1</code> and <code>b2</code>) are resolved in yet two other R processes (pids 28592 and 28594).</p>
+
+<p>For more details on working with nested futures and different evaluation strategies at each level, see Vignette '<a href="future-3-topologies.html">Futures in R: Future Topologies</a>'.</p>
+
+<h3>Checking A Future without Blocking</h3>
+
+<p>It is possible to check whether a future has been resolved or not without blocking. This can be done using the <code>resolved(f)</code> function, which takes an explicit future <code>f</code> as input. If we work with implicit futures (as in all the examples above), we can use the <code>f <- futureOf(a)</code> function to retrieve the explicit future from an implicit one. For example,</p>
+
+<pre><code class="r">> plan(multiprocess)
+> a %<-% {
++ cat("Resolving 'a' ...")
++ Sys.sleep(2)
++ cat("done\n")
++ Sys.getpid()
++ }
+> cat("Waiting for 'a' to be resolved ...\n")
+Waiting for 'a' to be resolved ...
+> f <- futureOf(a)
+> count <- 1
+> while (!resolved(f)) {
++ cat(count, "\n")
++ Sys.sleep(0.2)
++ count <- count + 1
++ }
+1
+2
+3
+4
+5
+> cat("Waiting for 'a' to be resolved ... DONE\n")
+Waiting for 'a' to be resolved ... DONE
+> a
+[1] 28595
+</code></pre>
+
+<h2>Failed Futures</h2>
+
+<p>Sometimes the future is not what you expected. If an error occurs while evaluating a future, the error is propagated and thrown as an error in the calling environment <em>when the future value is requested</em>. For example, if we use lazy evaluation on a future that generates an error, we might see something like</p>
+
+<pre><code class="r">> plan(sequential)
+> a %<-% {
++ cat("Resolving 'a' ...\n")
++ stop("Whoops!")
++ 42
++ } %lazy% TRUE
+> cat("Everything is still ok although we have created a future that will fail.\n")
+Everything is still ok although we have created a future that will fail.
+> a
+Resolving 'a' ...
+Error in eval(expr, envir, enclos) : Whoops!
+</code></pre>
+
+<p>The error is thrown each time the value is requested, that is, if we try to get the value again will generate the same error:</p>
+
+<pre><code class="r">> a
+Error in eval(expr, envir, enclos) : Whoops!
+In addition: Warning message:
+restarting interrupted promise evaluation
+</code></pre>
+
+<p>To see the list of calls (evaluated expressions) that lead up to the error, we can use the <code>backtrace()</code> function(*) on the future, i.e.</p>
+
+<pre><code class="r">> backtrace(a)
+[[1]]
+eval(quote({
+ cat("Resolving 'a' ...\\n")
+ stop("Whoops!")
+ 42
+}), new.env())
+[[2]]
+eval(quote({
+ cat("Resolving 'a' ...\\n")
+ stop("Whoops!")
+ 42
+}), new.env())
+[[3]]
+stop("Whoops!")
+</code></pre>
+
+<p>(*) The commonly used <code>traceback()</code> does not provide relevant information in the context of futures.</p>
+
+<h2>Globals</h2>
+
+<p>Whenever an R expression is to be evaluated asynchronously (in parallel) or sequentially via lazy evaluation, global (aka “free”) objects have to be identified and passed to the evaluator. They need to be passed exactly as they were at the time the future was created, because, for lazy evaluation, globals may otherwise change between when it is created and when it is resolved. For asynchronous processing, the reason globals need to be identified is so that they can be ex [...]
+
+<p>The future package tries to automate these tasks as far as possible. It does this with help of the <a href="https://cran.r-project.org/package=globals">globals</a> package, which uses static-code inspection to identify global variables. If a global variable is identified, it is captured and made available to the evaluating process.
+Moreover, if a global is defined in a package, then that global is not exported. Instead, it is made sure that the corresponding package is attached when the future is evaluated. This not only better reflects the setup of the main R session, but it also minimizes the need for exporting globals, which saves not only memory but also time and bandwidth, especially when using remote compute nodes.</p>
+
+<p>Finally, it should be clarified that identifying globals from static code inspection alone is a challenging problem. There will always be corner cases where automatic identification of globals fails so that either false globals are identified (less of a concern) or some of the true globals are missing (which will result in a run-time error or possibly the wrong results). Vignette '<a href="future-2-issues.html">Futures in R: Common Issues with Solutions</a>' provides example [...]
+
+<h2>Constraints when using Implicit Futures</h2>
+
+<p>There is one limitation with implicit futures that does not exist for explicit ones. Because an explicit future is just like any other object in R it can be assigned anywhere/to anything. For instance, we can create several of them in a loop and assign them to a list, e.g.</p>
+
+<pre><code class="r">> plan(multiprocess)
+> f <- list()
+> for (ii in 1:3) {
++ f[[ii]] <- future({
++ Sys.getpid()
++ })
++ }
+> v <- lapply(f, FUN = value)
+> str(v)
+List of 3
+ $ : int 28602
+ $ : int 28603
+ $ : int 28605
+</code></pre>
+
+<p>This is <em>not</em> possible to do when using implicit futures. This is because the <code>%<-%</code> assignment operator <em>cannot</em> be used in all cases where the regular <code><-</code> assignment operator can be used. It can only be used to assign future values to <em>environments</em> (including the calling environment) much like how <code>assign(name, value, envir)</code> works. However, we can assign implicit futures to environments using <em>named indices</em>, e.g.</p>
+
+<pre><code class="r">> plan(multiprocess)
+> v <- new.env()
+> for (name in c("a", "b", "c")) {
++ v[[name]] %<-% {
++ Sys.getpid()
++ }
++ }
+> v <- as.list(v)
+> str(v)
+List of 3
+ $ a: int 28606
+ $ b: int 28607
+ $ c: int 28608
+</code></pre>
+
+<p>Here <code>as.list(v)</code> blocks until all futures in the environment <code>v</code> have been resolved. Then their values are collected and returned as a regular list.</p>
+
+<p>If <em>numeric indices</em> are required, then <em>list environments</em> can be used. List environments, which are implemented by the <a href="https://cran.r-project.org/package=listenv">listenv</a> package, are regular environments with customized subsetting operators making it possible to index them much like how lists can be indexed. By using list environments where we otherwise would use lists, we can also assign implicit futures to list-like objects using numeric indices. For [...]
+
+<pre><code class="r">> library("listenv")
+> plan(multiprocess)
+> v <- listenv()
+> for (ii in 1:3) {
++ v[[ii]] %<-% {
++ Sys.getpid()
++ }
++ }
+> v <- as.list(v)
+> str(v)
+List of 3
+ $ : int 28609
+ $ : int 28610
+ $ : int 28611
+</code></pre>
+
+<p>As previously, <code>as.list(v)</code> blocks until all futures are resolved.</p>
+
+<h2>Demos</h2>
+
+<p>To see a live illustration how different types of futures are evaluated, run the Mandelbrot demo of this package. First, try with the sequential evaluation,</p>
+
+<pre><code class="r">library("future")
+plan(sequential)
+demo("mandelbrot", package = "future", ask = FALSE)
+</code></pre>
+
+<p>which resembles how the script would run if futures were not used. Then, try multiprocess evaluation, which calculates the different Mandelbrot planes using parallel R processes running in the background. Try,</p>
+
+<pre><code class="r">plan(multiprocess)
+demo("mandelbrot", package = "future", ask = FALSE)
+</code></pre>
+
+<p>Finally, if you have access to multiple machines you can try to set up a cluster of workers and use them, e.g.</p>
+
+<pre><code class="r">plan(cluster, workers = c("n2", "n5", "n6", "n6", "n9"))
+demo("mandelbrot", package = "future", ask = FALSE)
+</code></pre>
+
+<h2>Contributing</h2>
+
+<p>The goal of this package is to provide a standardized and unified API for using futures in R. What you are seeing right now is an early but sincere attempt to achieve this goal. If you have comments or ideas on how to improve the 'future' package, I would love to hear about them. The preferred way to get in touch is via the <a href="https://github.com/HenrikBengtsson/future/">GitHub repository</a>, where you also find the latest source code. I am also open to contributions [...]
+
+<hr/>
+
+<p>Copyright Henrik Bengtsson, 2015-2017</p>
+
+</body>
+
+</html>
diff --git a/README.md b/inst/doc/future-1-overview.md.rsp
similarity index 63%
rename from README.md
rename to inst/doc/future-1-overview.md.rsp
index db35c56..847e8b9 100644
--- a/README.md
+++ b/inst/doc/future-1-overview.md.rsp
@@ -1,9 +1,28 @@
-# future: A Future API for R
+<%@meta language="R-vignette" content="--------------------------------
+%\VignetteIndexEntry{A Future for R: A Comprehensive Overview}
+%\VignetteAuthor{Henrik Bengtsson}
+%\VignetteKeyword{R}
+%\VignetteKeyword{package}
+%\VignetteKeyword{vignette}
+%\VignetteKeyword{future}
+%\VignetteKeyword{promise}
+%\VignetteKeyword{lazy evaluation}
+%\VignetteKeyword{synchronous}
+%\VignetteKeyword{asynchronous}
+%\VignetteKeyword{parallel}
+%\VignetteKeyword{cluster}
+%\VignetteEngine{R.rsp::rsp}
+%\VignetteTangle{FALSE}
+
+Do not edit the *.md.rsp file. Instead edit the *.md.rsp.rsp (sic!)
+file found under inst/vignettes-static/ of the source package.
+--------------------------------------------------------------------"%>
+# A Future for R: A Comprehensive Overview
## Introduction
The purpose of the [future] package is to provide a very simple and uniform way of evaluating R expressions asynchronously using various resources available to the user.
-In programming, a _future_ is an abstraction for a _value_ that may be available at some point in the future. The state of a future can either be _unresolved_ or _resolved_. As soon as it is resolved, the value is available instantaneously. If the value is queried while the future is still unresolved, the current process is _blocked_ until the future is resolved. It is possible to check whether a future is resolved or not without blocking. Exactly how and when futures are resolved d [...]
+In programming, a _future_ is an abstraction for a _value_ that may be available at some point in the future. The state of a future can either be _unresolved_ or _resolved_. As soon as it is resolved, the value is available instantaneously. If the value is queried while the future is still unresolved, the current process is _blocked_ until the future is resolved. It is possible to check whether a future is resolved or not without blocking. Exactly how and when futures are resolved d [...]
Here is an example illustrating how the basics of futures work. First, consider the following code snippet that uses plain R code:
```r
@@ -89,9 +108,8 @@ The future package implements the following types of futures:
| Name | OSes | Description
|:----------------|:------------|:-----------------------------------------------------
| _synchronous:_ | | _non-parallel:_
-| `eager` | all |
-| `lazy` | all | lazy evaluation - happens only if the value is requested
-| `transparent` | all | as eager w/ early signaling and w/out local (for debugging)
+| `sequential` | all | sequentially and in the current R process
+| `transparent` | all | as sequential w/ early signaling and w/out local (for debugging)
| _asynchronous:_ | | _parallel_:
| `multiprocess` | all | multicore iff supported, otherwise multisession
| `multisession` | all | background R sessions (on current machine)
@@ -99,27 +117,29 @@ The future package implements the following types of futures:
| `cluster` | all | external R sessions on current, local, and/or remote machines
| `remote` | all | Simple access to remote R sessions
-The future package is designed such that support for additional strategies can be implemented as well. For instance, the [future.BatchJobs] package provides futures for all types of _cluster functions_ ("backends") that the [BatchJobs] package supports. Specifically, futures for evaluating R expressions via job schedulers such as Slurm, TORQUE/PBS, Oracle/Sun Grid Engine (SGE) and Load Sharing Facility (LSF) are also available.
+_Note_: Prior to future 1.3.0, `eager` and `lazy` were also options, but has since been deprecated and are both defunct as of 1.6.0. The reason for this is that whether a future should be resolved by lazy evaluation or not has to, in some cases, be in the control of the developer, and if the end user would be able change that, the code may not function as intended.
-By default, future expressions are evaluated eagerly (= instantaneously) and synchronously (in the current R session). This evaluation strategy is referred to as "eager" and we refer to futures using this strategy as "eager futures". In this section, we will go through each of these strategies and discuss what they have in common and how they differ.
+The future package is designed such that support for additional strategies can be implemented as well. For instance, the [future.batchtools] package provides futures for all types of _cluster functions_ ("backends") that the [batchtools] package supports. Specifically, futures for evaluating R expressions via job schedulers such as Slurm, TORQUE/PBS, Oracle/Sun Grid Engine (SGE) and Load Sharing Facility (LSF) are also available. (_Comment_: The [future.BatchJobs] package provides ana [...]
+
+By default, future expressions are evaluated eagerly (= instantaneously) and synchronously (in the current R session). This evaluation strategy is referred to as "sequential". In this section, we will go through each of these strategies and discuss what they have in common and how they differ.
### Consistent Behavior Across Futures
-Before going through each of the different future strategies, it is probably helpful to clarify the objectives the Future API (as defined by the future package). When programming with futures, it should not really matter what future strategy is used for executing code. This is because we cannot really know what computational resources the user has access to so the choice of evaluation strategy should be in the hand of the user and not the developer. In other words, the code should not [...]
+Before going through each of the different future strategies, it is probably helpful to clarify the objectives the Future API (as defined by the future package). When programming with futures, it should not really matter what future strategy is used for executing code. This is because we cannot really know what computational resources the user has access to so the choice of evaluation strategy should be in the hands of the user and not the developer. In other words, the code should no [...]
-One of the designs of the Future API was to encapsulate any differences such that all types of futures will appear to work the same. This despite expressions may be evaluated locally in the current R session or across the world in remote R sessions. Another obvious advantage of having a consistent API and behavior among different types of futures is that it helps while prototyping. Typically one would use eager evaluation while building up a script and, later, when the script is fully [...]
+One of the designs of the Future API was to encapsulate any differences such that all types of futures will appear to work the same. This despite expressions may be evaluated locally in the current R session or across the world in remote R sessions. Another obvious advantage of having a consistent API and behavior among different types of futures is that it helps while prototyping. Typically one would use sequential evaluation while building up a script and, later, when the script is [...]
Because of this, the defaults of the different strategies are such that the results and side effects of evaluating a future expression are as similar as possible. More specifically, the following is true for all futures:
* All _evaluation is done in a local environment_ (i.e. `local({ expr })`) so that assignments do not affect the calling environment. This is natural when evaluating in an external R process, but is also enforced when evaluating in the current R session.
-* When a future is constructed, _global variables are identified_. For asynchronous evaluation, globals are exported to the R process/session that will be evaluating the future expression. For lazy futures, globals are "frozen" (cloned to a local environment of the future). Also, in order to protect against exporting too large objects by mistake, there is a built-in assertion that the total size of all globals is less than a given threshold (controllable via an option, cf. `help("futu [...]
+* When a future is constructed, _global variables are identified_. For asynchronous evaluation, globals are exported to the R process/session that will be evaluating the future expression. For sequential futures with lazy evaluation (`lazy = TRUE`), globals are "frozen" (cloned to a local environment of the future). Also, in order to protect against exporting too large objects by mistake, there is a built-in assertion that the total size of all globals is less than a given threshold ( [...]
* Future _expressions are only evaluated once_. As soon as the value (or an error) has been collected it will be available for all succeeding requests.
Here is an example illustrating that all assignments are done to a local environment:
```r
-> plan(eager)
+> plan(sequential)
> a <- 1
> x %<-% {
+ a <- 2
@@ -137,16 +157,16 @@ Now we are ready to explore the different future strategies.
### Synchronous Futures
-Synchronous futures are resolved one after another and most commonly by the R process that creates them. When a synchronous future is being resolved it blocks the main process until resolved. There are two main types of synchronous futures in the future package, _eager_ and _lazy_ futures, which are described next.
+Synchronous futures are resolved one after another and most commonly by the R process that creates them. When a synchronous future is being resolved it blocks the main process until resolved. There are two types of synchronous futures in the future package, _sequential_ and _transparent_. (In future 1.2.0 and before, there was also _lazy_ futures, which has now been deprecated in favor of `f <- future(..., lazy = TRUE)` and `v %<-% { ... } %lazy% TRUE`.)
-#### Eager Futures
-Eager futures are the default unless otherwise specified. They were designed to behave as similar as possible to regular R evaluation while still fulfilling the Future API and its behaviors. Here is an example illustrating their properties:
+#### Sequential Futures
+Sequential futures are the default unless otherwise specified. They were designed to behave as similar as possible to regular R evaluation while still fulfilling the Future API and its behaviors. Here is an example illustrating their properties:
```r
-> plan(eager)
+> plan(sequential)
> pid <- Sys.getpid()
> pid
-[1] 10708
+[1] 28518
> a %<-% {
+ pid <- Sys.getpid()
+ cat("Resolving 'a' ...\n")
@@ -165,58 +185,19 @@ Resolving 'b' ...
+ }
Resolving 'c' ...
> b
-[1] 10708
+[1] 28518
> c
[1] 6.28
> a
[1] 3.14
> pid
-[1] 10708
+[1] 28518
```
-Since eager evaluation is taking place, each of the three futures is resolved instantaneously in the moment it is created. Note also how `pid` in the calling environment, which was assigned the process ID of the current process, is neither overwritten nor removed. This is because futures are evaluated in a local environment. Since synchronous (uni-)processing is used, future `b` is resolved by the main R process (still in a local environment), which is why the value of `b` and `pid` a [...]
-
-
-#### Lazy Futures
-A lazy future evaluates its expression only if its value is queried. Evaluation can also be triggered when the future is checked for being resolved or not. Here is the above example when using lazy evaluation:
-```r
-> plan(lazy)
-> pid <- Sys.getpid()
-> pid
-[1] 10708
-> a %<-% {
-+ pid <- Sys.getpid()
-+ cat("Resolving 'a' ...\n")
-+ 3.14
-+ }
-> b %<-% {
-+ rm(pid)
-+ cat("Resolving 'b' ...\n")
-+ Sys.getpid()
-+ }
-> c %<-% {
-+ cat("Resolving 'c' ...\n")
-+ 2 * a
-+ }
-Resolving 'a' ...
-> b
-Resolving 'b' ...
-[1] 10708
-> c
-Resolving 'c' ...
-[1] 6.28
-> a
-[1] 3.14
-> pid
-[1] 10708
-```
-As previously, variable `pid` is unaffected because all evaluation is done in a local environment. More interestingly, future `a` is no longer evaluated in the moment it is created, but instead when it is needed the first time, which happens when future `c` is created. This is because `a` is identified as a global variable that needs to be captured ("frozen" to `a == 3.14`) in order to set up future `c`. Later when `c` (the value of future `c`) is queried, `a` has already been resolve [...]
-
-_Comment_: Lazy evaluation is already used by R itself. Arguments are passed to functions using lazy evaluation. It is also possible to assign variables using lazy evaluation using `delayedAssign()`, but contrary to lazy futures this function does not freeze globals. For more information, see `help("delayedAssign", package="base")`.
-
-
-
+Since eager sequential evaluation is taking place, each of the three futures is resolved instantaneously in the moment it is created. Note also how `pid` in the calling environment, which was assigned the process ID of the current process, is neither overwritten nor removed. This is because futures are evaluated in a local environment. Since synchronous (uni-)processing is used, future `b` is resolved by the main R process (still in a local environment), which is why the value of `b` [...]
+#### Transparent Futures
+For troubleshooting, _transparent_ futures can be used by specifying `plan(transparent)`. A transparent future is technically a sequential future with instant signaling of conditions (including errors and warnings) and where evaluation, and therefore also assignments, take place in the calling environment. Transparent futures are particularly useful for troubleshooting errors that are otherwise hard to narrow down.
### Asynchronous Futures
@@ -229,7 +210,7 @@ We start with multisession futures because they are supported by all operating s
> plan(multisession)
> pid <- Sys.getpid()
> pid
-[1] 10708
+[1] 28518
> a %<-% {
+ pid <- Sys.getpid()
+ cat("Resolving 'a' ...\n")
@@ -245,13 +226,13 @@ We start with multisession futures because they are supported by all operating s
+ 2 * a
+ }
> b
-[1] 10729
+[1] 28539
> c
[1] 6.28
> a
[1] 3.14
> pid
-[1] 10708
+[1] 28518
```
The first thing we observe is that the values of `a`, `c` and `pid` are the same as previously. However, we notice that `b` is different from before. This is because future `b` is evaluated in a different R process and therefore it returns a different process ID. Another difference is that the messages, generated by `cat()`, are no longer displayed. This is because they are outputted to the background sessions and not the calling session.
@@ -259,10 +240,10 @@ The first thing we observe is that the values of `a`, `c` and `pid` are the same
When multisession evaluation is used, the package launches a set of R sessions in the background that will serve multisession futures by evaluating their expressions as they are created. If all background sessions are busy serving other futures, the creation of the next multisession future is _blocked_ until a background session becomes available again. The total number of background processes launched is decided by the value of `availableCores()`, e.g.
```r
> availableCores()
-mc.cores+1
- 3
+mc.cores
+ 2
```
-This particular result tells us that the `mc.cores` option was set such that we are allowed to use in total 3 processes including the main process. In other words, with these settings, there will be 2 background processes serving the multisession futures. The `availableCores()` is also agile to different options and system environment variables. For instance, if compute cluster schedulers are used (e.g. TORQUE/PBS and Slurm), they set specific environment variable specifying the numbe [...]
+This particular result tells us that the `mc.cores` option was set such that we are allowed to use in total 2 processes including the main process. In other words, with these settings, there will be 2 background processes serving the multisession futures. The `availableCores()` is also agile to different options and system environment variables. For instance, if compute cluster schedulers are used (e.g. TORQUE/PBS and Slurm), they set specific environment variable specifying the numbe [...]
#### Multicore Futures
@@ -291,7 +272,7 @@ Cluster futures evaluate expressions on an ad-hoc cluster (as implemented by the
> plan(cluster, workers = c("n1", "n2", "n3"))
> pid <- Sys.getpid()
> pid
-[1] 10708
+[1] 28518
> a %<-% {
+ pid <- Sys.getpid()
+ cat("Resolving 'a' ...\n")
@@ -307,58 +288,29 @@ Cluster futures evaluate expressions on an ad-hoc cluster (as implemented by the
+ 2 * a
+ }
> b
-[1] 10751
+[1] 28561
> c
[1] 6.28
> a
[1] 3.14
> pid
-[1] 10708
+[1] 28518
```
-Just as for the other asynchronous evaluation strategies, the output from `cat()` is not displayed on the current/calling machine.
+Just as for most other asynchronous evaluation strategies, the output from `cat()` is not displayed on the current/calling machine.
Any types of clusters that `parallel::makeCluster()` creates can be used for cluster futures. For instance, the above cluster can be explicitly set up as:
```r
cl <- parallel::makeCluster(c("n1", "n2", "n3"))
-plan(cluster, workers=cl)
+plan(cluster, workers = cl)
```
-Also, it is considered good style to shut down the cluster when it is no longer needed, that is, calling `parallel::stopCluster(cl)`. However, it will shut itself down if the main process is terminated, which will happen in the first example where the cluster in created internally. For more information on how to set up and manage such clusters, see `help("makeCluster", package="parallel")`.
+Also, it is considered good style to shut down cluster `cl` when it is no longer needed, that is, calling `parallel::stopCluster(cl)`. However, it will shut itself down if the main process is terminated. For more information on how to set up and manage such clusters, see `help("makeCluster", package = "parallel")`.
+Clusters created implicitly using `plan(cluster, workers = hosts)` where `hosts` is a character vector will also be shut down when the main R session terminates, or when the future strategy is changed, e.g. by calling `plan(sequential)`.
Note that with automatic authentication setup (e.g. SSH key pairs), there is nothing preventing us from using the same approach for using a cluster of remote machines.
-### Different Strategies for Different Futures
-Sometimes one may want to use an alternative evaluation strategy for a specific future. Although one can use `old <- plan(new)` and afterward `plan(old)` to temporarily switch strategies, a simpler approach is to use the `%plan%` operator, e.g.
-```r
-> plan(eager)
-> pid <- Sys.getpid()
-> pid
-[1] 10708
-> a %<-% {
-+ Sys.getpid()
-+ }
-> b %<-% {
-+ Sys.getpid()
-+ } %plan% multiprocess
-> c %<-% {
-+ Sys.getpid()
-+ } %plan% multiprocess
-> a
-[1] 10708
-> b
-[1] 10769
-> c
-[1] 10770
-```
-As seen by the different process IDs, future `a` is evaluated eagerly using the same process as the calling environment whereas the other two are evaluated using multiprocess futures.
-
-
-
-
-However, using different plans to individual futures this way has the drawback of hard coding the evaluation strategy. Doing so may prevent some users from using your script or your package, because they do not have the sufficient resources. It may also prevent users with a lot of resources from utilizing those because you assumed a less-powerful set of hardware. Because of this, we recommend against the use of `%plan%` other than for interactive prototyping.
-
### Nested Futures and Evaluation Topologies
This far we have discussed what can be referred to as "flat topology" of futures, that is, all futures are created in and assigned to the same environment. However, there is nothing stopping us from using a "nested topology" of futures, where one set of futures may, in turn, create another set of futures internally and so on.
@@ -384,71 +336,69 @@ For instance, here is an example of two "top" futures (`a` and `b`) that uses mu
+ c(b.pid = Sys.getpid(), b1.pid = b1, b2.pid = b2)
+ }
> pid
-[1] 10708
+[1] 28518
> a
-[1] 10771
+[1] 28584
> b
b.pid b1.pid b2.pid
- 10772 10772 10772
+ 28585 28585 28585
```
-By inspection the process IDs, we see that there are in total three different processes involved for resolving the futures. There is the main R process (pid 10708), and there are the two processes used by `a` (pid 10771) and `b` (pid 10772). However, the two futures (`b1` and `b2`) that is nested by `b` are evaluated by the same R process as `b`. This is because nested futures use eager evaluation unless otherwise specified. There are a few reasons for this, but the main reason is th [...]
+By inspection the process IDs, we see that there are in total three different processes involved for resolving the futures. There is the main R process (pid 28518), and there are the two processes used by `a` (pid 28584) and `b` (pid 28585). However, the two futures (`b1` and `b2`) that is nested by `b` are evaluated by the same R process as `b`. This is because nested futures use sequential evaluation unless otherwise specified. There are a few reasons for this, but the main reason [...]
-To specify a different type of _evaluation topology_, other than the first level of futures being resolved by multiprocess evaluation and the second level by eager evaluation, we can provide a list of evaluation strategies to `plan()`. First, the same evaluation strategies as above can be explicitly specified as:
+To specify a different type of _evaluation topology_, other than the first level of futures being resolved by multiprocess evaluation and the second level by sequential evaluation, we can provide a list of evaluation strategies to `plan()`. First, the same evaluation strategies as above can be explicitly specified as:
```r
-plan(list(multiprocess, eager))
+plan(list(multiprocess, sequential))
```
We would actually get the same behavior if we try with multiple levels of multiprocess evaluations;
```r
> plan(list(multiprocess, multiprocess))
[...]
> pid
-[1] 10708
+[1] 28518
> a
-[1] 10773
+[1] 28586
> b
b.pid b1.pid b2.pid
- 10774 10774 10774
+ 28587 28587 28587
```
-The reason for this is, also here, to protect us from launching more processes than what the machine can support. Internally, this is done by setting `mc.cores` to zero ([sic!](https://github.com/HenrikBengtsson/Wishlist-for-R/issues/7)) such that no _additional_ parallel processes can be launched. This is the case for both multisession and multicore evaluation.
+The reason for this is, also here, to protect us from launching more processes than what the machine can support. Internally, this is done by setting `mc.cores = 1` such that functions like `parallel::mclapply()` will fall back to run sequentially. This is the case for both multisession and multicore evaluation.
-Continuing, if we start off by eager evaluation and then use multiprocess evaluation for any nested futures, we get:
+Continuing, if we start off by sequential evaluation and then use multiprocess evaluation for any nested futures, we get:
```r
-> plan(list(eager, multiprocess))
+> plan(list(sequential, multiprocess))
[...]
Resolving 'a' ...
Resolving 'b' ...
> pid
-[1] 10708
+[1] 28518
> a
-[1] 10708
+[1] 28518
> b
b.pid b1.pid b2.pid
- 10708 10775 10776
+ 28518 28588 28589
```
-which clearly show that `a` and `b` are resolved in the calling process (pid 10708) whereas the two nested futures (`b1` and `b2`) are resolved in two separate R processes (pids 10775 and 10776).
+which clearly show that `a` and `b` are resolved in the calling process (pid 28518) whereas the two nested futures (`b1` and `b2`) are resolved in two separate R processes (pids 28588 and 28589).
Having said this, it is indeed possible to use nested multiprocess evaluation strategies, if we explicitly specify (read _force_) the number of cores available at each level. In order to do this we need to "tweak" the default settings, which can be done as follows:
```r
-> plan(list(tweak(multiprocess, workers = 3L), tweak(multiprocess,
-+ workers = 3L)))
+> plan(list(tweak(multiprocess, workers = 2L), tweak(multiprocess,
++ workers = 2L)))
[...]
> pid
-[1] 10708
+[1] 28518
> a
-[1] 10777
+[1] 28590
> b
b.pid b1.pid b2.pid
- 10778 10779 10840
+ 28591 28592 28594
```
-First, we see that both `a` and `b` are resolved in different processes (pids 10777 and 10778) than the calling process (pid 10708). Second, the two nested futures (`b1` and `b2`) are resolved in yet two other R processes (pids 10779 and 10840).
-
+First, we see that both `a` and `b` are resolved in different processes (pids 28590 and 28591) than the calling process (pid 28518). Second, the two nested futures (`b1` and `b2`) are resolved in yet two other R processes (pids 28592 and 28594).
-To clarify, when we set up the two levels of multiprocess evaluation, we specified that in total 3 processes may be used at each level. We choose three parallel processes, not just two, because one is always consumed by the calling process leaving two to be used for the asynchronous futures. This is why we see that `pid`, `a` and `b` are all resolved by the same process. If we had allowed only two cores at the top level, `a` and `b` would have been resolved by the same background proc [...]
For more details on working with nested futures and different evaluation strategies at each level, see Vignette '[Futures in R: Future Topologies]'.
@@ -480,19 +430,19 @@ Waiting for 'a' to be resolved ...
> cat("Waiting for 'a' to be resolved ... DONE\n")
Waiting for 'a' to be resolved ... DONE
> a
-[1] 10841
+[1] 28595
```
## Failed Futures
-Sometimes the future is not what you expected. If an error occurs while evaluating a future, the error is propagated and thrown as an error in the calling environment _when the future value is requested_. For example,
+Sometimes the future is not what you expected. If an error occurs while evaluating a future, the error is propagated and thrown as an error in the calling environment _when the future value is requested_. For example, if we use lazy evaluation on a future that generates an error, we might see something like
```r
-> plan(lazy)
+> plan(sequential)
> a %<-% {
+ cat("Resolving 'a' ...\n")
+ stop("Whoops!")
+ 42
-+ }
++ } %lazy% TRUE
> cat("Everything is still ok although we have created a future that will fail.\n")
Everything is still ok although we have created a future that will fail.
> a
@@ -511,12 +461,16 @@ To see the list of calls (evaluated expressions) that lead up to the error, we c
> backtrace(a)
[[1]]
eval(quote({
- cat("Resolving 'a' ...\n")
+ cat("Resolving 'a' ...\\n")
stop("Whoops!")
42
}), new.env())
[[2]]
-eval(expr, envir, enclos)
+eval(quote({
+ cat("Resolving 'a' ...\\n")
+ stop("Whoops!")
+ 42
+}), new.env())
[[3]]
stop("Whoops!")
```
@@ -524,13 +478,12 @@ stop("Whoops!")
## Globals
-Whenever an R expression is to be evaluated asynchronously (in parallel) or via lazy evaluation, global objects have to be identified and passed to the evaluator. They need to be passed exactly as they were at the time the future was created, because, for a lazy future, globals may otherwise change between when it is created and when it is resolved. For asynchronous processing, the reason globals need to be identified is so that they can be exported to the process that evaluates the future.
+Whenever an R expression is to be evaluated asynchronously (in parallel) or sequentially via lazy evaluation, global (aka "free") objects have to be identified and passed to the evaluator. They need to be passed exactly as they were at the time the future was created, because, for lazy evaluation, globals may otherwise change between when it is created and when it is resolved. For asynchronous processing, the reason globals need to be identified is so that they can be exported to the p [...]
-The future package tries to automate these tasks as far as possible. It does this with help of the [globals] package. If a global variable is identified, it is captured and made available to the evaluating process.
+The future package tries to automate these tasks as far as possible. It does this with help of the [globals] package, which uses static-code inspection to identify global variables. If a global variable is identified, it is captured and made available to the evaluating process.
Moreover, if a global is defined in a package, then that global is not exported. Instead, it is made sure that the corresponding package is attached when the future is evaluated. This not only better reflects the setup of the main R session, but it also minimizes the need for exporting globals, which saves not only memory but also time and bandwidth, especially when using remote compute nodes.
-Finally, it should be clarified that identifying globals from static code inspection alone is a challenging problem. There will always be corner cases where automatic identification of globals fails so that either false globals are identified (less of a concern) or some of the true globals are missing (which will result in a runtime error or possibly the wrong results). Vignette '[Futures in R: Common Issues with Solutions]' provides examples of common cases and explains how to avoid t [...]
-
+Finally, it should be clarified that identifying globals from static code inspection alone is a challenging problem. There will always be corner cases where automatic identification of globals fails so that either false globals are identified (less of a concern) or some of the true globals are missing (which will result in a run-time error or possibly the wrong results). Vignette '[Futures in R: Common Issues with Solutions]' provides examples of common cases and explains how to avoid [...]
## Constraints when using Implicit Futures
@@ -547,9 +500,9 @@ There is one limitation with implicit futures that does not exist for explicit o
> v <- lapply(f, FUN = value)
> str(v)
List of 3
- $ : int 10921
- $ : int 10922
- $ : int 10924
+ $ : int 28602
+ $ : int 28603
+ $ : int 28605
```
This is _not_ possible to do when using implicit futures. This is because the `%<-%` assignment operator _cannot_ be used in all cases where the regular `<-` assignment operator can be used. It can only be used to assign future values to _environments_ (including the calling environment) much like how `assign(name, value, envir)` works. However, we can assign implicit futures to environments using _named indices_, e.g.
```r
@@ -563,9 +516,9 @@ This is _not_ possible to do when using implicit futures. This is because the `
> v <- as.list(v)
> str(v)
List of 3
- $ a: int 10925
- $ b: int 10928
- $ c: int 10931
+ $ a: int 28606
+ $ b: int 28607
+ $ c: int 28608
```
Here `as.list(v)` blocks until all futures in the environment `v` have been resolved. Then their values are collected and returned as a regular list.
@@ -582,38 +535,30 @@ If _numeric indices_ are required, then _list environments_ can be used. List e
> v <- as.list(v)
> str(v)
List of 3
- $ : int 10932
- $ : int 10933
- $ : int 10934
+ $ : int 28609
+ $ : int 28610
+ $ : int 28611
```
As previously, `as.list(v)` blocks until all futures are resolved.
## Demos
-To see a live illustration how different types of futures are evaluated, run the Mandelbrot demo of this package. First, try with the eager evaluation,
+To see a live illustration how different types of futures are evaluated, run the Mandelbrot demo of this package. First, try with the sequential evaluation,
```r
library("future")
-plan(eager)
-demo("mandelbrot", package="future", ask=FALSE)
+plan(sequential)
+demo("mandelbrot", package = "future", ask = FALSE)
```
-which closely imitates how the script would run if futures were not used. Then try the same using lazy evaluation,
-```r
-plan(lazy)
-demo("mandelbrot", package="future", ask=FALSE)
-```
-and see if you can notice the difference in how and when statements are evaluated.
-You may also try multiprocess evaluation, which calculates the different Mandelbrot planes using parallel R processes running in the background. Try,
+which resembles how the script would run if futures were not used. Then, try multiprocess evaluation, which calculates the different Mandelbrot planes using parallel R processes running in the background. Try,
```r
plan(multiprocess)
-demo("mandelbrot", package="future", ask=FALSE)
+demo("mandelbrot", package = "future", ask = FALSE)
```
-This will use multicore processing if you are on a system where R supports process forking, otherwise (such as on Windows) it will use multisession processing.
-
Finally, if you have access to multiple machines you can try to set up a cluster of workers and use them, e.g.
```r
-plan(cluster, workers=c("n2", "n5", "n6", "n6", "n9"))
-demo("mandelbrot", package="future", ask=FALSE)
+plan(cluster, workers = c("n2", "n5", "n6", "n6", "n9"))
+demo("mandelbrot", package = "future", ask = FALSE)
```
@@ -623,33 +568,13 @@ The goal of this package is to provide a standardized and unified API for using
[BatchJobs]: https://cran.r-project.org/package=BatchJobs
+[batchtools]: https://cran.r-project.org/package=batchtools
[future]: https://cran.r-project.org/package=future
[future.BatchJobs]: https://cran.r-project.org/package=future.BatchJobs
+[future.batchtools]: https://cran.r-project.org/package=future.batchtools
[globals]: https://cran.r-project.org/package=globals
[listenv]: https://cran.r-project.org/package=listenv
[Futures in R: Common Issues with Solutions]: future-2-issues.html
[Futures in R: Future Topologies]: future-3-topologies.html
-
-## Installation
-R package future is available on [CRAN](https://cran.r-project.org/package=future) and can be installed in R as:
-```r
-install.packages('future')
-```
-
-### Pre-release version
-
-To install the pre-release version that is available in branch `develop`, use:
-```r
-source('http://callr.org/install#HenrikBengtsson/future@develop')
-```
-This will install the package from source.
-
-
-
-## Software status
-
-| Resource: | CRAN | Travis CI | Appveyor |
-| ------------- | ------------------- | --------------- | ---------------- |
-| _Platforms:_ | _Multiple_ | _Linux & macOS_ | _Windows_ |
-| R CMD check | <a href="https://cran.r-project.org/web/checks/check_results_future.html"><img border="0" src="http://www.r-pkg.org/badges/version/future" alt="CRAN version"></a> | <a href="https://travis-ci.org/HenrikBengtsson/future"><img src="https://travis-ci.org/HenrikBengtsson/future.svg" alt="Build status"></a> | <a href="https://ci.appveyor.com/project/HenrikBengtsson/future"><img src="https://ci.appveyor.com/api/projects/status/github/HenrikBengtsson/future?svg=true" alt="Bu [...]
-| Test coverage | | <a href="https://codecov.io/gh/HenrikBengtsson/future"><img src="https://codecov.io/gh/HenrikBengtsson/future/branch/develop/graph/badge.svg" alt="Coverage Status"/></a> | |
+---
+Copyright Henrik Bengtsson, 2015-2017
diff --git a/inst/doc/future-2-issues.html b/inst/doc/future-2-issues.html
new file mode 100644
index 0000000..f16660c
--- /dev/null
+++ b/inst/doc/future-2-issues.html
@@ -0,0 +1,454 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+
+<title>A Future for R: Common Issues with Solutions</title>
+
+<script type="text/javascript">
+window.onload = function() {
+ var imgs = document.getElementsByTagName('img'), i, img;
+ for (i = 0; i < imgs.length; i++) {
+ img = imgs[i];
+ // center an image if it is the only element of its parent
+ if (img.parentElement.childElementCount === 1)
+ img.parentElement.style.textAlign = 'center';
+ }
+};
+</script>
+
+<!-- Styles for R syntax highlighter -->
+<style type="text/css">
+ pre .operator,
+ pre .paren {
+ color: rgb(104, 118, 135)
+ }
+
+ pre .literal {
+ color: #990073
+ }
+
+ pre .number {
+ color: #099;
+ }
+
+ pre .comment {
+ color: #998;
+ font-style: italic
+ }
+
+ pre .keyword {
+ color: #900;
+ font-weight: bold
+ }
+
+ pre .identifier {
+ color: rgb(0, 0, 0);
+ }
+
+ pre .string {
+ color: #d14;
+ }
+</style>
+
+<!-- R syntax highlighter -->
+<script type="text/javascript">
+var hljs=new function(){function m(p){return p.replace(/&/gm,"&").replace(/</gm,"<")}function f(r,q,p){return RegExp(q,"m"+(r.cI?"i":"")+(p?"g":""))}function b(r){for(var p=0;p<r.childNodes.length;p++){var q=r.childNodes[p];if(q.nodeName=="CODE"){return q}if(!(q.nodeType==3&&q.nodeValue.match(/\s+/))){break}}}function h(t,s){var p="";for(var r=0;r<t.childNodes.length;r++){if(t.childNodes[r].nodeType==3){var q=t.childNodes[r].nodeValue;if(s){q=q.replace(/\n/g,"")}p+=q}else{if(t.chi [...]
+hljs.initHighlightingOnLoad();
+</script>
+
+
+
+<style type="text/css">
+body, td {
+ font-family: sans-serif;
+ background-color: white;
+ font-size: 13px;
+}
+
+body {
+ max-width: 800px;
+ margin: auto;
+ padding: 1em;
+ line-height: 20px;
+}
+
+tt, code, pre {
+ font-family: 'DejaVu Sans Mono', 'Droid Sans Mono', 'Lucida Console', Consolas, Monaco, monospace;
+}
+
+h1 {
+ font-size:2.2em;
+}
+
+h2 {
+ font-size:1.8em;
+}
+
+h3 {
+ font-size:1.4em;
+}
+
+h4 {
+ font-size:1.0em;
+}
+
+h5 {
+ font-size:0.9em;
+}
+
+h6 {
+ font-size:0.8em;
+}
+
+a:visited {
+ color: rgb(50%, 0%, 50%);
+}
+
+pre, img {
+ max-width: 100%;
+}
+pre {
+ overflow-x: auto;
+}
+pre code {
+ display: block; padding: 0.5em;
+}
+
+code {
+ font-size: 92%;
+ border: 1px solid #ccc;
+}
+
+code[class] {
+ background-color: #F8F8F8;
+}
+
+table, td, th {
+ border: none;
+}
+
+blockquote {
+ color:#666666;
+ margin:0;
+ padding-left: 1em;
+ border-left: 0.5em #EEE solid;
+}
+
+hr {
+ height: 0px;
+ border-bottom: none;
+ border-top-width: thin;
+ border-top-style: dotted;
+ border-top-color: #999999;
+}
+
+ at media print {
+ * {
+ background: transparent !important;
+ color: black !important;
+ filter:none !important;
+ -ms-filter: none !important;
+ }
+
+ body {
+ font-size:12pt;
+ max-width:100%;
+ }
+
+ a, a:visited {
+ text-decoration: underline;
+ }
+
+ hr {
+ visibility: hidden;
+ page-break-before: always;
+ }
+
+ pre, blockquote {
+ padding-right: 1em;
+ page-break-inside: avoid;
+ }
+
+ tr, img {
+ page-break-inside: avoid;
+ }
+
+ img {
+ max-width: 100% !important;
+ }
+
+ @page :left {
+ margin: 15mm 20mm 15mm 10mm;
+ }
+
+ @page :right {
+ margin: 15mm 10mm 15mm 20mm;
+ }
+
+ p, h2, h3 {
+ orphans: 3; widows: 3;
+ }
+
+ h2, h3 {
+ page-break-after: avoid;
+ }
+}
+</style>
+
+<meta name="keywords" content="R, package, vignette, future, promise">
+<meta name="author" content="Henrik Bengtsson">
+
+</head>
+
+<body>
+<h1>A Future for R: Common Issues with Solutions</h1>
+
+<p>In the ideal case, all it takes to start using futures in R is to replace select standard assignments (<code><-</code>) in your R code with future assignments (<code>%<-%</code>) and make sure the right-hand side (RHS) expressions are within curly brackets (<code>{ ... }</code>). Also, if you assign these to lists (e.g. in a for loop), you need to use a list environment (<code>listenv</code>) instead of a plain list.</p>
+
+<p>However, as show below, there are few cases where you might run into some hurdles, but, as also shown, they are often easy to overcome. These are often related to global variables. </p>
+
+<p><em>If you identify other cases, please consider <a href="https://github.com/HenrikBengtsson/future/issues/">reporting</a> them so they can be documented here and possibly even be fixed.</em></p>
+
+<h2>Issues with globals</h2>
+
+<h3>Missing or incorrect globals (false negatives)</h3>
+
+<h4>Missing globals due to self assignment</h4>
+
+<p>If a global variable is (only) part of a “self” assignment, such as <code>x <- x + 1</code>, then the global is not identified and therefore neither exported / frozen. This explains the error in the following example:</p>
+
+<pre><code class="r">> library("future")
+> plan(sequential)
+> x <- 0
+> y %<-% { x <- x + 1 } %lazy% TRUE
+> rm(x)
+> y
+Error: object 'x' not found
+</code></pre>
+
+<p>The workaround is to guide R to find the missing global by explicitly listing it, e.g.</p>
+
+<pre><code class="r">> library("future")
+> plan(sequential)
+> x <- 0
+> y %<-% { x; x <- x + 1 } %lazy% TRUE
+> rm(x)
+> y
+[1] 1
+</code></pre>
+
+<p><em>Comment:</em> The goal is to in a future version of the package detect globals also in such self-assignment constructs.</p>
+
+<h4>do.call() - function not found</h4>
+
+<p>When calling a function using <code>do.call()</code> make sure to specify the function as the object itself and not by name. This will help identify the function as a global object in the future expression. For instance, use</p>
+
+<pre><code class="r">do.call(file_ext, list("foo.txt"))
+</code></pre>
+
+<p>instead of</p>
+
+<pre><code class="r">do.call("file_ext", list("foo.txt"))
+</code></pre>
+
+<p>so that <code>file_ext()</code> is properly located and exported. Although you may not notice a difference when evaluating futures in the same R session, it may become a problem if you use a character string instead of a function object when futures are evaluated in external R sessions, such as on a cluster.
+It may also become a problem with futures evaluated with lazy evaluation if the intended function is redefined after the future is resolved. For example,</p>
+
+<pre><code class="r">> library("future")
+> library("listenv")
+> library("tools")
+> plan(sequential)
+> pathnames <- c("foo.txt", "bar.png", "yoo.md")
+> res <- listenv()
+> for (ii in seq_along(pathnames)) {
++ res[[ii]] %<-% do.call("file_ext", list(pathnames[ii])) %lazy% TRUE
++ }
+> file_ext <- function(...) "haha!"
+> unlist(res)
+[1] "haha!" "haha!" "haha!"
+</code></pre>
+
+<h2>Trying to pass an unresolved future to another future</h2>
+
+<p>It is not possible for a future to resolve another one unless it was created by the future trying to resolve it. For instance, the following gives an error:</p>
+
+<pre><code class="r">> library("future")
+> plan(multiprocess)
+> f1 <- future({ Sys.getpid() })
+> f2 <- future({ value(f1) })
+> v1 <- value(f1)
+[1] 7464
+> v2 <- value(f2)
+Error: Invalid usage of futures: A future whose value has not yet been collected
+ can only be queried by the R process (cdd013cb-e045-f4a5-3977-9f064c31f188; pid
+ 1276 on MyMachine) that created it, not by any other R processes (5579f789-e7b6
+ -bace-c50d-6c7a23ddb5a3; pid 2352 on MyMachine): {; Sys.getpid(); }
+</code></pre>
+
+<p>This is because the main R process creates two futures, but then the second future tries to retrieve the value of the first one. This is an invalid request because the second future has no channel to communicate with the first future; it is only the process that created a future who can communicate with it(*).</p>
+
+<p>Note that it is only <em>unresolved</em> futures that cannot be queried this way. Thus, the solution to the above problem is to make sure all futures are resolved before they are passed to other futures, e.g.</p>
+
+<pre><code class="r">> f1 <- future({ Sys.getpid() })
+> v1 <- value(f1)
+> v1
+[1] 7464
+> f2 <- future({ value(f1) })
+> v2 <- value(f2)
+> v2
+[1] 7464
+</code></pre>
+
+<p>This works because the value has already been collected and stored inside future <code>f1</code> before future <code>f2</code> is created. Since the value is already stored internally, <code>value(f1)</code> is readily available everywhere. Of course, instead of using <code>value(f1)</code> for the second future, it would be more readable and cleaner to simply use <code>v1</code>.</p>
+
+<p>The above is typically not a problem when future assignments are used. For example:</p>
+
+<pre><code class="r">> v1 %<-% { Sys.getpid() })
+> v2 %<-% { v1 }
+> v1
+[1] 2352
+> v2
+[1] 2352
+</code></pre>
+
+<p>The reason that this approach works out of the box is because in the second future assignment <code>v1</code> is identified as a global variable, which is retrieved. Up to this point, <code>v1</code> is a promise (“delayed assignment” in R), but when it is retrieved as a global variable its value is resolved and <code>v1</code> becomes a regular variable.</p>
+
+<p>However, there are cases where future assignments can be passed via global variables without being resolved. This can happen if the future assignment is done to an element of an environment (including list environments). For instance,</p>
+
+<pre><code class="r">> library("listenv")
+> x <- listenv()
+> x$a %<-% { Sys.getpid() }
+> x$b %<-% { x$a }
+> x$a
+[1] 2352
+> x$b
+Error: Invalid usage of futures: A future whose value has not yet been collected
+ can only be queried by the R process (cdd013cb-e045-f4a5-3977-9f064c31f188; pid
+ 1276 on localhost) that created it, not by any other R processes (2ce86ccd-5854
+ -7a05-1373-e1b20022e4d8; pid 7464 on localhost): {; Sys.getpid(); }
+</code></pre>
+
+<p>As previously, this can be avoided by making sure <code>x$a</code> is resolved first, which can be one in various ways, e.g. <code>dummy <- x$a</code>, <code>resolve(x$a)</code> and <code>force(x$a)</code>.</p>
+
+<p><em>Footnote</em>: (*) Although sequential futures could be passed on to other futures part of the same R process and be resolved there because they share the same evaluation process, by definition of the Future API it is invalid to do so regardless of future type. This conservative approach is taken in order to make future expressions behave consistently regardless of the type of future used.</p>
+
+<h2>Miscellaneous</h2>
+
+<h3>Clashes with other packages</h3>
+
+<p>Sometimes other packages have functions or operators with the same name as the future package, and if those packages are attached <em>after</em> the future package, their objects will mask the ones of the future package. For instance, the igraph package also defines a <code>%<-%</code> operator which clashes with the one in future <em>if used at the prompt or in a script</em> (it is not a problem inside package because there we explicitly import objects in a known order). Here is [...]
+
+<pre><code class="r">> library("future")
+> library("igraph")
+
+Attaching package: 'igraph'
+
+The following objects are masked from 'package:future':
+
+ %<-%, %->%
+
+The following objects are masked from 'package:stats':
+
+ decompose, spectrum
+
+The following object is masked from 'package:base':
+
+ union
+
+> y %<-% { 42 }
+Error in get(".igraph.from", parent.frame()) :
+ object '.igraph.from' not found
+</code></pre>
+
+<p>Here we get an error because <code>%<-%</code> is from igraph and not the future assignment operator as we wanted. This can be confirmed as:</p>
+
+<pre><code class="r">> environment(`%<-%`)
+<environment: namespace:igraph>
+</code></pre>
+
+<p>To avoid this problem, attach the two packages in opposite order such that future comes last and thereby overrides igraph, i.e.</p>
+
+<pre><code class="r">> library("igraph")
+> library("future")
+
+Attaching package: 'future'
+
+The following objects are masked from 'package:igraph':
+
+%<-%, %->%
+
+> y %<-% { 42 }
+> y
+[1] 42
+</code></pre>
+
+<p>An alternative is to detach the future package and re-attach it, which will achieve the same thing:</p>
+
+<pre><code class="r">> detach("package:future")
+> library("future")
+</code></pre>
+
+<p>Yet another alternative is to explicitly override the object by importing it to the global environment, e.g.</p>
+
+<pre><code class="r">> `%<-%` <- future::`%<-%`
+> y %<-% { 42 }
+> y
+[1] 42
+</code></pre>
+
+<p>In this case, it does not matter in what order the packages are attached because we will always use the copy of <code>future::`%<-%`</code>.</p>
+
+<h3>Syntax error: “non-numeric argument to binary operator”</h3>
+
+<p>The future assignment operator <code>%<-%</code> is a <em>binary infix operator</em>, which means it has higher precedence than most other binary operators but also higher than some of the unary operators in R. For instance, this explains why we get the following error:</p>
+
+<pre><code class="r">> x %<-% 2 * runif(1)
+Error in x %<-% 2 * runif(1) : non-numeric argument to binary operator
+</code></pre>
+
+<p>What effectively is happening here is that because of the higher priority of <code>%<-%</code>, we first create a future <code>x %<-% 2</code> and then we try to multiply the future (not its value) with the value of <code>runif(1)</code> - which makes no sense. In order to properly assign the future variable, we need to put the future expression within curly brackets;</p>
+
+<pre><code class="r">> x %<-% { 2 * runif(1) }
+> x
+[1] 1.030209
+</code></pre>
+
+<p>Parentheses will also do. For details on precedence on operators in R, see Section 'Infix and prefix operators' in the 'R Language Definition' document.</p>
+
+<h3>R CMD check NOTEs</h3>
+
+<p>The code inspection run by <code>R CMD check</code> will not recognize the future assignment operator <code>%<-%</code> as an assignment operator, which is not surprising because <code>%<-%</code> is technically an infix operator. This means that if you for instance use the following code in your package:</p>
+
+<pre><code class="r">foo <- function() {
+ b <- 3.14
+ a %<-% { b + 1 }
+ a
+}
+</code></pre>
+
+<p>then <code>R CMD check</code> will produce a NOTE saying:</p>
+
+<pre><code class="sh">* checking R code for possible problems ... NOTE
+foo: no visible binding for global variable 'a'
+Undefined global functions or variables:
+ a
+</code></pre>
+
+<p>In order to avoid this, we can add a dummy assignment of the missing global at the top of the function, i.e.</p>
+
+<pre><code class="r">foo <- function() {
+ a <- NULL ## To please R CMD check
+ b <- 3.14
+ a %<-% { b + 1 }
+ a
+}
+</code></pre>
+
+<hr/>
+
+<p>Copyright Henrik Bengtsson, 2015-2017</p>
+
+</body>
+
+</html>
diff --git a/vignettes/future-2-issues.md.rsp b/inst/doc/future-2-issues.md.rsp
similarity index 92%
copy from vignettes/future-2-issues.md.rsp
copy to inst/doc/future-2-issues.md.rsp
index d45beaf..7f99d60 100644
--- a/vignettes/future-2-issues.md.rsp
+++ b/inst/doc/future-2-issues.md.rsp
@@ -27,9 +27,9 @@ _If you identify other cases, please consider [reporting](https://github.com/Hen
If a global variable is (only) part of a "self" assignment, such as `x <- x + 1`, then the global is not identified and therefore neither exported / frozen. This explains the error in the following example:
```r
> library("future")
-> plan("lazy")
+> plan(sequential)
> x <- 0
-> y %<-% { x <- x + 1 }
+> y %<-% { x <- x + 1 } %lazy% TRUE
> rm(x)
> y
Error: object 'x' not found
@@ -38,9 +38,9 @@ Error: object 'x' not found
The workaround is to guide R to find the missing global by explicitly listing it, e.g.
```r
> library("future")
-> plan("lazy")
+> plan(sequential)
> x <- 0
-> y %<-% { x; x <- x + 1 }
+> y %<-% { x; x <- x + 1 } %lazy% TRUE
> rm(x)
> y
[1] 1
@@ -60,16 +60,16 @@ instead of
do.call("file_ext", list("foo.txt"))
```
so that `file_ext()` is properly located and exported. Although you may not notice a difference when evaluating futures in the same R session, it may become a problem if you use a character string instead of a function object when futures are evaluated in external R sessions, such as on a cluster.
-It may also become a problem with lazy futures if the intended function is redefined after the future is resolved. For example,
+It may also become a problem with futures evaluated with lazy evaluation if the intended function is redefined after the future is resolved. For example,
```r
> library("future")
> library("listenv")
> library("tools")
-> plan(lazy)
+> plan(sequential)
> pathnames <- c("foo.txt", "bar.png", "yoo.md")
> res <- listenv()
> for (ii in seq_along(pathnames)) {
-+ res[[ii]] %<-% do.call("file_ext", list(pathnames[ii]))
++ res[[ii]] %<-% do.call("file_ext", list(pathnames[ii])) %lazy% TRUE
+ }
> file_ext <- function(...) "haha!"
> unlist(res)
@@ -133,7 +133,7 @@ Error: Invalid usage of futures: A future whose value has not yet been collected
```
As previously, this can be avoided by making sure `x$a` is resolved first, which can be one in various ways, e.g. `dummy <- x$a`, `resolve(x$a)` and `force(x$a)`.
-_Footnote_: (*) Although eager futures could be passed on to other futures part of the same R process and be resolved there because they share the same evaluation process, by definition of the Future API it is invalid to do so regardless of future type. This conservative approach is taken in order to make future expressions behave consistently regardless of the type of future used.
+_Footnote_: (*) Although sequential futures could be passed on to other futures part of the same R process and be resolved there because they share the same evaluation process, by definition of the Future API it is invalid to do so regardless of future type. This conservative approach is taken in order to make future expressions behave consistently regardless of the type of future used.
## Miscellaneous
@@ -249,4 +249,4 @@ foo <- function() {
[listenv]: https://cran.r-project.org/package=listenv
---
-Copyright Henrik Bengtsson, 2015-2016
+Copyright Henrik Bengtsson, 2015-2017
diff --git a/inst/doc/future-3-topologies.html b/inst/doc/future-3-topologies.html
new file mode 100644
index 0000000..fbb1b32
--- /dev/null
+++ b/inst/doc/future-3-topologies.html
@@ -0,0 +1,369 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+
+<title>A Future for R: Future Topologies</title>
+
+<script type="text/javascript">
+window.onload = function() {
+ var imgs = document.getElementsByTagName('img'), i, img;
+ for (i = 0; i < imgs.length; i++) {
+ img = imgs[i];
+ // center an image if it is the only element of its parent
+ if (img.parentElement.childElementCount === 1)
+ img.parentElement.style.textAlign = 'center';
+ }
+};
+</script>
+
+<!-- Styles for R syntax highlighter -->
+<style type="text/css">
+ pre .operator,
+ pre .paren {
+ color: rgb(104, 118, 135)
+ }
+
+ pre .literal {
+ color: #990073
+ }
+
+ pre .number {
+ color: #099;
+ }
+
+ pre .comment {
+ color: #998;
+ font-style: italic
+ }
+
+ pre .keyword {
+ color: #900;
+ font-weight: bold
+ }
+
+ pre .identifier {
+ color: rgb(0, 0, 0);
+ }
+
+ pre .string {
+ color: #d14;
+ }
+</style>
+
+<!-- R syntax highlighter -->
+<script type="text/javascript">
+var hljs=new function(){function m(p){return p.replace(/&/gm,"&").replace(/</gm,"<")}function f(r,q,p){return RegExp(q,"m"+(r.cI?"i":"")+(p?"g":""))}function b(r){for(var p=0;p<r.childNodes.length;p++){var q=r.childNodes[p];if(q.nodeName=="CODE"){return q}if(!(q.nodeType==3&&q.nodeValue.match(/\s+/))){break}}}function h(t,s){var p="";for(var r=0;r<t.childNodes.length;r++){if(t.childNodes[r].nodeType==3){var q=t.childNodes[r].nodeValue;if(s){q=q.replace(/\n/g,"")}p+=q}else{if(t.chi [...]
+hljs.initHighlightingOnLoad();
+</script>
+
+
+
+<style type="text/css">
+body, td {
+ font-family: sans-serif;
+ background-color: white;
+ font-size: 13px;
+}
+
+body {
+ max-width: 800px;
+ margin: auto;
+ padding: 1em;
+ line-height: 20px;
+}
+
+tt, code, pre {
+ font-family: 'DejaVu Sans Mono', 'Droid Sans Mono', 'Lucida Console', Consolas, Monaco, monospace;
+}
+
+h1 {
+ font-size:2.2em;
+}
+
+h2 {
+ font-size:1.8em;
+}
+
+h3 {
+ font-size:1.4em;
+}
+
+h4 {
+ font-size:1.0em;
+}
+
+h5 {
+ font-size:0.9em;
+}
+
+h6 {
+ font-size:0.8em;
+}
+
+a:visited {
+ color: rgb(50%, 0%, 50%);
+}
+
+pre, img {
+ max-width: 100%;
+}
+pre {
+ overflow-x: auto;
+}
+pre code {
+ display: block; padding: 0.5em;
+}
+
+code {
+ font-size: 92%;
+ border: 1px solid #ccc;
+}
+
+code[class] {
+ background-color: #F8F8F8;
+}
+
+table, td, th {
+ border: none;
+}
+
+blockquote {
+ color:#666666;
+ margin:0;
+ padding-left: 1em;
+ border-left: 0.5em #EEE solid;
+}
+
+hr {
+ height: 0px;
+ border-bottom: none;
+ border-top-width: thin;
+ border-top-style: dotted;
+ border-top-color: #999999;
+}
+
+ at media print {
+ * {
+ background: transparent !important;
+ color: black !important;
+ filter:none !important;
+ -ms-filter: none !important;
+ }
+
+ body {
+ font-size:12pt;
+ max-width:100%;
+ }
+
+ a, a:visited {
+ text-decoration: underline;
+ }
+
+ hr {
+ visibility: hidden;
+ page-break-before: always;
+ }
+
+ pre, blockquote {
+ padding-right: 1em;
+ page-break-inside: avoid;
+ }
+
+ tr, img {
+ page-break-inside: avoid;
+ }
+
+ img {
+ max-width: 100% !important;
+ }
+
+ @page :left {
+ margin: 15mm 20mm 15mm 10mm;
+ }
+
+ @page :right {
+ margin: 15mm 10mm 15mm 20mm;
+ }
+
+ p, h2, h3 {
+ orphans: 3; widows: 3;
+ }
+
+ h2, h3 {
+ page-break-after: avoid;
+ }
+}
+</style>
+
+<meta name="keywords" content="R, package, vignette, future, promise">
+<meta name="author" content="Henrik Bengtsson">
+
+</head>
+
+<body>
+<h1>A Future for R: Future Topologies</h1>
+
+<p>Futures can be nested in R such that one future creates another set of futures and so on. This may, for instance, occur within nested for loops, e.g.</p>
+
+<pre><code class="r">library("future")
+library("listenv")
+x <- listenv()
+for (ii in 1:3) {
+ x[[ii]] %<-% {
+ y <- listenv()
+ for (jj in 1:3) {
+ y[[jj]] %<-% { ii + jj / 10 }
+ }
+ y
+ }
+}
+unlist(x)
+## [1] 1.1 1.2 1.3 2.1 2.2 2.3 3.1 3.2 3.3
+</code></pre>
+
+<p>The default is to use synchronous futures unless otherwise specified, which is also true for nested futures. If we for instance specify, <code>plan(multiprocess)</code>, the first layer of futures (<code>x[[ii]] %<-% { expr }</code>) will be processed asynchronously in background R processes, and the futures in the second layer of futures (<code>y[[jj]] %<-% { expr }</code>) will be processed synchronously in the separate background R processes. If we wish to be explicit about [...]
+
+<h2>Example: High-Throughput Sequencing</h2>
+
+<p>Consider a high-throughput sequencing (HT-Seq) project with 50 human DNA samples where we have one FASTQ file per sample containing the raw sequence reads as they come out of the sequencing machine. With this data, we wish to align each FASTQ to a reference genome such that we generate 24 individual BAM files per sample - one per chromosome.</p>
+
+<p>Here is the layout of what such an analysis could look like in R using futures.</p>
+
+<pre><code class="r">library("future")
+library("listenv")
+htseq_align <- function(fq, chr) { chr }
+
+fqs <- dir(pattern = "[.]fastq$")
+
+bams <- listenv()
+for (ss in seq_along(fqs)) {
+ fq <- fqs[ss]
+ bams[[ss]] %<-% {
+ bams_ss <- listenv()
+ for (cc in 1:24) {
+ bams_ss[[cc]] %<-% htseq_align(fq, chr = cc)
+ }
+ as.list(bams_ss)
+ }
+}
+bams <- as.list(bams)
+</code></pre>
+
+<p>The default is to use synchronous futures, so without further specifications, the above will process each sample and each chromosome sequentially. Next, we will consider what can be done with the following two computer setups:</p>
+
+<ul>
+<li>A single machine with 8 cores</li>
+<li>A compute cluster with 3 machines each with 16 cores</li>
+</ul>
+
+<h3>One multi-core machine</h3>
+
+<p>With a single machine of 8 cores, we could choose to process multiple samples at the same time while processing chromosomes sequentially. In other words, we would like to evaluate the outer layer of futures using multiprocess futures and the inner ones as sequential futures. This can be specified as:</p>
+
+<pre><code class="r">plan(list(multiprocess, sequential))
+</code></pre>
+
+<p>The internals for processing multiprocess future queries <code>availableCores()</code> to infer how many cores can be used simultaneously, so there is no need to explicitly specify that there are 8 cores available.</p>
+
+<p><em>Comment</em>: Since synchronous is the default future, we could skip trailing sequential futures in the setup, e.g. <code>plan(list(multiprocess))</code> or just <code>plan(multiprocess)</code>. However, it does not hurt to be explicit.</p>
+
+<p>If we instead would like to process the sample sequentially and the chromosomes in parallel, we can use:</p>
+
+<pre><code class="r">plan(list(sequential, multiprocess))
+</code></pre>
+
+<p>We could also process the data such that we allocate two cores for processing two samples in parallel each using four cores for processing four chromosomes in parallel:</p>
+
+<pre><code class="r">plan(list(tweak(multiprocess, workers = 2), tweak(multiprocess, workers = 4)))
+</code></pre>
+
+<h3>An ad-hoc compute cluster</h3>
+
+<p>With a compute cluster of 3 machines each with 16 cores, we can run up to 48 alignment processes in parallel. A natural setup is to have one machine process one sample in parallel. We could specify this as:</p>
+
+<pre><code class="r">nodes <- c("n1", "n2", "n3")
+plan(list(tweak(cluster, workers = nodes), multiprocess))
+</code></pre>
+
+<p><em>Comment:</em> Multiprocess futures are agile to its environment, that is, they will query the machine they are running on to find out how many parallel processes it can run at the same time.</p>
+
+<p>One possible downside to the above setup is that we might not utilize all available cores all the time. This is because the alignment of the shorter chromosomes will finish sooner than the longer ones, which means that we might at the end of each sample have only a few alignment processes running on each machine leaving the remaining cores idle/unused. An alternative set up is then to use the following setup:</p>
+
+<pre><code class="r">nodes <- rep(c("n1", "n2", "n3"), each = 8)
+plan(list(tweak(cluster, workers = nodes), multiprocess))
+</code></pre>
+
+<p>This will cause up to 24 (= 3*8) samples to be processed in parallel each processing two chromosomes at the same time.</p>
+
+<h2>Example: A remote compute cluster</h2>
+
+<p>Imagine we have access to a remote compute cluster, with login node <code>remote.server.org</code>, and that the cluster has three nodes <code>n1</code>, <code>n2</code>, and <code>n3</code>. Also, let us assume we have already set up the cluster such that we can log in via public key authentication via SSH, i.e. when we do <code>ssh remote.server.org</code> authentication is done automatically.</p>
+
+<p>With the above setup, we can use nested futures in our local R session to evaluate R expression on the remote compute cluster and its three nodes. Here is a proof of concept illustrating how the different nested futures are evaluated on different machines.</p>
+
+<pre><code class="r">library("future")
+library("listenv")
+
+## Set up access to remote login node
+login <- tweak(remote, workers = "remote.server.org")
+plan(login)
+
+## Set up cluster nodes on login node
+nodes %<-% { .keepme <- parallel::makeCluster(c("n1", "n2", "n3")) }
+
+## Specify future topology
+## login node -> { cluster nodes } -> { multiple cores }
+plan(list(
+ login,
+ tweak(cluster, workers = nodes),
+ multiprocess
+))
+
+
+## (a) This will be evaluated on the cluster login computer
+x %<-% {
+ thost <- Sys.info()[["nodename"]]
+ tpid <- Sys.getpid()
+ y <- listenv()
+ for (task in 1:4) {
+ ## (b) This will be evaluated on a compute node on the cluster
+ y[[task]] %<-% {
+ mhost <- Sys.info()[["nodename"]]
+ mpid <- Sys.getpid()
+ z <- listenv()
+ for (jj in 1:2) {
+ ## (c) These will be evaluated in separate processes on the same compute node
+ z[[jj]] %<-% data.frame(task = task,
+ top.host = thost, top.pid = tpid,
+ mid.host = mhost, mid.pid = mpid,
+ host = Sys.info()[["nodename"]],
+ pid = Sys.getpid())
+ }
+ Reduce(rbind, z)
+ }
+ }
+ Reduce(rbind, y)
+}
+
+print(x)
+## task top.host top.pid mid.host mid.pid host pid
+## 1 1 login 391547 n1 391878 n1 393943
+## 2 1 login 391547 n1 391878 n1 393951
+## 3 2 login 391547 n2 392204 n2 393971
+## 4 2 login 391547 n2 392204 n2 393978
+## 5 3 login 391547 n3 392527 n3 394040
+## 6 3 login 391547 n3 392527 n3 394048
+## 7 4 login 391547 n1 391878 n1 393959
+## 8 4 login 391547 n1 391878 n1 393966
+</code></pre>
+
+<p>Try the above <code>x %<-% { ... }</code> future with, say, <code>plan(list(sequential, multiprocess))</code> and see what the output will be.</p>
+
+<hr/>
+
+<p>Copyright Henrik Bengtsson, 2015-2017</p>
+
+</body>
+
+</html>
diff --git a/vignettes/future-3-topologies.md.rsp b/inst/doc/future-3-topologies.md.rsp
similarity index 81%
copy from vignettes/future-3-topologies.md.rsp
copy to inst/doc/future-3-topologies.md.rsp
index c8962f2..a19fd8f 100644
--- a/vignettes/future-3-topologies.md.rsp
+++ b/inst/doc/future-3-topologies.md.rsp
@@ -26,15 +26,15 @@ for (ii in 1:3) {
x[[ii]] %<-% {
y <- listenv()
for (jj in 1:3) {
- y[[jj]] %<-% { ii + jj/10 }
+ y[[jj]] %<-% { ii + jj / 10 }
}
- y
+ y
}
}
unlist(x)
## [1] 1.1 1.2 1.3 2.1 2.2 2.3 3.1 3.2 3.3
```
-The default is to use synchronous futures unless otherwise specified, which is also true for nested futures. If we for instance specify, `plan(multiprocess)`, the first layer of futures (`x[[ii]] %<-% { expr }`) will be processed asynchronously in background R processes, and the futures in the second layer of futures (`y[[jj]] %<-% { expr }`) will be processed synchronously in the current R processes (one of the background processes). If we wish to use a different strategy for the seco [...]
+The default is to use synchronous futures unless otherwise specified, which is also true for nested futures. If we for instance specify, `plan(multiprocess)`, the first layer of futures (`x[[ii]] %<-% { expr }`) will be processed asynchronously in background R processes, and the futures in the second layer of futures (`y[[jj]] %<-% { expr }`) will be processed synchronously in the separate background R processes. If we wish to be explicit about this, we can specify `plan(list(multiproc [...]
## Example: High-Throughput Sequencing
@@ -46,7 +46,7 @@ library("future")
library("listenv")
htseq_align <- function(fq, chr) { chr }
-fqs <- dir(pattern="[.]fastq$")
+fqs <- dir(pattern = "[.]fastq$")
bams <- listenv()
for (ss in seq_along(fqs)) {
@@ -54,7 +54,7 @@ for (ss in seq_along(fqs)) {
bams[[ss]] %<-% {
bams_ss <- listenv()
for (cc in 1:24) {
- bams_ss[[cc]] %<-% htseq_align(fq, chr=cc)
+ bams_ss[[cc]] %<-% htseq_align(fq, chr = cc)
}
as.list(bams_ss)
}
@@ -68,22 +68,22 @@ The default is to use synchronous futures, so without further specifications, th
* A compute cluster with 3 machines each with 16 cores
### One multi-core machine
-With a single machine of 8 cores, we could choose to process multiple samples at the same time while processing chromosomes sequentially. In other words, we would like to evaluate the outer layer of futures using multiprocess futures and the inner ones as eager synchronous futures. This can be specified as:
+With a single machine of 8 cores, we could choose to process multiple samples at the same time while processing chromosomes sequentially. In other words, we would like to evaluate the outer layer of futures using multiprocess futures and the inner ones as sequential futures. This can be specified as:
```r
-plan(list(multiprocess, eager))
+plan(list(multiprocess, sequential))
```
The internals for processing multiprocess future queries `availableCores()` to infer how many cores can be used simultaneously, so there is no need to explicitly specify that there are 8 cores available.
-_Comment_: Since synchronous is the default future, we could skip trailing synchronous eager futures in the setup, e.g. `plan(list(multiprocess))` or just `plan(eager)`. However, it does not hurt to be explicit.
+_Comment_: Since synchronous is the default future, we could skip trailing sequential futures in the setup, e.g. `plan(list(multiprocess))` or just `plan(multiprocess)`. However, it does not hurt to be explicit.
If we instead would like to process the sample sequentially and the chromosomes in parallel, we can use:
```r
-plan(list(eager, multiprocess))
+plan(list(sequential, multiprocess))
```
We could also process the data such that we allocate two cores for processing two samples in parallel each using four cores for processing four chromosomes in parallel:
```r
-plan(list(tweak(multiprocess, workers=2), tweak(multiprocess, workers=4)))
+plan(list(tweak(multiprocess, workers = 2), tweak(multiprocess, workers = 4)))
```
@@ -91,16 +91,16 @@ plan(list(tweak(multiprocess, workers=2), tweak(multiprocess, workers=4)))
With a compute cluster of 3 machines each with 16 cores, we can run up to 48 alignment processes in parallel. A natural setup is to have one machine process one sample in parallel. We could specify this as:
```r
nodes <- c("n1", "n2", "n3")
-plan(list(tweak(cluster, workers=nodes), multiprocess))
+plan(list(tweak(cluster, workers = nodes), multiprocess))
```
_Comment:_ Multiprocess futures are agile to its environment, that is, they will query the machine they are running on to find out how many parallel processes it can run at the same time.
One possible downside to the above setup is that we might not utilize all available cores all the time. This is because the alignment of the shorter chromosomes will finish sooner than the longer ones, which means that we might at the end of each sample have only a few alignment processes running on each machine leaving the remaining cores idle/unused. An alternative set up is then to use the following setup:
```r
-nodes <- rep(c("n1", "n2", "n3"), each=8)
-plan(list(tweak(cluster, workers=nodes), multiprocess))
+nodes <- rep(c("n1", "n2", "n3"), each = 8)
+plan(list(tweak(cluster, workers = nodes), multiprocess))
```
-This will cause up to 24 (=3*8) samples to be processed in parallel each processing two chromosomes at the same time.
+This will cause up to 24 (= 3*8) samples to be processed in parallel each processing two chromosomes at the same time.
## Example: A remote compute cluster
@@ -113,7 +113,7 @@ library("future")
library("listenv")
## Set up access to remote login node
-login <- tweak(remote, workers="remote.server.org")
+login <- tweak(remote, workers = "remote.server.org")
plan(login)
## Set up cluster nodes on login node
@@ -123,7 +123,7 @@ nodes %<-% { .keepme <- parallel::makeCluster(c("n1", "n2", "n3")) }
## login node -> { cluster nodes } -> { multiple cores }
plan(list(
login,
- tweak(cluster, workers=nodes),
+ tweak(cluster, workers = nodes),
multiprocess
))
@@ -141,9 +141,11 @@ x %<-% {
z <- listenv()
for (jj in 1:2) {
## (c) These will be evaluated in separate processes on the same compute node
- z[[jj]] %<-% data.frame(task=task, top.host=thost, top.pid=tpid,
- mid.host=mhost, mid.pid=mpid,
- host=Sys.info()[["nodename"]], pid=Sys.getpid())
+ z[[jj]] %<-% data.frame(task = task,
+ top.host = thost, top.pid = tpid,
+ mid.host = mhost, mid.pid = mpid,
+ host = Sys.info()[["nodename"]],
+ pid = Sys.getpid())
}
Reduce(rbind, z)
}
@@ -163,7 +165,7 @@ print(x)
## 8 4 login 391547 n1 391878 n1 393966
```
-Try the above `x %<-% { ... }` future with, say, `plan(list(lazy, multiprocess))` or `plan(list(eager, lazy, multiprocess))` and see what the output will be.
+Try the above `x %<-% { ... }` future with, say, `plan(list(sequential, multiprocess))` and see what the output will be.
[listenv]: https://cran.r-project.org/package=listenv
@@ -171,4 +173,4 @@ Try the above `x %<-% { ... }` future with, say, `plan(list(lazy, multiprocess))
[Futures in R: Common issues with solutions]: future-issues.html
---
-Copyright Henrik Bengtsson, 2015-2016
+Copyright Henrik Bengtsson, 2015-2017
diff --git a/inst/doc/future-4-startup.html b/inst/doc/future-4-startup.html
new file mode 100644
index 0000000..5b5588b
--- /dev/null
+++ b/inst/doc/future-4-startup.html
@@ -0,0 +1,268 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+
+<title>A Future for R: Controlling Default Future Strategy</title>
+
+<script type="text/javascript">
+window.onload = function() {
+ var imgs = document.getElementsByTagName('img'), i, img;
+ for (i = 0; i < imgs.length; i++) {
+ img = imgs[i];
+ // center an image if it is the only element of its parent
+ if (img.parentElement.childElementCount === 1)
+ img.parentElement.style.textAlign = 'center';
+ }
+};
+</script>
+
+<!-- Styles for R syntax highlighter -->
+<style type="text/css">
+ pre .operator,
+ pre .paren {
+ color: rgb(104, 118, 135)
+ }
+
+ pre .literal {
+ color: #990073
+ }
+
+ pre .number {
+ color: #099;
+ }
+
+ pre .comment {
+ color: #998;
+ font-style: italic
+ }
+
+ pre .keyword {
+ color: #900;
+ font-weight: bold
+ }
+
+ pre .identifier {
+ color: rgb(0, 0, 0);
+ }
+
+ pre .string {
+ color: #d14;
+ }
+</style>
+
+<!-- R syntax highlighter -->
+<script type="text/javascript">
+var hljs=new function(){function m(p){return p.replace(/&/gm,"&").replace(/</gm,"<")}function f(r,q,p){return RegExp(q,"m"+(r.cI?"i":"")+(p?"g":""))}function b(r){for(var p=0;p<r.childNodes.length;p++){var q=r.childNodes[p];if(q.nodeName=="CODE"){return q}if(!(q.nodeType==3&&q.nodeValue.match(/\s+/))){break}}}function h(t,s){var p="";for(var r=0;r<t.childNodes.length;r++){if(t.childNodes[r].nodeType==3){var q=t.childNodes[r].nodeValue;if(s){q=q.replace(/\n/g,"")}p+=q}else{if(t.chi [...]
+hljs.initHighlightingOnLoad();
+</script>
+
+
+
+<style type="text/css">
+body, td {
+ font-family: sans-serif;
+ background-color: white;
+ font-size: 13px;
+}
+
+body {
+ max-width: 800px;
+ margin: auto;
+ padding: 1em;
+ line-height: 20px;
+}
+
+tt, code, pre {
+ font-family: 'DejaVu Sans Mono', 'Droid Sans Mono', 'Lucida Console', Consolas, Monaco, monospace;
+}
+
+h1 {
+ font-size:2.2em;
+}
+
+h2 {
+ font-size:1.8em;
+}
+
+h3 {
+ font-size:1.4em;
+}
+
+h4 {
+ font-size:1.0em;
+}
+
+h5 {
+ font-size:0.9em;
+}
+
+h6 {
+ font-size:0.8em;
+}
+
+a:visited {
+ color: rgb(50%, 0%, 50%);
+}
+
+pre, img {
+ max-width: 100%;
+}
+pre {
+ overflow-x: auto;
+}
+pre code {
+ display: block; padding: 0.5em;
+}
+
+code {
+ font-size: 92%;
+ border: 1px solid #ccc;
+}
+
+code[class] {
+ background-color: #F8F8F8;
+}
+
+table, td, th {
+ border: none;
+}
+
+blockquote {
+ color:#666666;
+ margin:0;
+ padding-left: 1em;
+ border-left: 0.5em #EEE solid;
+}
+
+hr {
+ height: 0px;
+ border-bottom: none;
+ border-top-width: thin;
+ border-top-style: dotted;
+ border-top-color: #999999;
+}
+
+ at media print {
+ * {
+ background: transparent !important;
+ color: black !important;
+ filter:none !important;
+ -ms-filter: none !important;
+ }
+
+ body {
+ font-size:12pt;
+ max-width:100%;
+ }
+
+ a, a:visited {
+ text-decoration: underline;
+ }
+
+ hr {
+ visibility: hidden;
+ page-break-before: always;
+ }
+
+ pre, blockquote {
+ padding-right: 1em;
+ page-break-inside: avoid;
+ }
+
+ tr, img {
+ page-break-inside: avoid;
+ }
+
+ img {
+ max-width: 100% !important;
+ }
+
+ @page :left {
+ margin: 15mm 20mm 15mm 10mm;
+ }
+
+ @page :right {
+ margin: 15mm 10mm 15mm 20mm;
+ }
+
+ p, h2, h3 {
+ orphans: 3; widows: 3;
+ }
+
+ h2, h3 {
+ page-break-after: avoid;
+ }
+}
+</style>
+
+<meta name="keywords" content="R, package, vignette, future, promise">
+<meta name="author" content="Henrik Bengtsson">
+
+</head>
+
+<body>
+<h1>A Future for R: Controlling Default Future Strategy</h1>
+
+<p>The default is to use synchronous futures, but this <em>default</em> can be overridden via R options, system environment variables and command-line options as explained below as well as in <code>help("future.options", package = "future")</code>.</p>
+
+<h2>R options</h2>
+
+<p>The default strategy for resolving futures can be controlled via R option <code>future.plan</code>. For instance, if we add</p>
+
+<pre><code class="r">options(future.plan = "multiprocess")
+</code></pre>
+
+<p>to our <code>~/.Rprofile</code> startup script, the future package will resolve futures in parallel (asynchronously using all available cores), i.e.</p>
+
+<pre><code class="sh">$ Rscript -e "class(future::plan())"
+[1] "multiprocess" "future" "function"
+</code></pre>
+
+<p>Option <code>future.plan</code> is ignored if command-line option <code>--parallel</code> (<code>-p</code>) is specified.</p>
+
+<h2>Environment variables</h2>
+
+<p>An alternative to using <code>options()</code> for setting option <code>future.plan</code> is to specify system environment variable <code>R_FUTURE_PLAN</code>. If set, then the future package will set <code>future.plan</code> accordingly <em>when loaded</em>. For example,</p>
+
+<pre><code class="sh">$ export R_FUTURE_PLAN=multiprocess
+$ Rscript -e "class(future::plan())"
+[1] "multiprocess" "future" "function"
+</code></pre>
+
+<p>Environment variable <code>R_FUTURE_PLAN</code> is ignored if either option <code>future.plan</code> or command-line option <code>--parallel</code> (<code>-p</code>) is specified.</p>
+
+<h2>Command-line options</h2>
+
+<p>When loaded, the future package checks for the command-line option <code>--parallel=ncores</code> (short <code>-p ncores</code>) and sets the future strategy (via option <code>future.plan</code>) and the number of available cores (via option <code>mc.cores</code>) accordingly. This provides a convenient mechanism for specifying parallel future processing from the command line. For example, if we start R with</p>
+
+<pre><code class="sh">$ R --quiet --args --parallel=2
+</code></pre>
+
+<p>then future will interpret this as we wish to resolve futures in parallel using 2 cores. More specifically, we get that</p>
+
+<pre><code class="r">> future::availableCores()
+mc.cores
+ 2
+> class(future::plan())
+[1] "tweaked" "multiprocess" "future" "function"
+</code></pre>
+
+<p>We can use this command-line option also with <code>Rscript</code>, which provides a convenient mechanism for launching future-enhanced R scripts such that they run in parallel, e.g.</p>
+
+<pre><code class="sh">$ Rscript analysis.R --parallel=4
+</code></pre>
+
+<p>This does, of course, require that the script uses futures and the future package.</p>
+
+<p>If <code>--parallel=1</code> is specified, or equivalently <code>-p 1</code>, then futures are resolved using a single process.</p>
+
+<p>Specifying these command-line options override any other startup settings.</p>
+
+<hr/>
+
+<p>Copyright Henrik Bengtsson, 2015-2017</p>
+
+</body>
+
+</html>
diff --git a/vignettes/future-4-startup.md.rsp b/inst/doc/future-4-startup.md.rsp
similarity index 94%
copy from vignettes/future-4-startup.md.rsp
copy to inst/doc/future-4-startup.md.rsp
index e8637d8..ea59e45 100644
--- a/vignettes/future-4-startup.md.rsp
+++ b/inst/doc/future-4-startup.md.rsp
@@ -12,18 +12,18 @@
<%
library("R.utils")
`%<-%` <- future::`%<-%`
-options("withCapture/newline"=FALSE)
+options("withCapture/newline" = FALSE)
%>
# <%@meta name="title"%>
-The default is to use synchronous futures, but this _default_ can be overridden via R options, system environment variables and command-line options as explained below as well as in `help("future.options", package="future")`.
+The default is to use synchronous futures, but this _default_ can be overridden via R options, system environment variables and command-line options as explained below as well as in `help("future.options", package = "future")`.
## R options
The default strategy for resolving futures can be controlled via R option `future.plan`. For instance, if we add
```r
-options(future.plan="multiprocess")
+options(future.plan = "multiprocess")
```
to our `~/.Rprofile` startup script, the future package will resolve futures in parallel (asynchronously using all available cores), i.e.
```sh
@@ -53,8 +53,8 @@ $ R --quiet --args --parallel=2
then future will interpret this as we wish to resolve futures in parallel using 2 cores. More specifically, we get that
```r
> future::availableCores()
-mc.cores+1
- 2
+mc.cores
+ 2
> class(future::plan())
[1] "tweaked" "multiprocess" "future" "function"
```
@@ -73,4 +73,4 @@ Specifying these command-line options override any other startup settings.
[future]: https://cran.r-project.org/package=future
---
-Copyright Henrik Bengtsson, 2015-2016
+Copyright Henrik Bengtsson, 2015-2017
diff --git a/inst/vignettes-static/future-1-overview.md.rsp.rsp b/inst/vignettes-static/future-1-overview.md.rsp.rsp
index b7c68ce..899a85b 100644
--- a/inst/vignettes-static/future-1-overview.md.rsp.rsp
+++ b/inst/vignettes-static/future-1-overview.md.rsp.rsp
@@ -45,15 +45,15 @@ file found under inst/vignettes-static/ of the source package.
R.utils::use("R.utils")
use("future")
`%<-%` <- future::`%<-%`
-options("withCapture/newline"=FALSE)
-options(mc.cores=2L)
+options("withCapture/newline" = FALSE)
+options(mc.cores = 2L)
%>
# <%@meta name="title"%>
## Introduction
The purpose of the [future] package is to provide a very simple and uniform way of evaluating R expressions asynchronously using various resources available to the user.
-In programming, a _future_ is an abstraction for a _value_ that may be available at some point in the future. The state of a future can either be _unresolved_ or _resolved_. As soon as it is resolved, the value is available instantaneously. If the value is queried while the future is still unresolved, the current process is _blocked_ until the future is resolved. It is possible to check whether a future is resolved or not without blocking. Exactly how and when futures are resolved d [...]
+In programming, a _future_ is an abstraction for a _value_ that may be available at some point in the future. The state of a future can either be _unresolved_ or _resolved_. As soon as it is resolved, the value is available instantaneously. If the value is queried while the future is still unresolved, the current process is _blocked_ until the future is resolved. It is possible to check whether a future is resolved or not without blocking. Exactly how and when futures are resolved d [...]
Here is an example illustrating how the basics of futures work. First, consider the following code snippet that uses plain R code:
```r
@@ -139,9 +139,8 @@ The future package implements the following types of futures:
| Name | OSes | Description
|:----------------|:------------|:-----------------------------------------------------
| _synchronous:_ | | _non-parallel:_
-| `eager` | all |
-| `lazy` | all | lazy evaluation - happens only if the value is requested
-| `transparent` | all | as eager w/ early signaling and w/out local (for debugging)
+| `sequential` | all | sequentially and in the current R process
+| `transparent` | all | as sequential w/ early signaling and w/out local (for debugging)
| _asynchronous:_ | | _parallel_:
| `multiprocess` | all | multicore iff supported, otherwise multisession
| `multisession` | all | background R sessions (on current machine)
@@ -149,28 +148,30 @@ The future package implements the following types of futures:
| `cluster` | all | external R sessions on current, local, and/or remote machines
| `remote` | all | Simple access to remote R sessions
-The future package is designed such that support for additional strategies can be implemented as well. For instance, the [future.BatchJobs] package provides futures for all types of _cluster functions_ ("backends") that the [BatchJobs] package supports. Specifically, futures for evaluating R expressions via job schedulers such as Slurm, TORQUE/PBS, Oracle/Sun Grid Engine (SGE) and Load Sharing Facility (LSF) are also available.
+_Note_: Prior to future 1.3.0, `eager` and `lazy` were also options, but has since been deprecated and are both defunct as of 1.6.0. The reason for this is that whether a future should be resolved by lazy evaluation or not has to, in some cases, be in the control of the developer, and if the end user would be able change that, the code may not function as intended.
-By default, future expressions are evaluated eagerly (= instantaneously) and synchronously (in the current R session). This evaluation strategy is referred to as "eager" and we refer to futures using this strategy as "eager futures". In this section, we will go through each of these strategies and discuss what they have in common and how they differ.
+The future package is designed such that support for additional strategies can be implemented as well. For instance, the [future.batchtools] package provides futures for all types of _cluster functions_ ("backends") that the [batchtools] package supports. Specifically, futures for evaluating R expressions via job schedulers such as Slurm, TORQUE/PBS, Oracle/Sun Grid Engine (SGE) and Load Sharing Facility (LSF) are also available. (_Comment_: The [future.BatchJobs] package provides ana [...]
+
+By default, future expressions are evaluated eagerly (= instantaneously) and synchronously (in the current R session). This evaluation strategy is referred to as "sequential". In this section, we will go through each of these strategies and discuss what they have in common and how they differ.
### Consistent Behavior Across Futures
-Before going through each of the different future strategies, it is probably helpful to clarify the objectives the Future API (as defined by the future package). When programming with futures, it should not really matter what future strategy is used for executing code. This is because we cannot really know what computational resources the user has access to so the choice of evaluation strategy should be in the hand of the user and not the developer. In other words, the code should not [...]
+Before going through each of the different future strategies, it is probably helpful to clarify the objectives the Future API (as defined by the future package). When programming with futures, it should not really matter what future strategy is used for executing code. This is because we cannot really know what computational resources the user has access to so the choice of evaluation strategy should be in the hands of the user and not the developer. In other words, the code should no [...]
-One of the designs of the Future API was to encapsulate any differences such that all types of futures will appear to work the same. This despite expressions may be evaluated locally in the current R session or across the world in remote R sessions. Another obvious advantage of having a consistent API and behavior among different types of futures is that it helps while prototyping. Typically one would use eager evaluation while building up a script and, later, when the script is fully [...]
+One of the designs of the Future API was to encapsulate any differences such that all types of futures will appear to work the same. This despite expressions may be evaluated locally in the current R session or across the world in remote R sessions. Another obvious advantage of having a consistent API and behavior among different types of futures is that it helps while prototyping. Typically one would use sequential evaluation while building up a script and, later, when the script is [...]
Because of this, the defaults of the different strategies are such that the results and side effects of evaluating a future expression are as similar as possible. More specifically, the following is true for all futures:
* All _evaluation is done in a local environment_ (i.e. `local({ expr })`) so that assignments do not affect the calling environment. This is natural when evaluating in an external R process, but is also enforced when evaluating in the current R session.
-* When a future is constructed, _global variables are identified_. For asynchronous evaluation, globals are exported to the R process/session that will be evaluating the future expression. For lazy futures, globals are "frozen" (cloned to a local environment of the future). Also, in order to protect against exporting too large objects by mistake, there is a built-in assertion that the total size of all globals is less than a given threshold (controllable via an option, cf. `help("futu [...]
+* When a future is constructed, _global variables are identified_. For asynchronous evaluation, globals are exported to the R process/session that will be evaluating the future expression. For sequential futures with lazy evaluation (`lazy = TRUE`), globals are "frozen" (cloned to a local environment of the future). Also, in order to protect against exporting too large objects by mistake, there is a built-in assertion that the total size of all globals is less than a given threshold ( [...]
* Future _expressions are only evaluated once_. As soon as the value (or an error) has been collected it will be available for all succeeding requests.
Here is an example illustrating that all assignments are done to a local environment:
```r
<%=withCapture({
-plan(eager)
+plan(sequential)
a <- 1
x %<-% {
a <- 2
@@ -198,50 +199,26 @@ Now we are ready to explore the different future strategies.
### Synchronous Futures
-Synchronous futures are resolved one after another and most commonly by the R process that creates them. When a synchronous future is being resolved it blocks the main process until resolved. There are two main types of synchronous futures in the future package, _eager_ and _lazy_ futures, which are described next.
+Synchronous futures are resolved one after another and most commonly by the R process that creates them. When a synchronous future is being resolved it blocks the main process until resolved. There are two types of synchronous futures in the future package, _sequential_ and _transparent_. (In future 1.2.0 and before, there was also _lazy_ futures, which has now been deprecated in favor of `f <- future(..., lazy = TRUE)` and `v %<-% { ... } %lazy% TRUE`.)
-#### Eager Futures
-Eager futures are the default unless otherwise specified. They were designed to behave as similar as possible to regular R evaluation while still fulfilling the Future API and its behaviors. Here is an example illustrating their properties:
+#### Sequential Futures
+Sequential futures are the default unless otherwise specified. They were designed to behave as similar as possible to regular R evaluation while still fulfilling the Future API and its behaviors. Here is an example illustrating their properties:
```r
<%=withCapture({
-plan(eager)
+plan(sequential)
<%@include file="incl/future-1-overview-example2.R"%>
})%>
```
-Since eager evaluation is taking place, each of the three futures is resolved instantaneously in the moment it is created. Note also how `pid` in the calling environment, which was assigned the process ID of the current process, is neither overwritten nor removed. This is because futures are evaluated in a local environment. Since synchronous (uni-)processing is used, future `b` is resolved by the main R process (still in a local environment), which is why the value of `b` and `pid` a [...]
+Since eager sequential evaluation is taking place, each of the three futures is resolved instantaneously in the moment it is created. Note also how `pid` in the calling environment, which was assigned the process ID of the current process, is neither overwritten nor removed. This is because futures are evaluated in a local environment. Since synchronous (uni-)processing is used, future `b` is resolved by the main R process (still in a local environment), which is why the value of `b` [...]
<%
## Sanity checks
stopifnot(b == pid)
%>
-#### Lazy Futures
-A lazy future evaluates its expression only if its value is queried. Evaluation can also be triggered when the future is checked for being resolved or not. Here is the above example when using lazy evaluation:
-```r
-<%=withCapture({
-plan(lazy)
-<%@include file="incl/future-1-overview-example2.R"%>
-})%>
-```
-As previously, variable `pid` is unaffected because all evaluation is done in a local environment. More interestingly, future `a` is no longer evaluated in the moment it is created, but instead when it is needed the first time, which happens when future `c` is created. This is because `a` is identified as a global variable that needs to be captured ("frozen" to `a == 3.14`) in order to set up future `c`. Later when `c` (the value of future `c`) is queried, `a` has already been resolve [...]
-
-_Comment_: Lazy evaluation is already used by R itself. Arguments are passed to functions using lazy evaluation. It is also possible to assign variables using lazy evaluation using `delayedAssign()`, but contrary to lazy futures this function does not freeze globals. For more information, see `help("delayedAssign", package="base")`.
-
-<%
-## Sanity checks
-stopifnot(b == pid)
-%>
-
-
-<%---
#### Transparent Futures
-
-For troubleshooting, there is also a _transparent_ future, which can be specified as `plan(transparent)`. A transparent future is technically a eager future with instant signaling of conditions (including errors and warnings) and where evaluation, and therefore also assignments, take place in the calling environment. Transparent futures are particularly useful for troubleshooting errors.
-
----%>
-
-
+For troubleshooting, _transparent_ futures can be used by specifying `plan(transparent)`. A transparent future is technically a sequential future with instant signaling of conditions (including errors and warnings) and where evaluation, and therefore also assignments, take place in the calling environment. Transparent futures are particularly useful for troubleshooting errors that are otherwise hard to narrow down.
### Asynchronous Futures
@@ -273,7 +250,7 @@ When multisession evaluation is used, the package launches a set of R sessions i
availableCores()
})%>
```
-This particular result tells us that the `mc.cores` option was set such that we are allowed to use in total <%= availableCores() %> processes including the main process. In other words, with these settings, there will be <%= availableCores()-1 %> background processes serving the multisession futures. The `availableCores()` is also agile to different options and system environment variables. For instance, if compute cluster schedulers are used (e.g. TORQUE/PBS and Slurm), they set spec [...]
+This particular result tells us that the `mc.cores` option was set such that we are allowed to use in total <%= availableCores() %> processes including the main process. In other words, with these settings, there will be <%= availableCores() %> background processes serving the multisession futures. The `availableCores()` is also agile to different options and system environment variables. For instance, if compute cluster schedulers are used (e.g. TORQUE/PBS and Slurm), they set specif [...]
#### Multicore Futures
@@ -294,7 +271,7 @@ plan(multicore)
## multicore processing, let's fake the future based on
## the output of the multisession example
bfr <- gsub("multisession", "multicore", bfr)
-bfr <- gsub(b, b+5L, bfr)
+bfr <- gsub(b, b + 5L, bfr)
bfr
}%>
```
@@ -321,14 +298,14 @@ Cluster futures evaluate expressions on an ad-hoc cluster (as implemented by the
## The second best we can do is to use the local machine.
hosts <- c("localhost", "localhost", "localhost")
bfr <- withCapture({
-plan(cluster, workers=hosts)
+plan(cluster, workers = hosts)
<%@include file="incl/future-1-overview-example2.R"%>
})
bfr <- gsub('hosts', 'c("n1", "n2", "n3")', bfr)
bfr
%>
```
-Just as for the other asynchronous evaluation strategies, the output from `cat()` is not displayed on the current/calling machine.
+Just as for most other asynchronous evaluation strategies, the output from `cat()` is not displayed on the current/calling machine.
<%
## Sanity checks
@@ -338,19 +315,21 @@ stopifnot(b != pid)
Any types of clusters that `parallel::makeCluster()` creates can be used for cluster futures. For instance, the above cluster can be explicitly set up as:
```r
cl <- parallel::makeCluster(c("n1", "n2", "n3"))
-plan(cluster, workers=cl)
+plan(cluster, workers = cl)
```
-Also, it is considered good style to shut down the cluster when it is no longer needed, that is, calling `parallel::stopCluster(cl)`. However, it will shut itself down if the main process is terminated, which will happen in the first example where the cluster in created internally. For more information on how to set up and manage such clusters, see `help("makeCluster", package="parallel")`.
+Also, it is considered good style to shut down cluster `cl` when it is no longer needed, that is, calling `parallel::stopCluster(cl)`. However, it will shut itself down if the main process is terminated. For more information on how to set up and manage such clusters, see `help("makeCluster", package = "parallel")`.
+Clusters created implicitly using `plan(cluster, workers = hosts)` where `hosts` is a character vector will also be shut down when the main R session terminates, or when the future strategy is changed, e.g. by calling `plan(sequential)`.
Note that with automatic authentication setup (e.g. SSH key pairs), there is nothing preventing us from using the same approach for using a cluster of remote machines.
+<%---
### Different Strategies for Different Futures
Sometimes one may want to use an alternative evaluation strategy for a specific future. Although one can use `old <- plan(new)` and afterward `plan(old)` to temporarily switch strategies, a simpler approach is to use the `%plan%` operator, e.g.
```r
<%=withCapture({
-plan(eager)
+plan(sequential)
pid <- Sys.getpid()
pid
a %<-% { Sys.getpid() }
@@ -361,7 +340,7 @@ b
c
})%>
```
-As seen by the different process IDs, future `a` is evaluated eagerly using the same process as the calling environment whereas the other two are evaluated using multiprocess futures.
+As seen by the different process IDs, future `a` is evaluated sequentially using the same process as the calling environment whereas the other two are evaluated using multiprocess futures.
<%
## Sanity checks
@@ -378,7 +357,7 @@ if (availableCores() == 1L) {
However, using different plans to individual futures this way has the drawback of hard coding the evaluation strategy. Doing so may prevent some users from using your script or your package, because they do not have the sufficient resources. It may also prevent users with a lot of resources from utilizing those because you assumed a less-powerful set of hardware. Because of this, we recommend against the use of `%plan%` other than for interactive prototyping.
-
+---%>
### Nested Futures and Evaluation Topologies
This far we have discussed what can be referred to as "flat topology" of futures, that is, all futures are created in and assigned to the same environment. However, there is nothing stopping us from using a "nested topology" of futures, where one set of futures may, in turn, create another set of futures internally and so on.
@@ -393,7 +372,7 @@ a
b
})%>
```
-By inspection the process IDs, we see that there are in total three different processes involved for resolving the futures. There is the main R process (pid <%= pid %>), and there are the two processes used by `a` (pid <%= a %>) and `b` (pid <%= b[1] %>). However, the two futures (`b1` and `b2`) that is nested by `b` are evaluated by the same R process as `b`. This is because nested futures use eager evaluation unless otherwise specified. There are a few reasons for this, but the mai [...]
+By inspection the process IDs, we see that there are in total three different processes involved for resolving the futures. There is the main R process (pid <%= pid %>), and there are the two processes used by `a` (pid <%= a %>) and `b` (pid <%= b[1] %>). However, the two futures (`b1` and `b2`) that is nested by `b` are evaluated by the same R process as `b`. This is because nested futures use sequential evaluation unless otherwise specified. There are a few reasons for this, but th [...]
<%
## Sanity checks
@@ -407,9 +386,9 @@ if (availableCores() == 1L) {
%>
-To specify a different type of _evaluation topology_, other than the first level of futures being resolved by multiprocess evaluation and the second level by eager evaluation, we can provide a list of evaluation strategies to `plan()`. First, the same evaluation strategies as above can be explicitly specified as:
+To specify a different type of _evaluation topology_, other than the first level of futures being resolved by multiprocess evaluation and the second level by sequential evaluation, we can provide a list of evaluation strategies to `plan()`. First, the same evaluation strategies as above can be explicitly specified as:
```r
-plan(list(multiprocess, eager))
+plan(list(multiprocess, sequential))
```
We would actually get the same behavior if we try with multiple levels of multiprocess evaluations;
```r
@@ -426,7 +405,7 @@ a
b
})%>
```
-The reason for this is, also here, to protect us from launching more processes than what the machine can support. Internally, this is done by setting `mc.cores` to zero ([sic!](https://github.com/HenrikBengtsson/Wishlist-for-R/issues/7)) such that no _additional_ parallel processes can be launched. This is the case for both multisession and multicore evaluation.
+The reason for this is, also here, to protect us from launching more processes than what the machine can support. Internally, this is done by setting `mc.cores = 1` such that functions like `parallel::mclapply()` will fall back to run sequentially. This is the case for both multisession and multicore evaluation.
<%
## Sanity checks
@@ -439,10 +418,10 @@ if (availableCores() == 1L) {
}
%>
-Continuing, if we start off by eager evaluation and then use multiprocess evaluation for any nested futures, we get:
+Continuing, if we start off by sequential evaluation and then use multiprocess evaluation for any nested futures, we get:
```r
<%=withCapture({
-plan(list(eager, multiprocess))
+plan(list(sequential, multiprocess))
})%>
[...]
<%
@@ -473,8 +452,8 @@ Having said this, it is indeed possible to use nested multiprocess evaluation st
```r
<%=withCapture({
plan(list(
- tweak(multiprocess, workers=3L),
- tweak(multiprocess, workers=3L)
+ tweak(multiprocess, workers = 2L),
+ tweak(multiprocess, workers = 2L)
))
})%>
[...]
@@ -498,8 +477,6 @@ if (supportsMulticore()) {
}
%>
-To clarify, when we set up the two levels of multiprocess evaluation, we specified that in total 3 processes may be used at each level. We choose three parallel processes, not just two, because one is always consumed by the calling process leaving two to be used for the asynchronous futures. This is why we see that `pid`, `a` and `b` are all resolved by the same process. If we had allowed only two cores at the top level, `a` and `b` would have been resolved by the same background proc [...]
-
For more details on working with nested futures and different evaluation strategies at each level, see Vignette '[Futures in R: Future Topologies]'.
@@ -531,18 +508,18 @@ a
## Failed Futures
-Sometimes the future is not what you expected. If an error occurs while evaluating a future, the error is propagated and thrown as an error in the calling environment _when the future value is requested_. For example,
+Sometimes the future is not what you expected. If an error occurs while evaluating a future, the error is propagated and thrown as an error in the calling environment _when the future value is requested_. For example, if we use lazy evaluation on a future that generates an error, we might see something like
```r
<%=withCapture({
-plan(lazy)
+plan(sequential)
a %<-% {
cat("Resolving 'a' ...\n")
stop("Whoops!")
42
-}
+} %lazy% TRUE
cat("Everything is still ok although we have created a future that will fail.\n")
})%>
-<% captureOutput(try(resolve(a), silent=TRUE)) %>
+<% captureOutput(try(resolve(a), silent = TRUE)) %>
> a
Resolving 'a' ...
Error in eval(expr, envir, enclos) : Whoops!
@@ -564,11 +541,11 @@ backtrace(a)
## Globals
-Whenever an R expression is to be evaluated asynchronously (in parallel) or via lazy evaluation, global objects have to be identified and passed to the evaluator. They need to be passed exactly as they were at the time the future was created, because, for a lazy future, globals may otherwise change between when it is created and when it is resolved. For asynchronous processing, the reason globals need to be identified is so that they can be exported to the process that evaluates the future.
+Whenever an R expression is to be evaluated asynchronously (in parallel) or sequentially via lazy evaluation, global (aka "free") objects have to be identified and passed to the evaluator. They need to be passed exactly as they were at the time the future was created, because, for lazy evaluation, globals may otherwise change between when it is created and when it is resolved. For asynchronous processing, the reason globals need to be identified is so that they can be exported to the p [...]
-The future package tries to automate these tasks as far as possible. It does this with help of the [globals] package. If a global variable is identified, it is captured and made available to the evaluating process.
+The future package tries to automate these tasks as far as possible. It does this with help of the [globals] package, which uses static-code inspection to identify global variables. If a global variable is identified, it is captured and made available to the evaluating process.
<%---
-If it identifies a symbol that it believes is a global object, but it fails to locate it in the calling environment (or any the environment accessible from that one), an error is thrown immediately. This minimizing the risk for runtime errors occurring later (sometimes much later) and in a different process (possible in a remote R session), which otherwise may be hard to troubleshoot. For instance,
+If it identifies a symbol that it believes is a global object, but it fails to locate it in the calling environment (or any the environment accessible from that one), an error is thrown immediately. This minimizing the risk for run-time errors occurring later (sometimes much later) and in a different process (possible in a remote R session), which otherwise may be hard to troubleshoot. For instance,
```r
> rm(a)
> x <- 5.0
@@ -585,14 +562,13 @@ failed to locate the corresponding object in the relevant environments: 'a'
---%>
Moreover, if a global is defined in a package, then that global is not exported. Instead, it is made sure that the corresponding package is attached when the future is evaluated. This not only better reflects the setup of the main R session, but it also minimizes the need for exporting globals, which saves not only memory but also time and bandwidth, especially when using remote compute nodes.
<%---
-As mentioned previously, for consistency across evaluation strategies, all types of futures validate globals upon creation. This is also true for cases where it would not be necessary, e.g. for eager evaluation of multicore evaluation (which forks the calling process "as-is"). However, in order to make it as easy as possible to switch between strategies without being surprised by slightly different behaviors, the Future API is designed to check for globals the same way regardless of strategy.
+As mentioned previously, for consistency across evaluation strategies, all types of futures validate globals upon creation. This is also true for cases where it would not be necessary, e.g. for eager evaluation of multicore futures (which forks the calling process "as-is"). However, in order to make it as easy as possible to switch between strategies without being surprised by slightly different behaviors, the Future API is designed to check for globals the same way regardless of strategy.
---%>
<%---
-Having said this, it is possible to disable validation of globals by setting `globals=FALSE`. This could make sense if one know for sure that only eager or multicore futures will be used. This argument can be tweaked as `plan(tweak(eager, globals=FALSE))` and `plan(tweak(multicore, globals=FALSE))`. However, it is strongly advised not to do this. Instead, as a best practice, it is always good if the code/script works with any type of futures.
+Having said this, it is possible to disable validation of globals by setting `globals = FALSE`. This could make sense if one know for sure that only sequential or multicore futures will be used. This argument can be tweaked as `plan(tweak(sequential, globals=FALSE))` and `plan(tweak(multicore, globals=FALSE))`. However, it is strongly advised not to do this. Instead, as a best practice, it is always good if the code/script works with any type of futures.
---%>
-Finally, it should be clarified that identifying globals from static code inspection alone is a challenging problem. There will always be corner cases where automatic identification of globals fails so that either false globals are identified (less of a concern) or some of the true globals are missing (which will result in a runtime error or possibly the wrong results). Vignette '[Futures in R: Common Issues with Solutions]' provides examples of common cases and explains how to avoid t [...]
-
+Finally, it should be clarified that identifying globals from static code inspection alone is a challenging problem. There will always be corner cases where automatic identification of globals fails so that either false globals are identified (less of a concern) or some of the true globals are missing (which will result in a run-time error or possibly the wrong results). Vignette '[Futures in R: Common Issues with Solutions]' provides examples of common cases and explains how to avoid [...]
## Constraints when using Implicit Futures
@@ -605,7 +581,7 @@ f <- list()
for (ii in 1:3) {
f[[ii]] <- future({ Sys.getpid() })
}
-v <- lapply(f, FUN=value)
+v <- lapply(f, FUN = value)
str(v)
})%>
```
@@ -641,29 +617,21 @@ As previously, `as.list(v)` blocks until all futures are resolved.
## Demos
-To see a live illustration how different types of futures are evaluated, run the Mandelbrot demo of this package. First, try with the eager evaluation,
+To see a live illustration how different types of futures are evaluated, run the Mandelbrot demo of this package. First, try with the sequential evaluation,
```r
library("future")
-plan(eager)
-demo("mandelbrot", package="future", ask=FALSE)
+plan(sequential)
+demo("mandelbrot", package = "future", ask = FALSE)
```
-which closely imitates how the script would run if futures were not used. Then try the same using lazy evaluation,
-```r
-plan(lazy)
-demo("mandelbrot", package="future", ask=FALSE)
-```
-and see if you can notice the difference in how and when statements are evaluated.
-You may also try multiprocess evaluation, which calculates the different Mandelbrot planes using parallel R processes running in the background. Try,
+which resembles how the script would run if futures were not used. Then, try multiprocess evaluation, which calculates the different Mandelbrot planes using parallel R processes running in the background. Try,
```r
plan(multiprocess)
-demo("mandelbrot", package="future", ask=FALSE)
+demo("mandelbrot", package = "future", ask = FALSE)
```
-This will use multicore processing if you are on a system where R supports process forking, otherwise (such as on Windows) it will use multisession processing.
-
Finally, if you have access to multiple machines you can try to set up a cluster of workers and use them, e.g.
```r
-plan(cluster, workers=c("n2", "n5", "n6", "n6", "n9"))
-demo("mandelbrot", package="future", ask=FALSE)
+plan(cluster, workers = c("n2", "n5", "n6", "n6", "n9"))
+demo("mandelbrot", package = "future", ask = FALSE)
```
@@ -673,11 +641,13 @@ The goal of this package is to provide a standardized and unified API for using
[BatchJobs]: https://cran.r-project.org/package=BatchJobs
+[batchtools]: https://cran.r-project.org/package=batchtools
[future]: https://cran.r-project.org/package=future
[future.BatchJobs]: https://cran.r-project.org/package=future.BatchJobs
+[future.batchtools]: https://cran.r-project.org/package=future.batchtools
[globals]: https://cran.r-project.org/package=globals
[listenv]: https://cran.r-project.org/package=listenv
[Futures in R: Common Issues with Solutions]: future-2-issues.html
[Futures in R: Future Topologies]: future-3-topologies.html
---
-Copyright Henrik Bengtsson, 2015-2016
+Copyright Henrik Bengtsson, 2015-2017
diff --git a/inst/vignettes-static/incl/future-1-overview-example3.R b/inst/vignettes-static/incl/future-1-overview-example3.R
index 54527a9..b60dfc8 100644
--- a/inst/vignettes-static/incl/future-1-overview-example3.R
+++ b/inst/vignettes-static/incl/future-1-overview-example3.R
@@ -15,5 +15,5 @@ b %<-% {
cat("Resolving 'b2' ...\n")
Sys.getpid()
}
- c(b.pid=Sys.getpid(), b1.pid=b1, b2.pid=b2)
+ c(b.pid = Sys.getpid(), b1.pid = b1, b2.pid = b2)
}
diff --git a/man/ClusterFuture-class.Rd b/man/ClusterFuture-class.Rd
index 7b748d3..6dc4da7 100644
--- a/man/ClusterFuture-class.Rd
+++ b/man/ClusterFuture-class.Rd
@@ -1,22 +1,25 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/ClusterFuture-class.R
\name{ClusterFuture-class}
-\alias{ClusterFuture}
\alias{ClusterFuture-class}
+\alias{ClusterFuture}
\alias{MultisessionFuture}
\alias{MultisessionFuture-class}
-\title{A cluster future is a future whose value will be resolved asynchroneously in a parallel process}
+\title{A cluster future is a future whose value will be resolved asynchronously in a parallel process}
\usage{
ClusterFuture(expr = NULL, envir = parent.frame(), substitute = FALSE,
- local = !persistent, globals = TRUE, gc = FALSE, persistent = FALSE,
- workers = NULL, user = NULL, master = NULL, revtunnel = TRUE,
- homogeneous = TRUE, ...)
+ globals = TRUE, packages = NULL, local = !persistent, gc = FALSE,
+ persistent = FALSE, workers = NULL, user = NULL, master = NULL,
+ revtunnel = TRUE, homogeneous = TRUE, ...)
}
\arguments{
-\item{local}{If TRUE, the expression is evaluated such that
-all assignments are done to local temporary environment, otherwise
-the assignments are done to the global environment of the \R process
-evaluating the future.}
+\item{expr}{An R \link[base]{expression}.}
+
+\item{envir}{The \link{environment} in which the evaluation
+is done (or inherits from if \code{local} is TRUE).}
+
+\item{substitute}{If TRUE, argument \code{expr} is
+\code{\link[base]{substitute}()}:ed, otherwise not.}
\item{globals}{(optional) a logical, a character vector,
or a named list for controlling how globals are handled.
@@ -46,16 +49,19 @@ same path to \file{Rscript} as the main R session. If FALSE, the
it is assumed to be on the PATH for each node.}
\item{...}{Additional named elements passed to \code{\link{Future}()}.}
+
+\item{sessioninfo}{If TRUE, session information is collected for each
+cluster node, otherwise not. This also servers as testing that each
+node is working properly.}
}
\value{
An object of class \code{ClusterFuture}.
}
\description{
-A cluster future is a future whose value will be resolved asynchroneously in a parallel process
+A cluster future is a future whose value will be resolved asynchronously in a parallel process
}
\seealso{
To evaluate an expression using "cluster future", see function
\code{\link{cluster}()}.
}
\keyword{internal}
-
diff --git a/man/ConstantFuture-class.Rd b/man/ConstantFuture-class.Rd
index bd26f7e..25421d1 100644
--- a/man/ConstantFuture-class.Rd
+++ b/man/ConstantFuture-class.Rd
@@ -1,12 +1,12 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/ConstantFuture-class.R
\name{ConstantFuture-class}
-\alias{ConstantFuture}
\alias{ConstantFuture-class}
+\alias{ConstantFuture}
\title{A future with a constant value}
\usage{
ConstantFuture(expr = NULL, envir = emptyenv(), substitute = FALSE,
- local = FALSE, ...)
+ globals = NULL, packages = NULL, local = FALSE, ...)
}
\arguments{
\item{expr}{An R \link[base]{expression}.}
@@ -17,11 +17,16 @@ is done (or inherits from if \code{local} is TRUE).}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
+\item{globals}{(optional) a named list of global objects needed in order
+for the future to be resolved correctly.}
+
+\item{packages}{(optional) a character vector specifying packages
+to be attached in the R environment evaluating the future.}
+
\item{local}{If TRUE, the expression is evaluated such that
all assignments are done to local temporary environment, otherwise
-the assignments are done in the calling environment.}
-
-\item{...}{Additional named elements of the future.}
+the assignments are done to the global environment of the \R process
+evaluating the future.}
\item{\dots}{Not used.}
}
@@ -33,4 +38,3 @@ A constant future is a future whose expression is a constant
and therefore by definition already resolved upon creation.
}
\keyword{internal}
-
diff --git a/man/Future-class.Rd b/man/Future-class.Rd
index 244d0aa..191c8e4 100644
--- a/man/Future-class.Rd
+++ b/man/Future-class.Rd
@@ -1,11 +1,12 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/Future-class.R
\name{Future-class}
-\alias{Future}
\alias{Future-class}
+\alias{Future}
\title{A future represents a value that will be available at some point in the future}
\usage{
Future(expr = NULL, envir = parent.frame(), substitute = FALSE,
+ globals = NULL, packages = NULL, seed = NULL, lazy = FALSE,
local = TRUE, gc = FALSE, earlySignal = FALSE, label = NULL, ...)
}
\arguments{
@@ -17,9 +18,21 @@ is done (or inherits from if \code{local} is TRUE).}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
+\item{globals}{(optional) a named list of global objects needed in order
+for the future to be resolved correctly.}
+
+\item{packages}{(optional) a character vector specifying packages
+to be attached in the R environment evaluating the future.}
+
+\item{seed}{(optional) A L'Ecuyer-CMRG RNG seed.}
+
+\item{lazy}{If \code{FALSE} (default), the future is resolved
+eagerly (starting immediately), otherwise not.}
+
\item{local}{If TRUE, the expression is evaluated such that
all assignments are done to local temporary environment, otherwise
-the assignments are done in the calling environment.}
+the assignments are done to the global environment of the \R process
+evaluating the future.}
\item{gc}{If TRUE, the garbage collector run (in the process that
evaluated the future) after the value of the future is collected.
@@ -55,4 +68,3 @@ assignment operator, which creates a future from the
right-hand-side (RHS) R expression and assigns its future value
to a variable as a \emph{\link[base]{promise}}.
}
-
diff --git a/man/FutureError.Rd b/man/FutureError.Rd
index a119913..8b998a9 100644
--- a/man/FutureError.Rd
+++ b/man/FutureError.Rd
@@ -23,4 +23,3 @@ An \link[base:conditions]{error} object of class FutureError.
An error occurred while trying to evaluate a future
}
\keyword{internal}
-
diff --git a/man/FutureGlobals.Rd b/man/FutureGlobals.Rd
new file mode 100644
index 0000000..37a7f0c
--- /dev/null
+++ b/man/FutureGlobals.Rd
@@ -0,0 +1,36 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/FutureGlobals-class.R
+\name{FutureGlobals}
+\alias{FutureGlobals}
+\alias{as.FutureGlobals}
+\alias{as.FutureGlobals.FutureGlobals}
+\alias{as.FutureGlobals.Globals}
+\alias{as.FutureGlobals.list}
+\alias{[.FutureGlobals}
+\alias{c.FutureGlobals}
+\alias{unique.FutureGlobals}
+\title{A representation of a set of globals used with futures}
+\usage{
+FutureGlobals(object = list(), resolved = FALSE, total_size = NA_real_,
+ ...)
+}
+\arguments{
+\item{object}{A named list.}
+
+\item{resolved}{A logical indicating whether these globals
+have been scanned for and resolved futures or not.}
+
+\item{total_size}{The total size of all globals, if known.}
+
+\item{\dots}{Not used.}
+}
+\value{
+An object of class \code{FutureGlobals}.
+}
+\description{
+A representation of a set of globals used with futures
+}
+\details{
+This class extends the \link[globals]{Globals} class by add
+attribute \code{resolved}.
+}
diff --git a/man/MulticoreFuture-class.Rd b/man/MulticoreFuture-class.Rd
index 811aeef..762207d 100644
--- a/man/MulticoreFuture-class.Rd
+++ b/man/MulticoreFuture-class.Rd
@@ -1,25 +1,32 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/MulticoreFuture-class.R
\name{MulticoreFuture-class}
-\alias{MulticoreFuture}
\alias{MulticoreFuture-class}
-\title{An multicore future is a future whose value will be resolved asynchroneously in a parallel process}
+\alias{MulticoreFuture}
+\title{An multicore future is a future whose value will be resolved asynchronously in a parallel process}
\usage{
MulticoreFuture(expr = NULL, envir = parent.frame(), substitute = FALSE,
globals = TRUE, ...)
}
\arguments{
+\item{expr}{An R \link[base]{expression}.}
+
+\item{envir}{The \link{environment} in which the evaluation
+is done (or inherits from if \code{local} is TRUE).}
+
+\item{substitute}{If TRUE, argument \code{expr} is
+\code{\link[base]{substitute}()}:ed, otherwise not.}
+
\item{...}{Additional named elements passed to \code{\link{Future}()}.}
}
\value{
An object of class \code{MulticoreFuture}.
}
\description{
-An multicore future is a future whose value will be resolved asynchroneously in a parallel process
+An multicore future is a future whose value will be resolved asynchronously in a parallel process
}
\seealso{
To evaluate an expression using "multicore future", see function
\code{\link{multicore}()}.
}
\keyword{internal}
-
diff --git a/man/MultiprocessFuture-class.Rd b/man/MultiprocessFuture-class.Rd
index 0396a02..cca86d7 100644
--- a/man/MultiprocessFuture-class.Rd
+++ b/man/MultiprocessFuture-class.Rd
@@ -1,9 +1,9 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/MultiprocessFuture-class.R
\name{MultiprocessFuture-class}
-\alias{MultiprocessFuture}
\alias{MultiprocessFuture-class}
-\title{An multiprocess future is a future whose value will be resolved asynchroneously in a parallel process}
+\alias{MultiprocessFuture}
+\title{An multiprocess future is a future whose value will be resolved asynchronously in a parallel process}
\usage{
MultiprocessFuture(expr = NULL, envir = parent.frame(),
substitute = FALSE, ...)
@@ -17,24 +17,16 @@ is done (or inherits from if \code{local} is TRUE).}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
-\item{...}{Additional named elements of the future.}
-
-\item{local}{If TRUE, the expression is evaluated such that
-all assignments are done to local temporary environment, otherwise
-the assignments are done to the global environment of the \R process
-evaluating the future.}
-
\item{\dots}{Additional named elements passed to \code{\link{Future}()}.}
}
\value{
An object of class \code{MultiprocessFuture}.
}
\description{
-An multiprocess future is a future whose value will be resolved asynchroneously in a parallel process
+An multiprocess future is a future whose value will be resolved asynchronously in a parallel process
}
\seealso{
To evaluate an expression using "multiprocess future", see function
\code{\link{multiprocess}()}.
}
\keyword{internal}
-
diff --git a/man/UniprocessFuture-class.Rd b/man/UniprocessFuture-class.Rd
index e595388..2bfcace 100644
--- a/man/UniprocessFuture-class.Rd
+++ b/man/UniprocessFuture-class.Rd
@@ -1,20 +1,24 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/UniprocessFuture-class.R
-\name{UniprocessFuture-class}
+% Please edit documentation in R/DEPRECATED.R, R/UniprocessFuture-class.R
+\name{EagerFuture}
\alias{EagerFuture}
\alias{LazyFuture}
-\alias{UniprocessFuture}
\alias{UniprocessFuture-class}
-\title{An uniprocess future is a future whose value will be resolved synchroneously in the current process}
+\alias{UniprocessFuture}
+\alias{SequentialFuture}
+\title{An uniprocess future is a future whose value will be resolved synchronously in the current process}
\usage{
-UniprocessFuture(expr = NULL, envir = parent.frame(), substitute = FALSE,
- globals = TRUE, local = TRUE, lazy = FALSE, ...)
-
EagerFuture(expr = NULL, envir = parent.frame(), substitute = FALSE,
- globals = TRUE, local = TRUE, lazy = FALSE, ...)
+ lazy = FALSE, globals = TRUE, local = TRUE, ...)
LazyFuture(expr = NULL, envir = parent.frame(), substitute = FALSE,
- globals = TRUE, local = TRUE, lazy = FALSE, ...)
+ lazy = TRUE, globals = TRUE, local = TRUE, ...)
+
+UniprocessFuture(expr = NULL, envir = parent.frame(), substitute = FALSE,
+ globals = TRUE, packages = NULL, lazy = FALSE, local = TRUE, ...)
+
+SequentialFuture(expr = NULL, envir = parent.frame(), substitute = FALSE,
+ lazy = FALSE, globals = TRUE, local = TRUE, ...)
}
\arguments{
\item{expr}{An R \link[base]{expression}.}
@@ -25,26 +29,30 @@ is done (or inherits from if \code{local} is TRUE).}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
-\item{local}{If TRUE, the expression is evaluated such that
-all assignments are done to local temporary environment, otherwise
-the assignments are done in the calling environment.}
-
\item{lazy}{If \code{FALSE} (default), then the setup and validation of
global variables are done for eager evaluation, otherwise not.}
-\item{...}{Additional named elements of the future.}
+\item{globals}{(optional) a named list of global objects needed in order
+for the future to be resolved correctly.}
+
+\item{local}{If TRUE, the expression is evaluated such that
+all assignments are done to local temporary environment, otherwise
+the assignments are done to the global environment of the \R process
+evaluating the future.}
\item{\dots}{Additional named elements passed to \code{\link{Future}()}.}
+
+\item{packages}{(optional) a character vector specifying packages
+to be attached in the R environment evaluating the future.}
}
\value{
An object of class \code{UniprocessFuture}.
}
\description{
-An uniprocess future is a future whose value will be resolved synchroneously in the current process
+An uniprocess future is a future whose value will be resolved synchronously in the current process
}
\seealso{
To evaluate an expression using "uniprocess future", see functions
\code{\link{uniprocess}()}.
}
\keyword{internal}
-
diff --git a/man/as.cluster.Rd b/man/as.cluster.Rd
index 5016f50..9bd0ea2 100644
--- a/man/as.cluster.Rd
+++ b/man/as.cluster.Rd
@@ -2,10 +2,10 @@
% Please edit documentation in R/as.cluster.R
\name{as.cluster}
\alias{as.cluster}
-\alias{as.cluster.SOCK0node}
-\alias{as.cluster.SOCKnode}
\alias{as.cluster.cluster}
\alias{as.cluster.list}
+\alias{as.cluster.SOCKnode}
+\alias{as.cluster.SOCK0node}
\alias{c.cluster}
\title{Coerce an object to a cluster object}
\usage{
@@ -37,4 +37,3 @@ An object of class \code{cluster}.
\description{
Coerce an object to a cluster object
}
-
diff --git a/man/availableCores.Rd b/man/availableCores.Rd
index 62f8e00..61d0312 100644
--- a/man/availableCores.Rd
+++ b/man/availableCores.Rd
@@ -5,15 +5,15 @@
\title{Get number of available cores on current machine}
\usage{
availableCores(constraints = NULL,
- methods = getOption("future.availableCores.methods", c("system",
- "mc.cores+1", "_R_CHECK_LIMIT_CORES_", "PBS", "SGE", "Slurm")),
- na.rm = TRUE, default = c(current = 1L), which = c("min", "max", "all"))
+ methods = getOption("future.availableCores.methods", c("system", "mc.cores",
+ "_R_CHECK_LIMIT_CORES_", "PBS", "SGE", "Slurm", "fallback")), na.rm = TRUE,
+ default = c(current = 1L), which = c("min", "max", "all"))
}
\arguments{
\item{constraints}{An optional character specifying under what
constraints ("purposes") we are requesting the values.
For instance, on systems where multicore processing is not supported
-(i.e. Windows), using \code{constrains="multicore"} will force a
+(i.e. Windows), using \code{constrains = "multicore"} will force a
single core to be reported.}
\item{methods}{A character vector specifying how to infer the number
@@ -30,9 +30,9 @@ If \code{"max"}, the maximum value is returned (be careful!)
If \code{"all"}, all values are returned.}
}
\value{
-Return a positive (>=1) integer.
-If \code{which="all"}, then more than one value may be returned.
-Together with \code{na.rm=FALSE} missing values may also be returned.
+Return a positive (>= 1) integer.
+If \code{which = "all"}, then more than one value may be returned.
+Together with \code{na.rm = FALSE} missing values may also be returned.
}
\description{
The current/main R session counts as one, meaning the minimum
@@ -44,12 +44,12 @@ are supported:
\itemize{
\item \code{"system"} -
Query \code{\link[parallel]{detectCores}()}.
- \item \code{"mc.cores+1"} -
+ \item \code{"mc.cores"} -
If available, returns the value of option
- \code{\link[base:options]{mc.cores}} + 1.
+ \code{\link[base:options]{mc.cores}}.
Note that \option{mc.cores} is defined as the number of
\emph{additional} R processes that can be used in addition to the
- main R process. This means that with \code{mc.cores=0} all
+ main R process. This means that with \code{mc.cores = 0} all
calculations should be done in the main R process, i.e. we have
exactly one core available for our calculations.
The \option{mc.cores} option defaults to environment variable
@@ -57,16 +57,17 @@ are supported:
package is loaded). The \option{mc.cores} option is used by for
instance \code{\link[parallel]{mclapply}()}.
\item \code{"PBS"} -
- Query Torque/PBS environment variable \env{PBS_NUM_PPN}.
- Depending on PBS system configuration, this \emph{resource} parameter
- may or may not default to one. It can be specified when submitting
- a job as in, for instance, \code{qsub -l nodes=4:ppn=2}, which
- requests four nodes each with two cores.
+ Query TORQUE/PBS environment variable \env{PBS_NUM_PPN}.
+ Depending on PBS system configuration, this \emph{resource}
+ parameter may or may not default to one.
+ An example of a job submission that results in this is
+ \code{qsub -l nodes=1:ppn=2}, which requests one node with two cores.
\item \code{"SGE"} -
Query Sun/Oracle Grid Engine (SGE) environment variable
\env{NSLOTS}.
- It can be specified when submitting a job as in, for instance,
- \code{qsub -pe by_node 2}, which two cores on a single machine.
+ An example of a job submission that results in this is
+ \code{qsub -pe smp 2} (or \code{qsub -pe by_node 2}), which
+ requests two cores on a single machine.
\item \code{"Slurm"} -
Query Simple Linux Utility for Resource Management (Slurm)
environment variable \env{SLURM_CPUS_PER_TASK}.
@@ -81,15 +82,17 @@ variable is queried. If neither is set, a missing value is returned.
\section{Advanced usage}{
It is possible to override the maximum number of cores on the machine
-as reported by \code{availableCores(methods="system")}. This can be
+as reported by \code{availableCores(methods = "system")}. This can be
done by first specifying
-\code{options(future.availableCores.methods="mc.cores+1")} and
-then the number of cores to use (in addition to the main R process),
-e.g. \code{options(mc.cores=8)} will cause the value of
-\code{availableCores()} to be 9 (=8+1).
+\code{options(future.availableCores.methods = "mc.cores")} and
+then the number of cores to use, e.g. \code{options(mc.cores = 8)}.
Having said this, it is almost always better to do this by explicitly
setting the number of workers when specifying the future strategy,
-e.g. \code{plan(multiprocess, workers=9)}.
+e.g. \code{plan(multiprocess, workers = 8)}.
}
-\keyword{internal}
+\seealso{
+To get the number of available workers regardless of machine,
+see \code{\link{availableWorkers}()}.
+}
+\keyword{internal}
diff --git a/man/availableWorkers.Rd b/man/availableWorkers.Rd
new file mode 100644
index 0000000..9067744
--- /dev/null
+++ b/man/availableWorkers.Rd
@@ -0,0 +1,61 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/availableWorkers.R
+\name{availableWorkers}
+\alias{availableWorkers}
+\title{Get set of available workers}
+\usage{
+availableWorkers(methods = getOption("future.availableWorkers.methods",
+ c("mc.cores", "_R_CHECK_LIMIT_CORES_", "PBS", "SGE", "Slurm", "system",
+ "fallback")), na.rm = TRUE, default = "localhost", which = c("auto",
+ "min", "max", "all"))
+}
+\arguments{
+\item{methods}{A character vector specifying how to infer the number
+of available cores.}
+
+\item{na.rm}{If TRUE, only non-missing settings are considered/returned.}
+
+\item{default}{The default set of workers.}
+
+\item{which}{A character specifying which set / sets to return.
+If \code{"auto"}, the first non-empty set found.
+If \code{"min"}, the minimum value is returned.
+If \code{"max"}, the maximum value is returned (be careful!)
+If \code{"all"}, all values are returned.}
+}
+\value{
+Return a character vector of workers, which typically consists
+of names of machines / compute nodes, but may also be IP numbers.
+}
+\description{
+Get set of available workers
+}
+\details{
+The default set of workers for each method is
+\code{rep("localhost", times = availableCores(method))}, which means
+that each will at least use as many parallel workers on the current
+machine that \code{\link{availableCores}()} allows for that method.
+
+In addition, the following settings ("methods") are also acknowledged:
+\itemize{
+ \item \code{"PBS"} -
+ Query TORQUE/PBS environment variable \env{PBS_NODEFILE}.
+ If this is set and specifies an existing file, then the set
+ of workers is read from that file, where one worker (node)
+ is given per line.
+ An example of a job submission that results in this is
+ \code{qsub -l nodes = 4:ppn = 2}, which requests four nodes each
+ with two cores.
+ \item \code{"SGE"} -
+ Query Sun/Oracle Grid Engine (SGE) environment variable
+ \env{PE_HOSTFILE}.
+ An example of a job submission that results in this is
+ \code{qsub -pe mpi 8} (or \code{qsub -pe ompi 8}), which
+ requests eight cores on a any number of machines.
+}
+}
+\seealso{
+To get the number of available workers on the current machine,
+see \code{\link{availableCores}()}.
+}
+\keyword{internal}
diff --git a/man/backtrace.Rd b/man/backtrace.Rd
index 4796a0f..9022543 100644
--- a/man/backtrace.Rd
+++ b/man/backtrace.Rd
@@ -19,4 +19,3 @@ A list of calls.
\description{
Back trace the expressions evaluated before a condition was caught
}
-
diff --git a/man/cluster.Rd b/man/cluster.Rd
index 3ca2771..50551aa 100644
--- a/man/cluster.Rd
+++ b/man/cluster.Rd
@@ -2,29 +2,33 @@
% Please edit documentation in R/cluster.R
\name{cluster}
\alias{cluster}
-\title{Create a cluster future whose value will be resolved asynchroneously in a parallel process}
+\title{Create a cluster future whose value will be resolved asynchronously in a parallel process}
\usage{
-cluster(expr, envir = parent.frame(), substitute = TRUE, globals = TRUE,
- persistent = FALSE, workers = NULL, user = NULL, revtunnel = TRUE,
+cluster(expr, envir = parent.frame(), substitute = TRUE, lazy = FALSE,
+ seed = NULL, globals = TRUE, persistent = FALSE,
+ workers = availableWorkers(), user = NULL, revtunnel = TRUE,
homogeneous = TRUE, gc = FALSE, earlySignal = FALSE, label = NULL,
...)
}
\arguments{
-\item{expr}{An \R \link[base]{expression}.}
+\item{expr}{An \R \link[base]{expression} to be evaluated.}
\item{envir}{The \link{environment} from where global
-objects should be identified. Depending on "evaluator",
-it may also be the environment in which the expression
-is evaluated.}
+objects should be identified. Depending on the future
+strategy (the \code{evaluator}), it may also be the environment
+in which the expression is evaluated.}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
+\item{lazy}{Specifies whether a future should be resolved
+lazily or eagerly (default).}
+
+\item{seed}{(optional) A L'Ecuyer-CMRG RNG seed.}
+
\item{globals}{A logical, a character vector,
or a named list for controlling how globals are handled.
-For details, see below section.
-This argument can be specified via the \dots arguments
-for \code{future()} and \code{futureCall()}.}
+For details, see below section.}
\item{persistent}{If FALSE, the evaluation environment is cleared
from objects prior to the evaluation of the future.}
@@ -74,14 +78,11 @@ mechanism for all futures. After this \code{\link{future}()}
and \code{\link{\%<-\%}} will create \emph{cluster futures}.
}
\examples{
-## Cluster futures gives an error on R CMD check on
-## Windows (but not Linux or OS X) for unknown reasons.
-## The same code works in package tests.
\donttest{
## Use cluster futures
cl <- parallel::makeCluster(2L)
-plan(cluster, workers=cl)
+plan(cluster, workers = cl)
## A global variable
a <- 0
@@ -108,4 +109,3 @@ parallel::stopCluster(cl)
}
}
-
diff --git a/man/flapply.Rd b/man/flapply.Rd
deleted file mode 100644
index f127c62..0000000
--- a/man/flapply.Rd
+++ /dev/null
@@ -1,23 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/flapply.R
-\name{flapply}
-\alias{flapply}
-\title{Apply a Function over a List or Vector via Futures}
-\usage{
-flapply(x, FUN, ...)
-}
-\arguments{
-\item{x}{A vector-like object to iterate over.}
-
-\item{FUN}{A function taking at least one argument.}
-
-\item{...}{(optional) Additional arguments pass to \code{FUN()}.}
-}
-\value{
-A list.
-}
-\description{
-Apply a Function over a List or Vector via Futures
-}
-\keyword{internal}
-
diff --git a/man/future.Rd b/man/future.Rd
index fae73b7..3e2dc9b 100644
--- a/man/future.Rd
+++ b/man/future.Rd
@@ -1,37 +1,50 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/future.R, R/futureAssign.R, R/futureAssign_OP.R, R/futureCall.R
+% Please edit documentation in R/future.R, R/futureAssign.R,
+% R/futureAssign_OP.R, R/futureCall.R
\name{future}
-\alias{\%->\%}
-\alias{\%<-\%}
-\alias{\%<=\%}
-\alias{\%=>\%}
\alias{future}
+\alias{futureCall}
\alias{futureAssign}
+\alias{\%<-\%}
+\alias{\%->\%}
\alias{futureCall}
\title{Create a future}
\usage{
-future(expr, envir = parent.frame(), substitute = TRUE,
- evaluator = plan("next"), ...)
+future(expr, envir = parent.frame(), substitute = TRUE, globals = TRUE,
+ packages = NULL, lazy = FALSE, seed = NULL, evaluator = plan("next"),
+ ...)
-futureAssign(x, value, envir = parent.frame(), assign.env = envir,
- substitute = TRUE)
+futureAssign(x, value, envir = parent.frame(), substitute = TRUE,
+ lazy = FALSE, seed = NULL, globals = TRUE, ..., assign.env = envir)
x \%<-\% value
-futureCall(FUN, args = NULL, envir = parent.frame(), globals = TRUE,
- evaluator = plan("next"), ...)
+futureCall(FUN, args = NULL, envir = parent.frame(), lazy = FALSE,
+ seed = NULL, globals = TRUE, evaluator = plan("next"), ...)
}
\arguments{
-\item{expr}{An \R \link[base]{expression}.}
+\item{expr, value}{An \R \link[base]{expression} to be evaluated.}
\item{envir}{The \link{environment} from where global
-objects should be identified. Depending on "evaluator",
-it may also be the environment in which the expression
-is evaluated.}
+objects should be identified. Depending on the future
+strategy (the \code{evaluator}), it may also be the environment
+in which the expression is evaluated.}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
+\item{globals}{A logical, a character vector,
+or a named list for controlling how globals are handled.
+For details, see below section.}
+
+\item{packages}{(optional) a character vector specifying packages
+to be attached in the R environment evaluating the future.}
+
+\item{lazy}{Specifies whether a future should be resolved
+lazily or eagerly (default).}
+
+\item{seed}{(optional) A L'Ecuyer-CMRG RNG seed.}
+
\item{evaluator}{The actual function that evaluates
the future expression and returns a \link{Future}.
The evaluator function should accept all of the same
@@ -42,36 +55,32 @@ has specified via \code{\link{plan}()}.}
\item{...}{Additional arguments passed to the "evaluator".}
-\item{x}{the name of a future variable.}
-
-\item{value}{the \R \link[base]{expression} to be evaluated in
-the future and whose value will be assigned to the variable.}
+\item{x}{the name of a future variable, which will hold the value
+of the future expression (as a promise).}
\item{assign.env}{The \link[base]{environment} to which the variable
should be assigned.}
-\item{FUN}{A \link[base]{function} object.}
+\item{FUN}{A \link[base]{function} to be evaluated.}
\item{args}{A \link[base]{list} of arguments passed to function \code{FUN}.}
-
-\item{globals}{A logical, a character vector,
-or a named list for controlling how globals are handled.
-For details, see below section.
-This argument can be specified via the \dots arguments
-for \code{future()} and \code{futureCall()}.}
}
\value{
\code{f <- future(expr)} creates a \link{Future} \code{f} that evaluates expression \code{expr}, the value of the future is retrieved using \code{v <- value(f)}.
-\code{f <- futureCall(FUN, args)} creates a \link{Future} \code{f} that calls function \code{FUN} with arguments \code{args}, where the value of the future is retrieved using \code{v <- value(f)}.
-
-\code{futureAssign("v", expr)} and \code{v \%<-\% expr} (a future assignment) create a \link{Future} that evaluates expression \code{expr} and binds its value (as a \link[base]{promise}) to a variable \code{v}. The value of the future is automatically retrieved when the assigned variable (promise) is queried.
+\code{x \%<-\% value} (a future assignment) and
+\code{futureAssign("x", value)} create a \link{Future} that evaluates
+expression \code{expr} and binds its value (as a \link[base]{promise}) to
+a variable \code{x}. The value of the future is automatically retrieved
+when the assigned variable (promise) is queried.
The future itself is returned invisibly, e.g.
-\code{f <- futureAssign("v", expr)} and \code{f <- (v \%<-\% expr)}.
-Alternatively, the future of a future variable \code{v} can be retrieved
-without blocking using \code{f <- \link{futureOf}(v)}.
+\code{f <- futureAssign("x", expr)} and \code{f <- (x \%<-\% expr)}.
+Alternatively, the future of a future variable \code{x} can be retrieved
+without blocking using \code{f <- \link{futureOf}(x)}.
Both the future and the variable (promise) are assigned to environment
\code{assign.env} where the name of the future is \code{.future_<name>}.
+
+\code{f <- futureCall(FUN, args)} creates a \link{Future} \code{f} that calls function \code{FUN} with arguments \code{args}, where the value of the future is retrieved using \code{x <- value(f)}.
}
\description{
Creates a future that evaluates an \R expression or
@@ -80,8 +89,9 @@ How, when, and where these futures are evaluated can be configured
using \code{\link{plan}()} such that it is evaluated in parallel on,
for instance, the current machine, on a remote machine, or via a
job queue on a compute cluster.
-Importantly, \R code using futures remains the same regardless
-on these settings.
+Importantly, any \R code using futures remains the same regardless
+on these settings and there is no need to modify the code when
+switching from, say, sequential to parallel processing.
}
\details{
The state of a future is either unresolved or resolved.
@@ -91,23 +101,40 @@ until the future is resolved.
It is possible to check whether a future is resolved or not
without blocking by using \code{\link{resolved}(f)}.
-For a future created via a future assignment, the value is bound to
-a promise, which when queried will internally call \code{\link{value}()}
-on the future and which will then be resolved into a regular variable
-bound to that value. For example, with future assignment
-\code{v \%<-\% expr}, the first time variable \code{v} is queried
-the call blocks if (and only if) the future is not yet resolved. As soon
-as it is resolved, and any succeeding queries, querying \code{v} will
-immediately give the value.
-
-The future assignment construct \code{v \%<-\% expr} is not a formal
-assignment per se, but a binary infix operator on objects \code{v}
-and \code{expr}. However, by using non-standard evaluation, this
-constructs can emulate an assignment operator similar to
-\code{v <- expr}. Due to \R's precedence rules of operators,
-future expressions that contain multiple statements need to be
-explicitly bracketed, e.g. \code{v \%<-\% { a <- 2; a^2 }}.
+For a future created via a future assignment
+(\code{x \%<-\% value} or \code{futureAssign("x", value)}), the value
+is bound to a promise, which when queried will internally call
+\code{\link{value}()} on the future and which will then be resolved
+into a regular variable bound to that value. For example, with future
+assignment \code{x \%<-\% value}, the first time variable \code{x} is
+queried the call blocks if (and only if) the future is not yet resolved.
+As soon as it is resolved, and any succeeding queries, querying \code{x}
+will immediately give the value.
+
+The future assignment construct \code{x \%<-\% value} is not a formal
+assignment per se, but a binary infix operator on objects \code{x}
+and expression \code{value}. However, by using non-standard evaluation,
+this constructs can emulate an assignment operator similar to
+\code{x <- value}. Due to \R's precedence rules of operators,
+future expressions often needs to be explicitly bracketed, e.g.
+\code{x \%<-\% { a + b }}.
+}
+\section{Eager or lazy evaluation}{
+
+By default, a future is resolved using \emph{eager} evaluation
+(\code{lazy = FALSE}). This means that the expression starts to
+be evaluated as soon as the future is created.
+
+As an alternative, the future can be resolved using \emph{lazy}
+evaluation (\code{lazy = TRUE}). This means that the expression
+will only be evaluated when the value of the future is requested.
+\emph{Note that this means that the expression may not be evaluated
+at all - it is guaranteed to be evaluated if the value is requested}.
+
+For future assignments, lazy evaluation can be controlled via the
+\code{\%lazy\%} operator, e.g. \code{x \%<-\% { expr } \%lazy\% TRUE}.
}
+
\section{Globals used by future expressions}{
Global objects (short \emph{globals}) are objects (e.g. variables and
@@ -129,7 +156,7 @@ is that globals are automatically identified and gathered.
More precisely, globals are identified via code inspection of the
future expression \code{expr} and their values are retrieved with
environment \code{envir} as the starting point (basically via
-\code{get(global, envir=envir, inherits=TRUE)}).
+\code{get(global, envir = envir, inherits = TRUE)}).
\emph{In most cases, such automatic collection of globals is sufficient
and less tedious and error prone than if they are manually specified}.
@@ -155,7 +182,7 @@ or
Specifying globals explicitly avoids the overhead added from
automatically identifying the globals and gathering their values.
-Futhermore, if we know that the future expression does not make use
+Furthermore, if we know that the future expression does not make use
of any global variables, we can disable the automatic search for
globals by using
\preformatted{
@@ -209,10 +236,8 @@ using the \code{\link{\%globals\%}} operator, e.g.
y \%<-\% { median(x) } \%globals\% list(x = x, median = stats::median)
}
}
+
\examples{
-## Cluster futures gives an error on R CMD check on
-## Windows (but not Linux or OS X) for unknown reasons.
-## The same code works in package tests.
\donttest{
## Evaluate futures in parallel
plan(multiprocess)
@@ -220,8 +245,8 @@ plan(multiprocess)
## Data
x <- rnorm(100)
-y <- 2*x + 0.2 + rnorm(100)
-w <- 1 + x^2
+y <- 2 * x + 0.2 + rnorm(100)
+w <- 1 + x ^ 2
## (1) Regular assignments (evaluated sequentially)
@@ -276,7 +301,8 @@ print(fitB)
print(fitC)
}
\seealso{
-How, when and where futures are resolved is given by the future strategy,
-which can be set by the \code{\link{plan}()} function.
+How, when and where futures are resolved is given by the
+\emph{future strategy}, which can be set by the end user using the
+\code{\link{plan}()} function. The future strategy must not be
+set by the developer, e.g. it must not be called within a package.
}
-
diff --git a/man/future.options.Rd b/man/future.options.Rd
index 782c522..bb6529e 100644
--- a/man/future.options.Rd
+++ b/man/future.options.Rd
@@ -1,19 +1,15 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/options.R
\name{future.options}
-\alias{R_FUTURE_CORES}
-\alias{R_FUTURE_PLAN}
+\alias{future.options}
\alias{future.availableCores.methods}
\alias{future.cmdargs}
\alias{future.cores}
\alias{future.debug}
-\alias{future.demo.mandelbrot.nrow}
-\alias{future.demo.mandelbrot.region}
\alias{future.globals.maxSize}
\alias{future.globals.method}
\alias{future.globals.onMissing}
\alias{future.globals.resolve}
-\alias{future.options}
\alias{future.plan}
\alias{future.progress}
\alias{future.resolve.recursive}
@@ -21,6 +17,10 @@
\alias{future.wait.alpha}
\alias{future.wait.interval}
\alias{future.wait.times}
+\alias{R_FUTURE_PLAN}
+\alias{R_FUTURE_CORES}
+\alias{future.demo.mandelbrot.region}
+\alias{future.demo.mandelbrot.nrow}
\title{Options used for futures}
\description{
Below are all \R options that are currently used by the \pkg{future} package and packages enhancing it.\cr
@@ -30,15 +30,15 @@ Below are all \R options that are currently used by the \pkg{future} package and
\section{Options for controlling futures}{
\describe{
- \item{\option{future.plan}:}{Default future strategy plan used unless otherwise specified via \code{\link{plan}()}. This will also be the future plan set when calling \code{plan("default")}. If not specified, this option may be set when the \pkg{future} package is \emph{loaded} if command-line option \code{--parallel=ncores} (short \code{-p ncores}) is specified; if \code{ncores > 1}, then option \option{future.plan} is set to \code{multiprocess} otherwise \code{eager} (in addition to [...]
+ \item{\option{future.plan}:}{Default future strategy plan used unless otherwise specified via \code{\link{plan}()}. This will also be the future plan set when calling \code{plan("default")}. If not specified, this option may be set when the \pkg{future} package is \emph{loaded} if command-line option \code{--parallel=ncores} (short \code{-p ncores}) is specified; if \code{ncores > 1}, then option \option{future.plan} is set to \code{multiprocess} otherwise \code{sequential} (in additio [...]
\item{\option{future.globals.onMissing}:}{Action to take when non-existing global variables ("globals" or "unknowns") are identified when the future is created. If \code{"error"}, an error is generated immediately. If \code{"ignore"}, no action is taken and an attempt to evaluate the future expression will be made. The latter is useful when there is a risk for false-positive globals being identified, e.g. when future expression contains non-standard evaluation (NSE). (Default: \code [...]
\item{\option{future.globals.method}:}{Method used to identify globals. For details, see \code{\link[globals]{globalsOf}()}. (Default: \code{"ordered"})}
- \item{\option{future.globals.maxSize}:}{Maximum allowed total size (in bytes) of global variables identified. Used to prevent too large exports. (Default: \code{500*1024^2} = 500 MiB)}
+ \item{\option{future.globals.maxSize}:}{Maximum allowed total size (in bytes) of global variables identified. Used to prevent too large exports. (Default: \code{500 * 1024 ^ 2} = 500 MiB)}
\item{\option{future.globals.resolve}:}{If \code{TRUE}, globals that are \code{\link{Future}} objects (typically created as \emph{explicit} futures) will be resolved and have their values (using \code{value()}) collected. Because searching for unresolved futures among globals (including their content) can be expensive, the default is not to do it and instead leave it to the run-time checks that assert proper ownership when resolving futures and collecting their values. (Default: \code{ [...]
\item{\option{future.resolve.recursive}:}{An integer specifying the maximum recursive depth to which futures should be resolved. If negative, nothing is resolved. If \code{0}, only the future itself is resolved. If \code{1}, the future and any of its elements that are futures are resolved, and so on. If \code{+Inf}, infinite search depth is used. (Default: \code{0})}
- \item{\option{future.wait.times}:}{Maximum number of times a future is polled waiting for it to be resolved.}
- \item{\option{future.wait.interval}:}{Initial interval (in seconds) between polls.}
- \item{\option{future.wait.alpha}:}{Positive scale factor used to increase the interval after each poll. (Default: \code{0.01})}
+ \item{\option{future.wait.timeout}:}{Maximum waiting time (in seconds) for a free worker before a timeout error is generated. (Default: \code{30 * 24 * 60 * 60} (= 30 days))}
+ \item{\option{future.wait.interval}:}{Initial interval (in seconds) between polls. (Default: \code{0.2} (0.2 seconds))}
+ \item{\option{future.wait.alpha}:}{Positive scale factor used to increase the interval after each poll. (Default: \code{1.01})}
}
}
@@ -58,11 +58,16 @@ Below are all \R options that are currently used by the \pkg{future} package and
}
}
-\section{Options for configurating low-level system behaviors}{
+\section{Options for configuring low-level system behaviors}{
\describe{
- \item{\option{future.availableCores.methods}:}{Default lookup methods for \code{\link{availableCores}()}. (Default: \code{c("system", "mc.cores+1", "_R_CHECK_LIMIT_CORES_", "Slurm", "PBS", "SGE")})}
- \item{\option{future.availableCores.system}:}{Number of "system" cores used instead of what is reported by \code{\link{availableCores}(which="system")}. If not specified, this option is set according to system environment variable \env{R_FUTURE_AVAILABLECORES_SYSTEM} when the \pkg{future} package is \emph{loaded}.}
+ \item{\option{future.availableCores.methods}:}{Default lookup methods for \code{\link{availableCores}()}. (Default: \code{c("system", "mc.cores", "_R_CHECK_LIMIT_CORES_", "PBS", "SGE", "Slurm", "fallback")})}
+
+ \item{\option{future.availableWorkers.methods}:}{Default lookup methods for \code{\link{availableWorkers}()}. (Default: \code{c("mc.cores", "_R_CHECK_LIMIT_CORES_", "PBS", "SGE", "Slurm", "system", "fallback")})}
+
+ \item{\option{future.availableCores.fallback}:}{Number of cores to use when no core-specifying settings are detected other than \code{"system"}. If not specified, this option is set according to system environment variable \env{R_FUTURE_AVAILABLECORES_FALLBACK} when the \pkg{future} package is \emph{loaded}. This options makes it possible to set the default number of cores returned by \code{availableCores()} / \code{availableWorkers()} yet allow users and schedulers to override it. In H [...]
+
+ \item{\option{future.availableCores.system}:}{Number of "system" cores used instead of what is reported by \code{\link{availableCores}(which = "system")}. If not specified, this option is set according to system environment variable \env{R_FUTURE_AVAILABLECORES_SYSTEM} when the \pkg{future} package is \emph{loaded}. This option allows you to effectively override what \code{parallel::detectCores()} reports the system has.}
}
}
@@ -73,8 +78,8 @@ Below are all \R options that are currently used by the \pkg{future} package and
\item{\option{future.demo.mandelbrot.nrow}:}{Number of rows and columns of tiles. (Default: \code{3L})}
}
}
+
\seealso{
-To set \R options when \R starts (even before the \pkg{future} package is loaded), see the \link[base]{Startup} help page.
+To set \R options when \R starts (even before the \pkg{future} package is loaded), see the \link[base]{Startup} help page. The \href{https://cran.r-project.org/package=startup}{\pkg{startup}} package provides a friendly mechanism for configurating \R's startup process.
}
\keyword{internal}
-
diff --git a/man/futureOf.Rd b/man/futureOf.Rd
index c2bfebc..af47e52 100644
--- a/man/futureOf.Rd
+++ b/man/futureOf.Rd
@@ -52,16 +52,12 @@ env$c \%<-\% { 3 }
f <- futureOf(env$c)
print(f)
-f2 <- futureOf(c, envir=env)
+f2 <- futureOf(c, envir = env)
print(f2)
-f3 <- futureOf("c", envir=env)
+f3 <- futureOf("c", envir = env)
print(f3)
-fs <- futureOf(envir=env)
+fs <- futureOf(envir = env)
print(fs)
-
-
-
}
-
diff --git a/man/future_lapply.Rd b/man/future_lapply.Rd
new file mode 100644
index 0000000..0bc9074
--- /dev/null
+++ b/man/future_lapply.Rd
@@ -0,0 +1,112 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/future_lapply.R
+\name{future_lapply}
+\alias{future_lapply}
+\title{Apply a Function over a List or Vector via Futures}
+\usage{
+future_lapply(x, FUN, ..., future.globals = TRUE, future.packages = NULL,
+ future.seed = FALSE, future.lazy = FALSE, future.scheduling = 1)
+}
+\arguments{
+\item{x}{A vector-like object to iterate over.}
+
+\item{FUN}{A function taking at least one argument.}
+
+\item{...}{(optional) Additional arguments pass to \code{FUN()}.}
+
+\item{future.globals}{A logical, a character vector, or a named list for
+controlling how globals are handled. For details, see below section.}
+
+\item{future.packages}{(optional) a character vector specifying packages
+to be attached in the R environment evaluating the future.}
+
+\item{future.seed}{A logical or an integer (of length one or seven),
+or a list of \code{length(x)} with pre-generated random seeds.
+For details, see below section.}
+
+\item{future.lazy}{Specifies whether the futures should be resolved
+lazily or eagerly (default).}
+
+\item{future.scheduling}{Average number of futures ("chunks") per worker.
+If \code{0.0}, then a single future is used to process all elements
+of \code{x}.
+If \code{1.0} or \code{TRUE}, then one future per worker is used.
+If \code{2.0}, then each worker will process two futures
+(if there are enough elements in \code{x}).
+If \code{Inf} or \code{FALSE}, then one future per element of
+\code{x} is used.}
+}
+\value{
+A list with same length and names as \code{x}.
+}
+\description{
+Apply a Function over a List or Vector via Futures
+}
+\section{Global variables}{
+
+Argument \code{future.globals} may be used to control how globals
+should be handled similarly how the \code{globals} argument is used with
+\code{\link{future}()}.
+Since all function calls use the same set of globals, this function can do
+any gathering of globals upfront (once), which is more efficient than if
+it would be done for each future independently.
+If \code{TRUE}, \code{NULL} or not is specified (default), then globals
+are automatically identified and gathered.
+If a character vector of names is specified, then those globals are gathered.
+If a named list, then those globals are used as is.
+In all cases, \code{FUN} and any \code{...} arguments are automatically
+passed as globals to each future created as they are always needed.
+}
+
+\section{Reproducible random number generation (RNG)}{
+
+Unless \code{future.seed = FALSE}, this function guarantees to generate
+the exact same sequence of random numbers \emph{given the same initial
+seed / RNG state} - this regardless of type of futures and scheduling
+("chunking") strategy.
+
+RNG reproducibility is achieved by pregenerating the random seeds for all
+iterations (over \code{x}) by using L'Ecuyer-CMRG RNG streams. In each
+iteration, these seeds are set before calling \code{FUN(x[[ii]], ...)}.
+\emph{Note, for large \code{length(x)} this may introduce a large overhead.}
+As input (\code{future.seed}), a fixed seed (integer) may be given, either
+as a full L'Ecuyer-CMRG RNG seed (vector of 1+6 integers) or as a seed
+generating such a full L'Ecuyer-CMRG seed.
+If \code{future.seed = TRUE}, then \code{\link[base:Random]{.Random.seed}}
+is returned if it holds a L'Ecuyer-CMRG RNG seed, otherwise one is created
+randomly.
+If \code{future.seed = NA}, a L'Ecuyer-CMRG RNG seed is randomly created.
+If none of the function calls \code{FUN(x[[i]], ...)} uses random number
+generation, then \code{future.seed = FALSE} may be used.
+
+In addition to the above, it is possible to specify a pre-generated
+sequence of RNG seeds as a list such that
+\code{length(future.seed) == length(x)} and where each element is an
+integer seed that can be assigned to \code{\link[base:Random]{.Random.seed}}.
+Use this alternative with caution.
+\emph{Note that as.list(seq_along(x)) is \emph{not} a valid set of such
+\code{.Random.seed} values.}
+
+In all cases but \code{future.seed = FALSE}, the RNG state of the calling
+R processes after this function returns is guaranteed to be
+"forwarded one step" from the RNG state that was before the call and
+in the same way regardless of \code{future.seed}, \code{future.scheduling}
+and future strategy used. This is done in order to guarantee that an \R
+script calling \code{future_lapply()} multiple times should be numerically
+reproducible given the same initial seed.
+}
+
+\examples{
+## Regardless of the future plan, the number of workers,
+## and where they are, the random numbers will be identical
+plan(sequential)
+y1 <- future_lapply(1:5, FUN = rnorm, future.seed = 0xBEEF)
+str(y1)
+
+plan(multiprocess)
+y2 <- future_lapply(1:5, FUN = rnorm, future.seed = 0xBEEF)
+str(y2)
+
+stopifnot(all.equal(y1, y2))
+}
+\keyword{internal}
diff --git a/man/futures.Rd b/man/futures.Rd
index 8189631..4284831 100644
--- a/man/futures.Rd
+++ b/man/futures.Rd
@@ -26,4 +26,3 @@ future assignments (\code{\%<-\%}) and therefore stored as promises.
This function turns such promises into standard \code{Future}
objects.
}
-
diff --git a/man/getExpression.Rd b/man/getExpression.Rd
index 186cf8d..41bdb67 100644
--- a/man/getExpression.Rd
+++ b/man/getExpression.Rd
@@ -21,13 +21,12 @@ Inject code for the next type of future to use for nested futures
}
\details{
If no next future strategy is specified, the default is to
-use \link{eager} futures. This conservative approach protects
+use \link{sequential} futures. This conservative approach protects
against spawning off recursive futures by mistake, especially
\link{multicore} and \link{multisession} ones.
-The default will also set \code{options(mc.cores=0L)}, which
-means that no \emph{additional} R processes may be spawned off
-by functions such as \code{\link[parallel:mclapply]{mclapply}()}
-and friends (*).
+The default will also set \code{options(mc.cores = 1L)} (*) so that
+no parallel R processes are spawned off by functions such as
+\code{\link[parallel:mclapply]{mclapply}()} and friends.
Currently it is not possible to specify what type of nested
futures to be used, meaning the above default will always be
@@ -35,8 +34,8 @@ used.
See \href{https://github.com/HenrikBengtsson/future/issues/37}{Issue #37}
for plans on adding support for custom nested future types.
-(*) Note that using \code{mc.cores=0} will unfortunately cause
- \code{mclapply()} and friends to generate an error saying
+(*) Ideally we would set \code{mc.cores = 0} but that will unfortunately
+ cause \code{mclapply()} and friends to generate an error saying
"'mc.cores' must be >= 1". Ideally those functions should
fall back to using the non-multicore alternative in this
case, e.g. \code{mclapply(...)} => \code{lapply(...)}.
@@ -44,4 +43,3 @@ for plans on adding support for custom nested future types.
for a discussion on this.
}
\keyword{internal}
-
diff --git a/man/getGlobalsAndPackages.Rd b/man/getGlobalsAndPackages.Rd
index 6757ecb..e917774 100644
--- a/man/getGlobalsAndPackages.Rd
+++ b/man/getGlobalsAndPackages.Rd
@@ -32,4 +32,3 @@ Retrieves global variables of an expression and their associated packages
Internally, \code{\link[globals]{globalsOf}()} is used to identify globals and associated packages from the expression.
}
\keyword{internal}
-
diff --git a/man/grapes-globals-grapes.Rd b/man/grapes-globals-grapes.Rd
index be73a23..2395a93 100644
--- a/man/grapes-globals-grapes.Rd
+++ b/man/grapes-globals-grapes.Rd
@@ -18,7 +18,3 @@ in the help for \code{\link{future}()}.}
\description{
Specify globals for a future assignment
}
-\details{
-\code{a \%globals\% b} is short for \code{a \%tweak\% list(globals = b)}.
-}
-
diff --git a/man/grapes-label-grapes.Rd b/man/grapes-label-grapes.Rd
index 438a7ac..453ecb5 100644
--- a/man/grapes-label-grapes.Rd
+++ b/man/grapes-label-grapes.Rd
@@ -15,7 +15,3 @@ fassignment \%label\% label
\description{
Specify label for a future assignment
}
-\details{
-\code{a \%label\% b} is short for \code{a \%tweak\% list(label = b)}.
-}
-
diff --git a/man/grapes-lazy-grapes.Rd b/man/grapes-lazy-grapes.Rd
new file mode 100644
index 0000000..6a2d3c1
--- /dev/null
+++ b/man/grapes-lazy-grapes.Rd
@@ -0,0 +1,18 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/lazy_OP.R
+\name{\%lazy\%}
+\alias{\%lazy\%}
+\title{Control lazy / eager evaluation for a future assignment}
+\usage{
+fassignment \%lazy\% lazy
+}
+\arguments{
+\item{fassignment}{The future assignment, e.g.
+\code{x \%<-\% \{ expr \}}.}
+
+\item{lazy}{If \code{FALSE} (default), the future is resolved eagerly
+(immediately), otherwise not.}
+}
+\description{
+Control lazy / eager evaluation for a future assignment
+}
diff --git a/man/grapes-plan-grapes.Rd b/man/grapes-plan-grapes.Rd
index d304fbf..0e9f117 100644
--- a/man/grapes-plan-grapes.Rd
+++ b/man/grapes-plan-grapes.Rd
@@ -19,4 +19,3 @@ Use a specific plan for a future assignment
\seealso{
The \code{\link{plan}()} function sets the default plan for all futures.
}
-
diff --git a/man/grapes-seed-grapes.Rd b/man/grapes-seed-grapes.Rd
new file mode 100644
index 0000000..e320b5a
--- /dev/null
+++ b/man/grapes-seed-grapes.Rd
@@ -0,0 +1,17 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/seed_OP.R
+\name{\%seed\%}
+\alias{\%seed\%}
+\title{Set random seed for future assignment}
+\usage{
+fassignment \%seed\% seed
+}
+\arguments{
+\item{fassignment}{The future assignment, e.g.
+\code{x \%<-\% \{ expr \}}.}
+
+\item{seed}{(optional) A L'Ecuyer-CMRG RNG seed.}
+}
+\description{
+Set random seed for future assignment
+}
diff --git a/man/grapes-tweak-grapes.Rd b/man/grapes-tweak-grapes.Rd
index 3896ad2..81dd953 100644
--- a/man/grapes-tweak-grapes.Rd
+++ b/man/grapes-tweak-grapes.Rd
@@ -16,4 +16,3 @@ should be changed relative to the current strategy.}
\description{
Temporarily tweaks the arguments of the current strategy
}
-
diff --git a/man/makeClusterPSOCK.Rd b/man/makeClusterPSOCK.Rd
index c9646df..544c5b3 100644
--- a/man/makeClusterPSOCK.Rd
+++ b/man/makeClusterPSOCK.Rd
@@ -3,104 +3,215 @@
\name{makeClusterPSOCK}
\alias{makeClusterPSOCK}
\alias{makeNodePSOCK}
-\title{Create a Parallel Socket Cluster}
+\title{Create a cluster of \R workers for parallel processing}
\usage{
makeClusterPSOCK(workers, makeNode = makeNodePSOCK, port = c("auto",
"random"), ..., verbose = getOption("future.debug", FALSE))
makeNodePSOCK(worker = "localhost", master = NULL, port,
- connectTimeout = 2 * 60, timeout = 30 * 24 * 60 * 60, rscript = NULL,
- homogeneous = NULL, rscript_args = NULL, methods = TRUE,
- useXDR = TRUE, outfile = "/dev/null", renice = NA_integer_,
- rshcmd = "ssh", user = NULL, revtunnel = TRUE, rshopts = NULL,
- rank = 1L, manual = FALSE, dryrun = FALSE, verbose = FALSE)
+ connectTimeout = getOption("future.makeNodePSOCK.connectTimeout", 2 * 60),
+ timeout = getOption("future.makeNodePSOCK.timeout", 30 * 24 * 60 * 60),
+ rscript = NULL, homogeneous = NULL, rscript_args = NULL,
+ methods = TRUE, useXDR = TRUE, outfile = "/dev/null",
+ renice = NA_integer_, rshcmd = getOption("future.makeNodePSOCK.rshcmd",
+ NULL), user = NULL, revtunnel = TRUE,
+ rshopts = getOption("future.makeNodePSOCK.rshopts", NULL), rank = 1L,
+ manual = FALSE, dryrun = FALSE, verbose = FALSE)
}
\arguments{
-\item{workers}{The host names of workers (as a character vector) or
-the number of localhost workers (as a positive integer).}
+\item{workers}{The hostnames of workers (as a character vector) or the number
+of localhost workers (as a positive integer).}
-\item{makeNode}{A function that creates a \code{"SOCKnode"}
-or \code{"SOCK0node"} object, which represents a connection
-to a worker.}
+\item{makeNode}{A function that creates a \code{"SOCKnode"} or
+\code{"SOCK0node"} object, which represents a connection to a worker.}
-\item{port}{The port number of the master used to for communicating with all the workers (via socket connections). If an integer vector of ports, then a random one among those is chosen. If \code{"random"}, then a random port in \code{11000:11999} is chosen. If \code{"auto"} (default), then the default is taken from environment variable \env{R_PARALLEL_PORT}, otherwise \code{"random"} is used.}
+\item{port}{The port number of the master used to for communicating with all
+the workers (via socket connections). If an integer vector of ports, then a
+random one among those is chosen. If \code{"random"}, then a random port in
+\code{11000:11999} is chosen. If \code{"auto"} (default), then the default
+is taken from environment variable \env{R_PARALLEL_PORT}, otherwise
+\code{"random"} is used.}
-\item{...}{Optional arguments passed to \code{makeNode(workers[i], ..., rank=i)} where \code{i = seq_along{workers}}.}
+\item{...}{Optional arguments passed to
+\code{makeNode(workers[i], ..., rank = i)} where
+\code{i = seq_along(workers)}.}
\item{verbose}{If TRUE, informative messages are outputted.}
-\item{worker}{The host name or IP number of the machine where the worker should run.}
+\item{worker}{The hostname or IP number of the machine where the worker
+should run.}
-\item{master}{The host name or IP number of the master / calling machine, as known to the workers. If NULL (default), then the default is \code{Sys.info()[["nodename"]]} unless \code{worker} is the localhost (\code{"localhost"} or \code{"127.0.0.1"}) or \code{revtunnel = TRUE} in case it is \code{"localhost"}.}
+\item{master}{The hostname or IP number of the master / calling machine, as
+known to the workers. If NULL (default), then the default is
+\code{Sys.info()[["nodename"]]} unless \code{worker} is \emph{localhost} or
+\code{revtunnel = TRUE} in case it is \code{"localhost"}.}
-\item{connectTimeout}{The maximum time (in seconds) allowed for each sockect connection between the master and a worker to be established (defaults to 2 minutes). \emph{See note below on current lack of support on Linux and macOS systems.}}
+\item{connectTimeout}{The maximum time (in seconds) allowed for each socket
+connection between the master and a worker to be established (defaults to
+2 minutes). \emph{See note below on current lack of support on Linux and
+macOS systems.}}
-\item{timeout}{The maximum time (in seconds) allowed to pass without the master and a worker communicate with each other (defaults to 30 days).}
+\item{timeout}{The maximum time (in seconds) allowed to pass without the
+master and a worker communicate with each other (defaults to 30 days).}
-\item{rscript, homogeneous}{The system command for launching Rscript on the worker. If \code{NULL} (default), the default is \code{"Rscript"} unless \code{homogenenous} is TRUE, which in case it is \code{file.path(R.home("bin"), "Rscript")}. Argument \code{homogenenous} defaults to FALSE, unless \code{master} is the localhost (\code{"localhost"} or \code{"127.0.0.1"}).}
+\item{rscript, homogeneous}{The system command for launching \command{Rscript}
+on the worker and whether it is installed in the same path as the calling
+machine or not. For more details, see below.}
-\item{rscript_args}{Additional arguments to \code{Rscript} (as a character vector).}
+\item{rscript_args}{Additional arguments to \command{Rscript} (as a character
+vector).}
\item{methods}{If TRUE, then the \pkg{methods} package is also loaded.}
-\item{useXDR}{If TRUE, the communication between master and workers, which is binary, will be use big-endian (XDR).}
+\item{useXDR}{If TRUE, the communication between master and workers, which is
+binary, will use big-endian (XDR).}
-\item{outfile}{Where to direct the \link[base:stdout]{stdout} and \link[base:stderr]{stderr} connection output from the workers.}
+\item{outfile}{Where to direct the \link[base:stdout]{stdout} and
+\link[base:stderr]{stderr} connection output from the workers.}
-\item{renice}{A numerical 'niceness' (priority) to set for the worker processes.}
+\item{renice}{A numerical 'niceness' (priority) to set for the worker
+processes.}
-\item{rshcmd}{The command to be run on the master to launch a process on another host. Only applicable if \code{machine} is not localhost.}
+\item{rshcmd, rshopts}{The command (character vector) to be run on the master
+to launch a process on another host and any additional arguments (character
+vector). These arguments are only applied if \code{machine} is not
+\emph{localhost}. For more details, see below.}
-\item{user}{(optional) The user name to be used when communicating with another host.}
+\item{user}{(optional) The user name to be used when communicating with
+another host.}
-\item{revtunnel}{If TRUE, a reverse SSH tunneling is set up for each worker such that the worker R process sets up a socket connect to its local port \code{(port - rank + 1)} which then reaches the master on port \code{port}. If FALSE, then the worker will try to connect directly to port \code{port} on \code{master}.}
-
-\item{rshopts}{Additional arguments to \code{rshcmd} (as a character vector).}
+\item{revtunnel}{If TRUE, a reverse SSH tunnel is set up for each worker such
+that the worker R process sets up a socket connection to its local port
+\code{(port - rank + 1)} which then reaches the master on port \code{port}.
+If FALSE, then the worker will try to connect directly to port \code{port} on
+\code{master}. For more details, see below.}
\item{rank}{A unique one-based index for each worker (automatically set).}
-\item{manual}{If TRUE the workers will need to be run manually.}
+\item{manual}{If TRUE the workers will need to be run manually. The command
+to run will be displayed.}
-\item{dryrun}{If TRUE, nothing is set up, but a message suggesting how to launch the worker from the terminal is outputted. This is useful for troubleshooting.}
+\item{dryrun}{If TRUE, nothing is set up, but a message suggesting how to
+launch the worker from the terminal is outputted. This is useful for
+troubleshooting.}
}
\value{
An object of class \code{c("SOCKcluster", "cluster")} consisting
- of a list of \code{"SOCKnode"} or \code{"SOCK0node"} workers.
+of a list of \code{"SOCKnode"} or \code{"SOCK0node"} workers.
-\code{makeNodePSOCK()} returns a
- \code{"SOCKnode"} or \code{"SOCK0node"} object
- representing an established connection to a worker.
+\code{makeNodePSOCK()} returns a \code{"SOCKnode"} or
+\code{"SOCK0node"} object representing an established connection to a worker.
}
\description{
-Create a Parallel Socket Cluster
+The \code{makeClusterPSOCK()} function creates a cluster of \R workers
+for parallel processing. These \R workers may be background \R sessions
+on the current machine, \R sessions on external machines (local or remote),
+or a mix of such. For external workers, the default is to use SSH to connect
+to those external machines. This function works similarly to
+\code{\link[parallel:makePSOCKcluster]{makePSOCKcluster}} of the
+\pkg{parallel} package, but provides additional and more flexibility options
+for controlling the setup of the system calls that launch the background
+\R workers, and how to connect to external machines.
+}
+\section{Definition of \emph{localhost}}{
+
+A hostname is considered to be \emph{localhost} if it equals:
+\itemize{
+ \item \code{"localhost"},
+ \item \code{"127.0.0.1"}, or
+ \item \code{Sys.info()[["nodename"]]}.
+}
+It is also considered \emph{localhost} if it appears on the same line
+as the value of \code{Sys.info()[["nodename"]]} in file \file{/etc/hosts}.
}
-\details{
-The \code{makeClusterPSOCK()} function is similar to \code{\link[parallel:makePSOCKcluster]{makePSOCKcluster}} of the \pkg{parallel} package, but provides more flexibility in controlling the setup of the system calls that launch the background R workers and how to connect to external machines.
-
-The default is to use reverse SSH tunnelling for workers
-running on other machines. This avoids the complication of
-otherwise having to configure port forwarding in firewalls,
-which often requires static IP address but which also most
-users don't have priviligies to do themselves.
-It also has the advantage of not having to know the internal
-and / or the public IP address / host name of the master.
-
-If there is no communication between the master and a
-worker within the \code{timeout} limit, then the corresponding
-socket connection will be closed automatically. This will
-eventually result in an error in code trying to access the
-connection.
+
+\section{Default values of arguments \code{rshcmd} and \code{rshopts}}{
+
+Arguments \code{rshcmd} and \code{rshopts} are only used when connecting
+to an external host.
+
+The default method for connecting to an external host is via SSH and the
+system executable for this is given by argument \code{rshcmd}. The default
+is given by option \code{future.makeNodePSOCK.rshcmd} and if that is not
+set the default is either of \command{ssh} and \command{plink -ssh}.
+Most Unix-like systems, including macOS, have \command{ssh} preinstalled
+on the \code{PATH}. It is less common to find this command on Windows
+system, which are more likely to have the \command{PuTTY} software and
+its SSH client \command{plink} installed.
+If neither \command{ssh} nor \command{plink} is found, an informative error
+message is produced.
+It is also possible to specify the absolute path to the SSH client. To do
+this for PuTTY, specify the absolute path in the first element and option
+\command{-ssh} in the second as in
+\code{rshcmd = c("C:/Path/PuTTY/plink.exe", "-ssh")}.
+This is because all elements of \code{rshcmd} are individually "shell"
+quoted and element \code{rshcmd[1]} must be on the system \code{PATH}.
+
+Additional SSH options may be specified via argument \code{rshopts}, which
+defaults to option \code{future.makeNodePSOCK.rshopts}. For instance, a
+private SSH key can be provided as
+\code{rshopts = c("-i", "~/.ssh/my_private_key")}. PuTTY users should
+specify a PuTTY PPK file, e.g.
+\code{rshopts = c("-i", "C:/Users/joe/.ssh/my_keys.ppk")}.
+Contrary to \code{rshcmd}, elements of \code{rshopts} are not quoted.
+}
+
+\section{Reverse SSH tunneling}{
+
+The default is to use reverse SSH tunneling (\code{revtunnel = TRUE}) for
+workers running on other machines. This avoids the complication of
+otherwise having to configure port forwarding in firewalls, which often
+requires static IP address as well as privilieges to edit the firewall,
+something most users don't have.
+It also has the advantage of not having to know the internal and / or the
+public IP address / hostname of the master.
+Yet another advantage is that there will be no need for a DNS lookup by the
+worker machines to the master, which may not be configured or is disabled
+on some systems, e.g. compute clusters.
+}
+
+\section{Default value of argument \code{rscript}}{
+
+If \code{homogenenous} is FALSE, the \code{rscript} defaults to
+\code{"Rscript"}, i.e. it is assumed that the \command{Rscript} executable
+is available on the \code{PATH} of the worker.
+If \code{homogenenous} is TRUE, the \code{rscript} defaults to
+\code{file.path(R.home("bin"), "Rscript")}, i.e. it is basically assumed
+that the worker and the caller share the same file system and R installation.
+}
+
+\section{Default value of argument \code{homogeneous}}{
+
+The default value of \code{homogenenous} is TRUE if and only if either
+of the following is fullfilled:
+\itemize{
+ \item \code{worker} is \emph{localhost}
+ \item \code{revtunnel} is FALSE and \code{master} is \emph{localhost}
+ \item \code{worker} is neither an IP number nor a fully qualified domain
+ name (FQDN). A hostname is considered to be a FQDN if it contains
+ one or more periods
}
+}
+
\section{Connection time out}{
-Argument \code{connectTimeout} does \emph{not} work properly
-on Unix and macOS due to limitation in \R itself. For more details
-on this, please R devel thread 'BUG?: On Linux setTimeLimit() fails
-to propagate timeout error when it occurs (works on Windows)' on
-2016-10-26 (\url{https://stat.ethz.ch/pipermail/r-devel/2016-October/073309.html}). When used, the timeout will eventually trigger an error, but
-it won't happen until the socket connection timeout \code{timeout}
-itself happens.
+Argument \code{connectTimeout} does \emph{not} work properly on Unix and
+macOS due to limitation in \R itself. For more details on this, please see
+R-devel thread 'BUG?: On Linux setTimeLimit() fails to propagate timeout
+error when it occurs (works on Windows)' on 2016-10-26
+(\url{https://stat.ethz.ch/pipermail/r-devel/2016-October/073309.html}).
+When used, the timeout will eventually trigger an error, but it won't happen
+until the socket connection timeout \code{timeout} itself happens.
+}
+
+\section{Communication time out}{
+
+If there is no communication between the master and a worker within the
+\code{timeout} limit, then the corresponding socket connection will be
+closed automatically. This will eventually result in an error in code
+trying to access the connection.
}
+
\examples{
## Setup of three R workers on two remote machines are set up
workers <- c("n1.remote.org", "n2.remote.org", "n1.remote.org")
@@ -221,5 +332,13 @@ cl <- makeClusterPSOCK(
),
dryrun = TRUE
)
-}
+
+## Connect to remote Unix machine 'remote.server.org' on port 2200
+## as user 'bob' from a Windows machine with PuTTY installed
+cl <- makeClusterPSOCK(
+ "remote.server.org", user = "bob",
+ rshopts = c("-P", 2200, "-i", "C:/Users/bobby/.ssh/putty.ppk"),
+ dryrun = TRUE
+)
+}
diff --git a/man/mandelbrot.Rd b/man/mandelbrot.Rd
index b3bd019..ae12ca0 100644
--- a/man/mandelbrot.Rd
+++ b/man/mandelbrot.Rd
@@ -1,10 +1,10 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/mandelbrot.R
\name{mandelbrot}
-\alias{as.raster.Mandelbrot}
\alias{mandelbrot}
-\alias{mandelbrotTiles}
+\alias{as.raster.Mandelbrot}
\alias{plot.Mandelbrot}
+\alias{mandelbrot_tiles}
\title{Mandelbrot convergence counts}
\usage{
mandelbrot(...)
@@ -34,14 +34,14 @@ non-negative counts.
Mandelbrot convergence counts
}
\examples{
-counts <- mandelbrot(xmid=-0.75, ymid=0, side=3)
+counts <- mandelbrot(xmid = -0.75, ymid = 0, side = 3)
str(counts)
\dontrun{
plot(counts)
}
\dontrun{
-demo("mandelbrot", package="future", ask=FALSE)
+demo("mandelbrot", package = "future", ask = FALSE)
}
}
@@ -51,4 +51,3 @@ adopted from similar GPL code of Martin Maechler (available
from ftp://stat.ethz.ch/U/maechler/R/ on 2005-02-18 [sic!]).
}
\keyword{internal}
-
diff --git a/man/multicore.Rd b/man/multicore.Rd
index 8746d57..98e87f0 100644
--- a/man/multicore.Rd
+++ b/man/multicore.Rd
@@ -2,28 +2,31 @@
% Please edit documentation in R/multicore.R
\name{multicore}
\alias{multicore}
-\title{Create a multicore future whose value will be resolved asynchroneously in a forked parallel process}
+\title{Create a multicore future whose value will be resolved asynchronously in a forked parallel process}
\usage{
-multicore(expr, envir = parent.frame(), substitute = TRUE, globals = TRUE,
- workers = availableCores(constraints = "multicore"), earlySignal = FALSE,
- label = NULL, ...)
+multicore(expr, envir = parent.frame(), substitute = TRUE, lazy = FALSE,
+ seed = NULL, globals = TRUE, workers = availableCores(constraints =
+ "multicore"), earlySignal = FALSE, label = NULL, ...)
}
\arguments{
-\item{expr}{An \R \link[base]{expression}.}
+\item{expr}{An \R \link[base]{expression} to be evaluated.}
\item{envir}{The \link{environment} from where global
-objects should be identified. Depending on "evaluator",
-it may also be the environment in which the expression
-is evaluated.}
+objects should be identified. Depending on the future
+strategy (the \code{evaluator}), it may also be the environment
+in which the expression is evaluated.}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
+\item{lazy}{Specifies whether a future should be resolved
+lazily or eagerly (default).}
+
+\item{seed}{(optional) A L'Ecuyer-CMRG RNG seed.}
+
\item{globals}{A logical, a character vector,
or a named list for controlling how globals are handled.
-For details, see below section.
-This argument can be specified via the \dots arguments
-for \code{future()} and \code{futureCall()}.}
+For details, see below section.}
\item{workers}{The maximum number of multicore futures that can
be active at the same time before blocking.}
@@ -39,7 +42,7 @@ as soon as possible or not.}
A \link{MulticoreFuture}
If \code{workers == 1}, then all processing using done in the
current/main R session and we therefore fall back to using
-an eager future. This is also the case whenever multicore
+an sequential future. This is also the case whenever multicore
processing is not supported, e.g. on Windows.
}
\description{
@@ -57,7 +60,7 @@ cores available including the current/main R process, see
Not all systems support multicore futures. For instance,
it is not supported on Microsoft Windows. Trying to create
multicore futures on non-supported systems will silently
-fall back to using \link{eager} futures, which effectively
+fall back to using \link{sequential} futures, which effectively
corresponds to a multicore future that can handle one parallel
process (the current one) before blocking.
@@ -103,4 +106,3 @@ Use \code{\link{availableCores}("multicore") > 1L} to check
whether multicore futures are supported or not on the current
system.
}
-
diff --git a/man/multiprocess.Rd b/man/multiprocess.Rd
index 573c235..fbccf4a 100644
--- a/man/multiprocess.Rd
+++ b/man/multiprocess.Rd
@@ -2,23 +2,28 @@
% Please edit documentation in R/multiprocess.R
\name{multiprocess}
\alias{multiprocess}
-\title{Create a multiprocess future whose value will be resolved asynchroneously using multicore or a multisession evaluation}
+\title{Create a multiprocess future whose value will be resolved asynchronously using multicore or a multisession evaluation}
\usage{
multiprocess(expr, envir = parent.frame(), substitute = TRUE,
- globals = TRUE, workers = availableCores(), gc = FALSE,
- earlySignal = FALSE, label = NULL, ...)
+ lazy = FALSE, seed = NULL, globals = TRUE, workers = availableCores(),
+ gc = FALSE, earlySignal = FALSE, label = NULL, ...)
}
\arguments{
-\item{expr}{An \R \link[base]{expression}.}
+\item{expr}{An \R \link[base]{expression} to be evaluated.}
\item{envir}{The \link{environment} from where global
-objects should be identified. Depending on "evaluator",
-it may also be the environment in which the expression
-is evaluated.}
+objects should be identified. Depending on the future
+strategy (the \code{evaluator}), it may also be the environment
+in which the expression is evaluated.}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
+\item{lazy}{If \code{FALSE} (default), the future is resolved eagerly
+(immediately), otherwise not.}
+
+\item{seed}{(optional) A L'Ecuyer-CMRG RNG seed.}
+
\item{globals}{(optional) a logical, a character vector,
or a named list for controlling how globals are handled.
For details, see section 'Globals used by future expressions'
@@ -48,9 +53,6 @@ Regardless, its \emph{value is computed and resolved in
parallel in another process}.
}
\examples{
-## Multiprocess futures gives an error on R CMD check on
-## Windows (but not Linux or OS X) for unknown reasons.
-## The same code works in package tests.
\donttest{
## Use multiprocess futures
@@ -82,4 +84,3 @@ stopifnot(v == 0)
Internally \code{\link{multicore}()} and \code{\link{multisession}()}
are used.
}
-
diff --git a/man/multisession.Rd b/man/multisession.Rd
index 8f21e6f..6543c17 100644
--- a/man/multisession.Rd
+++ b/man/multisession.Rd
@@ -2,28 +2,32 @@
% Please edit documentation in R/multisession.R
\name{multisession}
\alias{multisession}
-\title{Create a multisession future whose value will be resolved asynchroneously in a parallel R session}
+\title{Create a multisession future whose value will be resolved asynchronously in a parallel R session}
\usage{
multisession(expr, envir = parent.frame(), substitute = TRUE,
- globals = TRUE, persistent = FALSE, workers = availableCores(),
- gc = FALSE, earlySignal = FALSE, label = NULL, ...)
+ lazy = FALSE, seed = NULL, globals = TRUE, persistent = FALSE,
+ workers = availableCores(), gc = FALSE, earlySignal = FALSE,
+ label = NULL, ...)
}
\arguments{
-\item{expr}{An \R \link[base]{expression}.}
+\item{expr}{An \R \link[base]{expression} to be evaluated.}
\item{envir}{The \link{environment} from where global
-objects should be identified. Depending on "evaluator",
-it may also be the environment in which the expression
-is evaluated.}
+objects should be identified. Depending on the future
+strategy (the \code{evaluator}), it may also be the environment
+in which the expression is evaluated.}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
+\item{lazy}{Specifies whether a future should be resolved
+lazily or eagerly (default).}
+
+\item{seed}{(optional) A L'Ecuyer-CMRG RNG seed.}
+
\item{globals}{A logical, a character vector,
or a named list for controlling how globals are handled.
-For details, see below section.
-This argument can be specified via the \dots arguments
-for \code{future()} and \code{futureCall()}.}
+For details, see below section.}
\item{persistent}{If FALSE, the evaluation environment is cleared
from objects prior to the evaluation of the future.}
@@ -77,10 +81,8 @@ The reason for this is that background sessions are currently
created using \code{\link[parallel:makeCluster]{makeCluster}()},
which requires that all R sessions are created at once.
}
+
\examples{
-## Multisession futures gives an error on R CMD check on
-## Windows (but not Linux or OS X) for unknown reasons.
-## The same code works in package tests.
\donttest{
## Use multisession futures
@@ -117,4 +119,3 @@ the former is not supported, see \link{multiprocess} futures.
Use \code{\link{availableCores}()} to see the total number of
cores that are available for the current R session.
}
-
diff --git a/man/nbrOfWorkers.Rd b/man/nbrOfWorkers.Rd
index 9a4eb0f..f3124c0 100644
--- a/man/nbrOfWorkers.Rd
+++ b/man/nbrOfWorkers.Rd
@@ -12,7 +12,7 @@ If NULL (default), the current evaluator as returned
by \code{\link{plan}()} is used.}
}
\value{
-A number in [1,Inf].
+A number in [1, Inf].
}
\description{
Gets the number of workers available
@@ -21,10 +21,9 @@ Gets the number of workers available
plan(multiprocess)
nbrOfWorkers() ## == availableCores()
-plan(multiprocess, workers=2)
+plan(multiprocess, workers = 2)
nbrOfWorkers() ## == 2
-plan(lazy)
+plan(sequential)
nbrOfWorkers() ## == 1
}
-
diff --git a/man/plan.Rd b/man/plan.Rd
index 64f186a..c4164e7 100644
--- a/man/plan.Rd
+++ b/man/plan.Rd
@@ -4,19 +4,27 @@
\alias{plan}
\title{Plan how to resolve a future}
\usage{
-plan(strategy = NULL, ..., substitute = TRUE, .call = TRUE)
+plan(strategy = NULL, ..., substitute = TRUE, .call = TRUE,
+ .cleanup = TRUE, .init = TRUE)
}
\arguments{
\item{strategy}{The evaluation function (or name of it) to use
for resolving a future. If NULL, then the current strategy is returned.}
+\item{\dots}{Additional arguments overriding the default arguments
+of the evaluation function. Which additional arguments are supported
+depends on what evaluation function is used, e.g. several support
+argument \code{workers} but not all. For details, see the individual
+functions of which some are linked to below.}
+
\item{substitute}{If TRUE, the \code{strategy} expression is
\code{substitute()}:d, otherwise not.}
\item{.call}{(internal) Used for recording the call to this function.}
-\item{\dots}{Additional arguments overriding the default arguments
-of the evaluation function.}
+\item{.cleanup}{(internal) Used to stop implicitly started clusters.}
+
+\item{.init}{(internal) Used to initiate workers.}
}
\value{
If a new strategy is chosen, then the previous one is returned
@@ -28,7 +36,7 @@ it specifies how \code{\link{future}()}:s are resolved,
e.g. sequentially or in parallel.
}
\details{
-The default strategy is \code{\link{eager}}, but the default can be
+The default strategy is \code{\link{sequential}}, but the default can be
configured by option \option{future.plan} and, if that is not set,
system environment variable \env{R_FUTURE_PLAN}.
To reset the strategy back to the default, use \code{plan("default")}.
@@ -36,17 +44,12 @@ To reset the strategy back to the default, use \code{plan("default")}.
\section{Implemented evaluation strategies}{
\itemize{
- \item{\code{\link{eager}}:}{
+ \item{\code{\link{sequential}}:}{
Resolves futures sequentially in the current R process.
}
- \item{\code{\link{lazy}}:}{
- Resolves futures synchronously (sequentially) in the current
- R process, but only if their values are requested. Futures for
- which the values are never requested will not be evaluated.
- }
\item{\code{\link{transparent}}:}{
- Resolves futures synchronously (sequentially) in the current
- R process and assignments will be done to the calling environment.
+ Resolves futures sequentially in the current R process and
+ assignments will be done to the calling environment.
Early stopping is enabled by default.
}
\item{\code{\link{multisession}}:}{
@@ -60,6 +63,7 @@ To reset the strategy back to the default, use \code{plan("default")}.
}
\item{\code{\link{multiprocess}}:}{
If multicore evaluation is supported, that will be used,
+ otherwise multisession evaluation will be used.
}
\item{\code{\link{cluster}}:}{
Resolves futures asynchronously (in parallel) in separate
@@ -79,11 +83,12 @@ that are typically available on high-performance compute
(HPC) clusters, e.g. LSF, Slurm, TORQUE/PBS, Sun Grid Engine,
and OpenLava.
}
+
\examples{
a <- b <- c <- NA_real_
-# A lazy uniprocess future
-plan(lazy)
+# An sequential future
+plan(sequential)
f <- future({
a <- 7
b <- 3
@@ -92,20 +97,20 @@ f <- future({
})
y <- value(f)
print(y)
-str(list(a=a, b=b, c=c)) ## All NAs
+str(list(a = a, b = b, c = c)) ## All NAs
-# An eager uniprocess future
-plan(eager)
+# A sequential future with lazy evaluation
+plan(sequential)
f <- future({
a <- 7
b <- 3
c <- 2
a * b * c
-})
+}) \%lazy\% TRUE
y <- value(f)
print(y)
-str(list(a=a, b=b, c=c)) ## All NAs
+str(list(a = a, b = b, c = c)) ## All NAs
# A multicore future
@@ -118,7 +123,7 @@ f <- future({
})
y <- value(f)
print(y)
-str(list(a=a, b=b, c=c)) ## All NAs
+str(list(a = a, b = b, c = c)) ## All NAs
## Multisession futures gives an error on R CMD check on
@@ -136,8 +141,7 @@ f <- future({
})
y <- value(f)
print(y)
-str(list(a=a, b=b, c=c)) ## All NAs
+str(list(a = a, b = b, c = c)) ## All NAs
}
}
-
diff --git a/man/private_length.Rd b/man/private_length.Rd
new file mode 100644
index 0000000..0761b2b
--- /dev/null
+++ b/man/private_length.Rd
@@ -0,0 +1,25 @@
+% Generated by roxygen2: do not edit by hand
+% Please edit documentation in R/utils.R
+\name{.length}
+\alias{.length}
+\title{Gets the length of an object without dispatching}
+\usage{
+.length(x)
+}
+\arguments{
+\item{x}{Any R object.}
+}
+\value{
+A non-negative integer.
+}
+\description{
+Gets the length of an object without dispatching
+}
+\details{
+This function returns \code{length(unclass(x))}, but tries to avoid
+calling \code{unclass(x)} unless necessary.
+}
+\seealso{
+\code{\link{.subset}()} and \code{\link{.subset2}()}.
+}
+\keyword{internal}
diff --git a/man/remote.Rd b/man/remote.Rd
index 2cc5feb..5578964 100644
--- a/man/remote.Rd
+++ b/man/remote.Rd
@@ -2,28 +2,32 @@
% Please edit documentation in R/remote.R
\name{remote}
\alias{remote}
-\title{Create a remote future whose value will be resolved asynchroneously in a remote process}
+\title{Create a remote future whose value will be resolved asynchronously in a remote process}
\usage{
-remote(expr, envir = parent.frame(), substitute = TRUE, globals = TRUE,
- persistent = TRUE, workers = NULL, user = NULL, revtunnel = TRUE,
- gc = FALSE, earlySignal = FALSE, myip = NULL, label = NULL, ...)
+remote(expr, envir = parent.frame(), substitute = TRUE, lazy = FALSE,
+ seed = NULL, globals = TRUE, persistent = TRUE, workers = NULL,
+ user = NULL, revtunnel = TRUE, gc = FALSE, earlySignal = FALSE,
+ myip = NULL, label = NULL, ...)
}
\arguments{
-\item{expr}{An \R \link[base]{expression}.}
+\item{expr}{An \R \link[base]{expression} to be evaluated.}
\item{envir}{The \link{environment} from where global
-objects should be identified. Depending on "evaluator",
-it may also be the environment in which the expression
-is evaluated.}
+objects should be identified. Depending on the future
+strategy (the \code{evaluator}), it may also be the environment
+in which the expression is evaluated.}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
+\item{lazy}{Specifies whether a future should be resolved
+lazily or eagerly (default).}
+
+\item{seed}{(optional) A L'Ecuyer-CMRG RNG seed.}
+
\item{globals}{A logical, a character vector,
or a named list for controlling how globals are handled.
-For details, see below section.
-This argument can be specified via the \dots arguments
-for \code{future()} and \code{futureCall()}.}
+For details, see below section.}
\item{persistent}{If FALSE, the evaluation environment is cleared
from objects prior to the evaluation of the future.}
@@ -61,13 +65,13 @@ which means that its \emph{value is computed and resolved
remotely in another process}.
}
\details{
-Note that remote futures use \code{persistent=TRUE} by default.
+Note that remote futures use \code{persistent = TRUE} by default.
}
\examples{
\dontrun{\donttest{
## Use a remote machine
-plan(remote, workers="remote.server.org")
+plan(remote, workers = "remote.server.org")
## Evaluate expression remotely
host \%<-\% { Sys.info()[["nodename"]] }
@@ -76,4 +80,3 @@ host
}}
}
-
diff --git a/man/requestCore.Rd b/man/requestCore.Rd
index b30c0ed..d1d21c0 100644
--- a/man/requestCore.Rd
+++ b/man/requestCore.Rd
@@ -5,8 +5,8 @@
\title{Request a core for multicore processing}
\usage{
requestCore(await, workers = availableCores(),
- times = getOption("future.wait.times", 1000),
- delta = getOption("future.wait.interval", 1),
+ timeout = getOption("future.wait.timeout", 30 * 24 * 60 * 60),
+ delta = getOption("future.wait.interval", 0.2),
alpha = getOption("future.wait.alpha", 1.01))
}
\arguments{
@@ -15,8 +15,8 @@ finished multicore subprocesses.}
\item{workers}{Total number of workers available.}
-\item{times}{Then maximum number of times subprocesses
-should be collected before timeout.}
+\item{timeout}{Maximum waiting time (in seconds) allowed
+before a timeout error is generated.}
\item{delta}{Then base interval (in seconds) to wait
between each try.}
@@ -33,4 +33,3 @@ If no cores are available, the current process
blocks until a core is available.
}
\keyword{internal}
-
diff --git a/man/resolve.Rd b/man/resolve.Rd
index c064150..f314bcd 100644
--- a/man/resolve.Rd
+++ b/man/resolve.Rd
@@ -2,28 +2,30 @@
% Please edit documentation in R/resolve.R
\name{resolve}
\alias{resolve}
-\title{Wait until all existing futures in an environment are resolved}
+\title{Resolve one or more futures synchronously}
\usage{
resolve(x, idxs = NULL, value = FALSE, recursive = 0, sleep = 1,
progress = getOption("future.progress", FALSE), ...)
}
\arguments{
-\item{x}{an environment holding futures.}
+\item{x}{a list, an environment, or a list environment holding futures
+that should be resolved. May also be a single \link{Future}.}
-\item{idxs}{subset of elements to check.}
+\item{idxs}{(optional) integer or logical index specifying the subset of
+elements to check.}
\item{value}{If TRUE, the values are retrieved, otherwise not.}
-\item{recursive}{A non-negative number specifying how deep of
-a recursion should be done. If TRUE, an infintive recursion
-is used. If FALSE or zero, no recursion is performed.}
+\item{recursive}{A non-negative number specifying how deep of a recursion
+should be done. If TRUE, an infinite recursion is used. If FALSE or zero,
+no recursion is performed.}
-\item{sleep}{Number of seconds to wait before checking
-if futures have been resolved since last time.}
+\item{sleep}{Number of seconds to wait before checking if futures have been
+resolved since last time.}
-\item{progress}{If TRUE textual progress summary is outputted.
-If a function, the it is called as \code{progress(done, total)}
-every time a future is resolved.}
+\item{progress}{If TRUE textual progress summary is outputted. If a
+function, the it is called as \code{progress(done, total)} every time a
+future is resolved.}
\item{\dots}{Not used}
}
@@ -31,14 +33,16 @@ every time a future is resolved.}
Returns \code{x} (regardless of subsetting or not).
}
\description{
-The environment is first scanned for futures and then the futures
-are polled until all are resolved. When a resolved future is
-detected its value is retrieved (optionally).
-This provides an efficient mechanism for waiting for a set of
-futures to be resolved and in the meanwhile retrieving values
-of already resolved futures.
+This function provides an efficient mechanism for waiting for multiple
+futures in a container (e.g. list or environment) to be resolved while in
+the meanwhile retrieving values of already resolved futures.
+}
+\details{
+This function is resolves synchronously, i.e. it blocks until \code{x} and
+any containing futures are resolved.
}
\seealso{
-futureOf
+To resolve a future \emph{variable}, first retrieve its
+\link{Future} object using \code{\link{futureOf}()}, e.g.
+\code{resolve(futureOf(x))}.
}
-
diff --git a/man/resolved.Rd b/man/resolved.Rd
index 4c3b03a..c0474ab 100644
--- a/man/resolved.Rd
+++ b/man/resolved.Rd
@@ -26,7 +26,6 @@ This method needs to be implemented by the class that implement
the Future API. The implementation must never throw an error,
but only return either TRUE or FALSE.
It should also be possible to use the method for polling the
-future until it is resolved (without having to wait infinitly long),
+future until it is resolved (without having to wait infinitely long),
e.g. \code{while (!resolved(future)) Sys.sleep(5)}.
}
-
diff --git a/man/run.Rd b/man/run.Rd
index a2452bc..0461ad7 100644
--- a/man/run.Rd
+++ b/man/run.Rd
@@ -1,8 +1,8 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/Future-class.R
\name{run.Future}
-\alias{run}
\alias{run.Future}
+\alias{run}
\title{Run a future}
\usage{
\method{run}{Future}(future, ...)
@@ -24,4 +24,4 @@ Further calls will result in an informative error.
If a future is not run when its value is queried,
then it is run at that point.
}
-
+\keyword{internal}
diff --git a/man/eager.Rd b/man/sequential.Rd
similarity index 51%
rename from man/eager.Rd
rename to man/sequential.Rd
index 7d7d6df..cb884a8 100644
--- a/man/eager.Rd
+++ b/man/sequential.Rd
@@ -1,38 +1,48 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/uniprocess.R
+% Please edit documentation in R/DEPRECATED.R, R/sequential.R
\name{eager}
\alias{eager}
\alias{lazy}
-\alias{transparent}
+\alias{sequential}
\alias{uniprocess}
-\title{Create a uniprocess future whose value will be in the current R session}
+\alias{transparent}
+\title{Create a sequential future whose value will be in the current R session}
\usage{
-eager(expr, envir = parent.frame(), substitute = TRUE, globals = TRUE,
- local = TRUE, earlySignal = FALSE, label = NULL, lazy = FALSE, ...)
+eager(expr, envir = parent.frame(), substitute = TRUE, lazy = FALSE,
+ seed = NULL, globals = TRUE, local = TRUE, earlySignal = FALSE,
+ label = NULL, ...)
-transparent(expr, envir = parent.frame(), substitute = TRUE,
- globals = FALSE, local = FALSE, earlySignal = TRUE, label = NULL,
- lazy = FALSE, ...)
+lazy(expr, envir = parent.frame(), substitute = TRUE, lazy = TRUE,
+ seed = NULL, globals = TRUE, local = TRUE, earlySignal = FALSE,
+ label = NULL, ...)
-lazy(expr, envir = parent.frame(), substitute = TRUE, globals = TRUE,
- local = TRUE, earlySignal = FALSE, label = NULL, lazy = TRUE, ...)
+sequential(expr, envir = parent.frame(), substitute = TRUE, lazy = FALSE,
+ seed = NULL, globals = TRUE, local = TRUE, earlySignal = FALSE,
+ label = NULL, ...)
+
+transparent(expr, envir = parent.frame(), substitute = TRUE, lazy = FALSE,
+ seed = NULL, globals = FALSE, local = FALSE, earlySignal = TRUE,
+ label = NULL, ...)
}
\arguments{
-\item{expr}{An \R \link[base]{expression}.}
+\item{expr}{An \R \link[base]{expression} to be evaluated.}
\item{envir}{The \link{environment} from where global
-objects should be identified. Depending on "evaluator",
-it may also be the environment in which the expression
-is evaluated.}
+objects should be identified. Depending on the future
+strategy (the \code{evaluator}), it may also be the environment
+in which the expression is evaluated.}
\item{substitute}{If TRUE, argument \code{expr} is
\code{\link[base]{substitute}()}:ed, otherwise not.}
+\item{lazy}{Specifies whether a future should be resolved
+lazily or eagerly (default).}
+
+\item{seed}{(optional) A L'Ecuyer-CMRG RNG seed.}
+
\item{globals}{A logical, a character vector,
or a named list for controlling how globals are handled.
-For details, see below section.
-This argument can be specified via the \dots arguments
-for \code{future()} and \code{futureCall()}.}
+For details, see below section.}
\item{local}{If TRUE, the expression is evaluated such that
all assignments are done to local temporary environment, otherwise
@@ -43,46 +53,42 @@ as soon as possible or not.}
\item{label}{An optional character string label attached to the future.}
-\item{lazy}{If \code{FALSE} (default), the future is resolved eagerly
-(immediately), otherwise not.}
-
\item{...}{Additional arguments passed to the "evaluator".}
}
\value{
-A \link{UniprocessFuture}.
+A \link{SequentialFuture}.
}
\description{
-A uniprocess future is a future that is evaluated sequentially in the
-current R session. The default is to resolve it eagerly, which means
-that its \emph{value is computed and resolved immediately}, which is
-how regular expressions are evaluated in R.
+A sequential future is a future that is evaluated sequentially in the
+current R session similarly to how \R expressions are evaluated in R.
The only difference to R itself is that globals are validated
by default just as for all other types of futures in this package.
}
\details{
-The preferred way to create a uniprocess future is not to call these functions
-directly, but to register them via \code{\link{plan}(eager)} such that it
-becomes the default mechanism for all futures. After this
+The preferred way to create a sequential future is not to call these functions
+directly, but to register them via \code{\link{plan}(sequential)} such that
+it becomes the default mechanism for all futures. After this
\code{\link{future}()} and \code{\link{\%<-\%}} will create
-\emph{eager uniprocess futures}.
+\emph{sequential futures}.
}
\section{transparent futures}{
-Transparent futures are eager uniprocess futures configured to emulate how R
+Transparent futures are sequential futures configured to emulate how R
evaluates expressions as far as possible. For instance, errors and
warnings are signaled immediately and assignments are done to the
calling environment (without \code{local()} as default for all other
types of futures). This makes transparent futures ideal for
troubleshooting, especially when there are errors.
}
+
\examples{
-## Use an eager uniprocess futures
-plan(eager)
+## Use sequential futures
+plan(sequential)
## A global variable
a <- 0
-## Create an eager uniprocess future (explicitly)
+## Create a sequential future
f <- future({
b <- 3
c <- 2
@@ -100,4 +106,3 @@ v <- value(f)
print(v)
stopifnot(v == 0)
}
-
diff --git a/man/sessionDetails.Rd b/man/sessionDetails.Rd
index 9258062..9d55002 100644
--- a/man/sessionDetails.Rd
+++ b/man/sessionDetails.Rd
@@ -16,7 +16,6 @@ Invisibly a list of all details.
Outputs details on the current R session
}
\details{
-Note that remote futures use \code{persistent=TRUE} by default.
+Note that remote futures use \code{persistent = TRUE} by default.
}
\keyword{internal}
-
diff --git a/man/supportsMulticore.Rd b/man/supportsMulticore.Rd
index 8d6afd9..5bc3588 100644
--- a/man/supportsMulticore.Rd
+++ b/man/supportsMulticore.Rd
@@ -17,4 +17,3 @@ except on Microsoft Windows.
\seealso{
To use multicore futures, use \code{\link{plan}(\link{multicore})}.
}
-
diff --git a/man/tweak.Rd b/man/tweak.Rd
index 008a85e..c0e1cb9 100644
--- a/man/tweak.Rd
+++ b/man/tweak.Rd
@@ -9,11 +9,11 @@ tweak(strategy, ..., penvir = parent.frame())
\arguments{
\item{strategy}{An existing future function or the name of one.}
-\item{penvir}{The environment used when searching for a future
-function by its name.}
-
\item{\dots}{Named arguments to replace the defaults of existing
arguments.}
+
+\item{penvir}{The environment used when searching for a future
+function by its name.}
}
\value{
a future function.
@@ -25,4 +25,3 @@ Tweaks a future function by adjusting its default arguments
Use \code{\link{plan}()} to set a future to become the
new default strategy.
}
-
diff --git a/man/usedCores.Rd b/man/usedCores.Rd
index 4a56d94..6a2618b 100644
--- a/man/usedCores.Rd
+++ b/man/usedCores.Rd
@@ -7,14 +7,13 @@
usedCores()
}
\value{
-A positive integer equal or greater than one.
+A non-negative integer.
}
\description{
-Get number of children plus one (for the current process)
+Get number of children (and don't count the current process)
used by the current R session. The number of children
is the total number of subprocesses launched by this
process that are still running and whose values have yet
not been collected.
}
\keyword{internal}
-
diff --git a/man/value.Rd b/man/value.Rd
index 04aa3b7..117c129 100644
--- a/man/value.Rd
+++ b/man/value.Rd
@@ -1,8 +1,8 @@
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/Future-class.R
\name{value.Future}
-\alias{value}
\alias{value.Future}
+\alias{value}
\title{The value of a future}
\usage{
\method{value}{Future}(future, signal = TRUE, ...)
@@ -26,4 +26,3 @@ the evaluation blocks until the future is resolved.
This method needs to be implemented by the class that implement
the Future API.
}
-
diff --git a/man/values.Rd b/man/values.Rd
index ea98f6b..d968960 100644
--- a/man/values.Rd
+++ b/man/values.Rd
@@ -22,4 +22,3 @@ All future elements are replaced by their corresponding
\code{value()} values. For all other elements, the existing
object is kept.
}
-
diff --git a/revdep/README.md b/revdep/README.md
deleted file mode 100644
index 8745595..0000000
--- a/revdep/README.md
+++ /dev/null
@@ -1,404 +0,0 @@
-# Setup
-
-## Platform
-
-|setting |value |
-|:--------|:----------------------------|
-|version |R version 3.3.2 (2016-10-31) |
-|system |x86_64, linux-gnu |
-|ui |X11 |
-|language |en |
-|collate |en_US.UTF-8 |
-|tz |Europe/Copenhagen |
-|date |2016-11-12 |
-
-## Packages
-
-|package |* |version |date |source |
-|:--------|:--|:----------|:----------|:---------------------------------|
-|digest | |0.6.10 |2016-08-02 |CRAN (R 3.3.1) |
-|future | |1.1.1-9000 |2016-11-12 |local (HenrikBengtsson/future at NA) |
-|globals | |0.7.1 |2016-10-14 |local |
-|listenv | |0.6.0 |2015-12-28 |CRAN (R 3.3.1) |
-|markdown | |0.7.7 |2015-04-22 |cran (@0.7.7) |
-|R.rsp | |0.30.0 |2016-05-15 |cran (@0.30.0) |
-
-# Check results
-
-19 packages
-
-|package |version | errors| warnings| notes|
-|:----------------|:-------|------:|--------:|-----:|
-|ACNE |0.8.1 | 0| 0| 0|
-|aroma.affymetrix |3.0.0 | 0| 0| 0|
-|aroma.cn |1.6.1 | 0| 0| 0|
-|aroma.core |3.0.0 | 0| 0| 1|
-|calmate |0.12.1 | 0| 0| 0|
-|doFuture |0.3.0 | 0| 0| 0|
-|fiery |0.2.1 | 0| 0| 0|
-|future.BatchJobs |0.13.1 | 0| 0| 0|
-|GeneBreak |1.4.0 | 0| 0| 1|
-|MPAgenomics |1.1.2 | 0| 0| 2|
-|NSA |0.0.32 | 0| 0| 6|
-|pbmcapply |1.1.1 | 0| 0| 0|
-|PECA |1.10.0 | 0| 0| 1|
-|PSCBS |0.62.0 | 0| 0| 0|
-|PureCN |1.2.3 | 0| 1| 1|
-|QDNAseq |1.10.0 | 0| 0| 0|
-|Repitools |1.20.0 | 0| 0| 2|
-|R.filesets |2.10.0 | 0| 0| 0|
-|TIN |1.6.0 | 0| 0| 2|
-
-## ACNE (0.8.1)
-Maintainer: Henrik Bengtsson <henrikb at braju.com>
-Bug reports: https://github.com/HenrikBengtsson/ACNE/issues
-
-0 errors | 0 warnings | 0 notes
-
-## aroma.affymetrix (3.0.0)
-Maintainer: Henrik Bengtsson <henrikb at braju.com>
-Bug reports: https://github.com/HenrikBengtsson/aroma.affymetrix/issues
-
-0 errors | 0 warnings | 0 notes
-
-## aroma.cn (1.6.1)
-Maintainer: Henrik Bengtsson <henrikb at braju.com>
-Bug reports: https://github.com/HenrikBengtsson/aroma.cn/issues
-
-0 errors | 0 warnings | 0 notes
-
-## aroma.core (3.0.0)
-Maintainer: Henrik Bengtsson <henrikb at braju.com>
-Bug reports: https://github.com/HenrikBengtsson/aroma.core/issues
-
-0 errors | 0 warnings | 1 note
-
-```
-checking package dependencies ... NOTE
-Packages suggested but not available for checking:
- ‘expectile’ ‘HaarSeg’ ‘mpcbs’
-```
-
-## calmate (0.12.1)
-Maintainer: Henrik Bengtsson <henrikb at braju.com>
-Bug reports: https://github.com/HenrikBengtsson/calmate/issues
-
-0 errors | 0 warnings | 0 notes
-
-## doFuture (0.3.0)
-Maintainer: Henrik Bengtsson <henrikb at braju.com>
-Bug reports: https://github.com/HenrikBengtsson/doFuture/issues
-
-0 errors | 0 warnings | 0 notes
-
-## fiery (0.2.1)
-Maintainer: Thomas Lin Pedersen <thomasp85 at gmail.com>
-Bug reports: https://github.com/thomasp85/fiery/issues
-
-0 errors | 0 warnings | 0 notes
-
-## future.BatchJobs (0.13.1)
-Maintainer: Henrik Bengtsson <henrikb at braju.com>
-Bug reports: https://github.com/HenrikBengtsson/future.BatchJobs/issues
-
-0 errors | 0 warnings | 0 notes
-
-## GeneBreak (1.4.0)
-Maintainer: Evert van den Broek <vandenbroek.evert at gmail.com>
-
-0 errors | 0 warnings | 1 note
-
-```
-checking R code for possible problems ... NOTE
-.glmbreak: no visible global function definition for ‘glm’
-.glmbreak: no visible global function definition for ‘predict’
-addGeneAnnotation,CopyNumberBreakPoints: no visible global function
- definition for ‘head’
-bpStats,CopyNumberBreakPoints: no visible global function definition
- for ‘sd’
-bpStats,CopyNumberBreakPoints: no visible global function definition
- for ‘p.adjust’
-Undefined global functions or variables:
- glm head p.adjust predict sd
-Consider adding
- importFrom("stats", "glm", "p.adjust", "predict", "sd")
- importFrom("utils", "head")
-to your NAMESPACE file.
-```
-
-## MPAgenomics (1.1.2)
-Maintainer: Samuel Blanck <samuel.blanck at inria.fr>
-
-0 errors | 0 warnings | 2 notes
-
-```
-checking dependencies in R code ... NOTE
-'library' or 'require' calls in package code:
- ‘R.devices’ ‘R.filesets’ ‘R.methodsS3’ ‘R.oo’ ‘aroma.affymetrix’
- ‘aroma.cn’ ‘aroma.core’ ‘aroma.light’ ‘matrixStats’ ‘snowfall’
- Please use :: or requireNamespace() instead.
- See section 'Suggested packages' in the 'Writing R Extensions' manual.
-Unexported object imported by a ':::' call: ‘cghseg:::segmeanCO’
- See the note in ?`:::` about the use of this operator.
-
-checking R code for possible problems ... NOTE
-.varregtimescount: no visible global function definition for ‘var’
-CGHSEGaroma: no visible global function definition for ‘read.csv’
-CGHSEGaroma : <anonymous>: no visible global function definition for
- ‘points’
-CGHSEGaroma : <anonymous>: no visible global function definition for
- ‘lines’
-CGHSEGaroma : <anonymous>: no visible global function definition for
- ‘write.table’
-CGHcall: no visible global function definition for ‘mad’
-... 43 lines ...
-tumorboostPlot: no visible global function definition for ‘par’
-tumorboostPlot: no visible global function definition for ‘axis’
-tumorboostPlot: no visible global function definition for ‘points’
-Undefined global functions or variables:
- axis head lines lm mad median optim par points read.csv sd var
- write.table
-Consider adding
- importFrom("graphics", "axis", "lines", "par", "points")
- importFrom("stats", "lm", "mad", "median", "optim", "sd", "var")
- importFrom("utils", "head", "read.csv", "write.table")
-to your NAMESPACE file.
-```
-
-## NSA (0.0.32)
-Maintainer: Maria Ortiz-Estevez <mortizest at gmail.com>
-
-0 errors | 0 warnings | 6 notes
-
-```
-checking package dependencies ... NOTE
-Depends: includes the non-default packages:
- ‘R.methodsS3’ ‘MASS’ ‘matrixStats’ ‘R.oo’ ‘R.utils’ ‘aroma.core’
- ‘aroma.affymetrix’ ‘DNAcopy’
-Adding so many packages to the search path is excessive and importing
-selectively is preferable.
-
-checking top-level files ... NOTE
-Non-standard file/directory found at top level:
- ‘incl’
-
-checking dependencies in R code ... NOTE
-Packages in Depends field not imported from:
- ‘DNAcopy’ ‘MASS’ ‘R.methodsS3’ ‘R.oo’ ‘aroma.affymetrix’ ‘aroma.core’
- ‘matrixStats’
- These packages need to be imported from (in the NAMESPACE file)
- for when this namespace is loaded but not attached.
-
-checking S3 generic/method consistency ... NOTE
-Found the following apparent S3 methods exported but not registered:
- NSAByTotalAndFracB.matrix allocateOutputDataSets.NSANormalization
- allocateOutputDataSets.SNPsNormalization
- allocateOutputDataSets.SampleNormalization
- as.character.NSANormalization as.character.SNPsNormalization
- as.character.SampleNormalization findArraysTodo.NSANormalization
- findArraysTodo.SampleNormalization findUnitsTodo.SNPsNormalization
- fitNSA.matrix fitNSAcnPs.matrix getDataSets.NSANormalization
- getDataSets.SNPsNormalization getDataSets.SampleNormalization
- getName.NSANormalization getName.SNPsNormalization
- getName.SampleNormalization getOutputDataSets.NSANormalization
- getOutputDataSets.SNPsNormalization
- getOutputDataSets.SampleNormalization getPath.NSANormalization
- getPath.SNPsNormalization getPath.SampleNormalization
- getRootPath.NSANormalization getRootPath.SNPsNormalization
- getRootPath.SampleNormalization process.NSANormalization
- process.SNPsNormalization process.SampleNormalization
- sampleNByTotalAndFracB.numeric snpsNByTotalAndFracB.matrix
-See section ‘Registering S3 methods’ in the ‘Writing R Extensions’
-manual.
-
-checking R code for possible problems ... NOTE
-NB: .First.lib is obsolete and will not be used in R >= 3.0.0
-
-.First.lib: no visible global function definition for
- ‘packageDescription’
-NSAByTotalAndFracB.matrix: no visible global function definition for
- ‘throw’
-NSAByTotalAndFracB.matrix: no visible global function definition for
- ‘str’
-NSANormalization: no visible global function definition for ‘throw’
-... 279 lines ...
- extractMatrix findUnitsTodo getAsteriskTags getChipType getFile
- getFullName getFullNames getGenomeInformation getName getNames
- getPath getPathname getPathnames getPositions getRam getRootPath
- getTags getUnitsOnChromosome hist median nbrOfFiles newInstance
- packageDescription rowAlls rowMedians segment setTags str throw trim
- verbose
-Consider adding
- importFrom("graphics", "hist")
- importFrom("stats", "approxfun", "median")
- importFrom("utils", "packageDescription", "str")
-to your NAMESPACE file.
-
-checking Rd line widths ... NOTE
-Rd file 'NSANormalization.Rd':
- \examples lines wider than 100 characters:
- by <- 50e3; # 50kb bins; you may want to try with other amounts of smoothing xOut <- seq(from=xRange[1], to=xRange[2], by=by);
- plot(getSignals(cnCNPS), getSignals(cnSNPS), xlim=Clim, ylim=Clim); abline(a=0, b=1, col="red", lwd=2);
-
-These lines will be truncated in the PDF manual.
-```
-
-## pbmcapply (1.1.1)
-Maintainer: Kevin kuang <kvn.kuang at mail.utoronto.ca>
-Bug reports: https://github.com/kvnkuang/pbmcapply/issues
-
-0 errors | 0 warnings | 0 notes
-
-## PECA (1.10.0)
-Maintainer: Tomi Suomi <tomi.suomi at utu.fi>
-
-0 errors | 0 warnings | 1 note
-
-```
-checking Rd line widths ... NOTE
-Rd file 'PECA.Rd':
- \usage lines wider than 90 characters:
- PECA_AffyBatch(affy=NULL, normalize=FALSE, test="t", type="median", paired=FALSE, progress=FALSE)
-
-These lines will be truncated in the PDF manual.
-```
-
-## PSCBS (0.62.0)
-Maintainer: Henrik Bengtsson <henrikb at braju.com>
-Bug reports: https://github.com/HenrikBengtsson/PSCBS/issues
-
-0 errors | 0 warnings | 0 notes
-
-## PureCN (1.2.3)
-Maintainer: Markus Riester <markus.riester at novartis.com>
-
-0 errors | 1 warning | 1 note
-
-```
-checking whether package ‘PureCN’ can be installed ... WARNING
-Found the following significant warnings:
- Warning: namespace ‘Matrix’ is not available and has been replaced
-See ‘/home/hb/repositories/future/revdep/checks/PureCN.Rcheck/00install.out’ for details.
-
-checking R code for possible problems ... NOTE
-Found the following calls to data() loading into the global environment:
-File ‘PureCN/R/bootstrapResults.R’:
- data(purecn.example.output)
-File ‘PureCN/R/callAlterations.R’:
- data(purecn.example.output)
- data(purecn.example.output)
-File ‘PureCN/R/callLOH.R’:
- data(purecn.example.output)
-File ‘PureCN/R/createCurationFile.R’:
- data(purecn.example.output)
-File ‘PureCN/R/curateResults.R’:
- data(purecn.example.output)
- data(purecn.example.output)
-File ‘PureCN/R/plotAbs.R’:
- data(purecn.example.output)
-File ‘PureCN/R/predictSomatic.R’:
- data(purecn.example.output)
-File ‘PureCN/R/readCurationFile.R’:
- data(purecn.example.output)
-See section ‘Good practice’ in ‘?data’.
-```
-
-## QDNAseq (1.10.0)
-Maintainer: Daoud Sie <d.sie at vumc.nl>
-Bug reports: https://github.com/ccagc/QDNAseq/issues
-
-0 errors | 0 warnings | 0 notes
-
-## Repitools (1.20.0)
-Maintainer: Mark Robinson <mark.robinson at imls.uzh.ch>
-
-0 errors | 0 warnings | 2 notes
-
-```
-checking R code for possible problems ... NOTE
-Found an obsolete/platform-specific call in the following function:
- ‘maskOut’
-Found the platform-specific device:
- ‘windows’
-dev.new() is the preferred way to open a new device, in the unlikely
-event one is needed.
-.cpgBoxplots: no visible global function definition for ‘pdf’
-.cpgBoxplots: no visible global function definition for ‘par’
-.cpgBoxplots: no visible global function definition for ‘dev.off’
-... 291 lines ...
- rainbow read.table rect str t.test text title verbose
-Consider adding
- importFrom("grDevices", "dev.off", "pdf", "rainbow")
- importFrom("graphics", "abline", "axis", "barplot", "bxp", "grid",
- "layout", "legend", "lines", "matlines", "matplot", "mtext",
- "par", "persp", "plot", "plot.new", "plot.window", "points",
- "polygon", "rect", "text", "title")
- importFrom("stats", "dbeta", "embed", "filter", "kmeans", "lm",
- "lowess", "p.adjust", "predict", "pt", "qnorm", "t.test")
- importFrom("utils", "read.table", "str")
-to your NAMESPACE file.
-
-checking Rd line widths ... NOTE
-Rd file 'ChromaBlocks.Rd':
- \usage lines wider than 90 characters:
- ChromaBlocks(rs.ip, rs.input, organism, chrs, ipWidth=100, inputWidth=500, preset=NULL, blockWidth=NULL, minBlocks=NULL, extend=NULL, c ... [TRUNCATED]
-
-Rd file 'GCbiasPlots.Rd':
- \usage lines wider than 90 characters:
- cex = 0.2, pch.col = "black", line.col = "red", lty = 1, lwd = 2, verbose = TRUE)
-
-Rd file 'absoluteCN.Rd':
-... 57 lines ...
-
-Rd file 'regionStats.Rd':
- \usage lines wider than 90 characters:
- regionStats(x, design = NULL, maxFDR=0.05, n.perm=5, window=600, mean.trim=.1, min.probes=10, max.gap=500, two.sides=TRUE, ndf, return. ... [TRUNCATED]
- regionStats(x, design = NULL, maxFDR=0.05, n.perm=5, window=600, mean.trim=.1, min.probes=10, max.gap=500, two.sides=TRUE, ind=NULL, re ... [TRUNCATED]
-
-Rd file 'writeWig.Rd':
- \usage lines wider than 90 characters:
- writeWig(rs, seq.len = NULL, design=NULL, sample=20, drop.zero=TRUE, normalize=TRUE, verbose=TRUE)
-
-These lines will be truncated in the PDF manual.
-```
-
-## R.filesets (2.10.0)
-Maintainer: Henrik Bengtsson <henrikb at braju.com>
-Bug reports: https://github.com/HenrikBengtsson/R.filesets/issues
-
-0 errors | 0 warnings | 0 notes
-
-## TIN (1.6.0)
-Maintainer: Bjarne Johannessen <bjajoh at rr-research.no>
-
-0 errors | 0 warnings | 2 notes
-
-```
-checking top-level files ... NOTE
-Non-standard file/directory found at top level:
- ‘doc’
-
-checking R code for possible problems ... NOTE
-aberrantExonUsage: no visible global function definition for ‘quantile’
-aberrantExonUsage: no visible global function definition for ‘ave’
-clusterPlot: no visible global function definition for ‘dist’
-clusterPlot: no visible global function definition for ‘hclust’
-clusterPlot: no visible global function definition for
- ‘colorRampPalette’
-clusterPlot: no visible global function definition for ‘par’
-clusterPlot: no visible global function definition for ‘png’
-clusterPlot: no visible global function definition for ‘jpeg’
-... 50 lines ...
- importFrom("stats", "ave", "dist", "hclust", "median", "quantile")
- importFrom("utils", "data", "read.table")
-to your NAMESPACE file.
-
-Found the following assignments to the global environment:
-File ‘TIN/R/aberrantExonUsage.R’:
- assign("quantiles", quantiles, envir = .GlobalEnv)
- assign("aberrantExons", aberrantExons, envir = .GlobalEnv)
-File ‘TIN/R/correlationPlot.R’:
- assign("randomGeneSetsDist", B, envir = .GlobalEnv)
- assign("traPermutationsDist", L, envir = .GlobalEnv)
-```
-
diff --git a/revdep/check.R b/revdep/check.R
deleted file mode 100644
index 1eefa00..0000000
--- a/revdep/check.R
+++ /dev/null
@@ -1,20 +0,0 @@
-library("devtools")
-
-availableCores <- function() {
- getenv <- function(name) {
- as.integer(Sys.getenv(name, NA_character_))
- }
- getopt <- function(name) {
- as.integer(getOption(name, NA_integer_))
- }
- if (is.finite(n <- getopt("mc.cores") + 1L)) return(n)
- if (is.finite(n <- getopt("Ncpus") + 1L)) return(n)
- if (is.finite(n <- getenv("PBS_NUM_PPN"))) return(n)
- if (is.finite(n <- getenv("SLURM_CPUS_PER_TASK"))) return(n)
- if (is.finite(n <- getenv("NSLOTS"))) return(n)
- 1L
-}
-
-revdep_check(bioconductor = TRUE, recursive = TRUE, threads = availableCores())
-revdep_check_save_summary()
-revdep_check_print_problems()
diff --git a/revdep/checks.rds b/revdep/checks.rds
deleted file mode 100644
index 325050b..0000000
Binary files a/revdep/checks.rds and /dev/null differ
diff --git a/revdep/problems.md b/revdep/problems.md
deleted file mode 100644
index cde0fc7..0000000
--- a/revdep/problems.md
+++ /dev/null
@@ -1,67 +0,0 @@
-# Setup
-
-## Platform
-
-|setting |value |
-|:--------|:----------------------------|
-|version |R version 3.3.2 (2016-10-31) |
-|system |x86_64, linux-gnu |
-|ui |X11 |
-|language |en |
-|collate |en_US.UTF-8 |
-|tz |Europe/Copenhagen |
-|date |2016-11-12 |
-
-## Packages
-
-|package |* |version |date |source |
-|:--------|:--|:----------|:----------|:---------------------------------|
-|digest | |0.6.10 |2016-08-02 |CRAN (R 3.3.1) |
-|future | |1.1.1-9000 |2016-11-12 |local (HenrikBengtsson/future at NA) |
-|globals | |0.7.1 |2016-10-14 |local |
-|listenv | |0.6.0 |2015-12-28 |CRAN (R 3.3.1) |
-|markdown | |0.7.7 |2015-04-22 |cran (@0.7.7) |
-|R.rsp | |0.30.0 |2016-05-15 |cran (@0.30.0) |
-
-# Check results
-
-1 packages with problems
-
-|package |version | errors| warnings| notes|
-|:-------|:-------|------:|--------:|-----:|
-|PureCN |1.2.3 | 0| 1| 1|
-
-## PureCN (1.2.3)
-Maintainer: Markus Riester <markus.riester at novartis.com>
-
-0 errors | 1 warning | 1 note
-
-```
-checking whether package ‘PureCN’ can be installed ... WARNING
-Found the following significant warnings:
- Warning: namespace ‘Matrix’ is not available and has been replaced
-See ‘/home/hb/repositories/future/revdep/checks/PureCN.Rcheck/00install.out’ for details.
-
-checking R code for possible problems ... NOTE
-Found the following calls to data() loading into the global environment:
-File ‘PureCN/R/bootstrapResults.R’:
- data(purecn.example.output)
-File ‘PureCN/R/callAlterations.R’:
- data(purecn.example.output)
- data(purecn.example.output)
-File ‘PureCN/R/callLOH.R’:
- data(purecn.example.output)
-File ‘PureCN/R/createCurationFile.R’:
- data(purecn.example.output)
-File ‘PureCN/R/curateResults.R’:
- data(purecn.example.output)
- data(purecn.example.output)
-File ‘PureCN/R/plotAbs.R’:
- data(purecn.example.output)
-File ‘PureCN/R/predictSomatic.R’:
- data(purecn.example.output)
-File ‘PureCN/R/readCurationFile.R’:
- data(purecn.example.output)
-See section ‘Good practice’ in ‘?data’.
-```
-
diff --git a/revdep/timing.md b/revdep/timing.md
deleted file mode 100644
index 8681cd1..0000000
--- a/revdep/timing.md
+++ /dev/null
@@ -1,25 +0,0 @@
-# Check times
-
-| |package |version | check_time|
-|:--|:----------------|:-------|----------:|
-|15 |PureCN |1.2.3 | 654|
-|17 |Repitools |1.20.0 | 441.5|
-|14 |PSCBS |0.62.0 | 281.6|
-|7 |fiery |0.2.1 | 259.8|
-|16 |QDNAseq |1.10.0 | 205.1|
-|19 |TIN |1.6.0 | 199.8|
-|13 |PECA |1.10.0 | 193.1|
-|9 |GeneBreak |1.4.0 | 178.4|
-|2 |aroma.affymetrix |3.0.0 | 170.2|
-|8 |future.BatchJobs |0.13.1 | 149.5|
-|6 |doFuture |0.3.0 | 135.7|
-|18 |R.filesets |2.10.0 | 131.4|
-|4 |aroma.core |3.0.0 | 124.7|
-|10 |MPAgenomics |1.1.2 | 73|
-|3 |aroma.cn |1.6.1 | 72.5|
-|1 |ACNE |0.8.1 | 58.6|
-|5 |calmate |0.12.1 | 49.4|
-|11 |NSA |0.0.32 | 36.8|
-|12 |pbmcapply |1.1.1 | 24|
-
-
diff --git a/tests/000.sessionDetails.R b/tests/000.sessionDetails.R
new file mode 100644
index 0000000..618ed6d
--- /dev/null
+++ b/tests/000.sessionDetails.R
@@ -0,0 +1,10 @@
+library("future")
+
+sd <- sessionDetails()
+print(sd)
+print(sd, output = "message")
+
+rm(list = "sd")
+
+
+
diff --git a/tests/ClusterRegistry.R b/tests/ClusterRegistry.R
index afa12b0..759d093 100644
--- a/tests/ClusterRegistry.R
+++ b/tests/ClusterRegistry.R
@@ -2,9 +2,9 @@ source("incl/start.R")
message("*** ClusterRegistry() ... ")
-for (cores in 1:min(3L, availableCores())) {
+for (cores in 1:availCores) {
message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
+ options(mc.cores = cores)
message("Available cores: ", availableCores())
@@ -16,27 +16,34 @@ for (cores in 1:min(3L, availableCores())) {
message("Stopping any running cluster ... DONE")
message("Starting cluster ...")
- res <- try({
- cluster <- ClusterRegistry("set", workers=availableCores()-1L)
+ res <- tryCatch({
+ cluster <- ClusterRegistry("set", workers = availableCores() - 1L)
str(cluster)
print(cluster)
- }, silent=TRUE)
- if (cores == 1) stopifnot(inherits(res, "try-error"))
+ }, error = identity)
+ if (cores == 1) stopifnot(inherits(res, "error"))
message("Starting cluster ... DONE")
+ message("Starting dual-worker cluster ...")
+ cluster <- ClusterRegistry(action = "start", workers = 2L)
+ str(cluster)
+ print(cluster)
+ stopifnot(length(cluster) == 2L)
+ message("Starting duel-worker cluster ... DONE")
+
message("Starting single-worker cluster ...")
- cluster <- ClusterRegistry(action="start", workers=1L)
+ cluster <- ClusterRegistry(action = "start", workers = 1L)
str(cluster)
print(cluster)
stopifnot(length(cluster) == 1L)
message("Starting single-worker cluster ... DONE")
- cluster <- ClusterRegistry(action="get")
+ cluster <- ClusterRegistry(action = "get")
print(cluster)
stopifnot(length(cluster) == 1L)
message("Stopping single-worker cluster ...")
- cluster <- ClusterRegistry(action="stop")
+ cluster <- ClusterRegistry(action = "stop")
print(cluster)
stopifnot(length(cluster) == 0L)
message("Stopping single-worker cluster ... DONE")
@@ -47,8 +54,8 @@ for (cores in 1:min(3L, availableCores())) {
message("*** ClusterRegistry() - exceptions ...")
-res <- try(ClusterRegistry(action="start", workers=TRUE))
-stopifnot(inherits(res, "try-error"))
+res <- tryCatch(ClusterRegistry(action = "start", workers = TRUE), error = identity)
+stopifnot(inherits(res, "error"))
message("*** ClusterRegistry() - exceptions ... DONE")
diff --git a/tests/DEPRECATED_DEFUNCT.R b/tests/DEPRECATED_DEFUNCT.R
new file mode 100644
index 0000000..1ed4bb4
--- /dev/null
+++ b/tests/DEPRECATED_DEFUNCT.R
@@ -0,0 +1,91 @@
+source("incl/start.R")
+
+message("*** Defunct and deprecated API ...")
+
+message("*** Defunct arguments ...")
+
+res <- tryCatch({
+ f <- ClusterFuture(42L, cluster = "localhost")
+}, error = function(ex) ex)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({
+ f <- cluster(42L, cluster = "localhost")
+}, error = function(ex) ex)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({
+ f <- multicore(42L, maxCores = 2L)
+}, error = function(ex) ex)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({
+ f <- multisession(42L, maxCores = 2L)
+}, error = function(ex) ex)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({
+ f <- multiprocess(42L, maxCores = 2L)
+}, error = function(ex) ex)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({
+ s <- tweak(multiprocess, maxCores = 2L)
+}, error = function(ex) ex)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({
+ s <- tweak(cluster, cluster = "localhost")
+}, error = function(ex) ex)
+stopifnot(inherits(res, "error"))
+
+message("*** Defunct arguments ... DONE")
+
+
+message("*** Defunct argument values ...")
+
+#res <- tryCatch({
+# n <- availableCores(methods = "mc.cores")
+#}, error = function(e) e)
+#stopifnot(inherits(res, "error"))
+
+message("*** Defunct argument values ... DONE")
+
+
+message("*** Defunct eager & lazy ...")
+
+res <- tryCatch({ f <- eager(42L) }, error = identity)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({ f <- lazy(42L) }, error = identity)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({ plan(eager) }, error = identity)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({ plan(lazy) }, error = identity)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({ plan(list(eager)) }, error = identity)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({ plan(list(lazy)) }, error = identity)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({ plan("eager") }, error = identity)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({ plan("lazy") }, error = identity)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({ f <- EagerFuture() }, error = identity)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({ f <- LazyFuture() }, error = identity)
+stopifnot(inherits(res, "error"))
+
+message("*** Defunct eager & lazy ... DONE")
+
+message("*** Defunct and deprecated API ... DONE")
+
+source("incl/end.R")
diff --git a/tests/Future-class.R b/tests/Future-class.R
index 3c26ea5..61c1441 100644
--- a/tests/Future-class.R
+++ b/tests/Future-class.R
@@ -6,16 +6,20 @@ message("*** Future class - exception ...")
f <- Future()
print(f)
-res <- try(value(f), silent=TRUE)
+res <- tryCatch(value(f), error = identity)
print(res)
-stopifnot(inherits(res, "try-error"))
+stopifnot(inherits(res, "error"))
## values() is an alias for value() for Future
-res <- try(values(f), silent=TRUE)
-stopifnot(inherits(res, "try-error"))
+res <- tryCatch(values(f), error = identity)
+stopifnot(inherits(res, "error"))
+
+## Invalid seed
+res <- tryCatch(f <- Future(42, seed = 1:2), error = identity)
+stopifnot(inherits(res, "error"))
## When no packages are exported
-foo <- structure(function(...) { Future(1) }, class="future")
+foo <- structure(function(...) { Future(1) }, class = "future")
plan(foo)
f <- Future()
expr <- getExpression(f)
@@ -23,19 +27,20 @@ print(expr)
stopifnot(is.call(expr))
clazzes <- list(
- uniprocess = UniprocessFuture,
- multisession = function(...) MultisessionFuture(..., workers=2L),
- eager = EagerFuture,
- lazy = LazyFuture
+ sequential = SequentialFuture,
+ multisession = function(...) MultisessionFuture(..., workers = 2L),
+ sequential = SequentialFuture
)
-if (supportsMulticore()) clazzes$multicore = function(...) MulticoreFuture(..., workers=2L)
+if (supportsMulticore()) {
+ clazzes$multicore = function(...) MulticoreFuture(..., workers = 2L)
+}
for (clazz in clazzes) {
## Calling run() more than once
f <- clazz({ 42L })
print(f)
run(f)
- res <- tryCatch(run(f), error=identity)
+ res <- tryCatch(run(f), error = identity)
stopifnot(inherits(res, "error"))
v <- value(f)
print(v)
diff --git a/tests/FutureError.R b/tests/FutureError.R
index 7441fd2..ec2028d 100644
--- a/tests/FutureError.R
+++ b/tests/FutureError.R
@@ -3,32 +3,37 @@ source("incl/start.R")
message("*** FutureError class ...")
## Minimal
-ex <- FutureError(message="Woops")
+ex <- FutureError(message = "Woops")
print(ex)
-f <- future({ 42L; stop("XXX") })
-v <- value(f, signal=FALSE)
+cond <- tryCatch(message("Woops", appendLF = FALSE), message = identity)
+ex2 <- FutureError(message = cond)
+print(ex2)
+stopifnot(conditionMessage(ex2) == conditionMessage(ex))
+
+f <- future({ 42L; stop("woops") })
+v <- value(f, signal = FALSE)
print(v)
-ex <- FutureError(message="Woops", future=f, output=c("Darn", "it"))
+ex <- FutureError(message = "Woops", future = f, output = c("Darn", "it"))
print(ex)
res <- getOutput(ex)
print(res)
stopifnot(all(res == c("Darn", "it")))
-res <- getOutput(ex, head=1L)
+res <- getOutput(ex, head = 1L)
print(res)
stopifnot(res == "Darn")
-res <- getOutput(ex, tail=1L)
+res <- getOutput(ex, tail = 1L)
print(res)
stopifnot(res == "it")
-res <- getOutput(ex, head=1L, tail=1L)
+res <- getOutput(ex, head = 1L, tail = 1L)
print(res)
stopifnot(res == c("Darn", "it"))
-res <- getOutput(ex, collapse="\n")
+res <- getOutput(ex, collapse = "\n")
print(res)
stopifnot(res == "Darn\nit")
diff --git a/tests/FutureGlobals.R b/tests/FutureGlobals.R
new file mode 100644
index 0000000..dd0fb0d
--- /dev/null
+++ b/tests/FutureGlobals.R
@@ -0,0 +1,38 @@
+source("incl/start.R")
+
+message("*** FutureGlobals() ...")
+
+fg1 <- FutureGlobals()
+print(fg1)
+
+fg2 <- FutureGlobals(fg1)
+print(fg2)
+
+g <- globals::as.Globals(list(a = 1, b = 1:3))
+print(g)
+
+fg3 <- FutureGlobals(g)
+print(fg3)
+
+fg4 <- as.FutureGlobals(g)
+print(fg4)
+
+print(resolved(fg3))
+
+fg <- fg4
+
+fg_unique <- unique(fg)
+print(fg_unique)
+
+fg_resolved <- resolve(fg)
+print(fg_resolved)
+
+message("- FutureGlobals() - exceptions ...")
+
+res <- tryCatch(g <- FutureGlobals(NULL), error = identity)
+print(res)
+stopifnot(inherits(res, "error"))
+
+message("*** FutureGlobals() ... DONE")
+
+source("incl/end.R")
diff --git a/tests/FutureRegistry.R b/tests/FutureRegistry.R
index 82d98de..2d0c959 100644
--- a/tests/FutureRegistry.R
+++ b/tests/FutureRegistry.R
@@ -4,7 +4,7 @@ message("*** FutureRegistry() ...")
for (where in c("multicore", "rscript")) {
message(sprintf("*** FutureRegistry('%s', 'list') ...", where))
- futures <- FutureRegistry(where, action="list")
+ futures <- FutureRegistry(where, action = "list")
print(futures)
stopifnot(length(futures) == 0L)
@@ -12,21 +12,21 @@ for (where in c("multicore", "rscript")) {
message(sprintf("*** FutureRegistry('%s', 'add') ...", where))
f <- future({ 1 })
print(f)
- FutureRegistry(where, action="add", future=f)
+ FutureRegistry(where, action = "add", future = f)
message(sprintf("*** FutureRegistry('%s', 'list') ...", where))
- futures <- FutureRegistry(where, action="list")
+ futures <- FutureRegistry(where, action = "list")
print(futures)
stopifnot(length(futures) == 1L)
message(sprintf("*** FutureRegistry('%s', 'remove') ...", where))
- FutureRegistry(where, action="remove", future=f)
+ FutureRegistry(where, action = "remove", future = f)
message(sprintf("*** FutureRegistry('%s', 'list') ...", where))
- futures <- FutureRegistry(where, action="list")
+ futures <- FutureRegistry(where, action = "list")
print(futures)
stopifnot(length(futures) == 0L)
@@ -34,34 +34,34 @@ for (where in c("multicore", "rscript")) {
message(sprintf("*** FutureRegistry('%s', 'add') ...", where))
f <- future({ 2 })
print(f)
- FutureRegistry(where, action="add", future=f)
+ FutureRegistry(where, action = "add", future = f)
message(sprintf("*** FutureRegistry('%s', 'list') ...", where))
- futures <- FutureRegistry(where, action="list")
+ futures <- FutureRegistry(where, action = "list")
print(futures)
stopifnot(length(futures) == 1L)
message(sprintf("*** FutureRegistry('%s', 'collect-first') ...", where))
- FutureRegistry(where, action="collect-first")
+ FutureRegistry(where, action = "collect-first")
- futures <- FutureRegistry(where, action="list")
+ futures <- FutureRegistry(where, action = "list")
print(futures)
stopifnot(length(futures) < 1L)
message(sprintf("*** FutureRegistry('%s', 'add') ...", where))
f <- future({ 2 })
print(f)
- FutureRegistry(where, action="add", future=f)
+ FutureRegistry(where, action = "add", future = f)
message(sprintf("*** FutureRegistry('%s', 'reset') ...", where))
- FutureRegistry(where, action="reset")
+ FutureRegistry(where, action = "reset")
message(sprintf("*** FutureRegistry('%s', 'list') ...", where))
- futures <- FutureRegistry(where, action="list")
+ futures <- FutureRegistry(where, action = "list")
print(futures)
stopifnot(length(futures) == 0L)
}
@@ -69,35 +69,35 @@ for (where in c("multicore", "rscript")) {
message("*** FutureRegistry() - exceptions ...")
-futures <- FutureRegistry(where="test", action="list")
+futures <- FutureRegistry(where = "test", action = "list")
stopifnot(length(futures) == 0)
f <- future(1)
-FutureRegistry(where="test", action="add", future=f)
-futures <- FutureRegistry(where="test", action="list")
+FutureRegistry(where = "test", action = "add", future = f)
+futures <- FutureRegistry(where = "test", action = "list")
stopifnot(length(futures) == 1)
-res <- try(FutureRegistry(where="test", action="add", future=f), silent=TRUE)
-stopifnot(inherits(res, "try-error"))
-futures <- FutureRegistry(where="test", action="list")
+res <- tryCatch(FutureRegistry(where = "test", action = "add", future = f), error = identity)
+stopifnot(inherits(res, "error"))
+futures <- FutureRegistry(where = "test", action = "list")
stopifnot(length(futures) == 1)
-FutureRegistry(where="test", action="remove", future=f)
-futures <- FutureRegistry(where="test", action="list")
+FutureRegistry(where = "test", action = "remove", future = f)
+futures <- FutureRegistry(where = "test", action = "list")
stopifnot(length(futures) == 0)
-res <- try(FutureRegistry(where="test", action="remove", future=f), silent=TRUE)
-stopifnot(inherits(res, "try-error"))
-futures <- FutureRegistry(where="test", action="list")
+res <- tryCatch(FutureRegistry(where = "test", action = "remove", future = f), error = identity)
+stopifnot(inherits(res, "error"))
+futures <- FutureRegistry(where = "test", action = "list")
stopifnot(length(futures) == 0)
-FutureRegistry(where="test", action="reset")
-futures <- FutureRegistry(where="test", action="list")
+FutureRegistry(where = "test", action = "reset")
+futures <- FutureRegistry(where = "test", action = "list")
stopifnot(length(futures) == 0)
-res <- try(FutureRegistry(where="test", action="<unknown>"), silent=TRUE)
-stopifnot(inherits(res, "try-error"))
+res <- tryCatch(FutureRegistry(where = "test", action = "<unknown>"), error = identity)
+stopifnot(inherits(res, "error"))
message("*** FutureRegistry() - exceptions ... DONE")
diff --git a/tests/as.cluster.R b/tests/as.cluster.R
index e5f90e5..fc2dd91 100644
--- a/tests/as.cluster.R
+++ b/tests/as.cluster.R
@@ -3,12 +3,13 @@ stopCluster <- parallel::stopCluster
message("*** cluster operations ...")
-message("*** cluster operations - as.cluster() ...")
-
local({
- cl <- makeClusterPSOCK(1L)
- on.exit(stopCluster(cl))
+ cl0 <- makeClusterPSOCK(1L)
+ on.exit(stopCluster(cl0))
+ cl <- cl0
print(cl)
+
+ message("*** cluster operations - as.cluster() ...")
cl1 <- as.cluster(cl)
print(cl1)
stopifnot(inherits(cl1, "cluster"), identical(cl1, cl))
@@ -18,43 +19,50 @@ local({
cl2 <- as.cluster(cl)
stopifnot(inherits(cl2, "cluster"), length(cl2) == 1L,
identical(cl2[[1]], node))
-})
-message("*** cluster operations - as.cluster() ... DONE")
-
-
-message("*** cluster operations - c(...) ...")
+ node <- cl[[1]]
+ print(node)
+ stopifnot(inherits(node, "SOCKnode"))
+ nodes <- list(node, node)
+ cl3 <- as.cluster(node)
+ print(cl3)
+ stopifnot(inherits(cl3, "cluster"), length(cl3) == 1L,
+ identical(cl3[[1]], node))
-local({
- cl1 <- makeClusterPSOCK(1L)
- on.exit(stopCluster(cl1))
- print(cl1)
+ cl4 <- as.cluster(nodes)
+ print(cl4)
+ stopifnot(inherits(cl4, "cluster"), length(cl4) == 2L,
+ identical(cl4[[1]], node), identical(cl4[[2]], node))
- cl2 <- makeClusterPSOCK(rep("localhost", times = 2L))
+ message("*** cluster operations - as.cluster() ... DONE")
+
+ message("*** cluster operations - c(...) ...")
+ cl2 <- makeClusterPSOCK("localhost")
on.exit(stopCluster(cl2), add = TRUE)
print(cl2)
cl <- c(cl1, cl2)
print(cl)
- stopifnot(inherits(cl, "cluster"), length(cl) == 3L)
+ stopifnot(inherits(cl, "cluster"), length(cl) == 2L)
stopifnot(identical(cl[1], cl1),
- identical(cl[2], cl2[1]), identical(cl[3], cl2[2]))
+ identical(cl[2], cl2[1]))
+ message("*** cluster operations - c(...) ... DONE")
})
-message("*** cluster operations - c(...) ... DONE")
-
message("*** cluster operations - makeClusterPSOCK(remotes) ...")
-remotes <- Sys.getenv("R_FUTURE_TESTS_REMOTES", "localhost,localhost")
-remotes <- gsub(" ", "", unlist(strsplit(remotes, split=",")))
+remotes <- Sys.getenv("R_FUTURE_TESTS_REMOTES", "")
+remotes <- gsub(" ", "", unlist(strsplit(remotes, split = ",")))
remotes <- remotes[nzchar(remotes)]
if (length(remotes) > 0) {
message("Remotes: ", paste(sQuote(remotes), collapse = ", "))
- cl <- makeClusterPSOCK(remotes, verbose = TRUE)
- print(cl)
- stopCluster(cl)
+ local({
+ cl <- makeClusterPSOCK(remotes, verbose = TRUE)
+ on.exit(stopCluster(cl))
+ print(cl)
+ })
}
message("*** cluster operations - makeClusterPSOCK(remotes) ... DONE")
diff --git a/tests/availableCores.R b/tests/availableCores.R
index 9a53bdf..c373c31 100644
--- a/tests/availableCores.R
+++ b/tests/availableCores.R
@@ -4,61 +4,53 @@ message("*** availableCores() ...")
## detectCores() may return NA_integer_
n <- parallel::detectCores()
-message(sprintf("detectCores()=%d", n))
+message(sprintf("detectCores() = %d", n))
stopifnot(length(n) == 1, is.numeric(n))
## Default
-print(availableCores())
-
n <- availableCores()
-message(sprintf("availableCores()=%d", n))
+message(sprintf("availableCores() = %d", n))
stopifnot(length(n) == 1, is.numeric(n), n >= 1)
## Minimium of all known settings (default)
-print(availableCores(which="min"))
+print(availableCores(which = "min"))
## Maximum of all known settings (should never be used)
-print(availableCores(which="max"))
+print(availableCores(which = "max"))
## All known settings
-print(availableCores(na.rm=FALSE, which="all"))
+print(availableCores(na.rm = FALSE, which = "all"))
## System settings
-n <- availableCores(methods="system")
+n <- availableCores(methods = "system")
print(n)
stopifnot(length(n) == 1, is.numeric(n), is.finite(n), n >= 1)
## Predefined ones for known cluster schedulers
-print(availableCores(methods="PBS"))
-print(availableCores(methods="SGE"))
-print(availableCores(methods="Slurm"))
+print(availableCores(methods = "PBS"))
+print(availableCores(methods = "SGE"))
+print(availableCores(methods = "Slurm"))
## Any R options and system environment variable
-print(availableCores(methods=c("width", "FOO_BAR_ENV"),
- na.rm=FALSE, which="all"))
+print(availableCores(methods = c("width", "FOO_BAR_ENV"),
+ na.rm = FALSE, which = "all"))
## Exception handling
-Sys.setenv("FOO_BAR_ENV"="0")
-res <- try(availableCores(methods="FOO_BAR_ENV"), silent=TRUE)
+Sys.setenv("FOO_BAR_ENV" = "0")
+res <- try(availableCores(methods = "FOO_BAR_ENV"), silent = TRUE)
stopifnot(inherits(res, "try-error"))
-## Deprecated
-res <- tryCatch(availableCores(methods="mc.cores"), warning=function(w) w)
-print(res)
-stopifnot(inherits(res, "warning"))
-
-
message("*** Internal detectCores() ...")
## Option 'future.availableCores.system'
env <- environment(future:::detectCores)
env$res <- NULL
-options(future.availableCores.system=2L)
+options(future.availableCores.system = 2L)
n <- detectCores()
print(n)
stopifnot(is.integer(n), is.finite(n), n >= 1, n == 2L)
-options(future.availableCores.system=NULL)
+options(future.availableCores.system = NULL)
## Reset
env <- environment(future:::detectCores)
@@ -67,11 +59,9 @@ n <- detectCores()
print(n)
stopifnot(is.integer(n), is.finite(n), n >= 1)
-
message("*** Internal detectCores() ... DONE")
message("*** availableCores() ... DONE")
source("incl/end.R")
-
diff --git a/tests/availableWorkers.R b/tests/availableWorkers.R
new file mode 100644
index 0000000..5066c11
--- /dev/null
+++ b/tests/availableWorkers.R
@@ -0,0 +1,154 @@
+source("incl/start.R")
+
+message("*** availableWorkers() ...")
+
+## The default
+w <- availableWorkers()
+print(w)
+stopifnot(is.character(w), length(w) >= 1)
+
+## Minimium of all known settings (default)
+print(availableWorkers(which = "min"))
+
+## Maximum of all known settings (should never be used)
+print(availableWorkers(which = "max"))
+
+## All known settings
+print(availableWorkers(na.rm = FALSE, which = "all"))
+
+## System settings
+w <- availableWorkers(methods = "system")
+print(w)
+stopifnot(is.character(w), length(w) >= 1)
+
+## Predefined ones for known cluster schedulers
+print(availableWorkers(methods = "PBS"))
+print(availableWorkers(methods = "SGE"))
+print(availableWorkers(methods = "Slurm"))
+
+
+
+message("*** HPC related ...")
+
+expand_nodes <- future:::expand_nodes
+read_pbs_nodefile <- future:::read_pbs_nodefile
+read_pe_hostfile <- future:::read_pe_hostfile
+
+workers0 <- c("n1", "n2", "n3", "n1", "n6", "n3", "n3", "n5")
+data0 <- as.data.frame(table(workers0), stringsAsFactors = FALSE)
+colnames(data0) <- c("node", "count")
+data0 <- data0[order(data0$node, data0$count), ]
+
+message("*** read_pbs_nodefile() ...")
+
+workers <- workers0
+pathname <- tempfile()
+writeLines(workers, con = pathname)
+
+data <- read_pbs_nodefile(pathname)
+str(data)
+stopifnot(
+ c("node") %in% colnames(data),
+ is.character(data$node),
+ !anyNA(data$node),
+ nrow(data$node) == length(workers),
+ all(sort(data$node) == sort(workers))
+)
+
+Sys.setenv(PBS_NODEFILE = pathname)
+Sys.setenv(PBS_NP = length(workers),
+ PBS_NUM_NODES = length(workers) / 2,
+ PBS_NUM_PPN = 2)
+workers <- availableWorkers(methods = "PBS")
+print(workers)
+stopifnot(length(workers) == length(workers0), all(workers == sort(workers0)))
+
+Sys.setenv(PBS_NUM_PPN = 3)
+res <- tryCatch({
+ workers <- availableWorkers(methods = "PBS")
+}, warning = identity)
+stopifnot(inherits(res, "warning"))
+
+Sys.setenv(PBS_NP = length(workers) + 1)
+res <- tryCatch({
+ workers <- availableWorkers(methods = "PBS")
+}, warning = identity)
+stopifnot(inherits(res, "warning"))
+
+## Exceptions
+workersE <- c(workers, "n 3")
+pathname <- tempfile()
+writeLines(workersE, con = pathname)
+res <- tryCatch(read_pbs_nodefile(pathname), error = identity)
+print(res)
+stopifnot(inherits(res, "error"))
+
+Sys.setenv(PBS_NODEFILE = "<non-existing-file>")
+res <- tryCatch({
+ workers <- availableWorkers(methods = "PBS")
+}, warning = identity)
+stopifnot(inherits(res, "warning"))
+
+
+message("*** read_pbs_nodefile() ... DONE")
+
+
+message("*** read_pe_hostfile() ...")
+
+workers <- workers0
+pathname <- tempfile()
+write.table(data0, file = pathname, quote = FALSE, row.names = FALSE, col.names = FALSE)
+lines <- readLines(pathname)
+print(lines)
+data <- read_pe_hostfile(pathname)
+print(data)
+stopifnot(
+ is.character(data$node),
+ !anyNA(data$node),
+ is.integer(data$count),
+ !anyNA(data$count),
+ all(is.finite(data$count)),
+ all(data$count > 0),
+ nrow(data) == nrow(data0),
+ all.equal(data[, c("node", "count")], data0[, c("node", "count")])
+)
+
+workers <- expand_nodes(data)
+stopifnot(length(workers) == length(workers0), all(workers == sort(workers0)))
+
+Sys.setenv(PE_HOSTFILE = pathname)
+Sys.setenv(NSLOTS = length(workers0)) ## Use to validate results
+workers <- availableWorkers(methods = "SGE")
+print(workers)
+stopifnot(length(workers) == length(workers0), all(workers == sort(workers0)))
+
+## Test validation
+Sys.setenv(NSLOTS = length(workers0) + 1L)
+workers <- tryCatch(availableWorkers(methods = "SGE"), warning = identity)
+print(workers)
+stopifnot(inherits(workers, "warning"))
+
+Sys.setenv(PE_HOSTFILE = "<non-existing-file>")
+res <- tryCatch({
+ workers <- availableWorkers(methods = "SGE")
+}, warning = identity)
+stopifnot(inherits(res, "warning"))
+
+message("*** read_pe_hostfile() ... DONE")
+
+
+message("*** HPC related ... DONE")
+
+
+## Any R options and system environment variable
+print(availableWorkers(methods = c("width", "FOO_BAR_ENV"),
+ na.rm = FALSE, which = "all"))
+
+## Exception handling
+Sys.setenv("FOO_BAR_ENV" = "0")
+res <- tryCatch(availableWorkers(methods = "FOO_BAR_ENV"), error = identity)
+stopifnot(inherits(res, "error"))
+
+message("*** availableWorkers() ... DONE")
+
+source("incl/end.R")
diff --git a/tests/backtrace.R b/tests/backtrace.R
index da9bec0..0446672 100644
--- a/tests/backtrace.R
+++ b/tests/backtrace.R
@@ -5,7 +5,7 @@ message("*** backtrace( ) ...")
message("*** backtrace( ) - explicit future ...")
f <- future({ 42L; stop("Woops") })
-v <- value(f, signal=FALSE)
+v <- value(f, signal = FALSE)
print(v)
calls <- backtrace(f)
print(calls)
@@ -26,25 +26,25 @@ message("*** backtrace( ) - exceptions ...")
message("- No condition ...")
f <- future(42L)
-res <- try(backtrace(f), silent=TRUE)
+res <- tryCatch(backtrace(f), error = identity)
print(res)
-stopifnot(inherits(res, "try-error"))
+stopifnot(inherits(res, "error"))
message("- No call stack ...")
f <- future({ 42L; stop("Woops") })
-v <- value(f, signal=FALSE)
+v <- value(f, signal = FALSE)
f$value$traceback <- NULL ## Remove call stack
-res <- try(backtrace(f), silent=TRUE)
+res <- tryCatch(backtrace(f), error = identity)
print(res)
-stopifnot(inherits(res, "try-error"))
+stopifnot(inherits(res, "error"))
if (availableCores() >= 2L) {
message("- Non-resolved future ...")
- plan(multiprocess, workers=2L)
- f <- future({ Sys.sleep(10); 42L; stop("Woops") })
- res <- try(backtrace(f), silent=TRUE)
+ plan(multiprocess, workers = 2L)
+ f <- future({ Sys.sleep(3); 42L; stop("Woops") })
+ res <- tryCatch(backtrace(f), error = identity)
print(res)
- stopifnot(inherits(res, "try-error"))
+ stopifnot(inherits(res, "error"))
}
message("*** backtrace( ) - exceptions ... DONE")
diff --git a/tests/cluster.R b/tests/cluster.R
index e0abff5..2af1cc7 100644
--- a/tests/cluster.R
+++ b/tests/cluster.R
@@ -1,223 +1,316 @@
source("incl/start.R")
library("listenv")
-
+options(future.debug = FALSE)
message("*** cluster() ...")
-message("Cluster type: ", parallel:::getClusterOption("type"))
-message("Library paths: ", paste(sQuote(.libPaths()), collapse=", "))
-message("Package path: ", sQuote(system.file(package="future")))
-
-for (cores in 1:2) {
- message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
-
- ## Set up a cluster with <cores> nodes (explicitly)
- cl <- parallel::makeCluster(cores)
- plan(cluster, workers=cl)
-
- ## No global variables
- f <- try(cluster({
- 42L
- }, workers=cl), silent=FALSE)
- print(f)
- stopifnot(inherits(f, "ClusterFuture"))
-
- print(resolved(f))
- y <- value(f)
- print(y)
- stopifnot(y == 42L)
-
-
- ## Set up a cluster with <cores> nodes (implicitly)
- plan(cluster, workers=cores)
-
- ## No global variables
- f <- try(cluster({
- 42L
- }, workers=cl), silent=FALSE)
- print(f)
- stopifnot(inherits(f, "ClusterFuture"))
-
- print(resolved(f))
- y <- value(f)
- print(y)
- stopifnot(y == 42L)
-
-
- ## A global variable
- a <- 0
- f <- try(future({
- b <- 3
- c <- 2
- a * b * c
- }))
- print(f)
-
-
- ## A cluster future is evaluated in a separate
- ## R session process. Changing the value of a global
- ## variable should not affect the result of the
- ## future.
- a <- 7 ## Make sure globals are frozen
- v <- value(f)
- print(v)
- stopifnot(v == 0)
-
-
- message("*** cluster() with globals and blocking")
- x <- listenv()
- for (ii in 1:4) {
- message(sprintf(" - Creating cluster future #%d ...", ii))
- x[[ii]] <- future({ ii })
- }
- message(sprintf(" - Resolving %d cluster futures", length(x)))
- v <- sapply(x, FUN=value)
- stopifnot(all(v == 1:4))
-
-
- message("*** cluster() and errors")
- f <- future({
- stop("Whoops!")
- 1
- })
- print(f)
- v <- value(f, signal=FALSE)
- print(v)
- stopifnot(inherits(v, "simpleError"))
-
- res <- try(value(f), silent=TRUE)
- print(res)
- stopifnot(inherits(res, "try-error"))
+message("Library paths: ", paste(sQuote(.libPaths()), collapse = ", "))
+message("Package path: ", sQuote(system.file(package = "future")))
- ## Error is repeated
- res <- try(value(f), silent=TRUE)
- print(res)
- stopifnot(inherits(res, "try-error"))
-
-
- message("*** cluster() - too large globals ...")
- ooptsT <- options(future.globals.maxSize=1024*4L)
-
- limit <- getOption("future.globals.maxSize")
- cat(sprintf("Max total size of globals: %g bytes\n", limit))
-
- ## A large object
- a <- 1:1014
- yTruth <- sum(a)
- size <- object.size(a)
- cat(sprintf("a: %g bytes\n", size))
- f <- future({ sum(a) })
- print(f)
- rm(list="a")
- v <- value(f)
- print(v)
- stopifnot(v == yTruth)
-
-
- ## A too large object
- a <- 1:1019 ## also on 32-bit platforms
- yTruth <- sum(a)
- size <- object.size(a)
- cat(sprintf("a: %g bytes\n", size))
- res <- try(f <- future({ sum(a) }), silent=TRUE)
- rm(list="a")
- stopifnot(inherits(res, "try-error"))
-
- ## Undo options changed in this test
- options(ooptsT)
-
- message("*** cluster() - too large globals ... DONE")
-
- message("*** cluster() - installed packages ...")
- f <- try(cluster({
- list(
- libPaths = .libPaths(),
- pkgs = installed.packages()
- )
- }, workers=cl), silent=FALSE)
- print(f)
- stopifnot(inherits(f, "ClusterFuture"))
- v <- value(f)
- message(paste(capture.output(str(v)), collapse="\n"))
- message("*** cluster() - installed packages ... DONE")
-
-
- message("*** cluster() - assert covr workaround ...")
- f <- try(cluster({
- future:::hpaste(1:100)
- }, workers=cl), silent=FALSE)
- print(f)
- stopifnot(inherits(f, "ClusterFuture"))
- v <- value(f)
- message(v)
- stopifnot(v == hpaste(1:100))
- message("*** cluster() - assert covr workaround ... DONE")
-
- message(sprintf("Testing with %d cores ... DONE", cores))
-} ## for (cores ...)
-
-
-message("*** cluster() - setDefaultCluster() ...")
+types <- "PSOCK"
+if (supportsMulticore()) types <- c(types, "FORK")
-library("parallel")
-setDefaultCluster(makeCluster(2))
-plan(cluster)
+## WORKAROUND: covr::package_coverage() -> merge_coverage() -> ... produces
+## "Error in readRDS(x) : error reading from connection" for type = "FORK".
+## Is this related to mcparallel() comments in help("package_coverage")?
+## /HB 2017-05-20
+if (covr_testing) types <- setdiff(types, "FORK")
pid <- Sys.getpid()
-message(pid)
-
-a %<-% Sys.getpid()
-b %<-% Sys.getpid()
-message(a)
-message(b)
-
-setDefaultCluster(NULL)
-
-message("*** cluster() - setDefaultCluster() ... DONE")
-
-
-message("*** cluster() - exceptions ...")
-
-res <- try(cluster(42L, workers=NA), silent=TRUE)
-print(res)
-stopifnot(inherits(res, "try-error"))
-
-message("*** cluster() - exceptions ... DONE")
-
-
-message("*** cluster() - crashed worker ...")
-
-cl <- parallel::makeCluster("localhost")
-plan(cluster, workers = cl)
-x %<-% 42L
-stopifnot(x == 42L)
-
-## Force R worker to quit
-x %<-% quit(save = "no")
-res <- tryCatch(y <- x, error = identity)
-print(res)
-stopifnot(
- inherits(res, "simpleError"),
- inherits(res, "FutureError")
-)
-
-## This is needed in order to reset the ClusterRegistry
-future:::ClusterRegistry("stop")
-
-## An alternative is to do:
-## plan(uniprocess); x %<-% NULL; print(x)
-
-## Verify that the reset worked
-cl <- parallel::makeCluster("localhost")
-plan(cluster, workers = cl)
-x %<-% 42L
-stopifnot(x == 42L)
+message("Main PID (original): ", pid)
+cl <- NULL
+for (type in types) {
+ message(sprintf("Test set #1 with cluster type %s ...", sQuote(type)))
+
+ for (cores in 1:availCores) {
+ message(sprintf("Testing with %d cores on type = %s ...",
+ cores, sQuote(type)))
+ options(mc.cores = cores)
+
+ ## Set up a cluster with <cores> nodes (explicitly)
+ cl <- parallel::makeCluster(cores, type = type)
+ print(cl)
+
+ plan(cluster, workers = cl)
+
+ ## No global variables
+ f <- try(cluster({
+ 42L
+ }, workers = cl), silent = FALSE)
+ print(f)
+ stopifnot(inherits(f, "ClusterFuture"))
+
+ print(resolved(f))
+ y <- value(f)
+ print(y)
+ stopifnot(y == 42L)
+
+
+ ## Set up a cluster with <cores> nodes (implicitly)
+ plan(cluster, workers = cores)
+
+ ## No global variables
+ f <- try(cluster({
+ 42L
+ }, workers = cl), silent = FALSE)
+ print(f)
+ stopifnot(inherits(f, "ClusterFuture"))
+
+ print(resolved(f))
+ y <- value(f)
+ print(y)
+ stopifnot(y == 42L)
+
+
+ ## A global variable
+ a <- 0
+ f <- try(future({
+ b <- 3
+ c <- 2
+ a * b * c
+ }))
+ print(f)
+
+
+ ## A cluster future is evaluated in a separate
+ ## R session process. Changing the value of a global
+ ## variable should not affect the result of the
+ ## future.
+ a <- 7 ## Make sure globals are frozen
+ v <- value(f)
+ print(v)
+ stopifnot(v == 0)
+
+
+ message("*** cluster() with globals and blocking")
+ x <- listenv()
+ for (ii in 1:3) {
+ message(sprintf(" - Creating cluster future #%d ...", ii))
+ x[[ii]] <- future({ ii })
+ }
+ message(sprintf(" - Resolving %d cluster futures", length(x)))
+ v <- sapply(x, FUN = value)
+ stopifnot(all(v == 1:3))
+
+
+ message("*** cluster() and errors")
+ f <- future({
+ stop("Whoops!")
+ 1
+ })
+ print(f)
+ v <- value(f, signal = FALSE)
+ print(v)
+ stopifnot(inherits(v, "simpleError"))
+
+ res <- tryCatch(value(f), error = identity)
+ print(res)
+ stopifnot(inherits(res, "error"))
+
+ ## Error is repeated
+ res <- tryCatch(value(f), error = identity)
+ print(res)
+ stopifnot(inherits(res, "error"))
+
+
+ message("*** cluster() - too large globals ...")
+ ooptsT <- options(future.globals.maxSize = object.size(1:1014))
+
+ limit <- getOption("future.globals.maxSize")
+ cat(sprintf("Max total size of globals: %g bytes\n", limit))
+
+ ## A large object
+ a <- 1:1014
+ yTruth <- sum(a)
+ size <- object.size(a)
+ cat(sprintf("a: %g bytes\n", size))
+ f <- future({ sum(a) })
+ print(f)
+ rm(list = "a")
+ v <- value(f)
+ print(v)
+ stopifnot(v == yTruth)
+
+
+ ## A too large object
+ a <- 1:1015
+ yTruth <- sum(a)
+ size <- object.size(a)
+ cat(sprintf("a: %g bytes\n", size))
+ res <- tryCatch(f <- future({ sum(a) }), error = identity)
+ rm(list = "a")
+ stopifnot(inherits(res, "error"))
+
+ ## Undo options changed in this test
+ options(ooptsT)
+
+ message("*** cluster() - too large globals ... DONE")
+
+ message("*** cluster() - installed libraries ...")
+ f <- try(cluster({
+ list(
+ libPaths = .libPaths()
+ )
+ }, workers = cl), silent = FALSE)
+ print(f)
+ stopifnot(inherits(f, "ClusterFuture"))
+ v <- value(f)
+ message(paste(capture.output(str(v)), collapse = "\n"))
+ message("*** cluster() - installed packages ... DONE")
+
+
+ message("*** cluster() - assert covr workaround ...")
+ f <- try(cluster({
+ future:::hpaste(1:100)
+ }, workers = cl), silent = FALSE)
+ print(f)
+ stopifnot(inherits(f, "ClusterFuture"))
+ v <- value(f)
+ message(v)
+ stopifnot(v == hpaste(1:100))
+ message("*** cluster() - assert covr workaround ... DONE")
+
+ message(sprintf("Testing with %d cores on type = %s ... DONE",
+ cores, sQuote(type)))
+ } ## for (cores ...)
+
+ message("*** cluster() - exceptions ...")
+
+ res <- tryCatch(cluster(42L, workers = NA), error = identity)
+ print(res)
+ stopifnot(inherits(res, "error"))
+
+ message("*** cluster() - exceptions ... DONE")
+
+ message("*** cluster() - assert registry behavior ...")
+
+ ## Explicitly created clusters are *not* added to the registry
+ cl <- parallel::makeCluster(cores, type = type)
+ plan(cluster, workers = cl)
+ clR <- ClusterRegistry("get")
+ stopifnot(is.null(clR))
+
+ ## ... and therefore changing plans shouldn't change anything
+ plan(sequential)
+ clR <- ClusterRegistry("get")
+ stopifnot(is.null(clR))
+
+ message("*** cluster() - assert registry behavior ... DONE")
+
+ ## Sanity checks
+ pid2 <- Sys.getpid()
+ message("Main PID (original): ", pid)
+ message("Main PID: ", pid2)
+ stopifnot(pid2 == pid)
+
+ ## Cleanup
+ print(cl)
+ str(cl)
+ parallel::stopCluster(cl)
+ plan(sequential)
+
+ message(sprintf("Test set #1 with cluster type %s ... DONE", sQuote(type)))
+} ## for (type ...)
-message("*** cluster() - crashed worker ... DONE")
+library("parallel")
+for (type in types) {
+ if (on_solaris) next
+
+ message(sprintf("Test set #2 with cluster type %s ...", sQuote(type)))
+
+ message("*** cluster() - setDefaultCluster() ...")
+
+ cl <- makeCluster(1L, type = type)
+ print(cl)
+
+ setDefaultCluster(cl)
+ ## FIXME: Make plan(cluster, workers = NULL) work such that
+ ## setDefaultCluster() is actually tested.
+ plan(cluster)
+
+ pid <- Sys.getpid()
+ message(pid)
+
+ a %<-% Sys.getpid()
+ message(a)
+
+ setDefaultCluster(NULL)
+
+ message("*** cluster() - setDefaultCluster() ... DONE")
+
+ ## Sanity checks
+ pid2 <- Sys.getpid()
+ message("Main PID (original): ", pid)
+ message("Main PID: ", pid2)
+ stopifnot(pid2 == pid)
+
+ ## Cleanup
+ print(cl)
+ str(cl)
+ parallel::stopCluster(cl)
+ plan(sequential)
+
+ message(sprintf("Test set #2 with cluster type %s ... DONE", sQuote(type)))
+} ## for (type ...)
+
+
+for (type in types) {
+ if (on_solaris) next
+
+ message(sprintf("Test set #3 with cluster type %s ...", sQuote(type)))
+
+ cl <- parallel::makeCluster(1L, type = type)
+ print(cl)
+
+ message("*** cluster() - crashed worker ...")
+
+ plan(cluster, workers = cl)
+ x %<-% 42L
+ stopifnot(x == 42L)
+
+ ## Force R worker to quit
+ x %<-% quit(save = "no")
+ res <- tryCatch(y <- x, error = identity)
+ print(res)
+ stopifnot(
+ inherits(res, "simpleError"),
+ inherits(res, "FutureError")
+ )
+
+ ## Cleanup
+ print(cl)
+ ## FIXME: Why doesn't this work here? It causes the below future to stall.
+ # parallel::stopCluster(cl)
+
+ ## Verify that the reset worked
+ cl <- parallel::makeCluster(1L, type = type)
+ print(cl)
+ plan(cluster, workers = cl)
+ x %<-% 42L
+ stopifnot(x == 42L)
+
+ message("*** cluster() - crashed worker ... DONE")
+
+ ## Sanity checks
+ pid2 <- Sys.getpid()
+ message("Main PID (original): ", pid)
+ message("Main PID: ", pid2)
+ stopifnot(pid2 == pid)
+
+ ## Cleanup
+ print(cl)
+ str(cl)
+ parallel::stopCluster(cl)
+
+ message(sprintf("Test set #3 with cluster type %s ... DONE", sQuote(type)))
+} ## for (type ...)
message("*** cluster() ... DONE")
-## Cleanup
-parallel::stopCluster(cl)
+## Sanity checks
+pid2 <- Sys.getpid()
+message("Main PID (original): ", pid)
+message("Main PID: ", pid2)
+stopifnot(pid2 == pid)
+
source("incl/end.R")
diff --git a/tests/constant.R b/tests/constant.R
index 710f971..4eb29e5 100644
--- a/tests/constant.R
+++ b/tests/constant.R
@@ -4,7 +4,7 @@ library("listenv")
message("*** constant() ...")
## No global variables
-f <- try(constant(42L), silent=FALSE)
+f <- try(constant(42L), silent = FALSE)
print(f)
stopifnot(inherits(f, "ConstantFuture"))
@@ -16,7 +16,7 @@ stopifnot(y == 42L)
plan(constant)
## No global variables
-f <- try(future(42L), silent=FALSE)
+f <- try(future(42L), silent = FALSE)
print(f)
stopifnot(inherits(f, "ConstantFuture"))
diff --git a/tests/demo.R b/tests/demo.R
index e242c0d..d65b570 100644
--- a/tests/demo.R
+++ b/tests/demo.R
@@ -1,20 +1,22 @@
source("incl/start.R")
-for (cores in 1:min(3L, availableCores())) {
- message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
+options(future.demo.mandelbrot.nrow = 2L)
+options(future.demo.mandelbrot.resolution = 50L)
+options(future.demo.mandelbrot.delay = FALSE)
- options("R_FUTURE_DEMO_MANDELBROT_PLANES"=4L)
+for (cores in 1:availCores) {
+ message(sprintf("Testing with %d cores ...", cores))
+ options(mc.cores = cores)
message("*** Demos ...")
message("*** Mandelbrot demo of the 'future' package ...")
if (getRversion() >= "3.2.0") {
- for (strategy in supportedStrategies()) {
+ for (strategy in supportedStrategies(cores)) {
message(sprintf("- plan('%s') ...", strategy))
plan(strategy)
- demo("mandelbrot", package="future", ask=FALSE)
+ demo("mandelbrot", package = "future", ask = FALSE)
message(sprintf("- plan('%s') ... DONE", strategy))
}
} else {
diff --git a/tests/deprecated.R b/tests/deprecated.R
deleted file mode 100644
index 9307aef..0000000
--- a/tests/deprecated.R
+++ /dev/null
@@ -1,64 +0,0 @@
-source("incl/start.R")
-
-message("*** Deprecated API ...")
-
-message("*** Deprecated arguments ...")
-
-res <- tryCatch({
- f <- ClusterFuture(42L, cluster="localhost")
-}, warning = function(w) w)
-stopifnot(inherits(res, "warning"))
-
-res <- tryCatch({
- f <- cluster(42L, cluster="localhost")
-}, warning = function(w) w)
-stopifnot(inherits(res, "warning"))
-
-res <- tryCatch({
- f <- multicore(42L, maxCores=2L)
-}, warning = function(w) w)
-stopifnot(inherits(res, "warning"))
-
-res <- tryCatch({
- f <- multisession(42L, maxCores=2L)
-}, warning = function(w) w)
-stopifnot(inherits(res, "warning"))
-
-res <- tryCatch({
- f <- multiprocess(42L, maxCores=2L)
-}, warning = function(w) w)
-stopifnot(inherits(res, "warning"))
-
-res <- tryCatch({
- s <- tweak(multiprocess, maxCores=2L)
-}, warning = function(w) w)
-stopifnot(inherits(res, "warning"))
-
-res <- tryCatch({
- s <- tweak(cluster, cluster="localhost")
-}, warning = function(w) w)
-stopifnot(inherits(res, "warning"))
-
-message("*** Deprecated arguments ... DONE")
-
-
-message("*** Deprecated argument values ...")
-
-res <- tryCatch({
- n <- availableCores(methods="mc.cores")
-}, warning = function(w) w)
-stopifnot(inherits(res, "warning"))
-
-message("*** Deprecated argument values ... DONE")
-
-
-message("*** Deprecated API ... DONE")
-
-
-message("*** Defunct API ...")
-
-
-message("*** Defunct API ... DONE")
-
-
-source("incl/end.R")
diff --git a/tests/dotdotdot.R b/tests/dotdotdot.R
index 7e16c33..21de078 100644
--- a/tests/dotdotdot.R
+++ b/tests/dotdotdot.R
@@ -1,16 +1,16 @@
source("incl/start.R")
library("listenv")
-for (cores in 1:min(3L, availableCores())) {
+for (cores in 1:availCores) {
message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
+ options(mc.cores = cores)
- message("*** Global argument '...' in lazy futures ...")
+ message("*** Global argument '...' ...")
sum_fcns <- list()
sum_fcns$A <- function(x, ...) {
- message("Arguments '...' exists: ", exists("...", inherits=TRUE))
+ message("Arguments '...' exists: ", exists("...", inherits = TRUE))
y %<-% { sum(x, ...) }
y
}
@@ -18,7 +18,7 @@ for (cores in 1:min(3L, availableCores())) {
sum_fcns$B <- function(x, ...) {
sumt <- function(x) {
- message("Arguments '...' exists: ", exists("...", inherits=TRUE))
+ message("Arguments '...' exists: ", exists("...", inherits = TRUE))
y %<-% { sum(x, ...) }
y
}
@@ -26,27 +26,27 @@ for (cores in 1:min(3L, availableCores())) {
}
sum_fcns$C <- function(x, y) {
- message("Arguments '...' exists: ", exists("...", inherits=TRUE))
+ message("Arguments '...' exists: ", exists("...", inherits = TRUE))
y %<-% { sum(x, y) }
y
}
sum_fcns$D <- function(x, y) {
- message("Arguments '...' exists: ", exists("...", inherits=TRUE))
+ message("Arguments '...' exists: ", exists("...", inherits = TRUE))
y %<-% { sum(x, y, ...) }
y
}
- for (strategy in supportedStrategies()) {
+ for (strategy in supportedStrategies(cores)) {
message(sprintf("- plan('%s') ...", strategy))
- plan(strategy, substitute=FALSE)
+ plan(strategy, substitute = FALSE)
for (name in names(sum_fcns)) {
message(sprintf("** Sum function '%s' with plan('%s') ...", name, strategy))
sum_fcn <- sum_fcns[[name]]
print(sum_fcn)
- y <- try(sum_fcn(1:2,3))
+ y <- try(sum_fcn(1:2, 3))
print(y)
if (name %in% c("D")) {
stopifnot(inherits(y, "try-error"))
@@ -59,6 +59,6 @@ for (cores in 1:min(3L, availableCores())) {
message(sprintf("Testing with %d cores ... DONE", cores))
} ## for (cores ...)
-message("*** Global argument '...' in lazy futures ... DONE")
+message("*** Global argument '...' ... DONE")
source("incl/end.R")
diff --git a/tests/eager.R b/tests/eager.R
deleted file mode 100644
index 7050175..0000000
--- a/tests/eager.R
+++ /dev/null
@@ -1,62 +0,0 @@
-source("incl/start.R")
-
-message("*** eager() ...")
-
-for (globals in c(FALSE, TRUE)) {
-
-message(sprintf("*** eager(..., globals=%s) without globals", globals))
-
-f <- eager({
- 42L
-}, globals=globals)
-stopifnot(inherits(f, "UniprocessFuture"), !f$lazy, inherits(f, "EagerFuture"))
-
-print(resolved(f))
-stopifnot(resolved(f))
-
-y <- value(f)
-print(y)
-stopifnot(y == 42L)
-
-
-message(sprintf("*** eager(..., globals=%s) with globals", globals))
-## A global variable
-a <- 0
-f <- eager({
- b <- 3
- c <- 2
- a * b * c
-}, globals=globals)
-print(f)
-
-## Since 'a' is a global variable in _eager_ future 'f',
-## it already has been resolved, and any changes to 'a'
-## at this point will _not_ affect the value of 'f'.
-a <- 7
-v <- value(f)
-print(v)
-stopifnot(v == 0)
-
-
-message(sprintf("*** eager(..., globals=%s) and errors", globals))
-f <- eager({
- stop("Whoops!")
- 1
-}, globals=globals)
-print(f)
-stopifnot(inherits(f, "UniprocessFuture"), !f$lazy, inherits(f, "EagerFuture"))
-
-res <- try(value(f), silent=TRUE)
-print(res)
-stopifnot(inherits(res, "try-error"))
-
-## Error is repeated
-res <- try(value(f), silent=TRUE)
-print(res)
-stopifnot(inherits(res, "try-error"))
-
-} # for (globals ...)
-
-message("*** eager() ... DONE")
-
-source("incl/end.R")
diff --git a/tests/early-signaling.R b/tests/early-signaling.R
index 8086152..31ff836 100644
--- a/tests/early-signaling.R
+++ b/tests/early-signaling.R
@@ -1,71 +1,62 @@
source("incl/start.R")
-options(future.debug=FALSE)
+options(future.debug = FALSE)
message("*** Early signaling of conditions ...")
-message("*** Early signaling of conditions with uniprocess futures ...")
+message("*** Early signaling of conditions with sequential futures ...")
-plan(uniprocess)
+plan(sequential)
f <- future({ stop("bang!") })
-Sys.sleep(1.0)
r <- resolved(f)
stopifnot(r)
-v <- try(value(f), silent=TRUE)
-stopifnot(inherits(v, "try-error"))
+v <- tryCatch(value(f), error = identity)
+stopifnot(inherits(v, "error"))
-plan(uniprocess, earlySignal=TRUE)
-f <- try(future({ stop("bang!") }), silent=TRUE)
-stopifnot(inherits(f, "try-error"))
-
-message("*** Early signaling of conditions with uniprocess futures ... DONE")
-
-
-message("*** Early signaling of conditions with lazy futures ...")
-
-plan(lazy)
-f <- future({ stop("bang!") })
-Sys.sleep(1.0)
+message("- with lazy evaluation ...")
+f <- future({ stop("bang!") }, lazy = TRUE)
r <- resolved(f)
stopifnot(r)
-v <- try(value(f), silent=TRUE)
-stopifnot(inherits(v, "try-error"))
+v <- tryCatch(value(f), error = identity)
+stopifnot(inherits(v, "error"))
-plan(lazy, earlySignal=TRUE)
+plan(sequential, earlySignal = TRUE)
+f <- tryCatch(future({ stop("bang!") }), error = identity)
+stopifnot(inherits(f, "error"))
+
+message("- with lazy evaluation ...")
## Errors
-f <- future({ stop("bang!") })
-Sys.sleep(1.0)
-r <- try(resolved(f), silent=TRUE)
-stopifnot(inherits(r, "try-error"))
-v <- try(value(f), silent=TRUE)
-stopifnot(inherits(v, "try-error"))
+f <- future({ stop("bang!") }, lazy = TRUE)
+r <- tryCatch(resolved(f), error = identity)
+stopifnot(inherits(r, "error"))
+v <- tryCatch(value(f), error = identity)
+stopifnot(inherits(v, "error"))
## Warnings
-f <- future({ warning("careful!") })
-Sys.sleep(1.0)
+f <- future({ warning("careful!") }, lazy = TRUE)
res <- tryCatch({
r <- resolved(f)
}, condition = function(w) w)
+str(res)
stopifnot(inherits(res, "warning"))
## Messages
-f <- future({ message("hey!") })
-Sys.sleep(1.0)
+f <- future({ message("hey!") }, lazy = TRUE)
res <- tryCatch({
r <- resolved(f)
}, condition = function(w) w)
stopifnot(inherits(res, "message"))
## Condition
-f <- future({ signalCondition(simpleCondition("hmm")) })
-Sys.sleep(1.0)
+f <- future({ signalCondition(simpleCondition("hmm")) }, lazy = TRUE)
res <- tryCatch({
r <- resolved(f)
}, condition = function(w) w)
stopifnot(inherits(res, "condition"))
-message("*** Early signaling of conditions with lazy futures ... DONE")
+message("*** Early signaling of conditions with sequential futures ... DONE")
+
message("Number of available cores: ", availableCores())
@@ -73,20 +64,20 @@ message("*** Early signaling of conditions with multisession futures ...")
plan(multisession)
f <- future({ stop("bang!") })
-Sys.sleep(1.0)
+Sys.sleep(0.5)
r <- resolved(f)
stopifnot(r)
-v <- try(value(f), silent=TRUE)
-stopifnot(inherits(v, "try-error"))
+v <- tryCatch(value(f), error = identity)
+stopifnot(inherits(v, "error"))
-plan(multisession, earlySignal=TRUE)
+plan(multisession, earlySignal = TRUE)
f <- future({ stop("bang!") })
-Sys.sleep(1.0)
+Sys.sleep(0.5)
print(f)
-r <- try(resolved(f), silent=TRUE)
-stopifnot(inherits(r, "try-error") || inherits(f, "UniprocessFuture"))
-v <- try(value(f), silent=TRUE)
-stopifnot(inherits(v, "try-error"))
+r <- tryCatch(resolved(f), error = identity)
+stopifnot(inherits(r, "error") || inherits(f, "SequentialFuture"))
+v <- tryCatch(value(f), error = identity)
+stopifnot(inherits(v, "error"))
message("*** Early signaling of conditions with multisession futures ... DONE")
@@ -96,23 +87,50 @@ message("*** Early signaling of conditions with multiprocess futures ...")
plan(multiprocess)
f <- future({ stop("bang!") })
-Sys.sleep(1.0)
+Sys.sleep(0.5)
r <- resolved(f)
stopifnot(r)
-v <- try(value(f), silent=TRUE)
-stopifnot(inherits(v, "try-error"))
+v <- tryCatch(value(f), error = identity)
+stopifnot(inherits(v, "error"))
-plan(multiprocess, earlySignal=TRUE)
+plan(multiprocess, earlySignal = TRUE)
f <- future({ stop("bang!") })
-Sys.sleep(1.0)
+Sys.sleep(0.5)
print(f)
-r <- try(resolved(f), silent=TRUE)
-stopifnot(inherits(r, "try-error") || inherits(f, "UniprocessFuture"))
-v <- try(value(f), silent=TRUE)
-stopifnot(inherits(v, "try-error"))
+r <- tryCatch(resolved(f), error = identity)
+stopifnot(inherits(r, "error") || inherits(f, "SequentialFuture"))
+v <- tryCatch(value(f), error = identity)
+stopifnot(inherits(v, "error"))
+
+## Errors
+f <- future({ stop("bang!") }, earlySignal = TRUE)
+Sys.sleep(0.5)
+r <- tryCatch(resolved(f), error = identity)
+stopifnot(inherits(r, "error"))
+v <- tryCatch(value(f), error = identity)
+stopifnot(inherits(v, "error"))
+
+## Warnings
+f <- future({ warning("careful!") }, earlySignal = TRUE)
+Sys.sleep(0.5)
+res <- tryCatch({ r <- resolved(f) }, condition = function(w) w)
+#stopifnot(inherits(res, "warning"))
+
+## Messages
+f <- future({ message("hey!") }, earlySignal = TRUE)
+Sys.sleep(0.5)
+res <- tryCatch({ r <- resolved(f) }, condition = function(w) w)
+#stopifnot(inherits(res, "message"))
+
+## Condition
+f <- future({ signalCondition(simpleCondition("hmm")) }, earlySignal = TRUE)
+Sys.sleep(0.5)
+res <- tryCatch({ r <- resolved(f) }, condition = function(w) w)
+#stopifnot(inherits(res, "condition"))
message("*** Early signaling of conditions with multiprocess futures ... DONE")
+
message("*** Early signaling of conditions ... DONE")
source("incl/end.R")
diff --git a/tests/flapply.R b/tests/flapply.R
deleted file mode 100644
index d1313fa..0000000
--- a/tests/flapply.R
+++ /dev/null
@@ -1,93 +0,0 @@
-source("incl/start.R")
-library("listenv")
-
-message("*** flapply() ...")
-
-for (cores in 1:min(3L, availableCores())) {
- message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
-
- message("- flapply(x, FUN=vector, ...) ...")
-
- x <- list(a="integer", b="numeric", c="character", c="list")
- str(list(x=x))
-
- y0 <- lapply(x, FUN=vector, length=2L)
- str(list(y0=y0))
-
- for (strategy in supportedStrategies()) {
- message(sprintf("- plan('%s') ...", strategy))
- plan(strategy)
- y <- flapply(x, FUN=vector, length=2L)
- str(list(y=y))
- stopifnot(identical(y, y0))
- }
-
-
- message("- flapply(x, FUN=base::vector, ...) ...")
-
- x <- list(a="integer", b="numeric", c="character", c="list")
- str(list(x=x))
-
- y0 <- lapply(x, FUN=base::vector, length=2L)
- str(list(y0=y0))
-
- for (strategy in supportedStrategies()) {
- message(sprintf("- plan('%s') ...", strategy))
- plan(strategy)
- y <- flapply(x, FUN=base::vector, length=2L)
- str(list(y=y))
- stopifnot(identical(y, y0))
- }
-
-
- message("- flapply(x, FUN=future:::hpaste, ...) ...")
-
- x <- list(a=c("hello", b=1:100))
- str(list(x=x))
-
- y0 <- lapply(x, FUN=future:::hpaste, collapse="; ", maxHead=3L)
- str(list(y0=y0))
-
- for (strategy in supportedStrategies()) {
- message(sprintf("- plan('%s') ...", strategy))
- plan(strategy)
- y <- flapply(x, FUN=future:::hpaste, collapse="; ", maxHead=3L)
- str(list(y=y))
- stopifnot(identical(y, y0))
- }
-
-
- message("- flapply(x, FUN=listenv::listenv, ...) ...")
-
- x <- list()
-
- y <- listenv()
- y$A <- 3L
- x$a <- y
-
- y <- listenv()
- y$A <- 3L
- y$B <- c("hello", b=1:100)
- x$b <- y
-
- print(x)
-
- y0 <- lapply(x, FUN=listenv::map)
- str(list(y0=y0))
-
- for (strategy in supportedStrategies()) {
- message(sprintf("- plan('%s') ...", strategy))
- plan(strategy)
- y <- flapply(x, FUN=listenv::map)
- str(list(y=y))
- stopifnot(identical(y, y0))
- }
-
- message(sprintf("Testing with %d cores ... DONE", cores))
-} ## for (cores ...)
-
-message("*** flapply() ... DONE")
-
-source("incl/end.R")
-
diff --git a/tests/future,labels.R b/tests/future,labels.R
index f1f6b62..1bc9151 100644
--- a/tests/future,labels.R
+++ b/tests/future,labels.R
@@ -2,15 +2,12 @@ source("incl/start.R")
message("*** Futures - labels ...")
-strategies <- supportedStrategies()
-strategies <- setdiff(strategies, "multiprocess")
-
-for (strategy in strategies) {
+for (strategy in supportedStrategies()) {
message(sprintf("- plan('%s') ...", strategy))
plan(strategy)
- for (label in list(NULL, sprintf("strategy=%s", strategy))) {
- fcn <- get(strategy, mode="function")
+ for (label in list(NULL, sprintf("strategy = %s", strategy))) {
+ fcn <- get(strategy, mode = "function")
stopifnot(inherits(fcn, strategy))
f <- fcn(42, label = label)
print(f)
diff --git a/tests/future.R b/tests/future.R
index 6f4e87b..5c6f180 100644
--- a/tests/future.R
+++ b/tests/future.R
@@ -1,11 +1,10 @@
source("incl/start.R")
-plan(lazy)
message("*** future() ...")
f <- future({
42L
-})
+}, lazy = TRUE)
print(resolved(f))
y <- value(f)
@@ -13,27 +12,22 @@ print(y)
stopifnot(y == 42L)
-message("*** future() w/ gc=TRUE ...")
+message("*** future() w/ gc = TRUE ...")
-f <- future(42L, gc=TRUE)
+f <- future(42L, gc = TRUE, lazy = TRUE)
print(f)
y <- value(f)
print(y)
stopifnot(y == 42L)
-message("*** future() w/ gc=TRUE ... DONE")
-
+message("*** future() w/ gc = TRUE ... DONE")
message("*** future() ... DONE")
-
-
-plan(lazy)
-
message("*** future() ...")
f <- future({
42L
-})
+}, lazy = TRUE)
print(resolved(f))
y <- value(f)
@@ -41,28 +35,28 @@ print(y)
stopifnot(y == 42L)
-message("*** future() w/ gc=TRUE ...")
+message("*** future() w/ gc = TRUE ...")
-f <- future(42L, gc=TRUE)
+f <- future(42L, gc = TRUE, lazy = TRUE)
print(f)
y <- value(f)
print(y)
stopifnot(y == 42L)
-message("*** future() w/ gc=TRUE ... DONE")
+message("*** future() w/ gc = TRUE ... DONE")
message("*** future() - exceptions ...")
-res <- try(future(42L, evaluator=TRUE))
-stopifnot(inherits(res, "try-error"))
+res <- tryCatch(future(42L, evaluator = TRUE, lazy = TRUE), error = identity)
+stopifnot(inherits(res, "error"))
-res <- try(future(42L, evaluator=function(...) TRUE))
-stopifnot(inherits(res, "try-error"))
+res <- tryCatch(future(42L, evaluator = function(...) TRUE, lazy = TRUE), error = identity)
+stopifnot(inherits(res, "error"))
-target <- list(name="<unknown>", envir=new.env(), code="Yo!", exists=TRUE)
-res <- try(get_future(target, mustExist=TRUE))
-stopifnot(inherits(res, "try-error"))
+target <- list(name = "<unknown>", envir = new.env(), code = "Yo!", exists = TRUE)
+res <- tryCatch(get_future(target, mustExist = TRUE), error = identity)
+stopifnot(inherits(res, "error"))
message("*** future() - exceptions ... DONE")
@@ -70,4 +64,3 @@ message("*** future() - exceptions ... DONE")
message("*** future() ... DONE")
source("incl/end.R")
-
diff --git a/tests/futureAssign.R b/tests/futureAssign.R
index 2f30387..ee7c4cf 100644
--- a/tests/futureAssign.R
+++ b/tests/futureAssign.R
@@ -1,7 +1,8 @@
source("incl/start.R")
message("*** futureAssign() ...")
-plan(lazy)
+
+message("*** futureAssign() - sequential w/ lazy evaluation ...")
delayedAssign("a", {
cat("Delayed assignment evaluated\n")
@@ -11,32 +12,76 @@ delayedAssign("a", {
futureAssign("b", {
cat("Future assignment evaluated\n")
2
-})
+}, lazy = TRUE)
## Because "lazy future" is used, the expression/value
## for 'b' will be resolved at the point. For other
## types of futures, it may already have been resolved
-cat(sprintf("b=%s\n", b))
+cat(sprintf("b = %s\n", b))
## The expression/value of 'a' is resolved at this point,
## because a delayed assignment (promise) was used.
-cat(sprintf("a=%s\n", a))
+cat(sprintf("a = %s\n", a))
stopifnot(identical(a, 1))
stopifnot(identical(b, 2))
+message("*** futureAssign() - sequential w/ lazy evaluation ... DONE")
+
+
+message("*** futureAssign() - lazy = TRUE / FALSE ...")
+
+for (cores in 1:availCores) {
+ message(sprintf("Testing with %d cores ...", cores))
+ options(mc.cores = cores)
+
+ for (strategy in supportedStrategies(cores)) {
+ message(sprintf("*** futureAssign() with %s futures ...", sQuote(strategy)))
+ plan(strategy)
+
+ ## Potential task name clashes
+ u <- new.env()
+ v <- new.env()
+ futureAssign("a", { 2 }, assign.env = u)
+ futureAssign("a", { 4 }, assign.env = v)
+
+ cat(sprintf("u$a = %s\n", u$a))
+ cat(sprintf("v$a = %s\n", v$a))
+
+ stopifnot(identical(u$a, 2))
+ stopifnot(identical(v$a, 4))
+
+
+ ## Global variables
+ a <- 1
+ futureAssign("b", { 2 * a })
+ a <- 2
+ stopifnot(b == 2)
+
+ ## Explicit lazy evaluation
+ for (lazy in c(FALSE, TRUE)) {
+ a <- 1
+ f <- futureAssign("b", { 2 * a }, lazy = lazy)
+ a <- 2
+ stopifnot(b == 2)
+ stopifnot(f$lazy == lazy || (strategy %in% c("multisession", "multiprocess") && cores == 1L))
+
+ ## Set 'lazy' via disposable option
+ options(future.disposable = list(lazy = lazy))
+ a <- 1
+ f <- futureAssign("b", { 2 * a })
+ a <- 2
+ stopifnot(b == 2)
+ stopifnot(f$lazy == lazy || (strategy %in% c("multisession", "multiprocess") && cores == 1L))
+ }
-## Potential task name clashes
-u <- new.env()
-v <- new.env()
-futureAssign("a", { 2 }, assign.env=u)
-futureAssign("a", { 4 }, assign.env=v)
+ message(sprintf("*** futureAssign() with %s futures ... DONE", sQuote(strategy)))
+ } # for (strategy in ...)
-cat(sprintf("u$a=%s\n", u$a))
-cat(sprintf("v$a=%s\n", v$a))
+ message(sprintf("Testing with %d cores ... DONE", cores))
+} ## for (cores ...)
-stopifnot(identical(u$a, 2))
-stopifnot(identical(v$a, 4))
+message("*** futureAssign() - lazy = TRUE / FALSE ... DONE")
message("*** futureAssign() ... DONE")
diff --git a/tests/futureAssign_OP.R b/tests/futureAssign_OP.R
index 2208740..0c4b6d1 100644
--- a/tests/futureAssign_OP.R
+++ b/tests/futureAssign_OP.R
@@ -2,24 +2,24 @@ source("incl/start.R")
message("*** %<-% ...")
-for (cores in 1:min(3L, availableCores())) {
+for (cores in 1:availCores) {
message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
+ options(mc.cores = cores)
- for (strategy in supportedStrategies()) {
+ for (strategy in supportedStrategies(cores)) {
message(sprintf("*** %%<-%% with %s futures ...", sQuote(strategy)))
plan(strategy)
- rm(list=intersect(c("x", "y"), ls()))
+ rm(list = intersect(c("x", "y"), ls()))
message("** Future evaluation without globals")
v1 %<-% { x <- 1 }
- stopifnot(!exists("x", inherits=FALSE), identical(v1, 1))
+ stopifnot(!exists("x", inherits = FALSE), identical(v1, 1))
message("** Future evaluation with globals")
a <- 2
v2 %<-% { x <- a }
- stopifnot(!exists("x", inherits=FALSE), identical(v2, a))
+ stopifnot(!exists("x", inherits = FALSE), identical(v2, a))
message("** Future evaluation with errors")
v3 %<-% {
@@ -27,65 +27,61 @@ for (cores in 1:min(3L, availableCores())) {
stop("Woops!")
x
}
- stopifnot(!exists("x", inherits=FALSE))
- res <- try(identical(v3, 3), silent=TRUE)
- stopifnot(inherits(res, "try-error"))
+ stopifnot(!exists("x", inherits = FALSE))
+ res <- tryCatch(identical(v3, 3), error = identity)
+ stopifnot(inherits(res, "error"))
y <- listenv::listenv()
- for (ii in 1:5) {
+ for (ii in 1:3) {
y[[ii]] %<-% {
if (ii %% 2 == 0) stop("Woops!")
ii
}
}
- res <- try(as.list(y), silent=TRUE)
- stopifnot(inherits(res, "try-error"))
- z <- y[c(1,3,5)]
+ res <- tryCatch(as.list(y), error = identity)
+ stopifnot(inherits(res, "error"))
+ z <- y[c(1, 3)]
z <- unlist(z)
- stopifnot(all(z == c(1,3,5)))
- res <- try(y[[2]], silent=TRUE)
- stopifnot(inherits(res, "try-error"))
- res <- try(y[[4]], silent=TRUE)
- stopifnot(inherits(res, "try-error"))
- res <- try(y[c(2,4)], silent=TRUE)
- stopifnot(inherits(res, "try-error"))
- res <- try(y[1:2], silent=TRUE)
- stopifnot(inherits(res, "try-error"))
+ stopifnot(all(z == c(1, 3)))
+ res <- tryCatch(y[[2]], error = identity)
+ stopifnot(inherits(res, "error"))
+ res <- tryCatch(y[1:2], error = identity)
+ stopifnot(inherits(res, "error"))
message("** Future evaluation with progress bar")
v4 %<-% {
cat("Processing: ")
- for (ii in 1:10) { cat("."); Sys.sleep(0.1) }
+ for (ii in 1:10) { cat(".") }
cat(" [100%]\n")
4
}
message("** Collecting results")
- printf("v1=%s\n", v1)
+ printf("v1 = %s\n", v1)
stopifnot(v1 == 1)
- printf("v2=%s\n", v2)
+ printf("v2 = %s\n", v2)
stopifnot(v2 == a)
stopifnot(tryCatch({
- printf("v3=%s\n", v3)
+ printf("v3 = %s\n", v3)
}, error = function(ex) {
printf("v3: <%s> (as expect)\n", class(ex)[1])
TRUE
}))
- printf("v4=%s\n", v4)
+ printf("v4 = %s\n", v4)
#stopifnot(v4 == 4)
message("** Left-to-right and right-to-left future assignments")
c %<-% 1
- printf("c=%s\n", c)
- 1 %=>% d
- printf("d=%s\n", d)
+ printf("c = %s\n", c)
+ 1 %->% d
+ printf("d = %s\n", d)
stopifnot(d == c)
@@ -95,14 +91,14 @@ for (cores in 1:min(3L, availableCores())) {
b <- 1
c %<-% 2
3 -> d
- 4 %=>% e
+ 4 %->% e
b + c + d + e
}
- printf("a=%s\n", a)
+ printf("a = %s\n", a)
stopifnot(a == 10)
- { a + 1 } %=>% b
- printf("b=%s\n", b)
+ { a + 1 } %->% b
+ printf("b = %s\n", b)
stopifnot(b == a + 1)
message(sprintf("*** %%<-%% with %s futures ... DONE", sQuote(strategy)))
diff --git a/tests/futureAssign_OP_with_environment.R b/tests/futureAssign_OP_with_environment.R
index 403f882..61b0efa 100644
--- a/tests/futureAssign_OP_with_environment.R
+++ b/tests/futureAssign_OP_with_environment.R
@@ -1,9 +1,8 @@
source("incl/start.R")
-plan(lazy)
## BACKWARD COMPATIBILITY
if (getRversion() < "3.2.0") {
- names <- function(x) if (is.environment(x)) ls(envir=x) else base::names(x)
+ names <- function(x) if (is.environment(x)) ls(envir = x) else base::names(x)
}
message("*** %<-% to environment ...")
@@ -15,11 +14,11 @@ z <- new.env()
stopifnot(length(names(z)) == 0L)
message("*** %<-% to environment: Assign by index (not allowed)")
-res <- try(z[[1]] %<-% { 2 }, silent=TRUE)
+res <- try(z[[1]] %<-% { 2 } %lazy% TRUE, silent = TRUE)
stopifnot(inherits(res, "try-error"))
message("*** %<-% to environment: Assign by name (new)")
-z$B %<-% TRUE
+z$B %<-% { TRUE } %lazy% TRUE
stopifnot(length(z) == 2) # sic!
stopifnot("B" %in% ls(z))
@@ -31,13 +30,13 @@ stopifnot(identical(names(y), "B"))
message("*** %<-% to environment: Potential task name clashes")
u <- new.env()
-u$a %<-% 1
+u$a %<-% { 1 } %lazy% TRUE
stopifnot(length(u) == 2)
stopifnot("a" %in% names(u))
fu <- futureOf(u$a)
v <- new.env()
-v$a %<-% 2
+v$a %<-% { 2 } %lazy% TRUE
stopifnot(length(v) == 2)
stopifnot("a" %in% names(v))
fv <- futureOf(v$a)
diff --git a/tests/futureAssign_OP_with_listenv.R b/tests/futureAssign_OP_with_listenv.R
index deab2a5..cd74387 100644
--- a/tests/futureAssign_OP_with_listenv.R
+++ b/tests/futureAssign_OP_with_listenv.R
@@ -1,6 +1,5 @@
source("incl/start.R")
library("listenv")
-plan(lazy)
message("*** %<-% to listenv ...")
@@ -11,15 +10,15 @@ z <- listenv()
stopifnot(length(names(z)) == 0)
message("*** %<-% to listenv: Assign by index")
-z[[1]] %<-% { 2 }
+z[[1]] %<-% { 2 } %lazy% TRUE
stopifnot(length(z) == 1)
stopifnot(length(names(z)) == 0)
-z[[1]] %<-% { 2 }
+z[[1]] %<-% { 2 } %lazy% TRUE
stopifnot(length(z) == 1)
stopifnot(length(names(z)) == 0)
-z[[4]] %<-% { "async!" }
+z[[4]] %<-% { "async!" } %lazy% TRUE
stopifnot(length(z) == 4)
stopifnot(length(names(z)) == 0)
@@ -29,7 +28,7 @@ stopifnot(identical(names(z), c("A", "B", "C", "D")))
message("*** %<-% to listenv: Assign by name (existing)")
-z$B %<-% TRUE
+z$B %<-% { TRUE } %lazy% TRUE
stopifnot(length(z) == 4)
stopifnot(identical(names(z), c("A", "B", "C", "D")))
@@ -41,12 +40,12 @@ stopifnot(identical(names(y), c("A", "B", "C", "D")))
message("*** %<-% to listenv: Asserting no name clashes among futures")
u <- listenv()
-u$a %<-% 1
+u$a %<-% { 1 } %lazy% TRUE
stopifnot(identical(names(u), "a"))
fu <- futureOf(u$a)
v <- listenv()
-v$a %<-% 2
+v$a %<-% { 2 } %lazy% TRUE
stopifnot(identical(names(v), "a"))
fv <- futureOf(v$a)
stopifnot(!identical(fu, fv))
@@ -62,16 +61,16 @@ message("*** %<-% to listenv: multiple dimensions ...")
x0 <- list()
length(x0) <- 6
-dim(x0) <- c(3,2)
+dim(x0) <- c(3, 2)
x <- listenv()
length(x) <- 6
-dim(x) <- c(3,2)
+dim(x) <- c(3, 2)
for (cc in 1:ncol(x)) {
for (rr in 1:nrow(x)) {
- x0[[rr,cc]] <- sprintf("(%s,%s)", rr, cc)
- x[[rr,cc]] %<-% sprintf("(%s,%s)", rr, cc)
+ x0[[rr, cc]] <- sprintf("(%s, %s)", rr, cc)
+ x[[rr, cc]] %<-% sprintf("(%s, %s)", rr, cc) %lazy% TRUE
}
}
diff --git a/tests/futureCall.R b/tests/futureCall.R
index fcb5dd0..9e58fc6 100644
--- a/tests/futureCall.R
+++ b/tests/futureCall.R
@@ -1,22 +1,21 @@
source("incl/start.R")
message("*** futureCall() ...")
-plan(lazy)
-f1 <- future(do.call(rnorm, args=list(n=100)))
-f2 <- futureCall(rnorm, args=list(n=100))
+f1 <- future(do.call(rnorm, args = list(n = 100)), lazy = TRUE)
+f2 <- futureCall(rnorm, args = list(n = 100), lazy = TRUE)
set.seed(42L)
-v0 <- rnorm(n=100)
-str(list(v0=v0))
+v0 <- rnorm(n = 100)
+str(list(v0 = v0))
set.seed(42L)
v1 <- value(f1)
-str(list(v1=v1))
+str(list(v1 = v1))
set.seed(42L)
v2 <- value(f2)
-str(list(v2=v2))
+str(list(v2 = v2))
## Because we use lazy futures and set the
## random seed just before they are resolved
diff --git a/tests/futureOf.R b/tests/futureOf.R
index 8a14688..9bc0cb4 100644
--- a/tests/futureOf.R
+++ b/tests/futureOf.R
@@ -1,10 +1,9 @@
source("incl/start.R")
library("listenv")
-plan(lazy)
message("*** futureOf() ...")
-a %<-% { 1 }
+a %<-% { 1 } %lazy% TRUE
f1 <- futureOf("a")
print(f1)
@@ -16,10 +15,9 @@ fs <- futureOf()
print(fs)
## Non-existing object
-res <- try(futureOf("non-exiting-object", mustExist=TRUE), silent=TRUE)
-stopifnot(inherits(res, "try-error"))
+res <- tryCatch(futureOf("non-exiting-object", mustExist = TRUE), error = identity)
+stopifnot(inherits(res, "error"))
message("*** futureOf() ... DONE")
source("incl/end.R")
-
diff --git a/tests/futureOf_with_environment.R b/tests/futureOf_with_environment.R
index 6feb406..dbf8449 100644
--- a/tests/futureOf_with_environment.R
+++ b/tests/futureOf_with_environment.R
@@ -1,6 +1,6 @@
source("incl/start.R")
-suppressWarnings(rm(list=c("x", "z")))
+suppressWarnings(rm(list = c("x", "z")))
message("*** futureOf() with environment ...")
@@ -9,22 +9,22 @@ message("*** futureOf() with environment - future assignments ...")
x <- new.env()
x$a %<-% { 1 }
-f1 <- futureOf("a", envir=x)
+f1 <- futureOf("a", envir = x)
print(f1)
-f2 <- futureOf(a, envir=x)
+f2 <- futureOf(a, envir = x)
f3 <- futureOf(x[["a"]])
f4 <- futureOf(x$a)
stopifnot(identical(f2, f1), identical(f3, f1), identical(f4, f1))
## Identify all futures
-fs <- futureOf(envir=x)
+fs <- futureOf(envir = x)
print(fs)
stopifnot(identical(names(fs), c("a")))
stopifnot(identical(fs$a, f1))
-fsD <- futureOf(envir=x, drop=TRUE)
+fsD <- futureOf(envir = x, drop = TRUE)
print(fsD)
-stopifnot(all(sapply(fsD, FUN=inherits, "Future")))
+stopifnot(all(sapply(fsD, FUN = inherits, "Future")))
stopifnot(identical(fsD, fs))
message("*** futureOf() with environment - future assignments ... DONE")
@@ -35,10 +35,10 @@ message("*** futureOf() with environment - futures ...")
x <- new.env()
x$a <- future({ 1 })
-f1 <- futureOf("a", envir=x)
+f1 <- futureOf("a", envir = x)
print(f1)
stopifnot(identical(f1, x$a))
-f2 <- futureOf(a, envir=x)
+f2 <- futureOf(a, envir = x)
stopifnot(identical(f2, x$a))
f3 <- futureOf(x[["a"]])
stopifnot(identical(f3, x$a))
@@ -46,14 +46,14 @@ f4 <- futureOf(x$a)
stopifnot(identical(f4, x$a))
## Identify all futures
-fs <- futureOf(envir=x)
+fs <- futureOf(envir = x)
print(fs)
stopifnot(identical(names(fs), c("a")))
stopifnot(identical(fs$a, f1))
-fsD <- futureOf(envir=x, drop=TRUE)
+fsD <- futureOf(envir = x, drop = TRUE)
print(fsD)
-stopifnot(all(sapply(fsD, FUN=inherits, "Future")))
+stopifnot(all(sapply(fsD, FUN = inherits, "Future")))
stopifnot(identical(fsD, fs))
message("*** futureOf() with environment - futures ... DONE")
@@ -62,17 +62,17 @@ message("*** futureOf() with environment - futures ... DONE")
message("*** futureOf() with environment - exceptions ...")
## Invalid subset
-res <- try(futureOf(x[[0]], mustExist=FALSE), silent=TRUE)
-stopifnot(inherits(res, "try-error"))
+res <- tryCatch(futureOf(x[[0]], mustExist = FALSE), error = identity)
+stopifnot(inherits(res, "error"))
-res <- try(futureOf(x[[0]], mustExist=TRUE), silent=TRUE)
-stopifnot(inherits(res, "try-error"))
+res <- tryCatch(futureOf(x[[0]], mustExist = TRUE), error = identity)
+stopifnot(inherits(res, "error"))
-res <- try(futureOf(x[[10]], mustExist=TRUE), silent=TRUE)
-stopifnot(inherits(res, "try-error"))
+res <- tryCatch(futureOf(x[[10]], mustExist = TRUE), error = identity)
+stopifnot(inherits(res, "error"))
-res <- try(futureOf(x[[1+2i]], mustExist=TRUE), silent=TRUE)
-stopifnot(inherits(res, "try-error"))
+res <- tryCatch(futureOf(x[[1 + 2i]], mustExist = TRUE), error = identity)
+stopifnot(inherits(res, "error"))
message("*** futureOf() with environment - exceptions ... DONE")
diff --git a/tests/futureOf_with_listenv.R b/tests/futureOf_with_listenv.R
index 7ae6736..701bbdc 100644
--- a/tests/futureOf_with_listenv.R
+++ b/tests/futureOf_with_listenv.R
@@ -1,39 +1,38 @@
source("incl/start.R")
library("listenv")
-plan(lazy)
message("*** futureOf() with listenv ...")
message("*** futureOf() with listenv - future assignments ...")
x <- listenv()
-x$a %<-% { 1 }
+x$a %<-% { 1 } %lazy% TRUE
-f1 <- futureOf("a", envir=x)
+f1 <- futureOf("a", envir = x)
print(f1)
-f2 <- futureOf(a, envir=x)
-f3 <- futureOf(1, envir=x)
+f2 <- futureOf(a, envir = x)
+f3 <- futureOf(1, envir = x)
f4 <- futureOf(x[["a"]])
f5 <- futureOf(x$a)
f6 <- futureOf(x[[1]])
stopifnot(identical(f2, f1), identical(f3, f2), identical(f4, f3),
identical(f5, f4), identical(f6, f5))
-x[[3]] %<-% { 3 }
-x$d %<-% { 4 }
+x[[3]] %<-% { 3 } %lazy% TRUE
+x$d %<-% { 4 } %lazy% TRUE
x[[5]] <- 5
## Identify all futures
-fs <- futureOf(envir=x)
+fs <- futureOf(envir = x)
print(fs)
stopifnot(identical(names(fs), names(x)))
stopifnot(identical(fs$a, f1))
-stopifnot(identical(fs[[3]], futureOf(3L, envir=x)))
-stopifnot(identical(fs$d, futureOf("d", envir=x)))
+stopifnot(identical(fs[[3]], futureOf(3L, envir = x)))
+stopifnot(identical(fs$d, futureOf("d", envir = x)))
-fsD <- futureOf(envir=x, drop=TRUE)
+fsD <- futureOf(envir = x, drop = TRUE)
print(fsD)
-stopifnot(all(sapply(fsD, FUN=inherits, "Future")))
+stopifnot(all(sapply(fsD, FUN = inherits, "Future")))
stopifnot(!identical(fsD, fs))
message("*** futureOf() with listenv - future assignments ... DONE")
@@ -42,14 +41,14 @@ message("*** futureOf() with listenv - future assignments ... DONE")
message("*** futureOf() with listenv - futures ...")
x <- listenv()
-x$a <- future({ 1 })
+x$a <- future({ 1 }, lazy = TRUE)
-f1 <- futureOf("a", envir=x)
+f1 <- futureOf("a", envir = x)
print(f1)
stopifnot(identical(f1, x$a))
-f2 <- futureOf(a, envir=x)
+f2 <- futureOf(a, envir = x)
stopifnot(identical(f2, x$a))
-f3 <- futureOf(1, envir=x)
+f3 <- futureOf(1, envir = x)
stopifnot(identical(f3, x$a))
f4 <- futureOf(x[["a"]])
stopifnot(identical(f4, x$a))
@@ -58,21 +57,21 @@ stopifnot(identical(f5, x$a))
f6 <- futureOf(x[[1]])
stopifnot(identical(f6, x$a))
-x[[3]] <- future({ 3 })
-x$d <- future({ 4 })
+x[[3]] <- future({ 3 }, lazy = TRUE)
+x$d <- future({ 4 }, lazy = TRUE)
x[[5]] <- 5
## Identify all futures
-fs <- futureOf(envir=x)
+fs <- futureOf(envir = x)
print(fs)
stopifnot(identical(names(fs), names(x)))
stopifnot(identical(fs$a, f1))
-stopifnot(identical(fs[[3]], futureOf(3L, envir=x)))
-stopifnot(identical(fs$d, futureOf("d", envir=x)))
+stopifnot(identical(fs[[3]], futureOf(3L, envir = x)))
+stopifnot(identical(fs$d, futureOf("d", envir = x)))
-fsD <- futureOf(envir=x, drop=TRUE)
+fsD <- futureOf(envir = x, drop = TRUE)
print(fsD)
-stopifnot(all(sapply(fsD, FUN=inherits, "Future")))
+stopifnot(all(sapply(fsD, FUN = inherits, "Future")))
stopifnot(!identical(fsD, fs))
message("*** futureOf() with listenv - futures ... DONE")
@@ -81,24 +80,24 @@ message("*** futureOf() with listenv - futures ... DONE")
message("*** futureOf() with listenv - exceptions ...")
## Invalid subset
-res <- try(futureOf(x[[0]], mustExist=FALSE), silent=TRUE)
-stopifnot(inherits(res, "try-error"))
+res <- tryCatch(futureOf(x[[0]], mustExist = FALSE), error = identity)
+stopifnot(inherits(res, "error"))
-res <- try(futureOf(x[[0]], mustExist=TRUE), silent=TRUE)
-stopifnot(inherits(res, "try-error"))
+res <- tryCatch(futureOf(x[[0]], mustExist = TRUE), error = identity)
+stopifnot(inherits(res, "error"))
## Out-of-bound subscript, cf lists
-stopifnot(is.na(futureOf(x[[10]], mustExist=FALSE)))
-res <- try(futureOf(x[[10]], mustExist=TRUE), silent=TRUE)
-stopifnot(inherits(res, "try-error"))
+stopifnot(is.na(futureOf(x[[10]], mustExist = FALSE)))
+res <- tryCatch(futureOf(x[[10]], mustExist = TRUE), error = identity)
+stopifnot(inherits(res, "error"))
## Invalid subscript
-res <- try(futureOf(x[[1+2i]], mustExist=TRUE), silent=TRUE)
-stopifnot(inherits(res, "try-error"))
+res <- tryCatch(futureOf(x[[1 + 2i]], mustExist = TRUE), error = identity)
+stopifnot(inherits(res, "error"))
## Non-existing object
-res <- try(futureOf(z[[1]], mustExist=TRUE), silent=TRUE)
-stopifnot(inherits(res, "try-error"))
+res <- tryCatch(futureOf(z[[1]], mustExist = TRUE), error = identity)
+stopifnot(inherits(res, "error"))
message("*** futureOf() with listenv - exceptions ... DONE")
diff --git a/tests/future_lapply,RNG.R b/tests/future_lapply,RNG.R
new file mode 100644
index 0000000..f0669f5
--- /dev/null
+++ b/tests/future_lapply,RNG.R
@@ -0,0 +1,160 @@
+source("incl/start.R")
+
+message("*** future_lapply() and RNGs ...")
+
+options(future.debug = FALSE)
+
+## Iterate of the same set in all tests
+x <- 1:5
+
+message("* future_lapply(x, ..., future.seed = FALSE) ...")
+
+y0 <- y0_nested <- seed00 <- NULL
+for (cores in 1:availCores) {
+ message(sprintf(" - Testing with %d cores ...", cores))
+ options(mc.cores = cores)
+
+ for (strategy in supportedStrategies(cores)) {
+ message(sprintf("* plan('%s') ...", strategy))
+ plan(strategy)
+
+ set.seed(0xBEEF)
+ seed0 <- get_random_seed()
+ y <- future_lapply(x, FUN = function(i) i, future.seed = FALSE)
+ y <- unlist(y)
+ seed <- get_random_seed()
+ if (is.null(y0)) {
+ y0 <- y
+ seed00 <- seed
+ }
+ str(list(y = y))
+ stopifnot(identical(seed, seed0), identical(seed, seed00))
+ ## NOTE: We cannot guarantee the same random numbers, because
+ ## future.seed = FALSE.
+
+ message(sprintf("* plan('%s') ... DONE", strategy))
+ } ## for (strategy ...)
+ message(sprintf(" - Testing with %d cores ... DONE", cores))
+} ## for (core ...)
+
+message("* future_lapply(x, ..., future.seed = FALSE) ... DONE")
+
+
+seed_sets <- list(
+ A = TRUE,
+ B = NA,
+ C = 42L,
+ D = future:::as_lecyer_cmrg_seed(42L),
+ E = list(),
+ F = vector("list", length = length(x))
+)
+
+## Generate sequence of seeds of the current RNGkind()
+## NOTE: This is NOT a good way to generate random seeds!!!
+seeds <- lapply(seq_along(x), FUN = function(i) {
+ set.seed(i)
+ globalenv()$.Random.seed
+})
+seed_sets$E <- seeds
+
+## Generate sequence of L'Ecyer CMRG seeds
+seeds <- seed_sets$F
+seeds[[1]] <- seed_sets$D
+for (kk in 2:length(x)) seeds[[kk]] <- parallel::nextRNGStream(seeds[[kk - 1]])
+seed_sets$F <- seeds
+
+rm(list = "seeds")
+
+for (name in names(seed_sets)) {
+ future.seed <- seed_sets[[name]]
+
+ if (is.list(future.seed)) {
+ label <- sprintf("<list of %d seeds each being a %d-int seed>",
+ length(future.seed), length(future.seed[[1]]))
+ } else {
+ label <- hpaste(future.seed)
+ }
+ message(sprintf("* future_lapply(x, ..., future.seed = %s) ...", label))
+
+ set.seed(0xBEEF)
+ y0 <- seed00 <- NULL
+
+ for (cores in 1:availCores) {
+ message(sprintf(" - Testing with %d cores ...", cores))
+ options(mc.cores = cores)
+
+ for (strategy in supportedStrategies(cores)) {
+ message(sprintf("* plan('%s') ...", strategy))
+ plan(strategy)
+
+ set.seed(0xBEEF)
+ seed0 <- get_random_seed()
+ y <- future_lapply(x, FUN = function(i) {
+ rnorm(1L)
+ }, future.seed = future.seed)
+ y <- unlist(y)
+ seed <- get_random_seed()
+ if (is.null(y0)) {
+ y0 <- y
+ seed00 <- seed
+ }
+ str(list(y = y))
+ stopifnot(!identical(seed, seed0), identical(seed, seed00),
+ identical(y, y0))
+
+ ## RNG-based results should also be identical regardless of
+ ## load-balance scheduling.
+ for (scheduling in list(FALSE, TRUE, 0, 0.5, 2.0, Inf)) {
+ set.seed(0xBEEF)
+ seed0 <- get_random_seed()
+ y <- future_lapply(x, FUN = function(i) {
+ rnorm(1L)
+ }, future.seed = future.seed, future.scheduling = scheduling)
+ seed <- get_random_seed()
+ y <- unlist(y)
+ str(list(y = y))
+ stopifnot(!identical(seed, seed0), identical(seed, seed00),
+ identical(y, y0))
+ }
+
+ ## Nested future_lapply():s
+ for (scheduling in list(FALSE, TRUE)) {
+ y <- future_lapply(x, FUN = function(i) {
+ .seed <- globalenv()$.Random.seed
+
+ z <- future_lapply(1:3, FUN = function(j) {
+ list(j = j, seed = globalenv()$.Random.seed)
+ }, future.seed = .seed)
+
+ ## Assert that all future seeds are unique
+ seeds <- lapply(z, FUN = function(x) x$seed)
+ for (kk in 2:length(seeds)) stopifnot(!all(seeds[[kk]] == seeds[[1]]))
+
+ list(i = i, seed = .seed, sample = rnorm(1L), z = z)
+ }, future.seed = 42L, future.scheduling = scheduling)
+
+ if (is.null(y0_nested)) y0_nested <- y
+ str(list(y = y))
+
+ ## Assert that all future seeds (also nested ones) are unique
+ seeds <- Reduce(c, lapply(y, FUN = function(x) {
+ c(list(seed = x$seed), lapply(x$z, FUN = function(x) x$seed))
+ }))
+ for (kk in 2:length(seeds)) stopifnot(!all(seeds[[kk]] == seeds[[1]]))
+
+ stopifnot(identical(y, y0_nested))
+ }
+
+ message(sprintf("* plan('%s') ... DONE", strategy))
+ } ## for (strategy ...)
+ message(sprintf(" - Testing with %d cores ... DONE", cores))
+ } ## for (cores ...)
+
+ message(sprintf("* future_lapply(x, ..., future.seed = %s) ... DONE", label))
+
+} ## for (name ...)
+
+
+message("*** future_lapply() and RNGs ... DONE")
+
+source("incl/end.R")
diff --git a/tests/future_lapply,globals.R b/tests/future_lapply,globals.R
new file mode 100644
index 0000000..4fa23ea
--- /dev/null
+++ b/tests/future_lapply,globals.R
@@ -0,0 +1,146 @@
+source("incl/start.R")
+
+message("*** future_lapply() - globals ...")
+
+plan(cluster, workers = "localhost")
+
+options(future.debug = FALSE)
+a <- 1
+b <- 2
+
+globals_set <- list(
+ A = FALSE,
+ B = TRUE,
+ C = c("a", "b"),
+ D = list(a = 2, b = 3)
+)
+
+x <- list(1)
+y_truth <- list(A = NULL, B = list(1), C = list(1), D = list(2))
+str(y_truth)
+
+for (name in names(globals_set)) {
+ globals <- globals_set[[name]]
+ message("Globals set ", sQuote(name))
+ y <- tryCatch({
+ future_lapply(x, FUN = function(x) {
+ median(c(x, a, b))
+ }, future.globals = globals, future.packages = "utils")
+ }, error = identity)
+ print(y)
+ stopifnot((name == "A" && inherits(y, "error")) ||
+ identical(y, y_truth[[name]]))
+}
+
+message("*** future_lapply() - globals ... DONE")
+
+
+
+## Test adopted from http://stackoverflow.com/questions/42561088/nested-do-call-within-a-foreach-dopar-environment-cant-find-function-passed-w
+
+message("*** future_lapply() - tricky globals ...")
+
+my_add <- function(a, b) a + b
+
+call_my_add <- function(a, b) {
+ do.call(my_add, args = list(a = a, b = b))
+}
+
+call_my_add_caller <- function(a, b, FUN = call_my_add) {
+ do.call(FUN, args = list(a = a, b = b))
+}
+
+main <- function(x = 1:2, caller = call_my_add_caller,
+ args = list(FUN = call_my_add)) {
+ results <- future_lapply(x, FUN = function(i) {
+ do.call(caller, args = c(list(a = i, b = i + 1L), args))
+ })
+ results
+}
+
+x <- list(list(1:2))
+z_length <- lapply(x, FUN = do.call, what = length)
+fun <- function(...) sum(...)
+z_fun <- lapply(x, FUN = do.call, what = fun)
+
+y0 <- NULL
+for (strategy in supportedStrategies()) {
+ plan(strategy)
+
+ y <- main(1:3)
+ if (is.null(y0)) y0 <- y
+ stopifnot(identical(y, y0))
+
+ message("- future_lapply(x, FUN = do.call, ...) ...")
+ z <- future_lapply(x, FUN = do.call, what = length)
+ stopifnot(identical(z, z_length))
+ z <- future_lapply(x, FUN = do.call, what = fun)
+ stopifnot(identical(z, z_fun))
+
+ message("- future_lapply(x, ...) - passing arguments via '...' ...")
+ ## typeof() == "list"
+ obj <- data.frame(a = 1:2)
+ stopifnot(typeof(obj) == "list")
+ y <- future_lapply(1L, function(a, b) typeof(b), b = obj)
+ stopifnot(identical(y[[1]], typeof(obj)))
+
+ ## typeof() == "environment"
+ obj <- new.env()
+ stopifnot(typeof(obj) == "environment")
+ y <- future_lapply(1L, function(a, b) typeof(b), b = obj)
+ stopifnot(identical(y[[1]], typeof(obj)))
+
+ ## typeof() == "S4"
+ if (requireNamespace("methods")) {
+ obj <- methods::getClass("MethodDefinition")
+ stopifnot(typeof(obj) == "S4")
+ y <- future_lapply(1L, function(a, b) typeof(b), b = obj)
+ stopifnot(identical(y[[1]], typeof(obj)))
+ }
+}
+
+message("*** future_lapply() - tricky globals ... DONE")
+
+
+message("*** future_lapply() - missing arguments ...")
+
+## Here 'abc' becomes missing, i.e. missing(abc) is TRUE
+foo <- function(x, abc) future_lapply(x, FUN = function(y) y)
+y <- foo(1:2)
+stopifnot(identical(y, as.list(1:2)))
+
+message("*** future_lapply() - missing arguments ... DONE")
+
+
+message("*** future_lapply() - false positives ...")
+
+## Here 'abc' becomes a promise, which fails to resolve
+## iff 'xyz' does not exist. (Issue #161)
+suppressWarnings(rm(list = "xyz"))
+foo <- function(x, abc) future_lapply(x, FUN = function(y) y)
+y <- foo(1:2, abc = (xyz >= 3.14))
+stopifnot(identical(y, as.list(1:2)))
+
+message("*** future_lapply() - false positives ... DONE")
+
+
+message("*** future_lapply() - globals exceptions ...")
+
+res <- tryCatch({
+ y <- future_lapply(1, FUN = function(x) x, future.globals = 42)
+}, error = identity)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({
+ y <- future_lapply(1, FUN = function(x) x, future.globals = list(1))
+}, error = identity)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({
+ y <- future_lapply(1, FUN = function(x) x, future.globals = "...future.FUN")
+}, error = identity)
+stopifnot(inherits(res, "error"))
+
+message("*** future_lapply() - globals exceptions ... DONE")
+
+source("incl/end.R")
diff --git a/tests/future_lapply.R b/tests/future_lapply.R
new file mode 100644
index 0000000..1896284
--- /dev/null
+++ b/tests/future_lapply.R
@@ -0,0 +1,72 @@
+source("incl/start.R")
+library("listenv")
+
+message("*** future_lapply() ...")
+
+x_a <- list(a = "integer", b = "numeric", c = "character", c = "list")
+str(list(x_a = x_a))
+y_a <- lapply(x_a, FUN = base::vector, length = 2L)
+str(list(y_a = y_a))
+
+x_b <- list(a = c("hello", b = 1:100))
+str(list(x_b = x_b))
+y_b <- lapply(x_b, FUN = future:::hpaste, collapse = "; ", maxHead = 3L)
+str(list(y_b = y_b))
+
+x_c <- list()
+y_c <- listenv()
+y_c$A <- 3L
+x_c$a <- y_c
+y_c<- listenv()
+y_c$A <- 3L
+y_c$B <- c("hello", b = 1:100)
+x_c$b <- y_c
+print(x_c)
+y_c <- lapply(x_c, FUN = listenv::map)
+str(list(y_c = y_c))
+
+for (cores in 1:availCores) {
+ message(sprintf("Testing with %d cores ...", cores))
+ options(mc.cores = cores)
+ strategies <- supportedStrategies(cores)
+
+ for (strategy in strategies) {
+ message(sprintf("- plan('%s') ...", strategy))
+ plan(strategy)
+
+ for (scheduling in list(FALSE, TRUE)) {
+ message("- future_lapply(x, FUN = vector, ...) ...")
+ y <- future_lapply(x_a, FUN = vector, length = 2L, future.scheduling = scheduling)
+ str(list(y = y))
+ stopifnot(identical(y, y_a))
+
+ message("- future_lapply(x, FUN = base::vector, ...) ...")
+ y <- future_lapply(x_a, FUN = base::vector, length = 2L, future.scheduling = scheduling)
+ str(list(y = y))
+ stopifnot(identical(y, y_a))
+
+ message("- future_lapply(x, FUN = future:::hpaste, ...) ...")
+ y <- future_lapply(x_b, FUN = future:::hpaste, collapse = "; ", maxHead = 3L, future.scheduling = scheduling)
+ str(list(y = y))
+ stopifnot(identical(y, y_b))
+
+ message("- future_lapply(x, FUN = listenv::listenv, ...) ...")
+ y <- future_lapply(x_c, FUN = listenv::map, future.scheduling = scheduling)
+ str(list(y = y))
+ stopifnot(identical(y, y_c))
+ } ## for (scheduling ...)
+
+ message("- future_lapply(x, FUN, ...) for large length(x) ...")
+ a <- 3.14
+ x_d <- 1:1e4
+ y <- future_lapply(x_d, FUN = function(z) sqrt(z + a))
+ y <- unlist(y, use.names = FALSE)
+ stopifnot(all.equal(y, sqrt(x_d + a)))
+ } ## for (strategy ...)
+
+ message(sprintf("Testing with %d cores ... DONE", cores))
+} ## for (cores ...)
+
+message("*** future_lapply() ... DONE")
+
+source("incl/end.R")
diff --git a/tests/futures.R b/tests/futures.R
index 915e7fa..3e12d0a 100644
--- a/tests/futures.R
+++ b/tests/futures.R
@@ -5,7 +5,7 @@ library("listenv")
if (getRversion() < "3.2.0") {
names <- function(x) {
if (class(x)[1] == "environment") {
- ls(envir=x, all.names=TRUE)
+ ls(envir = x, all.names = TRUE)
} else {
base::names(x)
}
@@ -14,23 +14,23 @@ if (getRversion() < "3.2.0") {
dims <- list(
NULL,
- c(1,6),
- c(2,3),
- c(2,3,1),
- c(2,1,3,1)
+ c(1, 6),
+ c(2, 3),
+ c(2, 3, 1),
+ c(2, 1, 3, 1)
)
message("*** futures() / resolved() / values() ...")
-for (cores in 1:min(3L, availableCores())) {
+for (cores in 1:availCores) {
message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
+ options(mc.cores = cores)
for (type in c("list", "environment", "listenv")) {
message(sprintf("Type of object: %s", type))
- for (strategy in supportedStrategies()) {
+ for (strategy in supportedStrategies(cores, excl = "multiprocess")) {
message("Type of future: ", strategy)
plan(strategy)
@@ -56,7 +56,7 @@ for (cores in 1:min(3L, availableCores())) {
if (type != "environment") {
names <- names(x)
dim(x) <- dim
- dimnames(x) <- lapply(dim, FUN=function(n) letters[1:n])
+ dimnames(x) <- lapply(dim, FUN = function(n) letters[1:n])
names(x) <- names
}
}
diff --git a/tests/globals,NSE.R b/tests/globals,NSE.R
index 6ca3e16..c770fa7 100644
--- a/tests/globals,NSE.R
+++ b/tests/globals,NSE.R
@@ -3,7 +3,7 @@ library("listenv")
message("*** Globals w/ non-standard evaluation (NSE) ...")
-data <- data.frame(x=1:5, y=1:5)
+data <- data.frame(x = 1:5, y = 1:5)
v0 <- subset(data, x < 3)$y
for (strategy in supportedStrategies()) {
@@ -12,29 +12,29 @@ for (strategy in supportedStrategies()) {
plan(strategy)
## Assert option is passed on to future
- options(future.globals.onMissing="error")
+ options(future.globals.onMissing = "error")
opt1 %<-% getOption("future.globals.onMissing")
stopifnot(identical(opt1, "error"))
- options(future.globals.onMissing="ignore")
+ options(future.globals.onMissing = "ignore")
opt2 %<-% getOption("future.globals.onMissing")
stopifnot(identical(opt2, "ignore"))
- options(future.globals.onMissing="error")
- res <- try({ v1 %<-% subset(data, x < 3)$y }, silent=TRUE)
+ options(future.globals.onMissing = "error")
+ res <- try({ v1 %<-% subset(data, x < 3)$y }, silent = TRUE)
stopifnot(inherits(res, "try-error"))
- options(future.globals.onMissing="ignore")
+ options(future.globals.onMissing = "ignore")
v2 %<-% subset(data, x < 3)$y
stopifnot(identical(v2, v0))
## Nested futures (requires option is passed on to future)
- plan(list(lazy, strategy))
- options(future.globals.onMissing="ignore")
+ plan(list(sequential, strategy))
+ options(future.globals.onMissing = "ignore")
v3 %<-% {
a %<-% subset(data, x < 3)$y
a
- }
+ } %lazy% TRUE
stopifnot(identical(v3, v0))
message(sprintf("- Strategy: %s ... DONE", strategy))
@@ -44,4 +44,3 @@ for (strategy in supportedStrategies()) {
message("*** Globals w/ non-standard evaluation (NSE) ... DONE")
source("incl/end.R")
-
diff --git a/tests/globals,formulas.R b/tests/globals,formulas.R
index b48abe3..627d2e2 100644
--- a/tests/globals,formulas.R
+++ b/tests/globals,formulas.R
@@ -3,133 +3,141 @@ source("incl/start.R")
library("datasets") ## cars data set
library("stats") ## lm(), poly(), xtabs()
-strategies <- supportedStrategies()
-strategies <- setdiff(strategies, "multiprocess")
-
message("*** Globals - formulas ...")
-message("*** Globals - lm(<formula>) ...")
-
-## From example("lm", package="stats")
-ctl <- c(4.17,5.58,5.18,6.11,4.50,4.61,5.17,4.53,5.33,5.14)
-trt <- c(4.81,4.17,4.41,3.59,5.87,3.83,6.03,4.89,4.32,4.69)
-group <- gl(2, 10, 20, labels = c("Ctl","Trt"))
+## (i) lm(<formula>):
+## From example("lm", package = "stats")
+ctl <- c(4.17, 5.58, 5.18, 6.11, 4.50, 4.61, 5.17, 4.53, 5.33, 5.14)
+trt <- c(4.81, 4.17, 4.41, 3.59, 5.87, 3.83, 6.03, 4.89, 4.32, 4.69)
+group <- gl(2, 10, 20, labels = c("Ctl", "Trt"))
weight <- c(ctl, trt)
+ctl <- trt <- NULL
+## Truth:
+fit_i <- lm(weight ~ group - 1)
+print(fit_i)
+## (ii) xtabs(~ x):
+x <- c(1, 1, 2, 2, 2)
## Truth:
-fit0 <- lm(weight ~ group - 1)
-print(fit0)
+tbl_ii <- xtabs(~ x)
+print(tbl_ii)
+
+## (iii) lm(<formula>, data = cars):
+exprs <- list(
+ # "remove-intercept-term" form of no-intercept
+ a = substitute({ lm(dist ~ . -1, data = cars) }),
+ # "make-intercept-zero" form of no-intercept
+ b = substitute({ lm(dist ~ . +0, data = cars) }),
+ # doesn't do what we want here
+ c = substitute({ lm(dist ~ speed + speed ^ 2, data = cars) }),
+ # gets us a quadratic term
+ d = substitute({ lm(dist ~ speed + I(speed ^ 2), data = cars) }),
+ # avoid potential multicollinearity
+ e = substitute({ lm(dist ~ poly(speed, 2), data = cars) })
+)
+
+## (iv) Globals - map(x, ~ expr):
+## A fake purrr::map() function with limited functionality
+map <- function(.x, .f, ...) {
+ if (inherits(.f, "formula")) {
+ expr <- .f[[-1]]
+ .f <- eval(bquote(function(...) {
+ .(expr)
+ }))
+ }
+ eval(lapply(.x, FUN = .f, ...))
+}
+
+inner_function <- function(x) { x + 1 }
+
+outer_function <- function(x) {
+ map(1:2, ~ inner_function(.x))
+}
+
+y_iv <- outer_function(1L)
+str(y_iv)
-for (cores in 1:min(3L, availableCores())) {
+
+for (cores in 1:availCores) {
message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
+ options(mc.cores = cores)
message("availableCores(): ", availableCores())
- for (strategy in strategies) {
+ for (strategy in supportedStrategies(cores)) {
message(sprintf("- plan('%s') ...", strategy))
plan(strategy)
+ message("- lm(<formula>) ...")
+
## Explicit future
f <- future({ lm(weight ~ group - 1) })
fit <- value(f)
print(fit)
- stopifnot(all.equal(fit, fit0))
+ stopifnot(all.equal(fit, fit_i))
+
+ ## Explicit future (lazy)
+ f <- future({ lm(weight ~ group - 1) }, lazy = TRUE)
+ fit <- value(f)
+ print(fit)
+ stopifnot(all.equal(fit, fit_i))
## Future assignment
fit %<-% { lm(weight ~ group - 1) }
print(fit)
- stopifnot(all.equal(fit, fit0))
- } ## for (strategy ...)
-
- message(sprintf("Testing with %d cores ... DONE", cores))
-} ## for (cores ...)
-
-message("*** Globals - lm(<formula>) ... DONE")
-
-
-message("*** Globals - one-side formulas, e.g. xtabs(~ x) ...")
-
-x <- c(1, 1, 2, 2, 2)
-
-## Truth:
-tbl0 <- xtabs(~ x)
-print(tbl0)
-
-for (cores in 1:min(3L, availableCores())) {
- message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
+ stopifnot(all.equal(fit, fit_i))
- message("availableCores(): ", availableCores())
+ ## Future assignment (non-lazy)
+ fit %<-% { lm(weight ~ group - 1) } %lazy% FALSE
+ print(fit)
+ stopifnot(all.equal(fit, fit_i))
- for (strategy in strategies) {
- message(sprintf("- plan('%s') ...", strategy))
- plan(strategy)
+ ## Future assignment (lazy)
+ fit %<-% { lm(weight ~ group - 1) } %lazy% TRUE
+ print(fit)
+ stopifnot(all.equal(fit, fit_i))
+ message("- Globals - one-side formulas, e.g. xtabs(~ x) ...")
## Explicit future
f <- future({ xtabs(~ x) })
tbl <- value(f)
print(tbl)
- stopifnot(all.equal(tbl, tbl0))
+ stopifnot(all.equal(tbl, tbl_ii))
## Future assignment
tbl %<-% { xtabs(~ x) }
print(tbl)
- stopifnot(all.equal(tbl, tbl0))
- } ## for (strategy ...)
-
- message(sprintf("Testing with %d cores ... DONE", cores))
-} ## for (cores ...)
-
-message("*** Globals - one-side formulas, e.g. xtabs(~ x) ... DONE")
-
-
-message("*** Globals - lm(<formula>, data=cars) ...")
-
-exprs <- list(
- # "remove-intercept-term" form of no-intercept
- a = substitute({ lm(dist ~ . -1, data=cars) }),
- # "make-intercept-zero" form of no-intercept
- b = substitute({ lm(dist ~ . +0, data=cars) }),
- # doesn't do what we want here
- c = substitute({ lm(dist ~ speed + speed^2, data=cars) }),
- # gets us a quadratic term
- d = substitute({ lm(dist ~ speed + I(speed^2), data=cars) }),
- # avoid potential multicollinearity
- e = substitute({ lm(dist ~ poly(speed,2), data=cars) })
-)
-
-for (kk in seq_along(exprs)) {
- expr <- exprs[[kk]]
- name <- names(exprs)[kk]
- message(sprintf("- Globals - lm(<formula #%d (%s)>, data=cars) ...", kk, sQuote(name)))
-
- fit0 <- eval(expr)
- print(fit0)
-
- for (cores in 1:min(3L, availableCores())) {
- message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
-
- message("availableCores(): ", availableCores())
-
- for (strategy in strategies) {
- message(sprintf("- plan('%s') ...", strategy))
- plan(strategy)
-
- f <- future(expr, substitute=FALSE)
+ stopifnot(all.equal(tbl, tbl_ii))
+
+ message("- Globals - lm(<formula>, data = cars) ...")
+ for (kk in seq_along(exprs)) {
+ expr <- exprs[[kk]]
+ name <- names(exprs)[kk]
+ message(sprintf("- Globals - lm(<formula #%d (%s)>, data = cars) ...",
+ kk, sQuote(name)))
+
+ fit_iii <- eval(expr)
+ print(fit_iii)
+
+ f <- future(expr, substitute = FALSE)
fit <- value(f)
print(fit)
- stopifnot(all.equal(fit, fit0))
- } ## for (strategy ...)
-
- message(sprintf("Testing with %d cores ... DONE", cores))
- } ## for (cores ...)
-} ## for (kk ...)
-
-message("*** Globals - lm(<formula>, data=cars) ... DONE")
-
+ stopifnot(all.equal(fit, fit_iii))
+ } ## for (kk ...)
+
+ message("- Globals - map(x, ~ expr) ...")
+ f <- future({ outer_function(1L) })
+ y <- value(f)
+ str(y)
+ stopifnot(all.equal(y, y_iv))
+
+ y %<-% { outer_function(1L) }
+ str(y)
+ stopifnot(all.equal(y, y_iv))
+ } ## for (strategy ...)
+ message(sprintf("Testing with %d cores ... DONE", cores))
+} ## for (cores ...)
message("*** Globals - formulas ... DONE")
diff --git a/tests/globals,manual.R b/tests/globals,manual.R
index c40e150..5c74f14 100644
--- a/tests/globals,manual.R
+++ b/tests/globals,manual.R
@@ -1,13 +1,30 @@
source("incl/start.R")
+message("*** getGlobalsAndPackages() ...")
+
+FutureGlobals <- future:::FutureGlobals
+
+globals <- structure(list(a = 1), where = list(a = globalenv()))
+globals <- FutureGlobals(globals, resolved = TRUE)
+gp <- getGlobalsAndPackages(expression(), globals = globals)
+
+message("- getGlobalsAndPackages() - exception ...")
+
+res <- tryCatch({
+ gp <- getGlobalsAndPackages(expression(), globals = 42)
+}, error = identity)
+stopifnot(inherits(res, "error"))
+
+message("*** getGlobalsAndPackages() - ... DONE")
+
message("*** Globals - manually ...")
message("*** Globals manually specified as named list ...")
globals <- list(
- a=1,
- b=2,
- sumtwo=function(x) x[1] + x[2]
+ a = 1,
+ b = 2,
+ sumtwo = function(x) x[1] + x[2]
)
## Assign 'globals' globally
@@ -16,7 +33,7 @@ attachLocally(globals)
## Truth
v0 <- local({
x <- 1:10
- sumtwo(a + b*x)
+ sumtwo(a + b * x)
})
@@ -27,23 +44,68 @@ for (strategy in supportedStrategies()) {
plan(strategy)
+ message("- Globals - automatic ...")
+
attachLocally(globals)
f <- future({
x <- 1:10
- sumtwo(a + b*x)
- }, globals=TRUE)
+ sumtwo(a + b * x)
+ }, globals = TRUE)
print(f)
- rm(list=names(globals))
+ rm(list = names(globals))
y <- value(f)
print(y)
stopifnot(all.equal(y, v0))
attachLocally(globals)
+ f <- futureAssign("y", {
+ x <- 1:10
+ sumtwo(a + b * x)
+ }, globals = TRUE)
+ print(f)
+ rm(list = names(globals))
+ z <- value(f)
+ print(z)
+ stopifnot(all.equal(z, y), all.equal(y, v0))
+
+ attachLocally(globals)
y %<-% {
x <- 1:10
- sumtwo(a + b*x)
+ sumtwo(a + b * x)
} %globals% TRUE
- rm(list=names(globals))
+ rm(list = names(globals))
+ print(y)
+ stopifnot(all.equal(y, v0))
+
+ attachLocally(globals)
+ f <- future({
+ x <- 1:10
+ sumtwo(a + b * x)
+ }, lazy = TRUE, globals = TRUE)
+ print(f)
+ rm(list = names(globals))
+ y <- value(f)
+ print(y)
+ stopifnot(all.equal(y, v0))
+
+ attachLocally(globals)
+ f <- futureAssign("y", {
+ x <- 1:10
+ sumtwo(a + b * x)
+ }, lazy = TRUE, globals = TRUE)
+ print(f)
+ rm(list = names(globals))
+ z <- value(f)
+ print(z)
+ stopifnot(all.equal(z, y), all.equal(y, v0))
+
+ ## Same with lazy evaluation
+ attachLocally(globals)
+ y %<-% {
+ x <- 1:10
+ sumtwo(a + b * x)
+ } %lazy% TRUE %globals% TRUE
+ rm(list = names(globals))
print(y)
stopifnot(all.equal(y, v0))
@@ -52,39 +114,42 @@ for (strategy in supportedStrategies()) {
print(y)
stopifnot(identical(y, 1))
+ ## Same with lazy evaluation
+ y %<-% { 1 } %lazy% TRUE %globals% FALSE
+ print(y)
+ stopifnot(identical(y, 1))
+
## Exception - missing global
attachLocally(globals)
f <- future({
x <- 1:10
- sumtwo(a + b*x)
- }, globals=FALSE)
+ sumtwo(a + b * x)
+ }, globals = FALSE)
print(f)
- rm(list=names(globals))
+ rm(list = names(globals))
y <- tryCatch(value(f), error = identity)
- if (!inherits(f, c("UniprocessFuture", "MulticoreFuture"))) {
+ if (!inherits(f, c("SequentialFuture", "UniprocessFuture", "MulticoreFuture"))) {
stopifnot(inherits(y, "simpleError"))
}
- message(sprintf("- Strategy: %s ... DONE", strategy))
-}
-
-message("*** Globals - automatic ... DONE")
-
-
-message("*** Globals manually specified as named list ...")
-
-for (strategy in supportedStrategies()) {
- message(sprintf("- Strategy: %s ...", strategy))
- plan(strategy)
-
+ message("- Globals manually specified as named list ...")
## Make sure globals do not exist
- rm(list=names(globals))
+ rm(list = names(globals))
f <- future({
x <- 1:10
- sumtwo(a + b*x)
- }, globals=globals)
+ sumtwo(a + b * x)
+ }, globals = globals)
+ print(f)
+ v <- value(f)
+ print(v)
+ stopifnot(all.equal(v, v0))
+
+ f <- future({
+ x <- 1:10
+ sumtwo(a + b * x)
+ }, lazy = TRUE, globals = globals)
print(f)
v <- value(f)
print(v)
@@ -92,31 +157,38 @@ for (strategy in supportedStrategies()) {
y %<-% {
x <- 1:10
- sumtwo(a + b*x)
+ sumtwo(a + b * x)
} %globals% globals
print(y)
stopifnot(all.equal(y, v0))
- message(sprintf("- Strategy: %s ... DONE", strategy))
-}
-
-message("*** Globals manually specified as named list ... DONE")
-
+ y %<-% {
+ x <- 1:10
+ sumtwo(a + b * x)
+ } %lazy% TRUE %globals% globals
+ print(y)
+ stopifnot(all.equal(y, v0))
-message("*** Globals manually specified by their names ...")
-for (strategy in supportedStrategies()) {
- message(sprintf("- Strategy: %s ...", strategy))
-
- plan(strategy)
+ message("- Globals manually specified by their names ...")
+ attachLocally(globals)
+ f <- future({
+ x <- 1:10
+ sumtwo(a + b * x)
+ }, globals = c("a", "b", "sumtwo"))
+ print(f)
+ rm(list = names(globals))
+ v <- value(f)
+ print(v)
+ stopifnot(all.equal(v, v0))
attachLocally(globals)
f <- future({
x <- 1:10
- sumtwo(a + b*x)
- }, globals=c("a", "b", "sumtwo"))
+ sumtwo(a + b * x)
+ }, lazy = TRUE, globals = c("a", "b", "sumtwo"))
print(f)
- rm(list=names(globals))
+ rm(list = names(globals))
v <- value(f)
print(v)
stopifnot(all.equal(v, v0))
@@ -124,18 +196,24 @@ for (strategy in supportedStrategies()) {
attachLocally(globals)
y %<-% {
x <- 1:10
- sumtwo(a + b*x)
+ sumtwo(a + b * x)
} %globals% c("a", "b", "sumtwo")
- rm(list=names(globals))
+ rm(list = names(globals))
+ print(y)
+ stopifnot(all.equal(y, v0))
+
+ attachLocally(globals)
+ y %<-% {
+ x <- 1:10
+ sumtwo(a + b * x)
+ } %lazy% TRUE %globals% c("a", "b", "sumtwo")
+ rm(list = names(globals))
print(y)
stopifnot(all.equal(y, v0))
message(sprintf("- Strategy: %s ... DONE", strategy))
}
-message("*** Globals manually specified by their names ... DONE")
-
-
message("*** Globals - manually ... DONE")
source("incl/end.R")
diff --git a/tests/globals,resolve.R b/tests/globals,resolve.R
index 12238ef..2d1da38 100644
--- a/tests/globals,resolve.R
+++ b/tests/globals,resolve.R
@@ -1,13 +1,13 @@
source("incl/start.R")
library("listenv")
-oopts <- c(oopts, options(future.globals.resolve=TRUE))
-setTimeLimit(cpu=10, elapsed=10, transient=TRUE)
+oopts <- c(oopts, options(future.globals.resolve = TRUE))
+setTimeLimit(cpu = 10, elapsed = 10, transient = TRUE)
message("*** Tricky use cases related to globals (part 2) ...")
-## Allow for two (sic!) background processes
-plan(multisession, workers=3L)
+## Allow for two background processes
+plan(multisession, workers = 2L)
env <- new.env()
@@ -18,7 +18,7 @@ env$a %<-% { 5 }
b %<-% { "a" }
## Resolve future #2 (frees up background process #2)
-message(sprintf("b=%s\n", sQuote(b)))
+message(sprintf("b = %s\n", sQuote(b)))
## Create future #3 (consumes background process #2)
## THIS IS THE TRICKY PART:
@@ -29,7 +29,7 @@ message(sprintf("b=%s\n", sQuote(b)))
y %<-% { env[[b]] }
## Resolve future #3
-message(sprintf("y=%s\n", y))
+message(sprintf("y = %s\n", y))
## Resolve future #1 if not already done
str(as.list(env))
@@ -38,7 +38,7 @@ str(as.list(env))
## Since future #1 is resolved it will work at this point
y %<-% { env[[b]] }
## Resolve future #4
-message(sprintf("y=%s\n", y))
+message(sprintf("y = %s\n", y))
message("*** Tricky use cases related to globals (part 2) ... DONE")
diff --git a/tests/globals,subassignment.R b/tests/globals,subassignment.R
index 59ec507..df9e252 100644
--- a/tests/globals,subassignment.R
+++ b/tests/globals,subassignment.R
@@ -1,13 +1,10 @@
source("incl/start.R")
oopts <- c(oopts, options(
- future.globals.resolve=TRUE,
- future.globals.onMissing="error"
+ future.globals.resolve = TRUE,
+ future.globals.onMissing = "error"
))
-strategies <- supportedStrategies()
-strategies <- setdiff(strategies, "multiprocess")
-
message("*** Globals - subassignments ...")
message("*** Globals - subassignments w/ x$a <- value ...")
@@ -15,7 +12,7 @@ message("*** Globals - subassignments w/ x$a <- value ...")
## Truth:
x <- x0 <- list()
y0 <- list(a = 1)
-str(list(x=x, y0=y0))
+str(list(x = x, y0 = y0))
y <- local({
x$a <- 1
@@ -37,13 +34,13 @@ stopifnot(identical(y, y0))
stopifnot(identical(x, list()))
-for (cores in 1:min(3L, availableCores())) {
+for (cores in 1:availCores) {
message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
+ options(mc.cores = cores)
message("availableCores(): ", availableCores())
- for (strategy in strategies) {
+ for (strategy in supportedStrategies(cores)) {
message(sprintf("- plan('%s') ...", strategy))
plan(strategy)
@@ -53,7 +50,18 @@ for (cores in 1:min(3L, availableCores())) {
x$a <- 1
x
})
- rm(list="x")
+ rm(list = "x")
+ y <- value(f)
+ print(y)
+ stopifnot(identical(y, y0))
+
+ ## Explicit future (lazy)
+ x <- list()
+ f <- future({
+ x$a <- 1
+ x
+ }, lazy = TRUE)
+ rm(list = "x")
y <- value(f)
print(y)
stopifnot(identical(y, y0))
@@ -64,7 +72,17 @@ for (cores in 1:min(3L, availableCores())) {
x$a <- 1
x
}
- rm(list="x")
+ rm(list = "x")
+ print(y)
+ stopifnot(identical(y, y0))
+
+ ## Same with forced lazy evaluation
+ x <- list()
+ y %<-% {
+ x$a <- 1
+ x
+ } %lazy% TRUE
+ rm(list = "x")
print(y)
stopifnot(identical(y, y0))
@@ -75,7 +93,7 @@ for (cores in 1:min(3L, availableCores())) {
x$a <- 1
x
}
- rm(list="x")
+ rm(list = "x")
print(y)
stopifnot(identical(y, list(b = 2, a = 1)))
@@ -85,7 +103,18 @@ for (cores in 1:min(3L, availableCores())) {
x[["a"]] <- 1
x
})
- rm(list="x")
+ rm(list = "x")
+ y <- value(f)
+ print(y)
+ stopifnot(identical(y, y0))
+
+ ## Explicit future (lazy)
+ x <- list()
+ f <- future({
+ x[["a"]] <- 1
+ x
+ }, lazy = TRUE)
+ rm(list = "x")
y <- value(f)
print(y)
stopifnot(identical(y, y0))
@@ -96,7 +125,7 @@ for (cores in 1:min(3L, availableCores())) {
x[["a"]] <- 1
x
}
- rm(list="x")
+ rm(list = "x")
print(y)
stopifnot(identical(y, y0))
@@ -106,7 +135,18 @@ for (cores in 1:min(3L, availableCores())) {
x["a"] <- list(1)
x
})
- rm(list="x")
+ rm(list = "x")
+ y <- value(f)
+ print(y)
+ stopifnot(identical(y, y0))
+
+ ## Explicit future (lazy)
+ x <- list()
+ f <- future({
+ x["a"] <- list(1)
+ x
+ }, lazy = TRUE)
+ rm(list = "x")
y <- value(f)
print(y)
stopifnot(identical(y, y0))
@@ -117,7 +157,7 @@ for (cores in 1:min(3L, availableCores())) {
x["a"] <- list(1)
x
}
- rm(list="x")
+ rm(list = "x")
print(y)
stopifnot(identical(y, y0))
@@ -128,7 +168,7 @@ for (cores in 1:min(3L, availableCores())) {
x[name] <- list(1)
x
}
- rm(list=c("x", "name"))
+ rm(list = c("x", "name"))
print(y)
stopifnot(identical(y, y0))
} ## for (strategy ...)
diff --git a/tests/globals,toolarge.R b/tests/globals,toolarge.R
index 63ba473..82da997 100644
--- a/tests/globals,toolarge.R
+++ b/tests/globals,toolarge.R
@@ -3,18 +3,18 @@ library("listenv")
message("*** Globals - too large ...")
-ooptsT <- options(future.globals.maxSize=1024L)
+ooptsT <- options(future.globals.maxSize = object.size(1:1000) - 1L)
limit <- getOption("future.globals.maxSize")
cat(sprintf("Max total size of globals: %g bytes\n", limit))
plan(multisession)
exprs <- list(
- A = substitute({ a }, env=list()),
- B = substitute({ a*b }, env=list()),
- C = substitute({ a*b*c }, env=list()),
- D = substitute({ a*b*c*d }, env=list()),
- E = substitute({ a*b*c*d*e }, env=list())
+ A = substitute({ a }, env = list()),
+ B = substitute({ a * b }, env = list()),
+ C = substitute({ a * b * c }, env = list()),
+ D = substitute({ a * b * c * d }, env = list()),
+ E = substitute({ a * b * c * d * e }, env = list())
)
a <- 1:1000
@@ -28,10 +28,12 @@ for (name in names(exprs)) {
expr <- exprs[[name]]
print(expr)
res <- tryCatch({
- f <- future(expr, substitute=FALSE)
+ f <- future(expr, substitute = FALSE)
}, error = function(ex) ex)
print(res)
stopifnot(inherits(res, "error"))
+ msg <- conditionMessage(res)
+ stopifnot(grepl("exceeds the maximum allowed size", msg))
}
message("*** Globals - too large ... DONE")
diff --git a/tests/globals,tricky.R b/tests/globals,tricky.R
index 144ce42..cdfcf24 100644
--- a/tests/globals,tricky.R
+++ b/tests/globals,tricky.R
@@ -1,55 +1,72 @@
source("incl/start.R")
library("listenv")
oopts <- c(oopts, options(
- future.globals.resolve=TRUE,
- future.globals.onMissing="error"
+ future.globals.resolve = TRUE,
+ future.globals.onMissing = "error"
))
message("*** Tricky use cases related to globals ...")
-strategies <- supportedStrategies()
-strategies <- setdiff(strategies, "multiprocess")
-
-for (cores in 1:min(3L, availableCores())) {
+for (cores in 1:availCores) {
message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
+ options(mc.cores = cores)
message("availableCores(): ", availableCores())
message("- Local variables with the same name as globals ...")
- methods <- c("conservative", "ordered")
-
- for (method in methods) {
- options(future.globals.method=method)
- message(sprintf("Method for identifying globals: '%s' ...", method))
+ for (strategy in supportedStrategies(cores)) {
+ message(sprintf("- plan('%s') ...", strategy))
+ plan(strategy)
- for (strategy in strategies) {
- message(sprintf("- plan('%s') ...", strategy))
- plan(strategy)
+ methods <- c("conservative", "ordered")
+ for (method in methods) {
+ options(future.globals.method = method)
+ message(sprintf("Method for identifying globals: '%s' ...", method))
a <- 3
yTruth <- local({
b <- a
a <- 2
- a*b
+ a * b
})
y %<-% {
b <- a
a <- 2
- a*b
+ a * b
+ }
+
+ rm(list = "a")
+
+ res <- try(y, silent = TRUE)
+ if (method == "conservative" && strategy %in% c("multisession", "cluster")) {
+ str(list(res = res))
+ stopifnot(inherits(res, "try-error"))
+ } else {
+ message(sprintf("y = %g", y))
+ stopifnot(identical(y, yTruth))
}
- rm(list="a")
- res <- try(y, silent=TRUE)
- if (method == "conservative" && strategy %in% c("lazy", "multisession")) {
- str(list(res=res))
+ ## Same with forced lazy evaluation
+ a <- 3
+
+ y %<-% {
+ b <- a
+ a <- 2
+ a * b
+ } %lazy% TRUE
+
+ rm(list = "a")
+
+ res <- try(y, silent = TRUE)
+ if (method == "conservative") {
+ str(list(res = res))
stopifnot(inherits(res, "try-error"))
} else {
- message(sprintf("y=%g", y))
+ message(sprintf("y = %g", y))
stopifnot(identical(y, yTruth))
}
@@ -58,16 +75,38 @@ for (cores in 1:min(3L, availableCores())) {
a <- 1
for (ii in 1:3) {
res[[ii]] %<-% {
- b <- a*ii
+ b <- a * ii
a <- 0
b
}
}
- rm(list="a")
+ rm(list = "a")
- res <- try(unlist(res), silent=TRUE)
- if (method == "conservative" && strategy %in% c("lazy", "multisession")) {
- str(list(res=res))
+ res <- try(unlist(res), silent = TRUE)
+ if (method == "conservative" && strategy %in% c("multisession", "cluster")) {
+ str(list(res = res))
+ stopifnot(inherits(res, "try-error"))
+ } else {
+ print(res)
+ stopifnot(all(res == 1:3))
+ }
+
+
+ ## Same with forced lazy evaluation
+ res <- listenv()
+ a <- 1
+ for (ii in 1:3) {
+ res[[ii]] %<-% {
+ b <- a * ii
+ a <- 0
+ b
+ } %lazy% TRUE
+ }
+ rm(list = "a")
+
+ res <- try(unlist(res), silent = TRUE)
+ if (method == "conservative") {
+ str(list(res = res))
stopifnot(inherits(res, "try-error"))
} else {
print(res)
@@ -77,25 +116,44 @@ for (cores in 1:min(3L, availableCores())) {
## Assert that `a` is resolved and turned into a constant future
## at the moment when future `b` is created.
- ## Requires options(future.globals.resolve=TRUE).
+ ## Requires options(future.globals.resolve = TRUE).
a <- future(1)
- b <- future(value(a)+1)
- rm(list="a")
- message(sprintf("value(b)=%g", value(b)))
+ b <- future(value(a) + 1)
+ rm(list = "a")
+ message(sprintf("value(b) = %g", value(b)))
stopifnot(value(b) == 2)
+ a <- future(1)
+ b <- future(value(a) + 1, lazy = TRUE)
+ rm(list = "a")
+ message(sprintf("value(b) = %g", value(b)))
+ stopifnot(value(b) == 2)
+
+ a <- future(1, lazy = TRUE)
+ b <- future(value(a) + 1)
+ rm(list = "a")
+ message(sprintf("value(b) = %g", value(b)))
+ stopifnot(value(b) == 2)
+
+ a <- future(1, lazy = TRUE)
+ b <- future(value(a) + 1, lazy = TRUE)
+ rm(list = "a")
+ message(sprintf("value(b) = %g", value(b)))
+ stopifnot(value(b) == 2)
+
+
## BUG FIX: In future (<= 1.0.0) a global 'pkg' would be
## overwritten by the name of the last package attached
## by the future.
pkg <- "foo"
- f <- uniprocess({ pkg })
+ f <- sequential({ pkg })
v <- value(f)
- message(sprintf("value(f)=%s", sQuote(v)))
+ message(sprintf("value(f) = %s", sQuote(v)))
stopifnot(pkg == "foo", v == "foo")
- } ## for (strategy ...)
-
- message(sprintf("Method for identifying globals: '%s' ... DONE", method))
- }
+
+ message(sprintf("Method for identifying globals: '%s' ... DONE", method))
+ }
+ } ## for (strategy ...)
message(sprintf("Testing with %d cores ... DONE", cores))
} ## for (cores ...)
diff --git a/tests/globals,tricky_recursive.R b/tests/globals,tricky_recursive.R
new file mode 100644
index 0000000..e1172af
--- /dev/null
+++ b/tests/globals,tricky_recursive.R
@@ -0,0 +1,117 @@
+source("incl/start.R")
+
+## Test adopted from http://stackoverflow.com/questions/42561088/nested-do-call-within-a-foreach-dopar-environment-cant-find-function-passed-w
+
+options(future.debug = FALSE)
+
+message("*** Tricky globals requiring recursive search ...")
+
+my_add <- function(a, b) a + b
+
+call_my_add <- function(a, b) {
+ do.call(my_add, args = list(a = a, b = b))
+}
+
+call_my_add_caller <- function(a, b, FUN = call_my_add) {
+ do.call(FUN, args = list(a = a, b = b))
+}
+
+main_future <- function(x = 1L, caller = call_my_add_caller,
+ args = list(FUN = call_my_add)) {
+ f <- future(caller(a = x, b = x + 1L, FUN = args$FUN))
+ value(f)
+}
+
+main_future_no_FUN <- function(x = 1L, caller = call_my_add_caller,
+ args = list(FUN = call_my_add)) {
+ f <- future(caller(a = x, b = x + 1L))
+ value(f)
+}
+
+main_futureCall <- function(x = 1L, caller = call_my_add_caller,
+ args = list(FUN = call_my_add)) {
+ f <- futureCall(caller, args = c(list(a = x, b = x+1L), args))
+ value(f)
+}
+
+main_futureCall_no_FUN <- function(x = 1L, caller = call_my_add_caller,
+ args = list(FUN = call_my_add)) {
+ f <- futureCall(caller, args = list(a = x, b = x+1L))
+ value(f)
+}
+
+main_lapply <- function(x = 1:2, caller = call_my_add_caller,
+ args = list(FUN = call_my_add)) {
+ lapply(x, FUN = function(i) {
+ do.call(caller, args = c(list(a = i, b = i+1L), args))
+ })
+}
+
+main_lapply_no_FUN <- function(x = 1:2, caller = call_my_add_caller,
+ args = list(FUN = call_my_add)) {
+ lapply(x, FUN = function(i) {
+ do.call(caller, args = list(a = i, b = i+1L))
+ })
+}
+
+main_future_lapply <- function(x = 1:2, caller = call_my_add_caller,
+ args = list(FUN = call_my_add)) {
+ future_lapply(x, FUN = function(i) {
+ do.call(caller, args = c(list(a = i, b = i + 1L), args))
+ })
+}
+
+main_future_lapply_no_FUN <- function(x = 1:2, caller = call_my_add_caller,
+ args = list(FUN = call_my_add)) {
+ future_lapply(x, FUN = function(i) {
+ do.call(caller, args = list(a = i, b = i + 1L))
+ })
+}
+
+x0 <- y0 <- z0 <- NULL
+for (strategy in supportedStrategies()) {
+ message(sprintf("*** strategy = %s ...", sQuote(strategy)))
+
+ plan(strategy)
+
+ x <- main_future()
+ str(list(x = x))
+ if (is.null(x0)) x0 <- x
+ stopifnot(identical(x, x0))
+
+ x2 <- main_future_no_FUN()
+ str(list(x2 = x2))
+ stopifnot(identical(x2, x0))
+
+ y <- main_futureCall()
+ str(list(y = y))
+ if (is.null(y0)) y0 <- y
+ stopifnot(identical(y, y0))
+
+ y2 <- main_futureCall_no_FUN()
+ str(list(y2 = y2))
+ stopifnot(identical(y2, y0))
+
+ z <- main_lapply()
+ str(list(z = z))
+ if (is.null(z0)) z0 <- z
+ stopifnot(identical(z, z0))
+
+ z2 <- main_lapply_no_FUN()
+ str(list(z2 = z2))
+ stopifnot(identical(z2, z0))
+
+ z3 <- main_future_lapply()
+ str(list(z3 = z3))
+ stopifnot(identical(z3, z0))
+
+ z4 <- main_future_lapply_no_FUN()
+ str(list(z4 = z4))
+ stopifnot(identical(z4, z0))
+
+ message(sprintf("*** strategy = %s ... DONE", sQuote(strategy)))
+}
+
+message("*** Tricky globals requiring recursive search ... DONE")
+
+source("incl/end.R")
diff --git a/tests/globalsOf,tweaks.R b/tests/globalsOf,tweaks.R
index 9a236b9..ca6c252 100644
--- a/tests/globalsOf,tweaks.R
+++ b/tests/globalsOf,tweaks.R
@@ -4,7 +4,7 @@ library("globals")
message("*** tweakExpression() ...")
-expr <- substitute({ a <<- 1; b <- 2; 3 ->> c }, env=list())
+expr <- substitute({ a <<- 1; b <- 2; 3 ->> c }, env = list())
print(expr)
exprT <- tweakExpression(expr)
print(exprT)
@@ -12,11 +12,11 @@ print(exprT)
b <- 2
exprs <- list(
- A = substitute({ a <- b; }, env=list()),
- B = substitute({ a <- b; b <- 1 }, env=list()),
- C = substitute({ a <- 1; a <- 2 }, env=list()),
- D = substitute({ a <<- 1; a <- 2 }, env=list()),
- E = substitute({ a <<- 1 }, env=list())
+ A = substitute({ a <- b; }, env = list()),
+ B = substitute({ a <- b; b <- 1 }, env = list()),
+ C = substitute({ a <- 1; a <- 2 }, env = list()),
+ D = substitute({ a <<- 1; a <- 2 }, env = list()),
+ E = substitute({ a <<- 1 }, env = list())
)
truth <- list(
@@ -33,7 +33,7 @@ for (kk in seq_along(exprs)) {
expr <- exprs[[kk]]
cat(sprintf("Expression #%d ('%s'):", kk, name))
print(expr)
- globals <- globalsOf(expr, tweak=tweakExpression)
+ globals <- globalsOf(expr, tweak = tweakExpression, recursive = TRUE)
globals <- cleanup(globals)
str(globals)
stopifnot(identical(names(globals), truth[[name]]))
diff --git a/tests/incl/end.R b/tests/incl/end.R
index 443ad87..1abd954 100644
--- a/tests/incl/end.R
+++ b/tests/incl/end.R
@@ -5,7 +5,7 @@ future::plan(oplan)
## Undo options
## (a) Added
added <- setdiff(names(options()), names(oopts0))
-opts <- vector("list", length=length(added))
+opts <- vector("list", length = length(added))
names(opts) <- added
options(opts)
## (b) Modified
@@ -17,21 +17,29 @@ stopifnot(identical(options(), oopts0))
## Undo system environment variables
## (a) Added
cenvs <- Sys.getenv()
-added <- setdiff(names(cenvs), names(oenvs))
+added <- setdiff(names(cenvs), names(oenvs0))
for (name in added) Sys.unsetenv(name)
-## (b) Modified?
-for (name in intersect(names(cenvs), names(oenvs))) {
+## (b) Missing
+missing <- setdiff(names(oenvs0), names(cenvs))
+if (length(missing) > 0) do.call(Sys.setenv, as.list(oenvs0[missing]))
+## (c) Modified?
+for (name in intersect(names(cenvs), names(oenvs0))) {
## WORKAROUND: On Linux Wine, base::Sys.getenv() may
## return elements with empty names. /HB 2016-10-06
if (nchar(name) == 0) next
- if (!identical(cenvs[[name]], oenvs[[name]])) {
- Sys.setenv(name, oenvs[[name]])
+ if (!identical(cenvs[[name]], oenvs0[[name]])) {
+ do.call(Sys.setenv, as.list(oenvs0[name]))
}
}
-## (c) Assert that everything was undone
-stopifnot(identical(Sys.getenv(), oenvs))
+## (d) Assert that everything was undone
+stopifnot(identical(Sys.getenv(), oenvs0))
## Undo variables
-rm(list=c(setdiff(ls(), ovars)))
+rm(list = c(setdiff(ls(), ovars)))
+
+## Travis CI specific: Explicit garbage collection because it
+## looks like Travis CI might run out of memory during 'covr'
+## testing and we now have so many tests. /HB 2017-01-11
+if ("covr" %in% loadedNamespaces()) gc()
diff --git a/tests/incl/start,load-only.R b/tests/incl/start,load-only.R
index acf9d8f..6958c4b 100644
--- a/tests/incl/start,load-only.R
+++ b/tests/incl/start,load-only.R
@@ -2,15 +2,39 @@
ovars <- ls()
oenvs <- oenvs0 <- Sys.getenv()
oopts0 <- options()
+
+covr_testing <- ("covr" %in% loadedNamespaces())
+on_solaris <- grepl("^solaris", R.version$os)
+
+## Default options
oopts <- options(
- warn=1L,
- mc.cores=2L,
- future.debug=TRUE
+ warn = 1L,
+ mc.cores = 2L,
+ future.debug = TRUE,
+ ## Reset the following during testing in case
+ ## they are set on the test system
+ future.availableCores.system = NULL,
+ future.availableCores.fallback = NULL
)
+
+
+## Reset the following during testing in case
+## they are set on the test system
+oenvs2 <- Sys.unsetenv(c(
+ "R_FUTURE_AVAILABLECORES_SYSTEM",
+ "R_FUTURE_AVAILABLECORES_FALLBACK",
+ ## SGE
+ "NSLOTS", "PE_HOSTFILE",
+ ## Slurm
+ "SLURM_CPUS_PER_TASK",
+ ## TORQUE / PBS
+ "PBS_NUM_PPN", "PBS_NODEFILE", "PBS_NP", "PBS_NUM_NODES"
+))
+
oplan <- future::plan()
## Use eager futures by default
-future::plan("eager")
+future::plan("sequential")
## Private future functions
.onLoad <- future:::.onLoad
@@ -18,9 +42,8 @@ future::plan("eager")
asIEC <- future:::asIEC
ClusterRegistry <- future:::ClusterRegistry
constant <- future:::constant
-uniprocess <- future:::uniprocess ## To become public
detectCores <- future:::detectCores
-flapply <- future:::flapply
+future_lapply <- future:::future_lapply
FutureRegistry <- future:::FutureRegistry
gassign <- future:::gassign
get_future <- future:::get_future
@@ -35,15 +58,28 @@ parseCmdArgs <- future:::parseCmdArgs
requestCore <- future:::requestCore
requestNode <- future:::requestNode
requirePackages <- future:::requirePackages
-supportedStrategies <- future:::supportedStrategies
tweakExpression <- future:::tweakExpression
whichIndex <- future:::whichIndex
+get_random_seed <- future:::get_random_seed
+
## Local functions for test scripts
printf <- function(...) cat(sprintf(...))
-mstr <- function(...) message(paste(capture.output(str(...)), collapse="\n"))
-attachLocally <- function(x, envir=parent.frame()) {
+mstr <- function(...) message(paste(capture.output(str(...)), collapse = "\n"))
+attachLocally <- function(x, envir = parent.frame()) {
for (name in names(x)) {
- assign(name, value=x[[name]], envir=envir)
+ assign(name, value = x[[name]], envir = envir)
}
}
+
+supportedStrategies <- function(cores = 1L, excl = c("multiprocess", "cluster"), ...) {
+ strategies <- future:::supportedStrategies(...)
+ strategies <- setdiff(strategies, excl)
+ if (cores > 1) {
+ strategies <- setdiff(strategies,
+ c("sequential", "uniprocess", "eager", "lazy"))
+ }
+ strategies
+}
+
+availCores <- min(2L, future::availableCores())
diff --git a/tests/invalid-owner.R b/tests/invalid-owner.R
index 5575f07..d7a0340 100644
--- a/tests/invalid-owner.R
+++ b/tests/invalid-owner.R
@@ -5,11 +5,11 @@ usedNodes <- function(future) {
## Number of unresolved cluster futures
workers <- future$workers
reg <- sprintf("workers-%s", attr(workers, "name"))
- c(used=length(future:::FutureRegistry(reg, action="list")), total=length(workers))
+ c(used = length(future:::FutureRegistry(reg, action = "list")), total = length(workers))
}
## This test requires at least two background processes
-plan(multisession, workers=3L)
+plan(multisession, workers = 2L)
message("*** future() - invalid ownership ...")
@@ -63,7 +63,7 @@ message("- Asserting ownership ... DONE")
message("- Trying with invalid ownership ...")
message("Creating future #1:")
-f1 <- future({ Sys.sleep(5); 42L })
+f1 <- future({ 42L })
print(f1)
cat(sprintf("Future #1 session: %d\n", f1$node))
stopifnot(identical(f1$owner, session_uuid))
@@ -77,9 +77,9 @@ stopifnot(identical(f2$owner, session_uuid))
print(usedNodes(f2))
message("Getting value of future #2:")
-res <- try(value(f2), silent=TRUE)
+res <- tryCatch(value(f2), error = identity)
print(res)
-stopifnot(inherits(res, "try-error"))
+stopifnot(inherits(res, "error"))
v1 <- value(f1)
print(v1)
diff --git a/tests/lazy.R b/tests/lazy.R
deleted file mode 100644
index c92a104..0000000
--- a/tests/lazy.R
+++ /dev/null
@@ -1,112 +0,0 @@
-source("incl/start.R")
-library("listenv")
-plan(lazy)
-
-message("*** lazy() ...")
-
-message("*** lazy() without globals")
-
-f <- lazy({
- 42L
-})
-stopifnot(inherits(f, "UniprocessFuture"), f$lazy)
-
-## Check whether a lazy future is resolved
-## or not will force evaluation
-print(resolved(f))
-stopifnot(resolved(f))
-
-y <- value(f)
-print(y)
-stopifnot(y == 42L)
-
-
-message("*** lazy() with globals")
-## A global variable
-a <- 0
-f <- lazy({
- b <- 3
- c <- 2
- a * b * c
-})
-print(f)
-
-## Although 'f' is a _lazy_ future and therefore
-## resolved/evaluates the future expression only
-## when the value is requested, any global variables
-## identified in the expression (here 'a') are
-## "frozen" at the time point when the future is
-## created. Because of this, 'a' preserved the
-## zero value although we reassign it below
-a <- 7 ## Make sure globals are frozen
-##if ("covr" %in% loadedNamespaces()) v <- 0 else ## WORKAROUND
-v <- value(f)
-print(v)
-stopifnot(v == 0)
-
-
-## A global variable (but without "freezing" it)
-a <- 0
-f <- lazy({
- b <- 3
- c <- 2
- a * b * c
-}, globals=FALSE)
-print(f)
-
-## Since 'a' is a global variable in _lazy_ future 'f',
-## which still hasn't been resolved, any changes to
-## 'a' until 'f' is resolved, will affect its value.
-a <- 7 ## ... but not in this case
-##if ("covr" %in% loadedNamespaces()) v <- 42 else ## WORKAROUND
-v <- value(f)
-print(v)
-stopifnot(v == 42)
-
-
-message("*** lazy() with globals (tricky)")
-x <- listenv()
-for (ii in 1:5) x[[ii]] <- lazy({ ii }, globals=TRUE)
-v <- sapply(x, FUN=value)
-stopifnot(all(v == 1:5)) ## Make sure globals are frozen
-
-x <- listenv()
-for (ii in 1:5) x[[ii]] <- lazy({ ii }, globals=FALSE)
-v <- sapply(x, FUN=value)
-stopifnot(all(v == 5L)) ## Make sure globals are not frozen
-
-
-message("*** lazy() and errors ...")
-f <- lazy({
- stop("Whoops!")
- 1
-})
-print(f)
-v <- value(f, signal=FALSE)
-print(v)
-stopifnot(inherits(v, "simpleError"))
-
-res <- try({ v <- value(f) }, silent=TRUE)
-print(res)
-stopifnot(inherits(res, "try-error"))
-
-## Error is repeated
-res <- try(value(f), silent=TRUE)
-print(res)
-stopifnot(inherits(res, "try-error"))
-
-message("*** lazy() and errors ... DONE")
-
-
-message("*** lazy() - exceptions ...")
-
-res <- try(lazy(42L, local=FALSE, globals=TRUE), silent=TRUE)
-print(res)
-stopifnot(inherits(res, "try-error"))
-
-message("*** lazy() - exceptions ... DONE")
-
-
-message("*** lazy() ... DONE")
-
-source("incl/end.R")
diff --git a/tests/makeClusterPSOCK.R b/tests/makeClusterPSOCK.R
new file mode 100644
index 0000000..065b47c
--- /dev/null
+++ b/tests/makeClusterPSOCK.R
@@ -0,0 +1,104 @@
+source("incl/start.R")
+
+is_fqdn <- future:::is_fqdn
+is_ip_number <- future:::is_ip_number
+is_localhost <- future:::is_localhost
+find_rshcmd <- future:::find_rshcmd
+
+message("*** makeClusterPSOCK() ...")
+
+message("- makeClusterPSOCK() - internal utility functions")
+
+stopifnot(
+ is_fqdn("a.b"),
+ is_fqdn("a.b.c"),
+ !is_fqdn("a")
+)
+
+stopifnot(
+ is_ip_number("1.2.3.4"),
+ !is_ip_number("a"),
+ !is_ip_number("1.2.3"),
+ !is_ip_number("1.2.3.256"),
+ !is_ip_number("1.2.3.-1"),
+ !is_ip_number("1.2.3.a")
+)
+
+## Reset internal cache
+stopifnot(is.na(is_localhost(worker = NULL, hostname = NULL)))
+stopifnot(
+ is_localhost("localhost"),
+ is_localhost("127.0.0.1"),
+ is_localhost(Sys.info()[["nodename"]]),
+ is_localhost(Sys.info()[["nodename"]]), ## cache hit
+ !is_localhost("not.a.localhost.hostname")
+)
+
+cmd <- find_rshcmd(must_work = FALSE)
+print(cmd)
+
+
+message("- makeClusterPSOCK()")
+
+cl <- makeClusterPSOCK("<a-non-existing-hostname>", user = "johndoe", master = NULL, revtunnel = FALSE, rshcmd = "my_ssh", renice = TRUE, manual = TRUE, dryrun = TRUE)
+print(cl)
+
+cl <- makeClusterPSOCK(1L, port = "random", dryrun = TRUE)
+print(cl)
+
+cl <- makeClusterPSOCK(1L)
+print(cl)
+parallel::stopCluster(cl)
+
+
+message("- makeClusterPSOCK() - exceptions")
+
+res <- tryCatch({
+ cl <- makeClusterPSOCK(1:2)
+}, error = identity)
+print(res)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({
+ cl <- makeClusterPSOCK(0L)
+}, error = identity)
+print(res)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({
+ cl <- makeClusterPSOCK(1L, rshcmd = character(0L))
+}, error = identity)
+print(res)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({
+ cl <- makeClusterPSOCK(1L, port = integer(0L))
+}, error = identity)
+print(res)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({
+ cl <- makeClusterPSOCK(1L, port = NA_integer_)
+}, error = identity)
+print(res)
+stopifnot(inherits(res, "error"))
+
+message("- makeClusterPSOCK() - exceptions")
+
+res <- tryCatch({
+ cl <- makeNodePSOCK("localhost", port = NA_integer_)
+}, error = identity)
+print(res)
+stopifnot(inherits(res, "error"))
+
+res <- tryCatch({
+ cl <- makeNodePSOCK("not.a.localhost.hostname", revtunnel = TRUE)
+}, error = identity)
+print(res)
+stopifnot(inherits(res, "error"))
+
+
+
+message("*** makeClusterPSOCK() ... DONE")
+
+source("incl/end.R")
diff --git a/tests/mandelbrot.R b/tests/mandelbrot.R
index c1a9d67..c8c6976 100644
--- a/tests/mandelbrot.R
+++ b/tests/mandelbrot.R
@@ -2,7 +2,7 @@ source("incl/start.R")
message("mandelbrot() ...")
-counts <- mandelbrot(xmid=-0.75, ymid=0.0, side=3.0, resolution=100L)
+counts <- mandelbrot(xmid = -0.75, ymid = 0.0, side = 3.0, resolution = 100L)
img <- as.raster(counts)
if (getRversion() >= "3.2.0") {
plot(img)
diff --git a/tests/multicore.R b/tests/multicore.R
index b514895..d9e7253 100644
--- a/tests/multicore.R
+++ b/tests/multicore.R
@@ -4,22 +4,22 @@ plan(multicore)
message("*** multicore() ...")
-for (cores in 1:min(3L, availableCores("multicore"))) {
+for (cores in 1:min(2L, availableCores("multicore"))) {
message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
+ options(mc.cores = cores)
if (!supportsMulticore()) {
- message(sprintf("Multicore futures are not supporting on '%s'. Falling back to use synchronous uniprocess futures", .Platform$OS.type))
+ message(sprintf("Multicore futures are not supporting on '%s'. Falling back to use synchronous sequential futures", .Platform$OS.type))
}
for (globals in c(FALSE, TRUE)) {
- message(sprintf("*** multicore(..., globals=%s) without globals", globals))
+ message(sprintf("*** multicore(..., globals = %s) without globals", globals))
f <- multicore({
42L
- }, globals=globals)
- stopifnot(inherits(f, "MulticoreFuture") || ((cores ==1 || !supportsMulticore()) && inherits(f, "UniprocessFuture")))
+ }, globals = globals)
+ stopifnot(inherits(f, "MulticoreFuture") || ((cores ==1 || !supportsMulticore()) && inherits(f, "SequentialFuture")))
print(resolved(f))
y <- value(f)
@@ -27,14 +27,14 @@ for (cores in 1:min(3L, availableCores("multicore"))) {
stopifnot(y == 42L)
- message(sprintf("*** multicore(..., globals=%s) with globals", globals))
+ message(sprintf("*** multicore(..., globals = %s) with globals", globals))
## A global variable
a <- 0
f <- multicore({
b <- 3
c <- 2
a * b * c
- }, globals=globals)
+ }, globals = globals)
print(f)
@@ -43,60 +43,58 @@ for (cores in 1:min(3L, availableCores("multicore"))) {
## variable should not affect the result of the
## future.
a <- 7 ## Make sure globals are frozen
-## if ("covr" %in% loadedNamespaces()) v <- 0 else ## WORKAROUND
v <- value(f)
print(v)
stopifnot(v == 0)
- message(sprintf("*** multicore(..., globals=%s) with globals and blocking", globals))
+ message(sprintf("*** multicore(..., globals = %s) with globals and blocking", globals))
x <- listenv()
for (ii in 1:4) {
message(sprintf(" - Creating multicore future #%d ...", ii))
- x[[ii]] <- multicore({ ii }, globals=globals)
+ x[[ii]] <- multicore({ ii }, globals = globals)
}
message(sprintf(" - Resolving %d multicore futures", length(x)))
-## if ("covr" %in% loadedNamespaces()) v <- 1:4 else ## WORKAROUND
- v <- sapply(x, FUN=value)
+ v <- sapply(x, FUN = value)
stopifnot(all(v == 1:4))
- message(sprintf("*** multicore(..., globals=%s) and errors", globals))
+ message(sprintf("*** multicore(..., globals = %s) and errors", globals))
f <- multicore({
stop("Whoops!")
1
- }, globals=globals)
+ }, globals = globals)
print(f)
- v <- value(f, signal=FALSE)
+ v <- value(f, signal = FALSE)
print(v)
stopifnot(inherits(v, "simpleError"))
- res <- try(value(f), silent=TRUE)
+ res <- try(value(f), silent = TRUE)
print(res)
stopifnot(inherits(res, "try-error"))
## Error is repeated
- res <- try(value(f), silent=TRUE)
+ res <- try(value(f), silent = TRUE)
print(res)
stopifnot(inherits(res, "try-error"))
} # for (globals ...)
- message("*** multicore(..., workers=1L) ...")
+ message("*** multicore(..., workers = 1L) ...")
a <- 2
b <- 3
yTruth <- a * b
- f <- multicore({ a * b }, workers=1L)
- rm(list=c("a", "b"))
+ f <- multicore({ a * b }, workers = 1L)
+ rm(list = c("a", "b"))
v <- value(f)
print(v)
stopifnot(v == yTruth)
- message("*** multicore(..., workers=1L) ... DONE")
+ message("*** multicore(..., workers = 1L) ... DONE")
message(sprintf("Testing with %d cores ... DONE", cores))
} ## for (cores ...)
diff --git a/tests/multiprocess.R b/tests/multiprocess.R
index eadc34a..ea207cb 100644
--- a/tests/multiprocess.R
+++ b/tests/multiprocess.R
@@ -4,16 +4,16 @@ plan(multiprocess)
message("*** multiprocess() ...")
-for (cores in 1:min(3L, availableCores())) {
+for (cores in 1:availCores) {
message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
+ options(mc.cores = cores)
## No global variables
f <- multiprocess({
42L
})
print(f)
- stopifnot(inherits(f, "MultiprocessFuture") || inherits(f, "UniprocessFuture"))
+ stopifnot(inherits(f, "MultiprocessFuture") || inherits(f, "SequentialFuture"))
print(resolved(f))
y <- value(f)
@@ -48,7 +48,7 @@ for (cores in 1:min(3L, availableCores())) {
x[[ii]] <- multiprocess({ ii })
}
message(sprintf(" - Resolving %d multiprocess futures", length(x)))
- v <- sapply(x, FUN=value)
+ v <- sapply(x, FUN = value)
stopifnot(all(v == 1:4))
@@ -58,22 +58,22 @@ for (cores in 1:min(3L, availableCores())) {
1
})
print(f)
- v <- value(f, signal=FALSE)
+ v <- value(f, signal = FALSE)
print(v)
stopifnot(inherits(v, "simpleError"))
- res <- try(value(f), silent=TRUE)
+ res <- try(value(f), silent = TRUE)
print(res)
stopifnot(inherits(res, "try-error"))
## Error is repeated
- res <- try(value(f), silent=TRUE)
+ res <- try(value(f), silent = TRUE)
print(res)
stopifnot(inherits(res, "try-error"))
message("*** multiprocess() - too large globals ...")
- ooptsT <- options(future.globals.maxSize=1024*4L)
+ ooptsT <- options(future.globals.maxSize = object.size(1:1014))
limit <- getOption("future.globals.maxSize")
cat(sprintf("Max total size of globals: %g bytes\n", limit))
@@ -86,9 +86,9 @@ for (cores in 1:min(3L, availableCores())) {
yTruth <- sum(a)
size <- object.size(a)
cat(sprintf("a: %g bytes\n", size))
- f <- multiprocess({ sum(a) }, workers=workers)
+ f <- multiprocess({ sum(a) }, workers = workers)
print(f)
- rm(list="a")
+ rm(list = "a")
v <- value(f)
print(v)
stopifnot(v == yTruth)
@@ -99,8 +99,8 @@ for (cores in 1:min(3L, availableCores())) {
yTruth <- sum(a)
size <- object.size(a)
cat(sprintf("a: %g bytes\n", size))
- res <- try(f <- multiprocess({ sum(a) }, workers=workers), silent=TRUE)
- rm(list="a")
+ res <- try(f <- multiprocess({ sum(a) }, workers = workers), silent = TRUE)
+ rm(list = "a")
stopifnot(inherits(res, "try-error"))
}
@@ -110,20 +110,20 @@ for (cores in 1:min(3L, availableCores())) {
message("*** multiprocess() - too large globals ... DONE")
- message("*** multiprocess(..., workers=1L) ...")
+ message("*** multiprocess(..., workers = 1L) ...")
a <- 2
b <- 3
yTruth <- a * b
- f <- multiprocess({ a * b }, workers=1L)
- rm(list=c("a", "b"))
+ f <- multiprocess({ a * b }, workers = 1L)
+ rm(list = c("a", "b"))
v <- value(f)
print(v)
stopifnot(v == yTruth)
- message("*** multiprocess(..., workers=1L) ... DONE")
+ message("*** multiprocess(..., workers = 1L) ... DONE")
message(sprintf("Testing with %d cores ... DONE", cores))
} ## for (cores ...)
diff --git a/tests/multisession.R b/tests/multisession.R
index 48b98c1..15d9172 100644
--- a/tests/multisession.R
+++ b/tests/multisession.R
@@ -4,16 +4,16 @@ plan(multisession)
message("*** multisession() ...")
-for (cores in 1:min(3L, availableCores())) {
+for (cores in 1:availCores) {
message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
+ options(mc.cores = cores)
## No global variables
f <- multisession({
42L
})
print(f)
- stopifnot(inherits(f, "ClusterFuture") || (inherits(f, "UniprocessFuture") && f$lazy))
+ stopifnot(inherits(f, "ClusterFuture") || (inherits(f, "SequentialFuture") && f$lazy))
print(resolved(f))
y <- value(f)
@@ -48,7 +48,7 @@ for (cores in 1:min(3L, availableCores())) {
x[[ii]] <- multisession({ ii })
}
message(sprintf(" - Resolving %d multisession futures", length(x)))
- v <- sapply(x, FUN=value)
+ v <- sapply(x, FUN = value)
stopifnot(all(v == 1:4))
@@ -58,22 +58,22 @@ for (cores in 1:min(3L, availableCores())) {
1
})
print(f)
- v <- value(f, signal=FALSE)
+ v <- value(f, signal = FALSE)
print(v)
stopifnot(inherits(v, "simpleError"))
- res <- try(value(f), silent=TRUE)
+ res <- try(value(f), silent = TRUE)
print(res)
stopifnot(inherits(res, "try-error"))
## Error is repeated
- res <- try(value(f), silent=TRUE)
+ res <- try(value(f), silent = TRUE)
print(res)
stopifnot(inherits(res, "try-error"))
message("*** multisession() - too large globals ...")
- ooptsT <- options(future.globals.maxSize=1024*4L)
+ ooptsT <- options(future.globals.maxSize = object.size(1:1014))
limit <- getOption("future.globals.maxSize")
cat(sprintf("Max total size of globals: %g bytes\n", limit))
@@ -86,21 +86,21 @@ for (cores in 1:min(3L, availableCores())) {
yTruth <- sum(a)
size <- object.size(a)
cat(sprintf("a: %g bytes\n", size))
- f <- multisession({ sum(a) }, workers=workers)
+ f <- multisession({ sum(a) }, workers = workers)
print(f)
- rm(list="a")
+ rm(list = "a")
v <- value(f)
print(v)
stopifnot(v == yTruth)
## A too large object
- a <- 1:1019 ## also on 32-bit platforms
+ a <- 1:1015
yTruth <- sum(a)
size <- object.size(a)
cat(sprintf("a: %g bytes\n", size))
- res <- try(f <- multisession({ sum(a) }, workers=workers), silent=TRUE)
- rm(list="a")
+ res <- try(f <- multisession({ sum(a) }, workers = workers), silent = TRUE)
+ rm(list = "a")
stopifnot(inherits(res, "try-error"))
}
@@ -110,23 +110,23 @@ for (cores in 1:min(3L, availableCores())) {
message("*** multisession() - too large globals ... DONE")
- message("*** multisession(..., workers=1L) ...")
+ message("*** multisession(..., workers = 1L) ...")
a <- 2
b <- 3
yTruth <- a * b
- f <- multisession({ a * b }, workers=1L)
- rm(list=c("a", "b"))
+ f <- multisession({ a * b }, workers = 1L)
+ rm(list = c("a", "b"))
v <- value(f)
print(v)
stopifnot(v == yTruth)
- message("*** multisession(..., workers=1L) ... DONE")
+ message("*** multisession(..., workers = 1L) ... DONE")
- message("*** multisession(..., gc=TRUE) ...")
- plan(multisession, workers=2L)
+ message("*** multisession(..., gc = TRUE) ...")
+ plan(multisession, workers = 2L)
f <- future({ gc() })
v <- value(f)
@@ -140,7 +140,7 @@ for (cores in 1:min(3L, availableCores())) {
v <- value(f)
print(v)
- f <- future({ integer(10e6) }, gc=TRUE)
+ f <- future({ integer(10e6) }, gc = TRUE)
v <- value(f)
str(v)
@@ -148,7 +148,20 @@ for (cores in 1:min(3L, availableCores())) {
v <- value(f)
print(v)
- message("*** multisession(..., gc=TRUE) ... TRUE")
+ message("*** multisession(..., gc = TRUE) ... TRUE")
+
+ message("*** multisession(...) - stopping with plan() change ...")
+
+ plan(multisession, workers = 2L)
+ f <- future(1L)
+ cl <- ClusterRegistry("get")
+ stopifnot(inherits(cl, "cluster"), length(cl) >= 1L)
+
+ plan(sequential)
+ cl <- ClusterRegistry("get")
+ stopifnot(is.null(cl), length(cl) == 0L)
+
+ message("*** multisession(...) - stopping with plan() change ... DONE")
message(sprintf("Testing with %d cores ... DONE", cores))
} ## for (cores ...)
diff --git a/tests/nbrOfWorkers.R b/tests/nbrOfWorkers.R
index b0cbf82..52242b9 100644
--- a/tests/nbrOfWorkers.R
+++ b/tests/nbrOfWorkers.R
@@ -2,11 +2,11 @@ source("incl/start.R")
message("*** nbrOfWorkers() ...")
-strategies <- c("uniprocess", "transparent", "eager", "lazy")
+strategies <- c("sequential", "transparent")
for (strategy in strategies) {
message("Type of future: ", strategy)
- evaluator <- get(strategy, mode="function")
+ evaluator <- get(strategy, mode = "function")
n <- nbrOfWorkers(evaluator)
message(sprintf("nbrOfWorkers: %d", n))
stopifnot(n == 1L)
@@ -18,39 +18,65 @@ for (strategy in strategies) {
} ## for (strategy ...)
-strategies <- c("multiprocess", "multisession", "multicore")
+strategies <- c("cluster", "multiprocess", "multisession", "multicore")
strategies <- intersect(strategies, supportedStrategies())
cores <- availableCores()
message("Number of available cores: ", cores)
+workers <- availableWorkers()
+nworkers <- length(workers)
+message(sprintf("Available workers: [n = %d] %s", nworkers, hpaste(sQuote(workers))))
for (strategy in strategies) {
message("Type of future: ", strategy)
- evaluator <- get(strategy, mode="function")
+ evaluator <- get(strategy, mode = "function")
n <- nbrOfWorkers(evaluator)
message(sprintf("nbrOfWorkers: %d", n))
- stopifnot(n == cores)
+ stopifnot(n == nworkers)
plan(strategy)
n <- nbrOfWorkers()
message(sprintf("nbrOfWorkers: %d", n))
- stopifnot(n == cores)
+ stopifnot(n == nworkers)
} ## for (strategy ...)
+message("Type of future: cluster")
+workers <- rep("localhost", times = 2L)
+plan(cluster, workers = workers)
+n <- nbrOfWorkers()
+message(sprintf("nbrOfWorkers: %d", n))
+stopifnot(n == length(workers))
+
+message("Type of future: remote")
+workers <- rep("localhost", times = 2L)
+plan(remote, workers = workers)
+n <- nbrOfWorkers()
+message(sprintf("nbrOfWorkers: %d", n))
+stopifnot(n == length(workers))
+
message("Type of future: constant")
n <- nbrOfWorkers(constant)
message(sprintf("nbrOfWorkers: %d", n))
stopifnot(n == 1)
-
message("Type of future: <future>")
-foo <- structure(function(...) NULL, class=c("future"))
+foo <- structure(function(...) NULL, class = c("future"))
n <- nbrOfWorkers(foo)
message(sprintf("nbrOfWorkers: %g", n))
stopifnot(n >= 0, is.infinite(n))
+message("Type of future: cluster with workers = <cluster object>")
+
+workers <- makeClusterPSOCK(2L)
+print(workers)
+plan(cluster, workers = workers)
+n <- nbrOfWorkers()
+message(sprintf("nbrOfWorkers: %g", n))
+stopifnot(n == length(workers))
+
+
message("*** nbrOfWorkers() ... DONE")
source("incl/end.R")
diff --git a/tests/nested_futures,mc.cores.R b/tests/nested_futures,mc.cores.R
index 5c61a2a..75908e9 100644
--- a/tests/nested_futures,mc.cores.R
+++ b/tests/nested_futures,mc.cores.R
@@ -1,5 +1,6 @@
source("incl/start.R")
library("listenv")
+options(future.debug = FALSE)
message("*** Nested futures - mc.cores ...")
@@ -13,18 +14,18 @@ cat("Available cores on this machine:\n")
cores <- availableCores()
print(cores)
-for (mc in 0:3) {
- message(sprintf("- mc.cores=%d ...", mc))
- options(mc.cores=mc)
+for (mc in 1:2) {
+ message(sprintf("- mc.cores = %d ...", mc))
+ options(mc.cores = mc)
mc2 <- min(mc, cores)
for (strategy in strategies) {
- message(sprintf("plan(list('uniprocess', '%s')):", strategy))
- plan(list('uniprocess', strategy))
+ message(sprintf("plan(list('sequential', '%s')):", strategy))
+ plan(list('sequential', strategy))
a %<-% {
b1 %<-% Sys.getpid()
b2 %<-% Sys.getpid()
- list(pid=Sys.getpid(), cores=availableCores(), pid1=b1, pid2=b2)
+ list(pid = Sys.getpid(), cores = availableCores(), pid1 = b1, pid2 = b2)
}
print(a)
stopifnot(a$pid == pid)
@@ -32,12 +33,12 @@ for (mc in 0:3) {
stopifnot((mc2 <= 1 && a$pid2 == pid) || (a$pid2 != pid))
stopifnot(((mc2 <= 1 || a$cores <= 2) && a$pid2 == a$pid1) || (a$pid2 != a$pid1))
- message(sprintf("plan(list('uniprocess', '%s':3)):", strategy))
- plan(list('uniprocess', tweak(strategy, workers=3)))
+ message(sprintf("plan(list('sequential', '%s':2)):", strategy))
+ plan(list('sequential', tweak(strategy, workers = 2)))
a %<-% {
b1 %<-% Sys.getpid()
b2 %<-% Sys.getpid()
- list(pid=Sys.getpid(), cores=availableCores(), pid1=b1, pid2=b2)
+ list(pid = Sys.getpid(), cores = availableCores(), pid1 = b1, pid2 = b2)
}
print(a)
stopifnot(a$pid == pid)
@@ -45,12 +46,12 @@ for (mc in 0:3) {
stopifnot((mc2 <= 1 && a$pid2 == pid) || (a$pid2 != pid))
stopifnot((mc2 <= 1 && a$pid2 == a$pid1) || (a$pid2 != a$pid1))
- message(sprintf("plan(list('%s', 'uniprocess')):", strategy))
- plan(list(strategy, 'uniprocess'))
+ message(sprintf("plan(list('%s', 'sequential')):", strategy))
+ plan(list(strategy, 'sequential'))
a %<-% {
b1 %<-% Sys.getpid()
b2 %<-% Sys.getpid()
- list(pid=Sys.getpid(), cores=availableCores(), pid1=b1, pid2=b2)
+ list(pid = Sys.getpid(), cores = availableCores(), pid1 = b1, pid2 = b2)
}
print(a)
stopifnot((mc2 <= 1 && a$pid == pid) || (a$pid != pid))
@@ -63,7 +64,7 @@ for (mc in 0:3) {
a %<-% {
b1 %<-% { Sys.sleep(1); Sys.getpid() }
b2 %<-% Sys.getpid()
- list(pid=Sys.getpid(), cores=availableCores(), pid1=b1, pid2=b2)
+ list(pid = Sys.getpid(), cores = availableCores(), pid1 = b1, pid2 = b2)
}
print(a)
stopifnot((mc2 <= 1 && a$pid == pid) || (a$pid != pid))
@@ -71,12 +72,12 @@ for (mc in 0:3) {
stopifnot((mc2 <= 1 && a$pid2 == pid) || (a$pid2 != pid))
stopifnot(a$pid2 == a$pid1)
- message(sprintf("plan(list('%s':3, '%s':3)):", strategy, strategy))
- plan(list(tweak(strategy, workers=3), tweak(strategy, workers=3)))
+ message(sprintf("plan(list('%s':2, '%s':2)):", strategy, strategy))
+ plan(list(tweak(strategy, workers = 2), tweak(strategy, workers = 2)))
a %<-% {
b1 %<-% Sys.getpid() ## This stalls
b2 %<-% Sys.getpid()
- list(pid=Sys.getpid(), cores=availableCores(), pid1=b1, pid2=b2)
+ list(pid = Sys.getpid(), cores = availableCores(), pid1 = b1, pid2 = b2)
}
print(a)
stopifnot(a$pid != pid)
@@ -85,9 +86,9 @@ for (mc in 0:3) {
stopifnot(a$pid2 != a$pid1)
} ## for (strategy ...)
- message(sprintf(" - mc.cores=%d ... DONE", mc))
+ message(sprintf(" - mc.cores = %d ... DONE", mc))
} ## for (mc ...)
message("*** Nested futures - mc.cores ... DONE")
-source("incl/end.R")
\ No newline at end of file
+source("incl/end.R")
diff --git a/tests/nested_futures.R b/tests/nested_futures.R
index 6ec56b9..646990a 100644
--- a/tests/nested_futures.R
+++ b/tests/nested_futures.R
@@ -1,14 +1,13 @@
source("incl/start.R")
-strategies <- future:::supportedStrategies()
-strategies <- setdiff(strategies, "multiprocess")
+strategies <- supportedStrategies()
message("*** Nested futures ...")
for (strategy1 in strategies) {
for (strategy2 in strategies) {
message(sprintf("- plan(list('%s', '%s')) ...", strategy1, strategy2))
- plan(list(a=strategy1, b=strategy2))
+ plan(list(a = strategy1, b = strategy2))
nested <- plan("list")
stopifnot(
@@ -29,26 +28,31 @@ for (strategy1 in strategies) {
length(nested_a) == 1L,
length(plan_a) == 1L,
inherits(plan_a[[1]], "future"),
- all.equal(plan_a, nested_a),
inherits(future::plan(), strategy2)
)
+ ## Attribute 'init' is modified at run time
+ for (kk in seq_along(plan_a)) attr(plan_a[[kk]], "init") <- NULL
+ for (kk in seq_along(nested_a)) attr(nested_a[[kk]], "init") <- NULL
+ stopifnot(all.equal(plan_a, nested_a))
+
y %<-% {
b <- 2L
-
+
## IMPORTANT: Use future::plan() - not just plan() - otherwise
## we're exporting the plan() function including its local stack!
plan_b <- future::plan("list")
nested_b <- nested_a[-1]
+
stopifnot(
length(nested_b) == 0L,
length(plan_b) == 1L,
inherits(plan_b[[1]], "future"),
- inherits(future::plan(), getOption("future.default", "uniprocess"))
+ inherits(future::plan(), getOption("future.default", "sequential"))
)
list(a = a, nested_a = nested_a, plan_a = plan_a,
- b = b, nested_b = nested_b, plan_b = plan_b)
+ b = b, nested_b = nested_b, plan_b = plan_b)
}
y
}
@@ -65,17 +69,21 @@ for (strategy1 in strategies) {
is.list(x$plan_a),
length(x$plan_a) == 1L,
inherits(x$plan_a[[1]], "future"),
- all.equal(x$plan_a, nested[-1L]),
x$b == 2L,
length(x$nested_b) == 0L,
is.list(x$plan_b),
length(x$plan_b) == 1L,
inherits(x$plan_b[[1]], "future"),
- inherits(x$plan_b[[1]], getOption("future.default", "uniprocess"))
+ inherits(x$plan_b[[1]], getOption("future.default", "sequential"))
)
- rm(list=c("nested", "x"))
+ ## Attribute 'init' is modified at run time
+ for (kk in seq_along(x$plan_a)) attr(x$plan_a[[kk]], "init") <- NULL
+ for (kk in seq_along(nested)) attr(nested[[kk]], "init") <- NULL
+ stopifnot(all.equal(x$plan_a, nested[-1L]))
+
+ rm(list = c("nested", "x"))
message(sprintf("- plan(list('%s', '%s')) ... DONE", strategy1, strategy2))
}
diff --git a/tests/objectSize.R b/tests/objectSize.R
new file mode 100644
index 0000000..4dd7d86
--- /dev/null
+++ b/tests/objectSize.R
@@ -0,0 +1,66 @@
+source("incl/start.R")
+
+objectSize <- future:::objectSize
+
+message("objectSize() ...")
+
+env <- new.env()
+env$a <- 3.14
+env$b <- 1:100
+
+env2 <- new.env()
+env2$env <- env
+
+## Namespaces will be skipped
+env3 <- getNamespace("utils")
+
+fcn <- function(...) TRUE
+
+objs <- list(
+ NULL,
+ TRUE,
+ 1L,
+ 3.14,
+ "hello",
+ 1:100,
+ 1:100 + 0.1,
+ letters,
+ list(a = 3.14, b = 1:100),
+ list(a = 3.14, b = 1:100, c = list(a = 3.14, b = 1:100)),
+ env,
+ env2,
+ env3,
+ fcn
+)
+
+for (kk in seq_along(objs)) {
+ obj <- objs[[kk]]
+ message(sprintf("objectSize(<%s>) ...", mode(obj)))
+ str(obj)
+
+ size0 <- object.size(obj)
+ str(size0)
+
+ size <- objectSize(obj)
+ str(size)
+
+ message(sprintf("objectSize(<%s>) ... DONE", mode(obj)))
+}
+
+message("*** objectSize() - globals with non-trustful length() ...")
+
+length.CantTrustLength <- function(x) length(unclass(x)) + 1L
+
+.length <- future:::.length
+
+x <- structure(as.list(1:3), class = c("CantTrustLength", "list"))
+str(list(n = length(x), n_true = .length(x)))
+stopifnot(length(x) > .length(x))
+size <- objectSize(x)
+print(size)
+
+message("*** objectSize() - globals with non-trustful length() ... DONE")
+
+message("objectSize() ... DONE")
+
+source("incl/end.R")
diff --git a/tests/plan.R b/tests/plan.R
index c37ee0c..cc5d65d 100644
--- a/tests/plan.R
+++ b/tests/plan.R
@@ -2,26 +2,26 @@ source("incl/start,load-only.R")
message("*** plan() ...")
-message("*** Set strategy via future::plan(future::lazy)")
-oplan <- future::plan(future::lazy)
+message("*** Set strategy via future::plan(future::multisession)")
+oplan <- future::plan(future::multisession)
print(future::plan())
future::plan(oplan)
print(future::plan())
-message("*** Set strategy via future::plan(future::lazy, local=FALSE)")
-oplan <- future::plan(future::lazy, local=FALSE)
+message("*** Set strategy via future::plan(future::multisession, globals = FALSE)")
+oplan <- future::plan(future::multisession, globals = FALSE)
print(future::plan())
future::plan(oplan)
print(future::plan())
-message("*** Set strategy via future::plan(future::lazy(local=FALSE)")
-oplan <- future::plan(future::lazy(local=FALSE))
+message("*** Set strategy via future::plan(future::multisession(globals = FALSE)")
+oplan <- future::plan(future::multisession(globals = FALSE))
print(future::plan())
future::plan(oplan)
print(future::plan())
-message("*** Set strategy via future::plan('lazy')")
-oplan <- future::plan("lazy")
+message("*** Set strategy via future::plan('multisession')")
+oplan <- future::plan("multisession")
print(future::plan())
future::plan(oplan)
print(future::plan())
@@ -36,29 +36,13 @@ print(future::plan())
library("future")
message("*** plan('unknown strategy')")
-res <- try(plan('unknown strategy'), silent=TRUE)
+res <- try(plan('unknown strategy'), silent = TRUE)
print(res)
stopifnot(inherits(res, "try-error"))
-message("*** plan() by (lazy) function")
-## Setting strategy by function
-plan(lazy)
-a <- 0
-f <- future({
- b <- 3
- c <- 2
- a * b * c
-})
-a <- 7 ## Make sure globals are frozen
-##if ("covr" %in% loadedNamespaces()) v <- 0 else ## WORKAROUND
-v <- value(f)
-print(v)
-stopifnot(v == 0)
-
-
-message("*** plan(uniprocess)")
-plan(uniprocess)
+message("*** plan(sequential)")
+plan(sequential)
a <- 0
f <- future({
b <- 3
@@ -71,15 +55,15 @@ print(v)
stopifnot(v == 0)
-message("*** plan('uniprocess')")
+message("*** plan('sequential')")
## Setting strategy by name
-plan("lazy")
+plan("multisession")
print(plan())
message("*** plan() and overriding defaults")
-message("*** plan(uniprocess)")
-plan(uniprocess)
+message("*** plan(sequential)")
+plan(sequential)
fcn <- plan()
print(fcn)
stopifnot(formals(fcn)$local == TRUE)
@@ -88,8 +72,8 @@ f <- future({ x <- 1 })
print(value(f))
stopifnot(x == 0)
-message("*** plan(uniprocess, local=FALSE)")
-plan(uniprocess, local=FALSE)
+message("*** plan(sequential, local = FALSE)")
+plan(sequential, local = FALSE)
fcn <- plan()
print(fcn)
stopifnot(formals(fcn)$local == FALSE)
@@ -98,22 +82,22 @@ f <- future({ x <- 1 })
print(value(f))
stopifnot(x == 1)
-message("*** plan(uniprocess, local=FALSE, abc=1, def=TRUE)")
-plan(uniprocess, local=FALSE, abc=1, def=TRUE)
+message("*** plan(sequential, local = FALSE, abc = 1, def = TRUE)")
+plan(sequential, local = FALSE, abc = 1, def = TRUE)
fcn <- plan()
print(fcn)
stopifnot(formals(fcn)$local == FALSE)
-message("*** plan(uniprocess(local=FALSE))")
-plan(lazy)
-plan(uniprocess(local=FALSE))
+message("*** plan(sequential(local = FALSE))")
+plan(multisession)
+plan(sequential(local = FALSE))
fcn <- plan()
print(fcn)
stopifnot(formals(fcn)$local == FALSE)
-message("*** plan(tweak(uniprocess, local=FALSE))")
-plan(lazy)
-plan(tweak(uniprocess, local=FALSE))
+message("*** plan(tweak(sequential, local = FALSE))")
+plan(multisession)
+plan(tweak(sequential, local = FALSE))
fcn <- plan()
print(fcn)
stopifnot(formals(fcn)$local == FALSE)
@@ -121,51 +105,51 @@ stopifnot(formals(fcn)$local == FALSE)
message("*** old <- plan(new)")
truth <- plan()
-old <- plan(lazy, local=FALSE)
+old <- plan(multisession, globals = FALSE)
stopifnot(identical(unclass(old), unclass(truth)))
-curr <- plan() ## curr == lazy(local=FALSE)
-prev <- plan(old) ## prev == lazy(local=FALSE)
+curr <- plan() ## curr == multisession(globals = FALSE)
+prev <- plan(old) ## prev == sequential(local = FALSE)
stopifnot(identical(unclass(curr), unclass(prev)))
curr <- plan() ## curr == old
stopifnot(identical(unclass(curr), unclass(old)))
stopifnot(identical(unclass(curr), unclass(truth)))
-message("*** %plan% 'uniprocess'")
-plan(lazy)
-x %<-% { a <- 1 } %plan% "uniprocess"
-stopifnot(identical(body(plan()), body(lazy)))
+message("*** %plan% 'sequential'")
+plan(multisession)
+x %<-% { a <- 1 } %plan% "sequential"
+stopifnot(identical(body(plan()), body(multisession)))
-message("*** %plan% uniprocess")
-plan(lazy)
+message("*** %plan% sequential")
+plan(multisession)
## %plan% can operate on any expression, so it
-## works just as an withPlan({ ... }, plan=...)
-fun <- { plan() } %plan% uniprocess
+## works just as an withPlan({ ... }, plan = ...)
+fun <- { plan() } %plan% sequential
f <- fun(1)
-stopifnot(inherits(f, "UniprocessFuture"), !f$lazy, inherits(f, "UniprocessFuture"))
+stopifnot(inherits(f, "SequentialFuture"), !f$lazy, inherits(f, "SequentialFuture"))
-x %<-% { a <- 1 } %plan% uniprocess
-stopifnot(identical(body(plan()), body(lazy)))
+x %<-% { a <- 1 } %plan% sequential
+stopifnot(identical(body(plan()), body(multisession)))
-message("*** %plan% uniprocess(local=FALSE) ")
-plan(lazy)
+message("*** %plan% sequential(local = FALSE) ")
+plan(multisession)
a <- 0
-x %<-% { a } %plan% uniprocess(local=FALSE)
+x %<-% { a } %plan% sequential(local = FALSE)
a <- 42
print(x)
stopifnot(x == 0)
-stopifnot(identical(body(plan()), body(lazy)))
+stopifnot(identical(body(plan()), body(multisession)))
+
+message("*** Nested futures with different plans")
-message("*** Nested futures with different plans ")
-plan(lazy)
c %<-% {
message("Resolving 'c'")
a %<-% {
message("Resolving 'a'")
2
- } %plan% uniprocess
+ } %plan% sequential
b %<-% {
message("Resolving 'b'")
-9 * a
@@ -173,7 +157,7 @@ c %<-% {
message("Local variable 'x'")
x <- b / 3
abs(x)
-}
+} %lazy% TRUE
d <- 42
print(d)
print(c)
@@ -181,35 +165,35 @@ stopifnot(c == 6)
message("*** plan() by functions and character names ... ")
-plan(uniprocess)
+plan(sequential)
a %<-% 42
stopifnot(a == 42)
-plan("uniprocess")
+plan("sequential")
a %<-% 42
stopifnot(a == 42)
-plan(list(uniprocess))
+plan(list(sequential))
a %<-% 42
stopifnot(a == 42)
-plan(list("uniprocess"))
+plan(list("sequential"))
a %<-% 42
stopifnot(a == 42)
-plan(list(uniprocess, lazy))
+plan(list(sequential, sequential))
a %<-% { b %<-% 42; b }
stopifnot(a == 42)
-plan(list("uniprocess", lazy))
+plan(list("sequential", sequential))
a %<-% { b %<-% 42; b }
stopifnot(a == 42)
-plan(list(uniprocess, "lazy"))
+plan(list(sequential, "sequential"))
a %<-% { b %<-% 42; b }
stopifnot(a == 42)
-plan(list("uniprocess", "lazy"))
+plan(list("sequential", "sequential"))
a %<-% { b %<-% 42; b }
stopifnot(a == 42)
@@ -218,7 +202,7 @@ message("*** plan() by functions and character names ... DONE")
message("*** plan() w/ commands ...")
-plan(list(uniprocess, uniprocess))
+plan(list(sequential, sequential))
res <- plan("list")
print(res)
stopifnot(length(res) == 2)
diff --git a/tests/remote.R b/tests/remote.R
index 44e4bf5..0c0496c 100644
--- a/tests/remote.R
+++ b/tests/remote.R
@@ -6,7 +6,7 @@ message("*** remote() ...")
## No global variables
f <- try(remote({
42L
-}, workers="localhost"), silent=FALSE)
+}, workers = "localhost"), silent = FALSE)
print(f)
stopifnot(inherits(f, "ClusterFuture"))
@@ -16,11 +16,11 @@ print(y)
stopifnot(y == 42L)
-plan(remote, workers="localhost")
+plan(remote, workers = "localhost")
## No global variables
f <- try(future({
42L
-}), silent=FALSE)
+}), silent = FALSE)
print(f)
stopifnot(inherits(f, "ClusterFuture"))
@@ -42,7 +42,7 @@ print(f)
message("*** remote() - exceptions ...")
-res <- try(remote(42L, workers=TRUE), silent=TRUE)
+res <- try(remote(42L, workers = TRUE), silent = TRUE)
print(res)
stopifnot(inherits(res, "try-error"))
diff --git a/tests/requestCore.R b/tests/requestCore.R
index c6edf73..31a08aa 100644
--- a/tests/requestCore.R
+++ b/tests/requestCore.R
@@ -4,10 +4,14 @@ message("*** requestCore() ...")
message("*** requestCore() - exceptions ...")
-res <- try(requestCore(function() {}, workers=1L), silent=TRUE)
+## There are no cores to choose from
+res <- try(requestCore(function() {}, workers = 0), silent = TRUE)
stopifnot(inherits(res, "try-error"))
-res <- try(requestCore(function() {}, times=0L), silent=TRUE)
+res <- try(requestCore(function() {}, timeout = -1.0), silent = TRUE)
+stopifnot(inherits(res, "try-error"))
+
+res <- try(requestCore(function() {}, alpha = 0.0), silent = TRUE)
stopifnot(inherits(res, "try-error"))
message("*** requestCore() - exceptions ... DONE")
@@ -15,9 +19,9 @@ message("*** requestCore() - exceptions ... DONE")
message("*** requestCore() - timeout ...")
-plan(multicore, workers=2L)
-a %<-% { Sys.sleep(3); 1 }
-res <- try(requestCore(function() {}, workers=2L, times=1L, delta=0.1))
+plan(multicore, workers = 2L)
+a %<-% { Sys.sleep(2); 1 }
+res <- try(requestCore(function() {}, workers = 1L, timeout = 0.5, delta = 0.1))
stopifnot(inherits(res, "try-error"))
stopifnot(a == 1)
diff --git a/tests/requestNode.R b/tests/requestNode.R
index 0195b86..2b020d0 100644
--- a/tests/requestNode.R
+++ b/tests/requestNode.R
@@ -4,21 +4,26 @@ message("*** requestNode() ...")
message("*** requestNode() - exceptions ...")
-res <- try(requestNode(function() {}, workers=1L), silent=TRUE)
+workers <- makeClusterPSOCK(2L)
+print(workers)
+
+res <- try(requestNode(function() {}, workers = workers, timeout = -1.0), silent = TRUE)
stopifnot(inherits(res, "try-error"))
-res <- try(requestNode(function() {}, times=0L), silent=TRUE)
+res <- try(requestNode(function() {}, workers = workers, alpha = 0), silent = TRUE)
stopifnot(inherits(res, "try-error"))
-message("*** requestNode() - exceptions ... DONE")
+parallel::stopCluster(workers)
+message("*** requestNode() - exceptions ... DONE")
message("*** requestNode() - timeout ...")
-plan(multisession, workers=2L)
-f <- future({ Sys.sleep(3); 1 })
+plan(multisession, workers = 2L)
+f <- future({ Sys.sleep(100); 1 })
+f2 <- future({ Sys.sleep(100); 2 })
-res <- try(requestNode(function() {}, workers=f$workers, times=1L, delta=0.1))
+res <- try(requestNode(function() { }, workers = f$workers, timeout = 0.5, delta = 0.1))
stopifnot(inherits(res, "try-error"))
message("*** requestNode() - timeout ... DONE")
diff --git a/tests/resolve.R b/tests/resolve.R
index 560e8bb..9dbd8cd 100644
--- a/tests/resolve.R
+++ b/tests/resolve.R
@@ -1,36 +1,53 @@
source("incl/start.R")
library("listenv")
-oopts <- c(oopts, options(future.progress=TRUE))
-plan(lazy)
+oopts <- c(oopts, options(future.progress = TRUE))
strategies <- supportedStrategies()
-strategies <- setdiff(strategies, "multiprocess")
message("*** resolve() ...")
+message("*** resolve() for default ...")
+
+x <- 1
+y <- resolve(x)
+stopifnot(identical(y, x))
+
+message("*** resolve() for default ... DONE")
+
message("*** resolve() for Future objects ...")
-plan(multisession, workers=2L)
+plan(multisession, workers = 2L)
for (value in c(FALSE, TRUE)) {
for (recursive in list(FALSE, TRUE, -1, 0, 1, 2, Inf)) {
- message(sprintf("- value=%s, recursive=%s ...", value, recursive))
+ message(sprintf("- value = %s, recursive = %s ...", value, recursive))
f <- future({
Sys.sleep(0.5)
- list(a=1, b=42L)
+ list(a = 1, b = 42L)
})
- res <- resolve(f, value=value, recursive=recursive)
+ res <- resolve(f, value = value, recursive = recursive)
+ stopifnot(identical(res, f))
+
+ f <- future({
+ Sys.sleep(0.5)
+ list(a = 1, b = 42L)
+ }, lazy = TRUE)
+ res <- resolve(f, value = value, recursive = recursive)
stopifnot(identical(res, f))
message("- w/ exception ...")
- f <- future(list(a=1, b=42L, c=stop("Nah!")))
- res <- resolve(f, value=value, recursive=recursive)
+ f <- future(list(a = 1, b = 42L, c = stop("Nah!")))
+ res <- resolve(f, value = value, recursive = recursive)
+ stopifnot(identical(res, f))
+
+ f <- future(list(a = 1, b = 42L, c = stop("Nah!")), lazy = TRUE)
+ res <- resolve(f, value = value, recursive = recursive)
stopifnot(identical(res, f))
- message(sprintf("- value=%s, recursive=%s ... DONE", value, recursive))
+ message(sprintf("- value = %s, recursive = %s ... DONE", value, recursive))
} ## for (resolve ...)
} ## for (value ...)
@@ -65,28 +82,51 @@ for (strategy in strategies) {
stopifnot(resolved(x[["b"]]))
x <- list()
+ x$a <- future(1, lazy = TRUE)
+ x$b <- future(2)
+ x[[3]] <- 3
+ y <- resolve(x)
+ stopifnot(identical(y, x))
+ stopifnot(resolved(x$a))
+ stopifnot(resolved(x[["b"]]))
+
+ x <- list()
+ x$a <- future(1, lazy = TRUE)
+ x$b <- future(2, lazy = TRUE)
+ x[[3]] <- 3
+ y <- resolve(x)
+ stopifnot(identical(y, x))
+ stopifnot(resolved(x$a))
+ stopifnot(resolved(x[["b"]]))
+
+ x <- list()
x$a <- future(1)
- x$b <- future({Sys.sleep(1); 2})
+ x$b <- future({Sys.sleep(0.5); 2})
x[[4]] <- 4
- dim(x) <- c(2,2)
- y <- resolve(x, idxs=1)
+ dim(x) <- c(2, 2)
+ y <- resolve(x, idxs = 1)
stopifnot(identical(y, x))
stopifnot(resolved(x[[1]]))
- y <- resolve(x, idxs=2)
+ y <- resolve(x, idxs = 2)
stopifnot(identical(y, x))
stopifnot(resolved(x[[2]]))
- y <- resolve(x, idxs=3)
+ y <- resolve(x, idxs = 3)
+ stopifnot(identical(y, x))
+ y <- resolve(x, idxs = seq_along(x))
stopifnot(identical(y, x))
- y <- resolve(x, idxs=seq_along(x))
+ y <- resolve(x, idxs = names(x))
stopifnot(identical(y, x))
- y <- resolve(x, idxs=names(x))
+
+ y <- resolve(x, idxs = matrix(c(1, 2), ncol = 2L), value = TRUE)
stopifnot(identical(y, x))
- y <- resolve(x, idxs=matrix(c(1,2), ncol=2L), value=TRUE)
+ x <- list()
+ for (kk in 1:3) x[[kk]] <- future({ Sys.sleep(0.1); kk })
+ y <- resolve(x)
stopifnot(identical(y, x))
x <- list()
- for (kk in 1:3) x[[kk]] <- future({ Sys.sleep(1); kk })
+ for (kk in 1:3) x[[kk]] <- future({ Sys.sleep(0.1); kk }, lazy = TRUE)
y <- resolve(x)
stopifnot(identical(y, x))
@@ -95,15 +135,15 @@ for (strategy in strategies) {
x$a <- 1
x$b <- 2
- res <- try(y <- resolve(x, idxs=0L), silent=TRUE)
- stopifnot(inherits(res, "try-error"))
+ res <- tryCatch(y <- resolve(x, idxs = 0L), error = identity)
+ stopifnot(inherits(res, "error"))
- res <- try(y <- resolve(x, idxs="unknown"), silent=TRUE)
- stopifnot(inherits(res, "try-error"))
+ res <- tryCatch(y <- resolve(x, idxs = "unknown"), error = identity)
+ stopifnot(inherits(res, "error"))
x <- list(1, 2)
- res <- try(x <- resolve(x, idxs="a"))
- stopifnot(inherits(res, "try-error"))
+ res <- tryCatch(x <- resolve(x, idxs = "a"), error = identity)
+ stopifnot(inherits(res, "error"))
message(sprintf("- plan('%s') ...", strategy))
} ## for (strategy ...)
@@ -127,52 +167,52 @@ for (strategy in strategies) {
x$b <- 2
y <- resolve(x)
stopifnot(identical(y, x))
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 0L)
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 0L)
x <- new.env()
x$a <- future(1)
x$b <- future(2)
x$c <- 3
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 2L)
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 2L)
y <- resolve(x)
stopifnot(identical(y, x))
stopifnot(resolved(x$a))
stopifnot(resolved(x$b))
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 2L)
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 2L)
x <- new.env()
x$a %<-% { 1 }
x$b %<-% { 2 }
x$c <- 3
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 2L)
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 2L)
y <- resolve(x) ## FIXME: Should not do value()!
stopifnot(identical(y, x))
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 2L)
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 2L)
x <- new.env()
x$a <- future({ 1 })
x$b %<-% { 2 }
x$c <- 3
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 2L)
- y <- resolve(x, idxs="a")
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 2L)
+ y <- resolve(x, idxs = "a")
stopifnot(identical(y, x))
stopifnot(resolved(x$a))
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 2L)
- y <- resolve(x, idxs="b")
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 2L)
+ y <- resolve(x, idxs = "b")
stopifnot(identical(y, x))
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 2L)
- y <- resolve(x, idxs="c")
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 2L)
+ y <- resolve(x, idxs = "c")
stopifnot(identical(y, x))
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 2L)
- y <- resolve(x, idxs=names(x), value=TRUE)
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 2L)
+ y <- resolve(x, idxs = names(x), value = TRUE)
stopifnot(identical(y, x))
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 2L)
- y <- resolve(x, recursive=TRUE, value=TRUE)
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 2L)
+ y <- resolve(x, recursive = TRUE, value = TRUE)
stopifnot(identical(y, x))
## Exceptions
- res <- try(y <- resolve(x, idxs="unknown"), silent=TRUE)
- stopifnot(inherits(res, "try-error"))
+ res <- tryCatch(y <- resolve(x, idxs = "unknown"), error = identity)
+ stopifnot(inherits(res, "error"))
message(sprintf("- plan('%s') ...", strategy))
} ## for (strategy ...)
@@ -186,11 +226,11 @@ for (strategy in strategies) {
message(sprintf("- plan('%s') ...", strategy))
plan(strategy)
- options(future.progress=function(done, total) {
- msg <- sprintf("Wohoo: %.0f%% (%d/%d)", 100*done/total, done, total)
+ options(future.progress = function(done, total) {
+ msg <- sprintf("Wohoo: %.0f%% (%d/%d)", 100 * done / total, done, total)
if (done < total) {
- bs <- paste(rep("\b", times=nchar(msg)), collapse="")
- message(paste(msg, bs, sep=""), appendLF=FALSE)
+ bs <- paste(rep("\b", times = nchar(msg)), collapse = "")
+ message(paste(msg, bs, sep = ""), appendLF = FALSE)
} else {
message(msg)
}
@@ -204,7 +244,7 @@ for (strategy in strategies) {
x <- listenv()
x$a <- 1
x$b <- 2
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 0L)
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 0L)
y <- resolve(x)
stopifnot(identical(y, x))
@@ -213,65 +253,65 @@ for (strategy in strategies) {
x$b <- future(2)
x$c <- 3
names <- names(x)
- dim(x) <- c(1,3)
+ dim(x) <- c(1, 3)
names(x) <- names
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 2L)
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 2L)
y <- resolve(x)
stopifnot(identical(y, x))
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 2L)
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 2L)
x <- listenv()
x$a %<-% { 1 }
x$b %<-% { 2 }
x$c <- 3
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 2L)
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 2L)
y <- resolve(x) ## FIXME: Should not do value()!
stopifnot(identical(y, x))
- #stopifnot(is.na(futureOf(x$a, mustExist=FALSE)))
- #stopifnot(is.na(futureOf(x$b, mustExist=FALSE)))
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 2L)
+ #stopifnot(is.na(futureOf(x$a, mustExist = FALSE)))
+ #stopifnot(is.na(futureOf(x$b, mustExist = FALSE)))
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 2L)
x <- listenv()
x$a <- future({ 1 })
- x$b %<-% { Sys.sleep(1); 2 }
+ x$b %<-% { Sys.sleep(0.5); 2 }
x$c %<-% { 3 }
x$d <- 4
names <- names(x)
- dim(x) <- c(2,2)
+ dim(x) <- c(2, 2)
names(x) <- names
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 3L)
- y <- resolve(x, idxs="a")
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 3L)
+ y <- resolve(x, idxs = "a")
stopifnot(identical(y, x))
- stopifnot(identical(futureOf(x$a, mustExist=FALSE), x$a))
+ stopifnot(identical(futureOf(x$a, mustExist = FALSE), x$a))
stopifnot(resolved(x$a))
- y <- resolve(x, idxs="b")
+ y <- resolve(x, idxs = "b")
stopifnot(identical(y, x))
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 3L)
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 3L)
- idxs <- matrix(c(1,2), ncol=2L)
- y <- resolve(x, idxs=idxs)
+ idxs <- matrix(c(1, 2), ncol = 2L)
+ y <- resolve(x, idxs = idxs)
stopifnot(identical(y, x))
- #stopifnot(is.na(futureOf(x$c, mustExist=FALSE)))
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 3L)
+ #stopifnot(is.na(futureOf(x$c, mustExist = FALSE)))
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 3L)
- y <- resolve(x, idxs=4L)
+ y <- resolve(x, idxs = 4L)
stopifnot(identical(y, x))
- #stopifnot(is.na(futureOf(x[[4L]], mustExist=FALSE)))
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 3L)
+ #stopifnot(is.na(futureOf(x[[4L]], mustExist = FALSE)))
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 3L)
- y <- resolve(x, idxs=names(x), value=TRUE)
+ y <- resolve(x, idxs = names(x), value = TRUE)
stopifnot(identical(y, x))
- stopifnot(length(futureOf(envir=x, drop=TRUE)) == 3L)
+ stopifnot(length(futureOf(envir = x, drop = TRUE)) == 3L)
- y <- resolve(x, recursive=TRUE, value=TRUE)
+ y <- resolve(x, recursive = TRUE, value = TRUE)
stopifnot(identical(y, x))
## Exceptions
- res <- try(y <- resolve(x, idxs=0L), silent=TRUE)
- stopifnot(inherits(res, "try-error"))
+ res <- tryCatch(y <- resolve(x, idxs = 0L), error = identity)
+ stopifnot(inherits(res, "error"))
- res <- try(y <- resolve(x, idxs="unknown"), silent=TRUE)
- stopifnot(inherits(res, "try-error"))
+ res <- tryCatch(y <- resolve(x, idxs = "unknown"), error = identity)
+ stopifnot(inherits(res, "error"))
message(sprintf("- plan('%s') ...", strategy))
} ## for (strategy ...)
@@ -279,6 +319,19 @@ for (strategy in strategies) {
message("*** resolve() for list environments ... DONE")
+message("*** resolve() - globals with non-trustful length() ...")
+
+length.CantTrustLength <- function(x) length(unclass(x)) + 1L
+
+.length <- future:::.length
+
+x <- structure(as.list(1:3), class = c("CantTrustLength", "list"))
+str(list(n = length(x), n_true = .length(x)))
+stopifnot(length(x) > .length(x))
+x <- resolve(x)
+
+message("*** resolve() - globals with non-trustful length() ... DONE")
+
message("*** resolved() - default ...")
diff --git a/tests/rng.R b/tests/rng.R
index c49ee93..51d675e 100644
--- a/tests/rng.R
+++ b/tests/rng.R
@@ -1,11 +1,18 @@
source("incl/start.R")
-library("future")
message("*** rng ...")
+## A valid L'Ecuyer-CMRG RNG seed
+seed <- c(407L, 1420090545L, 65713854L, -990249945L,
+ 1780737596L, -1213437427L, 1082168682L)
+f <- Future(42, seed = seed)
+print(f)
+
## See Section 6 on 'Random-number generation' in
-## vignette("parallel", package="parallel")
-fsample <- function(x, size=4L, seed=NULL) {
+## vignette("parallel", package = "parallel")
+fsample <- function(x, size = 4L, seed = NULL, what = c("future", "%<-%")) {
+ what <- match.arg(what)
+
## Must use session-specific '.GlobalEnv' here
.GlobalEnv <- globalenv()
@@ -17,76 +24,85 @@ fsample <- function(x, size=4L, seed=NULL) {
## Reset state of random seed afterwards?
on.exit({
if (is.null(oseed)) {
- rm(list=".Random.seed", envir=.GlobalEnv, inherits=FALSE)
+ rm(list = ".Random.seed", envir = .GlobalEnv, inherits = FALSE)
} else {
.GlobalEnv$.Random.seed <- oseed
}
- }, add=TRUE)
+ }, add = TRUE)
set.seed(seed)
}
.seed <- .Random.seed
- res <- listenv::listenv()
- for (ii in seq_len(size)) {
- .seed <- parallel::nextRNGStream(.seed)
- res[[ii]] %<-% {
- ## Must use session-specific '.GlobalEnv' here
- .GlobalEnv <- globalenv()
- .GlobalEnv$.Random.seed <- .seed
- sample(x, size=1L)
+
+ if (what == "future") {
+ fs <- list()
+ for (ii in seq_len(size)) {
+ .seed <- parallel::nextRNGStream(.seed)
+ fs[[ii]] <- future({ sample(x, size = 1L) }, seed = .seed)
+ }
+ res <- values(fs)
+ } else {
+ res <- listenv::listenv()
+ for (ii in seq_len(size)) {
+ .seed <- parallel::nextRNGStream(.seed)
+ res[[ii]] %<-% { sample(x, size = 1L) } %seed% .seed
}
+ res <- as.list(res)
}
- unlist(res)
+
+ res
} # fsample()
-dummy <- sample(0:9, size=1L)
+dummy <- sample(0:9, size = 1L)
seed0 <- .Random.seed
## Reference sample with fixed random seed
-plan("uniprocess")
-y0 <- fsample(0:9, seed=42L)
+plan("sequential")
+y0 <- fsample(0:9, seed = 42L)
## Assert that random seed is reset
stopifnot(identical(.GlobalEnv$.Random.seed, seed0))
-for (cores in 1:min(3L, availableCores())) {
+for (cores in 1:availCores) {
message(sprintf("Testing with %d cores ...", cores))
- options(mc.cores=cores-1L)
+ options(mc.cores = cores)
- for (strategy in supportedStrategies()) {
+ for (strategy in supportedStrategies(cores)) {
message(sprintf("%s ...", strategy))
- .GlobalEnv$.Random.seed <- seed0
-
plan(strategy)
- ## Fixed random seed
- y1 <- fsample(0:9, seed=42L)
- print(y1)
- stopifnot(identical(y1, y0))
-
- ## Assert that random seed is reset
- stopifnot(identical(.GlobalEnv$.Random.seed, seed0))
-
- ## Fixed random seed
- y2 <- fsample(0:9, seed=42L)
- print(y2)
- stopifnot(identical(y2, y1))
- stopifnot(identical(y2, y0))
+ for (what in c("future", "%<-%")) {
+ .GlobalEnv$.Random.seed <- seed0
- ## Assert that random seed is reset
- stopifnot(identical(.GlobalEnv$.Random.seed, seed0))
-
- ## No seed
- y3 <- fsample(0:9)
- print(y3)
-
- ## No seed
- y4 <- fsample(0:9)
- print(y4)
+ ## Fixed random seed
+ y1 <- fsample(0:9, seed = 42L, what = what)
+ print(y1)
+ stopifnot(identical(y1, y0))
+
+ ## Assert that random seed is reset
+ stopifnot(identical(.GlobalEnv$.Random.seed, seed0))
+
+ ## Fixed random seed
+ y2 <- fsample(0:9, seed = 42L, what = what)
+ print(y2)
+ stopifnot(identical(y2, y1))
+ stopifnot(identical(y2, y0))
+
+ ## Assert that random seed is reset
+ stopifnot(identical(.GlobalEnv$.Random.seed, seed0))
+
+ ## No seed
+ y3 <- fsample(0:9, what = what)
+ print(y3)
+
+ ## No seed
+ y4 <- fsample(0:9, what = what)
+ print(y4)
+ }
message(sprintf("%s ... done", strategy))
}
diff --git a/tests/sequential.R b/tests/sequential.R
new file mode 100644
index 0000000..3b0bee5
--- /dev/null
+++ b/tests/sequential.R
@@ -0,0 +1,28 @@
+source("incl/start.R")
+
+message("*** sequential() ...")
+
+message("- sequential() w/ required packages ...")
+
+f <- future(median(1:3), lazy = TRUE)
+print(f)
+
+## Doesn't work if covr that depends on stats is loaded
+try(unloadNamespace("stats"))
+
+v <- value(f)
+print(v)
+stopifnot(identical(v, 2L))
+
+stopifnot("stats" %in% loadedNamespaces())
+
+message("- SequentialFuture() - exceptions ...")
+
+res <- tryCatch({
+ f <- SequentialFuture(42, lazy = TRUE, local = FALSE)
+}, error = identity)
+stopifnot(inherits(res, "error"))
+
+message("*** sequential() ... DONE")
+
+source("incl/end.R")
diff --git a/tests/startup.R b/tests/startup.R
index e6a102e..00f9f03 100644
--- a/tests/startup.R
+++ b/tests/startup.R
@@ -1,6 +1,6 @@
source("incl/start.R")
-maxCores <- min(2L, availableCores(methods="system"))
+maxCores <- min(2L, availableCores(methods = "system"))
plan("default")
@@ -11,34 +11,34 @@ message("*** parseCmdArgs() ...")
args <- parseCmdArgs()
str(args)
-options(future.plan=NULL, future.cmdargs=c("-p", 1L))
+options(future.plan = NULL, future.cmdargs = c("-p", 1L))
args <- parseCmdArgs()
str(args)
stopifnot(args$p == 1L)
-options(future.plan=NULL, future.cmdargs=c(sprintf("--parallel=%d", maxCores)))
+options(future.plan = NULL, future.cmdargs = c(sprintf("--parallel=%d", maxCores)))
args <- parseCmdArgs()
str(args)
stopifnot(args$p == maxCores)
-options(future.plan=NULL, future.cmdargs=c("-p", 1L, sprintf("--parallel=%d", maxCores)))
+options(future.plan = NULL, future.cmdargs = c("-p", 1L, sprintf("--parallel=%d", maxCores)))
args <- parseCmdArgs()
str(args)
stopifnot(args$p == maxCores)
-options(future.plan=NULL, future.cmdargs=c("-p", 0L))
+options(future.plan = NULL, future.cmdargs = c("-p", 0L))
args <- parseCmdArgs()
stopifnot(is.null(args$p))
-res <- tryCatch(parseCmdArgs(), warning=function(w) w)
+res <- tryCatch(parseCmdArgs(), warning = function(w) w)
stopifnot(inherits(res, "warning"))
-options(future.plan=NULL, future.cmdargs=c("-p", .Machine$integer.max))
+options(future.plan = NULL, future.cmdargs = c("-p", .Machine$integer.max))
args <- parseCmdArgs()
stopifnot(is.null(args$p))
-res <- tryCatch(parseCmdArgs(), warning=function(w) w)
+res <- tryCatch(parseCmdArgs(), warning = function(w) w)
stopifnot(inherits(res, "warning"))
-options(future.plan=NULL, future.cmdargs=NULL)
+options(future.plan = NULL, future.cmdargs = NULL)
message("*** parseCmdArgs() ... DONE")
@@ -49,7 +49,7 @@ plan("default")
pkgname <- "future"
message("- .onLoad() w/out command-line options ...")
-options(future.plan=NULL, future.cmdargs=NULL)
+options(future.plan = NULL, future.cmdargs = NULL)
.onLoad(pkgname, pkgname)
strategy <- plan()
print(strategy)
@@ -58,26 +58,26 @@ plan("default")
message("- .onLoad() w/out command-line options ... DONE")
message("- .onLoad() w/ -p 1 ...")
-options(future.plan=NULL, future.cmdargs=c("-p", 1))
+options(future.plan = NULL, future.cmdargs = c("-p", 1))
.onLoad(pkgname, pkgname)
strategy <- plan()
print(strategy)
-stopifnot(inherits(strategy, "uniprocess"))
+stopifnot(inherits(strategy, "sequential"))
plan("default")
message("- .onLoad() w/ -p 1 ... DONE")
message("- .onLoad() w/ --parallel=1 ...")
plan("default")
-options(future.plan=NULL, future.cmdargs="-parallel=1")
+options(future.plan = NULL, future.cmdargs = "-parallel=1")
.onLoad(pkgname, pkgname)
strategy <- plan()
print(strategy)
-stopifnot(inherits(strategy, "uniprocess"))
+stopifnot(inherits(strategy, "sequential"))
plan("default")
message("- .onLoad() w/ --parallel=1 ... DONE")
message("- .onLoad() w/ -p 2 ...")
-options(future.plan=NULL, future.cmdargs=c("-p", 2))
+options(future.plan = NULL, future.cmdargs = c("-p", 2))
.onLoad(pkgname, pkgname)
strategy <- plan()
print(strategy)
@@ -90,7 +90,7 @@ plan("default")
message("- .onLoad() w/ -p 2 ... DONE")
message("- .onLoad() w/ -p 0 ...")
-options(future.plan=NULL, future.cmdargs=c("-p", 0))
+options(future.plan = NULL, future.cmdargs = c("-p", 0))
.onLoad(pkgname, pkgname)
strategy <- plan()
print(strategy)
@@ -99,7 +99,7 @@ plan("default")
message("- .onLoad() w/ -p 0 ... DONE")
message("- .onLoad() w/ -p -1 ...")
-options(future.plan=NULL, future.cmdargs=c("-p", -1))
+options(future.plan = NULL, future.cmdargs = c("-p", -1))
.onLoad(pkgname, pkgname)
strategy <- plan()
print(strategy)
@@ -108,7 +108,7 @@ plan("default")
message("- .onLoad() w/ -p -1 ... DONE")
message("- .onLoad() w/ -p foo ...")
-options(future.plan=NULL, future.cmdargs=c("-p", "foo"))
+options(future.plan = NULL, future.cmdargs = c("-p", "foo"))
.onLoad(pkgname, pkgname)
strategy <- plan()
print(strategy)
@@ -116,93 +116,125 @@ stopifnot(all(class(strategy) == class(strategy0)))
plan("default")
message("- .onLoad() w/ -p foo ... DONE")
-message("- .onLoad() w/ R_FUTURE_PLAN='lazy' ...")
-Sys.setenv("R_FUTURE_PLAN"="lazy")
-options(future.plan=NULL, future.cmdargs=NULL)
+message("- .onLoad() w/ R_FUTURE_PLAN = 'multisession' ...")
+Sys.setenv(R_FUTURE_PLAN = "multisession")
+options(future.plan = NULL, future.cmdargs = NULL)
.onLoad(pkgname, pkgname)
strategy <- plan()
print(strategy)
-stopifnot(inherits(strategy, "lazy"))
+stopifnot(inherits(strategy, "multisession"))
plan("default")
-Sys.setenv("R_FUTURE_PLAN"="")
-message("- .onLoad() w/ R_FUTURE_PLAN='lazy' ... DONE")
+Sys.setenv(R_FUTURE_PLAN = "")
+message("- .onLoad() w/ R_FUTURE_PLAN = 'multisession' ... DONE")
-message("- .onLoad() w/ future.plan='lazy' ...")
-options(future.plan=NULL, future.plan='lazy', future.cmdargs=NULL)
+message("- .onLoad() w/ future.plan = 'multisession' ...")
+options(future.plan = NULL, future.plan = 'multisession', future.cmdargs = NULL)
.onLoad(pkgname, pkgname)
strategy <- plan()
print(strategy)
-stopifnot(inherits(strategy, "lazy"))
+stopifnot(inherits(strategy, "multisession"))
plan("default")
-message("- .onLoad() w/ future.plan='lazy' ... DONE")
+message("- .onLoad() w/ future.plan = 'multisession' ... DONE")
-message("- .onLoad() w/ R_FUTURE_PLAN='lazy' & -p 1 ...")
-Sys.setenv("R_FUTURE_PLAN"="lazy")
-options(future.plan=NULL, future.cmdargs=c("-p", 1))
+message("- .onLoad() w/ R_FUTURE_PLAN = 'multisession' & -p 1 ...")
+Sys.setenv(R_FUTURE_PLAN = "multisession")
+options(future.plan = NULL, future.cmdargs = c("-p", 1))
.onLoad(pkgname, pkgname)
strategy <- plan()
print(strategy)
-stopifnot(inherits(strategy, "lazy"))
+stopifnot(inherits(strategy, "multisession"))
plan("default")
-Sys.setenv("R_FUTURE_PLAN"="")
-message("- .onLoad() w/ R_FUTURE_PLAN='lazy' & -p 1 ... DONE")
+Sys.setenv(R_FUTURE_PLAN = "")
+message("- .onLoad() w/ R_FUTURE_PLAN = 'multisession' & -p 1 ... DONE")
-message("- .onLoad() w/ future.plan='lazy' & -p 1 ...")
-options(future.plan='lazy', future.cmdargs=c("-p", "1"))
+message("- .onLoad() w/ future.plan = 'multisession' & -p 1 ...")
+options(future.plan = 'multisession', future.cmdargs = c("-p", "1"))
.onLoad(pkgname, pkgname)
strategy <- plan()
print(strategy)
-stopifnot(inherits(strategy, "lazy"))
+stopifnot(inherits(strategy, "multisession"))
plan("default")
-message("- .onLoad() w/ future.plan='lazy' & -p 1 ... DONE")
+message("- .onLoad() w/ future.plan = 'multisession' & -p 1 ... DONE")
-message("- .onLoad() w/ future.plan='lazy' & -p 1 ...")
-options(future.plan=lazy, future.cmdargs=c("-p", "1"))
+message("- .onLoad() w/ future.plan = 'multisession' & -p 1 ...")
+options(future.plan = multisession, future.cmdargs = c("-p", "1"))
.onLoad(pkgname, pkgname)
strategy <- plan()
print(strategy)
-stopifnot(inherits(strategy, "lazy"))
+stopifnot(inherits(strategy, "multisession"))
plan("default")
-message("- .onLoad() w/ future.plan='lazy' & -p 1 ... DONE")
+message("- .onLoad() w/ future.plan = 'multisession' & -p 1 ... DONE")
-message("- .onLoad() w/ future.availableCores.system=1L ...")
-options(future.availableCores.system=1L)
+message("- .onLoad() w/ future.availableCores.system = 1L ...")
+options(future.availableCores.system = 1L)
.onLoad(pkgname, pkgname)
-options(future.availableCores.system=NULL)
-message("- .onLoad() w/ future.availableCores.system=1L ... DONE")
+options(future.availableCores.system = NULL)
+message("- .onLoad() w/ future.availableCores.system = 1L ... DONE")
message("- .onLoad() w/ R_FUTURE_AVAILABLECORES_SYSTEM ...")
-Sys.setenv("R_FUTURE_AVAILABLECORES_SYSTEM"="1")
+Sys.setenv(R_FUTURE_AVAILABLECORES_SYSTEM = "1")
.onLoad(pkgname, pkgname)
ncores <- getOption("future.availableCores.system")
print(ncores)
stopifnot(is.integer(ncores), ncores == 1L)
Sys.unsetenv("R_FUTURE_AVAILABLECORES_SYSTEM")
-options(future.availableCores.system=NULL)
+options(future.availableCores.system = NULL)
-Sys.setenv("R_FUTURE_AVAILABLECORES_SYSTEM"="NA")
+Sys.setenv(R_FUTURE_AVAILABLECORES_SYSTEM = "NA")
.onLoad(pkgname, pkgname)
ncores <- getOption("future.availableCores.system")
print(ncores)
stopifnot(is.integer(ncores), is.na(ncores))
Sys.unsetenv("R_FUTURE_AVAILABLECORES_SYSTEM")
-options(future.availableCores.system=NULL)
+options(future.availableCores.system = NULL)
-Sys.setenv("R_FUTURE_AVAILABLECORES_SYSTEM"="NA_real_")
+Sys.setenv(R_FUTURE_AVAILABLECORES_SYSTEM = "NA_real_")
.onLoad(pkgname, pkgname)
ncores <- getOption("future.availableCores.system")
print(ncores)
stopifnot(is.integer(ncores), is.na(ncores))
Sys.unsetenv("R_FUTURE_AVAILABLECORES_SYSTEM")
-options(future.availableCores.system=NULL)
+options(future.availableCores.system = NULL)
message("- .onLoad() w/ R_FUTURE_AVAILABLECORES_SYSTEM ... DONE")
-options(future.plan=NULL, future.cmdargs=NULL, future.availableCores.system=1L)
+message("- .onLoad() w/ future.availableCores.fallback = 1L ...")
+options(future.availableCores.fallback = 1L)
+.onLoad(pkgname, pkgname)
+options(future.availableCores.fallback = NULL)
+message("- .onLoad() w/ future.availableCores.fallback = 1L ... DONE")
+
+message("- .onLoad() w/ R_FUTURE_AVAILABLECORES_FALLBACK ...")
+Sys.setenv(R_FUTURE_AVAILABLECORES_FALLBACK = "1")
+.onLoad(pkgname, pkgname)
+ncores <- getOption("future.availableCores.fallback")
+print(ncores)
+stopifnot(is.integer(ncores), ncores == 1L)
+Sys.unsetenv("R_FUTURE_AVAILABLECORES_FALLBACK")
+options(future.availableCores.fallback = NULL)
+
+Sys.setenv(R_FUTURE_AVAILABLECORES_FALLBACK = "NA")
+.onLoad(pkgname, pkgname)
+ncores <- getOption("future.availableCores.fallback")
+print(ncores)
+stopifnot(is.integer(ncores), is.na(ncores))
+Sys.unsetenv("R_FUTURE_AVAILABLECORES_FALLBACK")
+options(future.availableCores.fallback = NULL)
+
+Sys.setenv(R_FUTURE_AVAILABLECORES_FALLBACK = "NA_real_")
+.onLoad(pkgname, pkgname)
+ncores <- getOption("future.availableCores.fallback")
+print(ncores)
+stopifnot(is.integer(ncores), is.na(ncores))
+Sys.unsetenv("R_FUTURE_AVAILABLECORES_FALLBACK")
+options(future.availableCores.fallback = NULL)
+
+message("- .onLoad() w/ R_FUTURE_AVAILABLECORES_FALLBACK ... DONE")
+
-options(future.plan=NULL, future.cmdargs=NULL, future.availableCores.system=NULL)
+options(future.plan = NULL, future.cmdargs = NULL, future.availableCores.system = NULL, future.availableCores.fallback = NULL)
message("*** .onLoad() ... DONE")
@@ -214,7 +246,7 @@ pkgname <- "future"
message("- .onAttach() w/ option future.startup.loadScript ...")
for (value in list(NULL, FALSE, TRUE)) {
- options(future.startup.loadScript=value)
+ options(future.startup.loadScript = value)
.onAttach(pkgname, pkgname)
}
@@ -224,7 +256,7 @@ message("- .onAttach() with ./.future.R ...")
pathname <- ".future.R"
xyz <- 0L
-cat("xyz <- 42L; cat('ping\n')\n", file=pathname)
+cat("xyz <- 42L; cat('ping\n')\n", file = pathname)
.onAttach(pkgname, pkgname)
print(xyz)
stopifnot(is.integer(xyz), xyz >= 0, xyz == 42L)
diff --git a/tests/transparent.R b/tests/transparent.R
index 3e8bcd8..763afdb 100644
--- a/tests/transparent.R
+++ b/tests/transparent.R
@@ -6,9 +6,9 @@ message("*** transparent() ...")
## No global variables
f <- try(transparent({
42L
-}), silent=FALSE)
+}), silent = FALSE)
print(f)
-stopifnot(inherits(f, "UniprocessFuture"), !f$lazy)
+stopifnot(inherits(f, "SequentialFuture"), !f$lazy)
print(resolved(f))
y <- value(f)
@@ -20,9 +20,9 @@ plan(transparent)
## No global variables
f <- try(future({
42L
-}), silent=FALSE)
+}), silent = FALSE)
print(f)
-stopifnot(inherits(f, "UniprocessFuture"), !f$lazy)
+stopifnot(inherits(f, "SequentialFuture"), !f$lazy)
print(resolved(f))
y <- value(f)
diff --git a/tests/tweak.R b/tests/tweak.R
index 4f644ef..ac4c0a2 100644
--- a/tests/tweak.R
+++ b/tests/tweak.R
@@ -2,87 +2,87 @@ source("incl/start,load-only.R")
message("*** Tweaking future strategies ...")
-message("*** y <- tweak(future::lazy) ...")
-lazy2 <- future::tweak(future::lazy)
-print(args(lazy2))
-stopifnot(identical(lazy2, future::lazy))
-stopifnot(!inherits(lazy2, "tweaked"))
+message("*** y <- tweak(future::sequential) ...")
+sequential2 <- future::tweak(future::sequential)
+print(args(sequential2))
+stopifnot(identical(sequential2, future::sequential))
+stopifnot(!inherits(sequential2, "tweaked"))
-message("*** y <- tweak(future::lazy, local=FALSE) ...")
-lazy2 <- future::tweak(future::lazy, local=FALSE)
-print(args(lazy2))
-stopifnot(!identical(lazy2, future::lazy))
-stopifnot(inherits(lazy2, "tweaked"))
-stopifnot(identical(formals(lazy2)$local, FALSE))
+message("*** y <- tweak(future::sequential, local = FALSE) ...")
+sequential2 <- future::tweak(future::sequential, local = FALSE)
+print(args(sequential2))
+stopifnot(!identical(sequential2, future::sequential))
+stopifnot(inherits(sequential2, "tweaked"))
+stopifnot(identical(formals(sequential2)$local, FALSE))
-message("*** y <- tweak('lazy', local=FALSE) ...")
-lazy2 <- future::tweak("lazy", local=FALSE)
-print(args(lazy2))
-stopifnot(!identical(lazy2, future::lazy))
-stopifnot(inherits(lazy2, "tweaked"))
-stopifnot(identical(formals(lazy2)$local, FALSE))
+message("*** y <- tweak('sequential', local = FALSE) ...")
+sequential2 <- future::tweak("sequential", local = FALSE)
+print(args(sequential2))
+stopifnot(!identical(sequential2, future::sequential))
+stopifnot(inherits(sequential2, "tweaked"))
+stopifnot(identical(formals(sequential2)$local, FALSE))
library("future")
-message("*** y <- tweak(lazy, local=FALSE) ...")
-lazy2 <- future::tweak(lazy, local=FALSE)
-print(args(lazy2))
-stopifnot(!identical(lazy2, future::lazy))
-stopifnot(inherits(lazy2, "tweaked"))
-stopifnot(identical(formals(lazy2)$local, FALSE))
-
-message("*** y <- tweak('lazy', local=FALSE) ...")
-lazy2 <- future::tweak('lazy', local=FALSE)
-print(args(lazy2))
-stopifnot(!identical(lazy2, future::lazy))
-stopifnot(inherits(lazy2, "tweaked"))
-stopifnot(identical(formals(lazy2)$local, FALSE))
-
-message("*** y <- tweak('lazy', local=FALSE, abc=1, def=TRUE) ...")
+message("*** y <- tweak(sequential, local = FALSE) ...")
+sequential2 <- future::tweak(sequential, local = FALSE)
+print(args(sequential2))
+stopifnot(!identical(sequential2, future::sequential))
+stopifnot(inherits(sequential2, "tweaked"))
+stopifnot(identical(formals(sequential2)$local, FALSE))
+
+message("*** y <- tweak('sequential', local = FALSE) ...")
+sequential2 <- future::tweak('sequential', local = FALSE)
+print(args(sequential2))
+stopifnot(!identical(sequential2, future::sequential))
+stopifnot(inherits(sequential2, "tweaked"))
+stopifnot(identical(formals(sequential2)$local, FALSE))
+
+message("*** y <- tweak('sequential', local = FALSE, abc = 1, def = TRUE) ...")
res <- tryCatch({
- lazy2 <- future::tweak('lazy', local=FALSE, abc=1, def=TRUE)
-}, warning=function(w) {
+ sequential2 <- future::tweak('sequential', local = FALSE, abc = 1, def = TRUE)
+}, warning = function(w) {
w
})
stopifnot(inherits(res, "warning"))
-lazy2 <- future::tweak('lazy', local=FALSE, abc=1, def=TRUE)
-print(args(lazy2))
-stopifnot(!identical(lazy2, future::lazy))
-stopifnot(inherits(lazy2, "tweaked"))
-stopifnot(identical(formals(lazy2)$local, FALSE))
+sequential2 <- future::tweak('sequential', local = FALSE, abc = 1, def = TRUE)
+print(args(sequential2))
+stopifnot(!identical(sequential2, future::sequential))
+stopifnot(inherits(sequential2, "tweaked"))
+stopifnot(identical(formals(sequential2)$local, FALSE))
message("*** y %<-% { expr } %tweak% tweaks ...")
-plan(uniprocess)
+plan(sequential)
a <- 0
x %<-% { a <- 1; a }
print(x)
stopifnot(a == 0, x == 1)
-x %<-% { a <- 2; a } %tweak% list(local=FALSE)
+x %<-% { a <- 2; a } %tweak% list(local = FALSE)
print(x)
stopifnot(a == 2, x == 2)
-plan(uniprocess, local=FALSE)
+plan(sequential, local = FALSE)
a <- 0
x %<-% { a <- 1; a }
print(x)
stopifnot(a == 1, x == 1)
-x %<-% { a <- 2; a } %tweak% list(local=TRUE)
+x %<-% { a <- 2; a } %tweak% list(local = TRUE)
print(x)
stopifnot(a == 1, x == 2)
# Preserve nested futures
-plan(list(A=uniprocess, B=tweak(uniprocess, local=FALSE)))
+plan(list(A = sequential, B = tweak(sequential, local = FALSE)))
a <- 0
x %<-% {
@@ -97,7 +97,7 @@ x %<-% {
stopifnot(identical(names(plan("list")), "B"))
a <- 2
a
-} %tweak% list(local=FALSE)
+} %tweak% list(local = FALSE)
print(x)
stopifnot(a == 2, x == 2)
@@ -105,28 +105,41 @@ stopifnot(a == 2, x == 2)
message("*** y %<-% { expr } %tweak% tweaks ... DONE")
-message("*** tweak() - gc=TRUE ...")
+message("*** tweak() - gc = TRUE ...")
-res <- tryCatch(tweak(multisession, gc=TRUE), condition=identity)
+res <- tryCatch(tweak(multisession, gc = TRUE), condition = identity)
stopifnot(inherits(res, "tweaked"))
## Argument 'gc' is unknown
-res <- tryCatch(tweak(uniprocess, gc=TRUE), condition=identity)
+res <- tryCatch(tweak(sequential, gc = TRUE), condition = identity)
stopifnot(inherits(res, "warning"))
-res <- tryCatch(tweak(multicore, gc=TRUE), condition=identity)
+res <- tryCatch(tweak(multicore, gc = TRUE), condition = identity)
stopifnot(inherits(res, "warning"))
-message("*** tweak() - gc=TRUE ... DONE")
+message("*** tweak() - gc = TRUE ... DONE")
message("*** tweak() - exceptions ...")
-res <- try(tweak("<unknown-future-strategy>"), silent=TRUE)
+res <- try(tweak("<unknown-future-strategy>"), silent = TRUE)
stopifnot(inherits(res, "try-error"))
-res <- try(tweak(uniprocess, "unnamed-argument"), silent=TRUE)
+res <- try(tweak(base::eval), silent = TRUE)
+stopifnot(inherits(res, "try-error"))
+
+res <- try(tweak(sequential, "unnamed-argument"), silent = TRUE)
+stopifnot(inherits(res, "try-error"))
+
+## Arguments that must not be tweaked
+res <- try(tweak(sequential, lazy = TRUE), silent = TRUE)
+stopifnot(inherits(res, "try-error"))
+
+res <- try(tweak(sequential, asynchronous = FALSE), silent = TRUE)
+stopifnot(inherits(res, "try-error"))
+
+res <- try(tweak(sequential, seed = 42L), silent = TRUE)
stopifnot(inherits(res, "try-error"))
message("*** tweak() - exceptions ... DONE")
diff --git a/tests/utils.R b/tests/utils.R
index 864fa2a..8b9addb 100644
--- a/tests/utils.R
+++ b/tests/utils.R
@@ -15,34 +15,34 @@ z <- LETTERS[x]
printf("x = %s.\n", hpaste(x))
## x = 1, 2, 3, ..., 6.
-printf("x = %s.\n", hpaste(x, maxHead=2))
+printf("x = %s.\n", hpaste(x, maxHead = 2))
## x = 1, 2, ..., 6.
-printf("x = %s.\n", hpaste(x), maxHead=3) # Default
+printf("x = %s.\n", hpaste(x), maxHead = 3) # Default
## x = 1, 2, 3, ..., 6.
# It will never output 1, 2, 3, 4, ..., 6
-printf("x = %s.\n", hpaste(x, maxHead=4))
+printf("x = %s.\n", hpaste(x, maxHead = 4))
## x = 1, 2, 3, 4, 5 and 6.
# Showing the tail
-printf("x = %s.\n", hpaste(x, maxHead=1, maxTail=2))
+printf("x = %s.\n", hpaste(x, maxHead = 1, maxTail = 2))
## x = 1, ..., 5, 6.
# Turning off abbreviation
-printf("y = %s.\n", hpaste(y, maxHead=Inf))
+printf("y = %s.\n", hpaste(y, maxHead = Inf))
## y = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1
## ...or simply
-printf("y = %s.\n", paste(y, collapse=", "))
+printf("y = %s.\n", paste(y, collapse = ", "))
## y = 10, 9, 8, 7, 6, 5, 4, 3, 2, 1
# Change last separator
-printf("x = %s.\n", hpaste(x, lastCollapse=" and "))
+printf("x = %s.\n", hpaste(x, lastCollapse = " and "))
## x = 1, 2, 3, 4, 5 and 6.
# No collapse
-stopifnot(all(hpaste(x, collapse=NULL) == x))
+stopifnot(all(hpaste(x, collapse = NULL) == x))
# Empty input
stopifnot(identical(hpaste(character(0)), character(0)))
@@ -55,7 +55,7 @@ message("*** hpaste() ...")
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
message("*** asIEC() ...")
-for (size in c(0, 10^(0:20))) {
+for (size in c(0, 10 ^ (0:20))) {
cat(sprintf("Size: %.f bytes = %s\n", size, asIEC(size)))
}
@@ -68,9 +68,9 @@ message("*** asIEC() ... DONE")
message("*** mdebug() ...")
mdebug("Hello #1")
-options(future.debug=TRUE)
+options(future.debug = TRUE)
mdebug("Hello #2")
-options(future.debug=FALSE)
+options(future.debug = FALSE)
mdebug("Hello #3")
message("*** mdebug() ... DONE")
@@ -81,31 +81,31 @@ message("*** mdebug() ... DONE")
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
message("*** geval() et al. ...")
-gls <- function(..., envir=.GlobalEnv) ls(..., envir=envir)
+gls <- function(..., envir = .GlobalEnv) ls(..., envir = envir)
message("- gls() ...")
-genv <- new.env(parent=globalenv())
-vars <- gls(envir=genv)
+genv <- new.env(parent = globalenv())
+vars <- gls(envir = genv)
print(vars)
stopifnot(length(vars) == 0)
message("- gassign() ...")
-gassign("a", 1, envir=genv)
-vars <- gls(envir=genv)
+gassign("a", 1, envir = genv)
+vars <- gls(envir = genv)
print(vars)
stopifnot(length(vars) == 1)
message("- grmall() ...")
-grmall(envir=genv)
-vars <- gls(envir=genv)
+grmall(envir = genv)
+vars <- gls(envir = genv)
print(vars)
stopifnot(length(vars) == 0)
message("- geval() ...")
-gassign("a", 1, envir=genv)
-res <- geval(substitute(a), envir=genv)
+gassign("a", 1, envir = genv)
+res <- geval(substitute(a), envir = genv)
print(res)
-vars <- gls(envir=genv)
+vars <- gls(envir = genv)
print(vars)
stopifnot(length(vars) == 1)
@@ -121,7 +121,7 @@ message("*** requirePackages() ...")
res <- requirePackages("future")
res <- requirePackages(c("future", "listenv"))
-res <- try(requirePackages("<unknown package>"), silent=TRUE)
+res <- try(requirePackages("<unknown package>"), silent = TRUE)
stopifnot(inherits(res, "try-error"))
message("*** requirePackages() ... DONE")
@@ -136,15 +136,15 @@ mclapply <- importParallel("mclapply")
stopifnot(identical(mclapply, parallel::mclapply))
ns <- getNamespace("parallel")
-if (exists("sendCall", envir=ns, mode="function")) {
+if (exists("sendCall", envir = ns, mode = "function")) {
sendCall <- importParallel("sendCall")
stopifnot(identical(sendCall, parallel:::sendCall))
} else {
- res <- try(importParallel("sendCall"), silent=TRUE)
+ res <- try(importParallel("sendCall"), silent = TRUE)
stopifnot(inherits(res, "try-error"))
}
-res <- try(importParallel("<unknown function>"), silent=TRUE)
+res <- try(importParallel("<unknown function>"), silent = TRUE)
stopifnot(inherits(res, "try-error"))
message("*** importParallel() ... DONE")
@@ -153,24 +153,24 @@ message("*** importParallel() ... DONE")
# myInternalIP() and myExternalIP()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
message("*** myInternalIP() ...")
-ips <- myInternalIP(mustWork=FALSE)
-message("myInternalIP(): ", paste(ips, collapse=", "))
+ips <- myInternalIP(mustWork = FALSE)
+message("myInternalIP(): ", paste(ips, collapse = ", "))
message("*** myInternalIP() ... DONE")
-ips <- myInternalIP(force=TRUE, which="first", mustWork=FALSE)
-message("myInternalIP(which='first'): ", paste(ips, collapse=", "))
+ips <- myInternalIP(force = TRUE, which = "first", mustWork = FALSE)
+message("myInternalIP(which = 'first'): ", paste(ips, collapse = ", "))
message("*** myInternalIP() ... DONE")
-ips <- myInternalIP(force=TRUE, which="last", mustWork=FALSE)
-message("myInternalIP(which='last'): ", paste(ips, collapse=", "))
+ips <- myInternalIP(force = TRUE, which = "last", mustWork = FALSE)
+message("myInternalIP(which = 'last'): ", paste(ips, collapse = ", "))
message("*** myInternalIP() ... DONE")
-ips <- myInternalIP(force=TRUE, which="all", mustWork=FALSE)
-message("myInternalIP(which='all'): ", paste(ips, collapse=", "))
+ips <- myInternalIP(force = TRUE, which = "all", mustWork = FALSE)
+message("myInternalIP(which = 'all'): ", paste(ips, collapse = ", "))
message("*** myInternalIP() ... DONE")
message("*** myExternalIP() ...")
-ip <- myExternalIP(mustWork=FALSE)
+ip <- myExternalIP(mustWork = FALSE)
message("myExternalIP(): ", ip)
message("*** myExternalIP() ... DONE")
diff --git a/tests/uuid.R b/tests/uuid.R
index 67cbb0d..b9774ab 100644
--- a/tests/uuid.R
+++ b/tests/uuid.R
@@ -1,5 +1,6 @@
source("incl/start.R")
session_uuid <- future:::session_uuid
+add_cluster_uuid <- future:::add_cluster_uuid
message("*** session_uuid() ...")
@@ -15,4 +16,35 @@ stopifnot(id != id0)
message("*** session_uuid() ... DONE")
+message("*** add_cluster_uuid() ...")
+
+cl <- parallel::makeCluster(1L, type = "PSOCK")
+cl <- add_cluster_uuid(cl)
+str(cl)
+parallel::stopCluster(cl)
+uuid <- as.vector(attr(cl[[1]]$con, "uuid"))
+print(uuid)
+stopifnot(is.character(uuid), nzchar(uuid))
+
+
+if (supportsMulticore()) {
+ cl <- parallel::makeCluster(1L, type = "FORK")
+ cl <- add_cluster_uuid(cl)
+ str(cl)
+ parallel::stopCluster(cl)
+ uuid <- as.vector(attr(cl[[1]]$con, "uuid"))
+ print(uuid)
+ stopifnot(is.character(uuid), nzchar(uuid))
+}
+
+cl <- structure(list(
+ structure(list(
+ rank = 1L, RECVTAG = 33, SENDTAG = 22, comm = 1
+ ), class = "MPInode")
+ ), class = c("spawnedMPIcluster", "MPIcluster", "cluster")
+)
+cl <- add_cluster_uuid(cl)
+
+message("*** add_cluster_uuid() ... DONE")
+
source("incl/end.R")
diff --git a/tests/whichIndex.R b/tests/whichIndex.R
index 4bc0e55..4948b2d 100644
--- a/tests/whichIndex.R
+++ b/tests/whichIndex.R
@@ -4,72 +4,72 @@ message("*** whichIndex() ...")
dims <- list(
c(1),
- c(1,1),
+ c(1, 1),
c(2),
- c(2,1),
- c(3,4,3)
+ c(2, 1),
+ c(3, 4, 3)
)
for (dim in dims) {
- printf("Dimensions: (%s)\n", paste(dim, collapse=", "))
- dimnames <- lapply(dim, FUN=function(n) letters[seq_len(n)])
- X <- array(seq_len(prod(dim)), dim=dim, dimnames=dimnames)
+ printf("Dimensions: (%s)\n", paste(dim, collapse = ", "))
+ dimnames <- lapply(dim, FUN = function(n) letters[seq_len(n)])
+ X <- array(seq_len(prod(dim)), dim = dim, dimnames = dimnames)
print(X)
- idxs <- unique(round(seq(from=1L, to=length(X), length.out=5)))
+ idxs <- unique(round(seq(from = 1L, to = length(X), length.out = 5)))
print(idxs)
stopifnot(all(X[idxs] == idxs))
## (a) By matrix and dimindices
- I <- arrayInd(idxs, .dim=dim(X))
+ I <- arrayInd(idxs, .dim = dim(X))
print(I)
- idxs2 <- whichIndex(I, dim=dim(X))
+ idxs2 <- whichIndex(I, dim = dim(X))
print(idxs2)
stopifnot(all(idxs2 == idxs))
## (b) By matrix and dimnames
- N <- array(NA_character_, dim=dim(I))
+ N <- array(NA_character_, dim = dim(I))
for (kk in seq_len(ncol(N))) {
- N[,kk] <- dimnames[[kk]][I[,kk]]
+ N[, kk] <- dimnames[[kk]][I[, kk]]
}
print(N)
- idxs3 <- whichIndex(N, dim=dim(X), dimnames=dimnames(X))
+ idxs3 <- whichIndex(N, dim = dim(X), dimnames = dimnames(X))
print(idxs3)
stopifnot(all(idxs3 == idxs))
## (b) By data.frame
D <- as.data.frame(I)
- for (cc in seq(from=1L, to=ncol(D))) D[[cc]] <- N[,cc]
+ for (cc in seq(from = 1L, to = ncol(D))) D[[cc]] <- N[, cc]
print(D)
- idxs4 <- whichIndex(D, dim=dim(X), dimnames=dimnames(X))
+ idxs4 <- whichIndex(D, dim = dim(X), dimnames = dimnames(X))
print(idxs4)
stopifnot(all(idxs4 == idxs))
}
## Exceptions
-dim <- c(2,3)
+dim <- c(2, 3)
ndim <- length(dim)
-dimnames <- lapply(dim, FUN=function(n) letters[seq_len(n)])
+dimnames <- lapply(dim, FUN = function(n) letters[seq_len(n)])
-I <- matrix(c(1,1,2,4), ncol=ndim)
-res <- try(idxs <- whichIndex(I, dim=dim, dimnames=dimnames), silent=TRUE)
+I <- matrix(c(1, 1, 2, 4), ncol = ndim)
+res <- try(idxs <- whichIndex(I, dim = dim, dimnames = dimnames), silent = TRUE)
stopifnot(inherits(res, "try-error"))
-I <- matrix(c(0,0), ncol=ndim)
-res <- try(idxs <- whichIndex(I, dim=dim, dimnames=dimnames), silent=TRUE)
+I <- matrix(c(0, 0), ncol = ndim)
+res <- try(idxs <- whichIndex(I, dim = dim, dimnames = dimnames), silent = TRUE)
stopifnot(inherits(res, "try-error"))
-I <- matrix(c("a","q"), ncol=ndim)
-res <- try(idxs <- whichIndex(I, dim=dim, dimnames=dimnames), silent=TRUE)
+I <- matrix(c("a", "q"), ncol = ndim)
+res <- try(idxs <- whichIndex(I, dim = dim, dimnames = dimnames), silent = TRUE)
stopifnot(inherits(res, "try-error"))
-D <- data.frame(a=c(1,1), b=c(2,4))
-res <- try(idxs <- whichIndex(D, dim=dim, dimnames=dimnames), silent=TRUE)
+D <- data.frame(a = c(1, 1), b = c(2, 4))
+res <- try(idxs <- whichIndex(D, dim = dim, dimnames = dimnames), silent = TRUE)
stopifnot(inherits(res, "try-error"))
-D <- data.frame(a=c("a","q"), b=c(1,2))
-res <- try(idxs <- whichIndex(D, dim=dim, dimnames=dimnames), silent=TRUE)
+D <- data.frame(a = c("a", "q"), b = c(1, 2))
+res <- try(idxs <- whichIndex(D, dim = dim, dimnames = dimnames), silent = TRUE)
stopifnot(inherits(res, "try-error"))
message("*** whichIndex() ... DONE")
diff --git a/vignettes/future-1-overview.md.rsp b/vignettes/future-1-overview.md.rsp
index 40c9e01..847e8b9 100644
--- a/vignettes/future-1-overview.md.rsp
+++ b/vignettes/future-1-overview.md.rsp
@@ -22,7 +22,7 @@ file found under inst/vignettes-static/ of the source package.
## Introduction
The purpose of the [future] package is to provide a very simple and uniform way of evaluating R expressions asynchronously using various resources available to the user.
-In programming, a _future_ is an abstraction for a _value_ that may be available at some point in the future. The state of a future can either be _unresolved_ or _resolved_. As soon as it is resolved, the value is available instantaneously. If the value is queried while the future is still unresolved, the current process is _blocked_ until the future is resolved. It is possible to check whether a future is resolved or not without blocking. Exactly how and when futures are resolved d [...]
+In programming, a _future_ is an abstraction for a _value_ that may be available at some point in the future. The state of a future can either be _unresolved_ or _resolved_. As soon as it is resolved, the value is available instantaneously. If the value is queried while the future is still unresolved, the current process is _blocked_ until the future is resolved. It is possible to check whether a future is resolved or not without blocking. Exactly how and when futures are resolved d [...]
Here is an example illustrating how the basics of futures work. First, consider the following code snippet that uses plain R code:
```r
@@ -108,9 +108,8 @@ The future package implements the following types of futures:
| Name | OSes | Description
|:----------------|:------------|:-----------------------------------------------------
| _synchronous:_ | | _non-parallel:_
-| `eager` | all |
-| `lazy` | all | lazy evaluation - happens only if the value is requested
-| `transparent` | all | as eager w/ early signaling and w/out local (for debugging)
+| `sequential` | all | sequentially and in the current R process
+| `transparent` | all | as sequential w/ early signaling and w/out local (for debugging)
| _asynchronous:_ | | _parallel_:
| `multiprocess` | all | multicore iff supported, otherwise multisession
| `multisession` | all | background R sessions (on current machine)
@@ -118,27 +117,29 @@ The future package implements the following types of futures:
| `cluster` | all | external R sessions on current, local, and/or remote machines
| `remote` | all | Simple access to remote R sessions
-The future package is designed such that support for additional strategies can be implemented as well. For instance, the [future.BatchJobs] package provides futures for all types of _cluster functions_ ("backends") that the [BatchJobs] package supports. Specifically, futures for evaluating R expressions via job schedulers such as Slurm, TORQUE/PBS, Oracle/Sun Grid Engine (SGE) and Load Sharing Facility (LSF) are also available.
+_Note_: Prior to future 1.3.0, `eager` and `lazy` were also options, but has since been deprecated and are both defunct as of 1.6.0. The reason for this is that whether a future should be resolved by lazy evaluation or not has to, in some cases, be in the control of the developer, and if the end user would be able change that, the code may not function as intended.
-By default, future expressions are evaluated eagerly (= instantaneously) and synchronously (in the current R session). This evaluation strategy is referred to as "eager" and we refer to futures using this strategy as "eager futures". In this section, we will go through each of these strategies and discuss what they have in common and how they differ.
+The future package is designed such that support for additional strategies can be implemented as well. For instance, the [future.batchtools] package provides futures for all types of _cluster functions_ ("backends") that the [batchtools] package supports. Specifically, futures for evaluating R expressions via job schedulers such as Slurm, TORQUE/PBS, Oracle/Sun Grid Engine (SGE) and Load Sharing Facility (LSF) are also available. (_Comment_: The [future.BatchJobs] package provides ana [...]
+
+By default, future expressions are evaluated eagerly (= instantaneously) and synchronously (in the current R session). This evaluation strategy is referred to as "sequential". In this section, we will go through each of these strategies and discuss what they have in common and how they differ.
### Consistent Behavior Across Futures
-Before going through each of the different future strategies, it is probably helpful to clarify the objectives the Future API (as defined by the future package). When programming with futures, it should not really matter what future strategy is used for executing code. This is because we cannot really know what computational resources the user has access to so the choice of evaluation strategy should be in the hand of the user and not the developer. In other words, the code should not [...]
+Before going through each of the different future strategies, it is probably helpful to clarify the objectives the Future API (as defined by the future package). When programming with futures, it should not really matter what future strategy is used for executing code. This is because we cannot really know what computational resources the user has access to so the choice of evaluation strategy should be in the hands of the user and not the developer. In other words, the code should no [...]
-One of the designs of the Future API was to encapsulate any differences such that all types of futures will appear to work the same. This despite expressions may be evaluated locally in the current R session or across the world in remote R sessions. Another obvious advantage of having a consistent API and behavior among different types of futures is that it helps while prototyping. Typically one would use eager evaluation while building up a script and, later, when the script is fully [...]
+One of the designs of the Future API was to encapsulate any differences such that all types of futures will appear to work the same. This despite expressions may be evaluated locally in the current R session or across the world in remote R sessions. Another obvious advantage of having a consistent API and behavior among different types of futures is that it helps while prototyping. Typically one would use sequential evaluation while building up a script and, later, when the script is [...]
Because of this, the defaults of the different strategies are such that the results and side effects of evaluating a future expression are as similar as possible. More specifically, the following is true for all futures:
* All _evaluation is done in a local environment_ (i.e. `local({ expr })`) so that assignments do not affect the calling environment. This is natural when evaluating in an external R process, but is also enforced when evaluating in the current R session.
-* When a future is constructed, _global variables are identified_. For asynchronous evaluation, globals are exported to the R process/session that will be evaluating the future expression. For lazy futures, globals are "frozen" (cloned to a local environment of the future). Also, in order to protect against exporting too large objects by mistake, there is a built-in assertion that the total size of all globals is less than a given threshold (controllable via an option, cf. `help("futu [...]
+* When a future is constructed, _global variables are identified_. For asynchronous evaluation, globals are exported to the R process/session that will be evaluating the future expression. For sequential futures with lazy evaluation (`lazy = TRUE`), globals are "frozen" (cloned to a local environment of the future). Also, in order to protect against exporting too large objects by mistake, there is a built-in assertion that the total size of all globals is less than a given threshold ( [...]
* Future _expressions are only evaluated once_. As soon as the value (or an error) has been collected it will be available for all succeeding requests.
Here is an example illustrating that all assignments are done to a local environment:
```r
-> plan(eager)
+> plan(sequential)
> a <- 1
> x %<-% {
+ a <- 2
@@ -156,16 +157,16 @@ Now we are ready to explore the different future strategies.
### Synchronous Futures
-Synchronous futures are resolved one after another and most commonly by the R process that creates them. When a synchronous future is being resolved it blocks the main process until resolved. There are two main types of synchronous futures in the future package, _eager_ and _lazy_ futures, which are described next.
+Synchronous futures are resolved one after another and most commonly by the R process that creates them. When a synchronous future is being resolved it blocks the main process until resolved. There are two types of synchronous futures in the future package, _sequential_ and _transparent_. (In future 1.2.0 and before, there was also _lazy_ futures, which has now been deprecated in favor of `f <- future(..., lazy = TRUE)` and `v %<-% { ... } %lazy% TRUE`.)
-#### Eager Futures
-Eager futures are the default unless otherwise specified. They were designed to behave as similar as possible to regular R evaluation while still fulfilling the Future API and its behaviors. Here is an example illustrating their properties:
+#### Sequential Futures
+Sequential futures are the default unless otherwise specified. They were designed to behave as similar as possible to regular R evaluation while still fulfilling the Future API and its behaviors. Here is an example illustrating their properties:
```r
-> plan(eager)
+> plan(sequential)
> pid <- Sys.getpid()
> pid
-[1] 10708
+[1] 28518
> a %<-% {
+ pid <- Sys.getpid()
+ cat("Resolving 'a' ...\n")
@@ -184,58 +185,19 @@ Resolving 'b' ...
+ }
Resolving 'c' ...
> b
-[1] 10708
-> c
-[1] 6.28
-> a
-[1] 3.14
-> pid
-[1] 10708
-```
-Since eager evaluation is taking place, each of the three futures is resolved instantaneously in the moment it is created. Note also how `pid` in the calling environment, which was assigned the process ID of the current process, is neither overwritten nor removed. This is because futures are evaluated in a local environment. Since synchronous (uni-)processing is used, future `b` is resolved by the main R process (still in a local environment), which is why the value of `b` and `pid` a [...]
-
-
-#### Lazy Futures
-A lazy future evaluates its expression only if its value is queried. Evaluation can also be triggered when the future is checked for being resolved or not. Here is the above example when using lazy evaluation:
-```r
-> plan(lazy)
-> pid <- Sys.getpid()
-> pid
-[1] 10708
-> a %<-% {
-+ pid <- Sys.getpid()
-+ cat("Resolving 'a' ...\n")
-+ 3.14
-+ }
-> b %<-% {
-+ rm(pid)
-+ cat("Resolving 'b' ...\n")
-+ Sys.getpid()
-+ }
-> c %<-% {
-+ cat("Resolving 'c' ...\n")
-+ 2 * a
-+ }
-Resolving 'a' ...
-> b
-Resolving 'b' ...
-[1] 10708
+[1] 28518
> c
-Resolving 'c' ...
[1] 6.28
> a
[1] 3.14
> pid
-[1] 10708
+[1] 28518
```
-As previously, variable `pid` is unaffected because all evaluation is done in a local environment. More interestingly, future `a` is no longer evaluated in the moment it is created, but instead when it is needed the first time, which happens when future `c` is created. This is because `a` is identified as a global variable that needs to be captured ("frozen" to `a == 3.14`) in order to set up future `c`. Later when `c` (the value of future `c`) is queried, `a` has already been resolve [...]
-
-_Comment_: Lazy evaluation is already used by R itself. Arguments are passed to functions using lazy evaluation. It is also possible to assign variables using lazy evaluation using `delayedAssign()`, but contrary to lazy futures this function does not freeze globals. For more information, see `help("delayedAssign", package="base")`.
-
-
-
+Since eager sequential evaluation is taking place, each of the three futures is resolved instantaneously in the moment it is created. Note also how `pid` in the calling environment, which was assigned the process ID of the current process, is neither overwritten nor removed. This is because futures are evaluated in a local environment. Since synchronous (uni-)processing is used, future `b` is resolved by the main R process (still in a local environment), which is why the value of `b` [...]
+#### Transparent Futures
+For troubleshooting, _transparent_ futures can be used by specifying `plan(transparent)`. A transparent future is technically a sequential future with instant signaling of conditions (including errors and warnings) and where evaluation, and therefore also assignments, take place in the calling environment. Transparent futures are particularly useful for troubleshooting errors that are otherwise hard to narrow down.
### Asynchronous Futures
@@ -248,7 +210,7 @@ We start with multisession futures because they are supported by all operating s
> plan(multisession)
> pid <- Sys.getpid()
> pid
-[1] 10708
+[1] 28518
> a %<-% {
+ pid <- Sys.getpid()
+ cat("Resolving 'a' ...\n")
@@ -264,13 +226,13 @@ We start with multisession futures because they are supported by all operating s
+ 2 * a
+ }
> b
-[1] 10729
+[1] 28539
> c
[1] 6.28
> a
[1] 3.14
> pid
-[1] 10708
+[1] 28518
```
The first thing we observe is that the values of `a`, `c` and `pid` are the same as previously. However, we notice that `b` is different from before. This is because future `b` is evaluated in a different R process and therefore it returns a different process ID. Another difference is that the messages, generated by `cat()`, are no longer displayed. This is because they are outputted to the background sessions and not the calling session.
@@ -278,10 +240,10 @@ The first thing we observe is that the values of `a`, `c` and `pid` are the same
When multisession evaluation is used, the package launches a set of R sessions in the background that will serve multisession futures by evaluating their expressions as they are created. If all background sessions are busy serving other futures, the creation of the next multisession future is _blocked_ until a background session becomes available again. The total number of background processes launched is decided by the value of `availableCores()`, e.g.
```r
> availableCores()
-mc.cores+1
- 3
+mc.cores
+ 2
```
-This particular result tells us that the `mc.cores` option was set such that we are allowed to use in total 3 processes including the main process. In other words, with these settings, there will be 2 background processes serving the multisession futures. The `availableCores()` is also agile to different options and system environment variables. For instance, if compute cluster schedulers are used (e.g. TORQUE/PBS and Slurm), they set specific environment variable specifying the numbe [...]
+This particular result tells us that the `mc.cores` option was set such that we are allowed to use in total 2 processes including the main process. In other words, with these settings, there will be 2 background processes serving the multisession futures. The `availableCores()` is also agile to different options and system environment variables. For instance, if compute cluster schedulers are used (e.g. TORQUE/PBS and Slurm), they set specific environment variable specifying the numbe [...]
#### Multicore Futures
@@ -310,7 +272,7 @@ Cluster futures evaluate expressions on an ad-hoc cluster (as implemented by the
> plan(cluster, workers = c("n1", "n2", "n3"))
> pid <- Sys.getpid()
> pid
-[1] 10708
+[1] 28518
> a %<-% {
+ pid <- Sys.getpid()
+ cat("Resolving 'a' ...\n")
@@ -326,58 +288,29 @@ Cluster futures evaluate expressions on an ad-hoc cluster (as implemented by the
+ 2 * a
+ }
> b
-[1] 10751
+[1] 28561
> c
[1] 6.28
> a
[1] 3.14
> pid
-[1] 10708
+[1] 28518
```
-Just as for the other asynchronous evaluation strategies, the output from `cat()` is not displayed on the current/calling machine.
+Just as for most other asynchronous evaluation strategies, the output from `cat()` is not displayed on the current/calling machine.
Any types of clusters that `parallel::makeCluster()` creates can be used for cluster futures. For instance, the above cluster can be explicitly set up as:
```r
cl <- parallel::makeCluster(c("n1", "n2", "n3"))
-plan(cluster, workers=cl)
+plan(cluster, workers = cl)
```
-Also, it is considered good style to shut down the cluster when it is no longer needed, that is, calling `parallel::stopCluster(cl)`. However, it will shut itself down if the main process is terminated, which will happen in the first example where the cluster in created internally. For more information on how to set up and manage such clusters, see `help("makeCluster", package="parallel")`.
+Also, it is considered good style to shut down cluster `cl` when it is no longer needed, that is, calling `parallel::stopCluster(cl)`. However, it will shut itself down if the main process is terminated. For more information on how to set up and manage such clusters, see `help("makeCluster", package = "parallel")`.
+Clusters created implicitly using `plan(cluster, workers = hosts)` where `hosts` is a character vector will also be shut down when the main R session terminates, or when the future strategy is changed, e.g. by calling `plan(sequential)`.
Note that with automatic authentication setup (e.g. SSH key pairs), there is nothing preventing us from using the same approach for using a cluster of remote machines.
-### Different Strategies for Different Futures
-Sometimes one may want to use an alternative evaluation strategy for a specific future. Although one can use `old <- plan(new)` and afterward `plan(old)` to temporarily switch strategies, a simpler approach is to use the `%plan%` operator, e.g.
-```r
-> plan(eager)
-> pid <- Sys.getpid()
-> pid
-[1] 10708
-> a %<-% {
-+ Sys.getpid()
-+ }
-> b %<-% {
-+ Sys.getpid()
-+ } %plan% multiprocess
-> c %<-% {
-+ Sys.getpid()
-+ } %plan% multiprocess
-> a
-[1] 10708
-> b
-[1] 10769
-> c
-[1] 10770
-```
-As seen by the different process IDs, future `a` is evaluated eagerly using the same process as the calling environment whereas the other two are evaluated using multiprocess futures.
-
-
-
-
-However, using different plans to individual futures this way has the drawback of hard coding the evaluation strategy. Doing so may prevent some users from using your script or your package, because they do not have the sufficient resources. It may also prevent users with a lot of resources from utilizing those because you assumed a less-powerful set of hardware. Because of this, we recommend against the use of `%plan%` other than for interactive prototyping.
-
### Nested Futures and Evaluation Topologies
This far we have discussed what can be referred to as "flat topology" of futures, that is, all futures are created in and assigned to the same environment. However, there is nothing stopping us from using a "nested topology" of futures, where one set of futures may, in turn, create another set of futures internally and so on.
@@ -403,71 +336,69 @@ For instance, here is an example of two "top" futures (`a` and `b`) that uses mu
+ c(b.pid = Sys.getpid(), b1.pid = b1, b2.pid = b2)
+ }
> pid
-[1] 10708
+[1] 28518
> a
-[1] 10771
+[1] 28584
> b
b.pid b1.pid b2.pid
- 10772 10772 10772
+ 28585 28585 28585
```
-By inspection the process IDs, we see that there are in total three different processes involved for resolving the futures. There is the main R process (pid 10708), and there are the two processes used by `a` (pid 10771) and `b` (pid 10772). However, the two futures (`b1` and `b2`) that is nested by `b` are evaluated by the same R process as `b`. This is because nested futures use eager evaluation unless otherwise specified. There are a few reasons for this, but the main reason is th [...]
+By inspection the process IDs, we see that there are in total three different processes involved for resolving the futures. There is the main R process (pid 28518), and there are the two processes used by `a` (pid 28584) and `b` (pid 28585). However, the two futures (`b1` and `b2`) that is nested by `b` are evaluated by the same R process as `b`. This is because nested futures use sequential evaluation unless otherwise specified. There are a few reasons for this, but the main reason [...]
-To specify a different type of _evaluation topology_, other than the first level of futures being resolved by multiprocess evaluation and the second level by eager evaluation, we can provide a list of evaluation strategies to `plan()`. First, the same evaluation strategies as above can be explicitly specified as:
+To specify a different type of _evaluation topology_, other than the first level of futures being resolved by multiprocess evaluation and the second level by sequential evaluation, we can provide a list of evaluation strategies to `plan()`. First, the same evaluation strategies as above can be explicitly specified as:
```r
-plan(list(multiprocess, eager))
+plan(list(multiprocess, sequential))
```
We would actually get the same behavior if we try with multiple levels of multiprocess evaluations;
```r
> plan(list(multiprocess, multiprocess))
[...]
> pid
-[1] 10708
+[1] 28518
> a
-[1] 10773
+[1] 28586
> b
b.pid b1.pid b2.pid
- 10774 10774 10774
+ 28587 28587 28587
```
-The reason for this is, also here, to protect us from launching more processes than what the machine can support. Internally, this is done by setting `mc.cores` to zero ([sic!](https://github.com/HenrikBengtsson/Wishlist-for-R/issues/7)) such that no _additional_ parallel processes can be launched. This is the case for both multisession and multicore evaluation.
+The reason for this is, also here, to protect us from launching more processes than what the machine can support. Internally, this is done by setting `mc.cores = 1` such that functions like `parallel::mclapply()` will fall back to run sequentially. This is the case for both multisession and multicore evaluation.
-Continuing, if we start off by eager evaluation and then use multiprocess evaluation for any nested futures, we get:
+Continuing, if we start off by sequential evaluation and then use multiprocess evaluation for any nested futures, we get:
```r
-> plan(list(eager, multiprocess))
+> plan(list(sequential, multiprocess))
[...]
Resolving 'a' ...
Resolving 'b' ...
> pid
-[1] 10708
+[1] 28518
> a
-[1] 10708
+[1] 28518
> b
b.pid b1.pid b2.pid
- 10708 10775 10776
+ 28518 28588 28589
```
-which clearly show that `a` and `b` are resolved in the calling process (pid 10708) whereas the two nested futures (`b1` and `b2`) are resolved in two separate R processes (pids 10775 and 10776).
+which clearly show that `a` and `b` are resolved in the calling process (pid 28518) whereas the two nested futures (`b1` and `b2`) are resolved in two separate R processes (pids 28588 and 28589).
Having said this, it is indeed possible to use nested multiprocess evaluation strategies, if we explicitly specify (read _force_) the number of cores available at each level. In order to do this we need to "tweak" the default settings, which can be done as follows:
```r
-> plan(list(tweak(multiprocess, workers = 3L), tweak(multiprocess,
-+ workers = 3L)))
+> plan(list(tweak(multiprocess, workers = 2L), tweak(multiprocess,
++ workers = 2L)))
[...]
> pid
-[1] 10708
+[1] 28518
> a
-[1] 10777
+[1] 28590
> b
b.pid b1.pid b2.pid
- 10778 10779 10840
+ 28591 28592 28594
```
-First, we see that both `a` and `b` are resolved in different processes (pids 10777 and 10778) than the calling process (pid 10708). Second, the two nested futures (`b1` and `b2`) are resolved in yet two other R processes (pids 10779 and 10840).
-
+First, we see that both `a` and `b` are resolved in different processes (pids 28590 and 28591) than the calling process (pid 28518). Second, the two nested futures (`b1` and `b2`) are resolved in yet two other R processes (pids 28592 and 28594).
-To clarify, when we set up the two levels of multiprocess evaluation, we specified that in total 3 processes may be used at each level. We choose three parallel processes, not just two, because one is always consumed by the calling process leaving two to be used for the asynchronous futures. This is why we see that `pid`, `a` and `b` are all resolved by the same process. If we had allowed only two cores at the top level, `a` and `b` would have been resolved by the same background proc [...]
For more details on working with nested futures and different evaluation strategies at each level, see Vignette '[Futures in R: Future Topologies]'.
@@ -499,19 +430,19 @@ Waiting for 'a' to be resolved ...
> cat("Waiting for 'a' to be resolved ... DONE\n")
Waiting for 'a' to be resolved ... DONE
> a
-[1] 10841
+[1] 28595
```
## Failed Futures
-Sometimes the future is not what you expected. If an error occurs while evaluating a future, the error is propagated and thrown as an error in the calling environment _when the future value is requested_. For example,
+Sometimes the future is not what you expected. If an error occurs while evaluating a future, the error is propagated and thrown as an error in the calling environment _when the future value is requested_. For example, if we use lazy evaluation on a future that generates an error, we might see something like
```r
-> plan(lazy)
+> plan(sequential)
> a %<-% {
+ cat("Resolving 'a' ...\n")
+ stop("Whoops!")
+ 42
-+ }
++ } %lazy% TRUE
> cat("Everything is still ok although we have created a future that will fail.\n")
Everything is still ok although we have created a future that will fail.
> a
@@ -530,12 +461,16 @@ To see the list of calls (evaluated expressions) that lead up to the error, we c
> backtrace(a)
[[1]]
eval(quote({
- cat("Resolving 'a' ...\n")
+ cat("Resolving 'a' ...\\n")
stop("Whoops!")
42
}), new.env())
[[2]]
-eval(expr, envir, enclos)
+eval(quote({
+ cat("Resolving 'a' ...\\n")
+ stop("Whoops!")
+ 42
+}), new.env())
[[3]]
stop("Whoops!")
```
@@ -543,13 +478,12 @@ stop("Whoops!")
## Globals
-Whenever an R expression is to be evaluated asynchronously (in parallel) or via lazy evaluation, global objects have to be identified and passed to the evaluator. They need to be passed exactly as they were at the time the future was created, because, for a lazy future, globals may otherwise change between when it is created and when it is resolved. For asynchronous processing, the reason globals need to be identified is so that they can be exported to the process that evaluates the future.
+Whenever an R expression is to be evaluated asynchronously (in parallel) or sequentially via lazy evaluation, global (aka "free") objects have to be identified and passed to the evaluator. They need to be passed exactly as they were at the time the future was created, because, for lazy evaluation, globals may otherwise change between when it is created and when it is resolved. For asynchronous processing, the reason globals need to be identified is so that they can be exported to the p [...]
-The future package tries to automate these tasks as far as possible. It does this with help of the [globals] package. If a global variable is identified, it is captured and made available to the evaluating process.
+The future package tries to automate these tasks as far as possible. It does this with help of the [globals] package, which uses static-code inspection to identify global variables. If a global variable is identified, it is captured and made available to the evaluating process.
Moreover, if a global is defined in a package, then that global is not exported. Instead, it is made sure that the corresponding package is attached when the future is evaluated. This not only better reflects the setup of the main R session, but it also minimizes the need for exporting globals, which saves not only memory but also time and bandwidth, especially when using remote compute nodes.
-Finally, it should be clarified that identifying globals from static code inspection alone is a challenging problem. There will always be corner cases where automatic identification of globals fails so that either false globals are identified (less of a concern) or some of the true globals are missing (which will result in a runtime error or possibly the wrong results). Vignette '[Futures in R: Common Issues with Solutions]' provides examples of common cases and explains how to avoid t [...]
-
+Finally, it should be clarified that identifying globals from static code inspection alone is a challenging problem. There will always be corner cases where automatic identification of globals fails so that either false globals are identified (less of a concern) or some of the true globals are missing (which will result in a run-time error or possibly the wrong results). Vignette '[Futures in R: Common Issues with Solutions]' provides examples of common cases and explains how to avoid [...]
## Constraints when using Implicit Futures
@@ -566,9 +500,9 @@ There is one limitation with implicit futures that does not exist for explicit o
> v <- lapply(f, FUN = value)
> str(v)
List of 3
- $ : int 10921
- $ : int 10922
- $ : int 10924
+ $ : int 28602
+ $ : int 28603
+ $ : int 28605
```
This is _not_ possible to do when using implicit futures. This is because the `%<-%` assignment operator _cannot_ be used in all cases where the regular `<-` assignment operator can be used. It can only be used to assign future values to _environments_ (including the calling environment) much like how `assign(name, value, envir)` works. However, we can assign implicit futures to environments using _named indices_, e.g.
```r
@@ -582,9 +516,9 @@ This is _not_ possible to do when using implicit futures. This is because the `
> v <- as.list(v)
> str(v)
List of 3
- $ a: int 10925
- $ b: int 10928
- $ c: int 10931
+ $ a: int 28606
+ $ b: int 28607
+ $ c: int 28608
```
Here `as.list(v)` blocks until all futures in the environment `v` have been resolved. Then their values are collected and returned as a regular list.
@@ -601,38 +535,30 @@ If _numeric indices_ are required, then _list environments_ can be used. List e
> v <- as.list(v)
> str(v)
List of 3
- $ : int 10932
- $ : int 10933
- $ : int 10934
+ $ : int 28609
+ $ : int 28610
+ $ : int 28611
```
As previously, `as.list(v)` blocks until all futures are resolved.
## Demos
-To see a live illustration how different types of futures are evaluated, run the Mandelbrot demo of this package. First, try with the eager evaluation,
+To see a live illustration how different types of futures are evaluated, run the Mandelbrot demo of this package. First, try with the sequential evaluation,
```r
library("future")
-plan(eager)
-demo("mandelbrot", package="future", ask=FALSE)
+plan(sequential)
+demo("mandelbrot", package = "future", ask = FALSE)
```
-which closely imitates how the script would run if futures were not used. Then try the same using lazy evaluation,
-```r
-plan(lazy)
-demo("mandelbrot", package="future", ask=FALSE)
-```
-and see if you can notice the difference in how and when statements are evaluated.
-You may also try multiprocess evaluation, which calculates the different Mandelbrot planes using parallel R processes running in the background. Try,
+which resembles how the script would run if futures were not used. Then, try multiprocess evaluation, which calculates the different Mandelbrot planes using parallel R processes running in the background. Try,
```r
plan(multiprocess)
-demo("mandelbrot", package="future", ask=FALSE)
+demo("mandelbrot", package = "future", ask = FALSE)
```
-This will use multicore processing if you are on a system where R supports process forking, otherwise (such as on Windows) it will use multisession processing.
-
Finally, if you have access to multiple machines you can try to set up a cluster of workers and use them, e.g.
```r
-plan(cluster, workers=c("n2", "n5", "n6", "n6", "n9"))
-demo("mandelbrot", package="future", ask=FALSE)
+plan(cluster, workers = c("n2", "n5", "n6", "n6", "n9"))
+demo("mandelbrot", package = "future", ask = FALSE)
```
@@ -642,11 +568,13 @@ The goal of this package is to provide a standardized and unified API for using
[BatchJobs]: https://cran.r-project.org/package=BatchJobs
+[batchtools]: https://cran.r-project.org/package=batchtools
[future]: https://cran.r-project.org/package=future
[future.BatchJobs]: https://cran.r-project.org/package=future.BatchJobs
+[future.batchtools]: https://cran.r-project.org/package=future.batchtools
[globals]: https://cran.r-project.org/package=globals
[listenv]: https://cran.r-project.org/package=listenv
[Futures in R: Common Issues with Solutions]: future-2-issues.html
[Futures in R: Future Topologies]: future-3-topologies.html
---
-Copyright Henrik Bengtsson, 2015-2016
+Copyright Henrik Bengtsson, 2015-2017
diff --git a/vignettes/future-2-issues.md.rsp b/vignettes/future-2-issues.md.rsp
index d45beaf..7f99d60 100644
--- a/vignettes/future-2-issues.md.rsp
+++ b/vignettes/future-2-issues.md.rsp
@@ -27,9 +27,9 @@ _If you identify other cases, please consider [reporting](https://github.com/Hen
If a global variable is (only) part of a "self" assignment, such as `x <- x + 1`, then the global is not identified and therefore neither exported / frozen. This explains the error in the following example:
```r
> library("future")
-> plan("lazy")
+> plan(sequential)
> x <- 0
-> y %<-% { x <- x + 1 }
+> y %<-% { x <- x + 1 } %lazy% TRUE
> rm(x)
> y
Error: object 'x' not found
@@ -38,9 +38,9 @@ Error: object 'x' not found
The workaround is to guide R to find the missing global by explicitly listing it, e.g.
```r
> library("future")
-> plan("lazy")
+> plan(sequential)
> x <- 0
-> y %<-% { x; x <- x + 1 }
+> y %<-% { x; x <- x + 1 } %lazy% TRUE
> rm(x)
> y
[1] 1
@@ -60,16 +60,16 @@ instead of
do.call("file_ext", list("foo.txt"))
```
so that `file_ext()` is properly located and exported. Although you may not notice a difference when evaluating futures in the same R session, it may become a problem if you use a character string instead of a function object when futures are evaluated in external R sessions, such as on a cluster.
-It may also become a problem with lazy futures if the intended function is redefined after the future is resolved. For example,
+It may also become a problem with futures evaluated with lazy evaluation if the intended function is redefined after the future is resolved. For example,
```r
> library("future")
> library("listenv")
> library("tools")
-> plan(lazy)
+> plan(sequential)
> pathnames <- c("foo.txt", "bar.png", "yoo.md")
> res <- listenv()
> for (ii in seq_along(pathnames)) {
-+ res[[ii]] %<-% do.call("file_ext", list(pathnames[ii]))
++ res[[ii]] %<-% do.call("file_ext", list(pathnames[ii])) %lazy% TRUE
+ }
> file_ext <- function(...) "haha!"
> unlist(res)
@@ -133,7 +133,7 @@ Error: Invalid usage of futures: A future whose value has not yet been collected
```
As previously, this can be avoided by making sure `x$a` is resolved first, which can be one in various ways, e.g. `dummy <- x$a`, `resolve(x$a)` and `force(x$a)`.
-_Footnote_: (*) Although eager futures could be passed on to other futures part of the same R process and be resolved there because they share the same evaluation process, by definition of the Future API it is invalid to do so regardless of future type. This conservative approach is taken in order to make future expressions behave consistently regardless of the type of future used.
+_Footnote_: (*) Although sequential futures could be passed on to other futures part of the same R process and be resolved there because they share the same evaluation process, by definition of the Future API it is invalid to do so regardless of future type. This conservative approach is taken in order to make future expressions behave consistently regardless of the type of future used.
## Miscellaneous
@@ -249,4 +249,4 @@ foo <- function() {
[listenv]: https://cran.r-project.org/package=listenv
---
-Copyright Henrik Bengtsson, 2015-2016
+Copyright Henrik Bengtsson, 2015-2017
diff --git a/vignettes/future-3-topologies.md.rsp b/vignettes/future-3-topologies.md.rsp
index c8962f2..a19fd8f 100644
--- a/vignettes/future-3-topologies.md.rsp
+++ b/vignettes/future-3-topologies.md.rsp
@@ -26,15 +26,15 @@ for (ii in 1:3) {
x[[ii]] %<-% {
y <- listenv()
for (jj in 1:3) {
- y[[jj]] %<-% { ii + jj/10 }
+ y[[jj]] %<-% { ii + jj / 10 }
}
- y
+ y
}
}
unlist(x)
## [1] 1.1 1.2 1.3 2.1 2.2 2.3 3.1 3.2 3.3
```
-The default is to use synchronous futures unless otherwise specified, which is also true for nested futures. If we for instance specify, `plan(multiprocess)`, the first layer of futures (`x[[ii]] %<-% { expr }`) will be processed asynchronously in background R processes, and the futures in the second layer of futures (`y[[jj]] %<-% { expr }`) will be processed synchronously in the current R processes (one of the background processes). If we wish to use a different strategy for the seco [...]
+The default is to use synchronous futures unless otherwise specified, which is also true for nested futures. If we for instance specify, `plan(multiprocess)`, the first layer of futures (`x[[ii]] %<-% { expr }`) will be processed asynchronously in background R processes, and the futures in the second layer of futures (`y[[jj]] %<-% { expr }`) will be processed synchronously in the separate background R processes. If we wish to be explicit about this, we can specify `plan(list(multiproc [...]
## Example: High-Throughput Sequencing
@@ -46,7 +46,7 @@ library("future")
library("listenv")
htseq_align <- function(fq, chr) { chr }
-fqs <- dir(pattern="[.]fastq$")
+fqs <- dir(pattern = "[.]fastq$")
bams <- listenv()
for (ss in seq_along(fqs)) {
@@ -54,7 +54,7 @@ for (ss in seq_along(fqs)) {
bams[[ss]] %<-% {
bams_ss <- listenv()
for (cc in 1:24) {
- bams_ss[[cc]] %<-% htseq_align(fq, chr=cc)
+ bams_ss[[cc]] %<-% htseq_align(fq, chr = cc)
}
as.list(bams_ss)
}
@@ -68,22 +68,22 @@ The default is to use synchronous futures, so without further specifications, th
* A compute cluster with 3 machines each with 16 cores
### One multi-core machine
-With a single machine of 8 cores, we could choose to process multiple samples at the same time while processing chromosomes sequentially. In other words, we would like to evaluate the outer layer of futures using multiprocess futures and the inner ones as eager synchronous futures. This can be specified as:
+With a single machine of 8 cores, we could choose to process multiple samples at the same time while processing chromosomes sequentially. In other words, we would like to evaluate the outer layer of futures using multiprocess futures and the inner ones as sequential futures. This can be specified as:
```r
-plan(list(multiprocess, eager))
+plan(list(multiprocess, sequential))
```
The internals for processing multiprocess future queries `availableCores()` to infer how many cores can be used simultaneously, so there is no need to explicitly specify that there are 8 cores available.
-_Comment_: Since synchronous is the default future, we could skip trailing synchronous eager futures in the setup, e.g. `plan(list(multiprocess))` or just `plan(eager)`. However, it does not hurt to be explicit.
+_Comment_: Since synchronous is the default future, we could skip trailing sequential futures in the setup, e.g. `plan(list(multiprocess))` or just `plan(multiprocess)`. However, it does not hurt to be explicit.
If we instead would like to process the sample sequentially and the chromosomes in parallel, we can use:
```r
-plan(list(eager, multiprocess))
+plan(list(sequential, multiprocess))
```
We could also process the data such that we allocate two cores for processing two samples in parallel each using four cores for processing four chromosomes in parallel:
```r
-plan(list(tweak(multiprocess, workers=2), tweak(multiprocess, workers=4)))
+plan(list(tweak(multiprocess, workers = 2), tweak(multiprocess, workers = 4)))
```
@@ -91,16 +91,16 @@ plan(list(tweak(multiprocess, workers=2), tweak(multiprocess, workers=4)))
With a compute cluster of 3 machines each with 16 cores, we can run up to 48 alignment processes in parallel. A natural setup is to have one machine process one sample in parallel. We could specify this as:
```r
nodes <- c("n1", "n2", "n3")
-plan(list(tweak(cluster, workers=nodes), multiprocess))
+plan(list(tweak(cluster, workers = nodes), multiprocess))
```
_Comment:_ Multiprocess futures are agile to its environment, that is, they will query the machine they are running on to find out how many parallel processes it can run at the same time.
One possible downside to the above setup is that we might not utilize all available cores all the time. This is because the alignment of the shorter chromosomes will finish sooner than the longer ones, which means that we might at the end of each sample have only a few alignment processes running on each machine leaving the remaining cores idle/unused. An alternative set up is then to use the following setup:
```r
-nodes <- rep(c("n1", "n2", "n3"), each=8)
-plan(list(tweak(cluster, workers=nodes), multiprocess))
+nodes <- rep(c("n1", "n2", "n3"), each = 8)
+plan(list(tweak(cluster, workers = nodes), multiprocess))
```
-This will cause up to 24 (=3*8) samples to be processed in parallel each processing two chromosomes at the same time.
+This will cause up to 24 (= 3*8) samples to be processed in parallel each processing two chromosomes at the same time.
## Example: A remote compute cluster
@@ -113,7 +113,7 @@ library("future")
library("listenv")
## Set up access to remote login node
-login <- tweak(remote, workers="remote.server.org")
+login <- tweak(remote, workers = "remote.server.org")
plan(login)
## Set up cluster nodes on login node
@@ -123,7 +123,7 @@ nodes %<-% { .keepme <- parallel::makeCluster(c("n1", "n2", "n3")) }
## login node -> { cluster nodes } -> { multiple cores }
plan(list(
login,
- tweak(cluster, workers=nodes),
+ tweak(cluster, workers = nodes),
multiprocess
))
@@ -141,9 +141,11 @@ x %<-% {
z <- listenv()
for (jj in 1:2) {
## (c) These will be evaluated in separate processes on the same compute node
- z[[jj]] %<-% data.frame(task=task, top.host=thost, top.pid=tpid,
- mid.host=mhost, mid.pid=mpid,
- host=Sys.info()[["nodename"]], pid=Sys.getpid())
+ z[[jj]] %<-% data.frame(task = task,
+ top.host = thost, top.pid = tpid,
+ mid.host = mhost, mid.pid = mpid,
+ host = Sys.info()[["nodename"]],
+ pid = Sys.getpid())
}
Reduce(rbind, z)
}
@@ -163,7 +165,7 @@ print(x)
## 8 4 login 391547 n1 391878 n1 393966
```
-Try the above `x %<-% { ... }` future with, say, `plan(list(lazy, multiprocess))` or `plan(list(eager, lazy, multiprocess))` and see what the output will be.
+Try the above `x %<-% { ... }` future with, say, `plan(list(sequential, multiprocess))` and see what the output will be.
[listenv]: https://cran.r-project.org/package=listenv
@@ -171,4 +173,4 @@ Try the above `x %<-% { ... }` future with, say, `plan(list(lazy, multiprocess))
[Futures in R: Common issues with solutions]: future-issues.html
---
-Copyright Henrik Bengtsson, 2015-2016
+Copyright Henrik Bengtsson, 2015-2017
diff --git a/vignettes/future-4-startup.md.rsp b/vignettes/future-4-startup.md.rsp
index e8637d8..ea59e45 100644
--- a/vignettes/future-4-startup.md.rsp
+++ b/vignettes/future-4-startup.md.rsp
@@ -12,18 +12,18 @@
<%
library("R.utils")
`%<-%` <- future::`%<-%`
-options("withCapture/newline"=FALSE)
+options("withCapture/newline" = FALSE)
%>
# <%@meta name="title"%>
-The default is to use synchronous futures, but this _default_ can be overridden via R options, system environment variables and command-line options as explained below as well as in `help("future.options", package="future")`.
+The default is to use synchronous futures, but this _default_ can be overridden via R options, system environment variables and command-line options as explained below as well as in `help("future.options", package = "future")`.
## R options
The default strategy for resolving futures can be controlled via R option `future.plan`. For instance, if we add
```r
-options(future.plan="multiprocess")
+options(future.plan = "multiprocess")
```
to our `~/.Rprofile` startup script, the future package will resolve futures in parallel (asynchronously using all available cores), i.e.
```sh
@@ -53,8 +53,8 @@ $ R --quiet --args --parallel=2
then future will interpret this as we wish to resolve futures in parallel using 2 cores. More specifically, we get that
```r
> future::availableCores()
-mc.cores+1
- 2
+mc.cores
+ 2
> class(future::plan())
[1] "tweaked" "multiprocess" "future" "function"
```
@@ -73,4 +73,4 @@ Specifying these command-line options override any other startup settings.
[future]: https://cran.r-project.org/package=future
---
-Copyright Henrik Bengtsson, 2015-2016
+Copyright Henrik Bengtsson, 2015-2017
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/r-cran-future.git
More information about the debian-med-commit
mailing list