diff --git a/DESCRIPTION b/DESCRIPTION
index 1d0d036..518cda1 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,6 +1,6 @@
Package: simpleCache
-Version: 0.2.1
-Date: 2017-08-03
+Version: 0.3.0
+Date: 2017-08-21
Title: Simply Caching R Objects
Description: Provides intuitive functions for caching R objects, encouraging
reproducible, restartable, and distributed R analysis. The user selects a
diff --git a/NAMESPACE b/NAMESPACE
index aa89046..6462911 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -1,7 +1,7 @@
# Generated by roxygen2: do not edit by hand
export(addCacheSearchEnvironment)
-export(availCaches)
+export(listCaches)
export(loadCaches)
export(resetCacheSearchEnvironment)
export(setCacheBuildDir)
diff --git a/NEWS b/NEWS
index 5dcd18d..a162b38 100644
--- a/NEWS
+++ b/NEWS
@@ -1,19 +1,23 @@
# Change log
All notable changes to this project will be documented in this file.
-## [0.2.1] -- 2017-07-30
+## [0.3.0] -- 2017-08-21
-### Added
+ - Switched default cache dir to tempdir()
+ - changed availCaches() to listCaches()
+ - changes cache building to happen in parent.frame(), so that any loaded
+ packages are available for cache building
- - Added examples.
-## [0.2.0] -- 2017-07-30
+## [0.2.1] -- 2017-07-30
-### Added
+ - Added examples
+
+## [0.2.0] -- 2017-07-30
- support for batchjobs parallel processing
- docs, prep for submission to CRAN
## [0.0.1]
-Long-term stable version
+ - Long-term stable version
diff --git a/R/examples/example.R b/R/examples/example.R
index 8e59882..c15a7ec 100644
--- a/R/examples/example.R
+++ b/R/examples/example.R
@@ -1,5 +1,5 @@
# choose location to store caches
-cacheDir = system.file("cache", package="simpleCache")
+cacheDir = tempdir()
cacheDir
setCacheDir(cacheDir)
@@ -13,7 +13,7 @@ normSample2 = rnorm(10, 0, 1)
storeCache("normSample2")
# what's available?
-availCaches()
+listCaches()
# load a cache
simpleCache("normSample")
diff --git a/R/listCaches.R b/R/listCaches.R
new file mode 100644
index 0000000..dff1b03
--- /dev/null
+++ b/R/listCaches.R
@@ -0,0 +1,14 @@
+#' Show available caches.
+#'
+#' Lists any cache files in the cache directory.
+#'
+#' @param cacheSubDir Optional parameter to specify a subdirectory of the cache folder.
+#' @return \code{character} vector in which each element is the path to a file that
+#' represents an available cache (within \code{getOption("RCACHE.DIR")})
+#' @export
+#' @example
+#' R/examples/example.R
+listCaches = function(cacheSubDir="") {
+ list.files(paste0(getOption("RCACHE.DIR"), cacheSubDir))
+}
+
diff --git a/R/loadCaches.R b/R/loadCaches.R
index 2d95930..b20981e 100644
--- a/R/loadCaches.R
+++ b/R/loadCaches.R
@@ -16,17 +16,3 @@ loadCaches = function(cacheNames, ...) {
simpleCache(cacheNames[i], loadEnvir=parent.frame(n=2), ...)
}
}
-
-
-#' Show available caches.
-#'
-#' Lists any cache files in the cache directory.
-#'
-#' @param cacheSubDir Optional parameter to specify a subdirectory of the cache folder.
-#' @export
-#' @example
-#' R/examples/example.R
-availCaches = function(cacheSubDir="") {
- list.files(paste0(getOption("RCACHE.DIR"), cacheSubDir))
-}
-
diff --git a/R/simpleCache.R b/R/simpleCache.R
index 9703350..93f5909 100755
--- a/R/simpleCache.R
+++ b/R/simpleCache.R
@@ -112,9 +112,11 @@ simpleCache = function(cacheName, instruction=NULL, buildEnvir=NULL,
cacheDir = file.path(cacheDir, cacheSubDir)
}
if (is.null(cacheDir)) {
- message(strwrap("You must set global option RCACHE.DIR with setSharedCacheDir(),
- or specify a cacheDir parameter directly to simpleCache()."))
- return(NA)
+ message(strwrap("No cacheDir specified. You should set global option
+ RCACHE.DIR with setCacheDir(), or specify a cacheDir parameter directly
+ to simpleCache(). With no other option, simpleCache will use tempdir():
+ ", initial="", prefix=" "), tempdir())
+ cacheDir = tempdir()
}
if (!"character" %in% class(cacheName)) {
stop("simpleCache expects the cacheName variable to be a character
@@ -245,23 +247,43 @@ simpleCache = function(cacheName, instruction=NULL, buildEnvir=NULL,
# No cluster submission request, so just run it here!
# "ret," for return, is the name the cacheName is stored under.
if (parse) {
- ret = eval(parse(text=instruction))
+ ret = eval(parse(text=instruction), envir=parent.frame())
} else {
- ret = eval( instruction )
+ # Here we do the evaluation in the parent frame so that
+ # it will have access to any packages the user has loaded
+ # that may be required to run the code. Otherwise, it will
+ # run in the simpleCache namespace which could lack these
+ # packages (or have a different search path hierarchy),
+ # leading to failures. The `substitute` call here ensures
+ # the code isn't evaluated at argument stage, but is retained
+ # until it makes it to the `eval` call.
+ ret = eval(instruction, envir=parent.frame())
}
}
if (timer) { toc() }
} else {
# Build environment was provided.
+ # we must place the instruction in the environment to build from
+ if (exists("instruction", buildEnvir)) {
+ stop("Can't provide a variable named 'instruction' in buildEnvir")
+ }
+ buildEnvir$instruction = instruction
+ be = as.environment(buildEnvir)
+ # As described above, this puts global package functions into
+ # scope so instructions can use them.
+ parent.env(be) = parent.frame()
if (timer) { tic() }
if (parse) {
- ret = with(buildEnvir, eval(parse(text=instruction)))
+ ret = with(be, eval(parse(text=instruction)))
} else {
- ret = with(buildEnvir, eval(instruction))
+ #ret = with(buildEnvir, evalq(instruction))
+ ret = with(be, eval(instruction))
+
}
if (timer) { toc() }
}
}
+
# tryCatch
}, error = function(e) { if (nofail) warning(e) else stop(e) })
diff --git a/README.md b/README.md
index 57dd3f4..2ae50cc 100644
--- a/README.md
+++ b/README.md
@@ -1,16 +1,33 @@
simpleCache: R caching for restartable analysis
-----------------------------------------------
-<a href="https://travis-ci.org/databio/simpleCache"><img src="https://travis-ci.org/databio/simpleCache.svg?branch=master" alt="Travis CI status"></img>
-</a>
+<a href="https://travis-ci.org/databio/simpleCache"><img src="https://travis-ci.org/databio/simpleCache.svg?branch=master" alt="Travis CI status"></img></a>
-`simpleCache` is an R package providing functions for caching R objects. Its purpose is to encourage writing reusable, restartable, and reproducible analysis pipelines for projects with massive data and computational requirements.
+`simpleCache` is an R package providing functions for caching R objects. Its
+purpose is to encourage writing reusable, restartable, and reproducible analysis
+pipelines for projects with massive data and computational requirements.
-Like its name indicates, `simpleCache` is intended to be simple. You choose a location to store your caches, and then provide the function with nothing more than a cache name and instructions (R code) for how to produce the R object. While simple, `simpleCache` also provides some advanced options like environment assignments, recreating caches, reloading caches, and even cluster compute bindings (using the `batchtools` package) making it flexible enough for use in large-scale data analysis projects.
+Like its name indicates, `simpleCache` is intended to be simple. You choose a
+location to store your caches, and then provide the function with nothing more
+than a cache name and instructions (R code) for how to produce the R object.
+While simple, `simpleCache` also provides some advanced options like environment
+assignments, recreating caches, reloading caches, and even cluster compute
+bindings (using the `batchtools` package) making it flexible enough for use in
+large-scale data analysis projects.
--------------------------------------------------------------------------------
### Installing simpleCache
-Install the development version directly from github with devtools
+
+`simpleCache` is on
+[CRAN](https://cran.r-project.org/web/packages/simpleCache/index.html) and can
+be installed as usual:
+
+```
+install.packages("simpleCache")
+```
+
+If you like, you may install the development version directly from github with
+devtools
```
devtools::install_github("databio/simpleCache")
@@ -18,22 +35,22 @@ devtools::install_github("databio/simpleCache")
To install a local copy:
```
-packageFolder = "~/R/simpleCache";
-install.packages(packageFolder, repos=NULL)
+packageFolder = "~/R/simpleCache"; install.packages(packageFolder, repos=NULL)
```
--------------------------------------------------------------------------------
### Running simpleCache
-`simpleCache` comes with a single primary function that will do almost everything you need. In short, you run it with a few lines like this:
+`simpleCache` comes with a single primary function that will do almost
+everything you need. In short, you run it with a few lines like this:
```
-library(simpleCache)
-setCacheDir("~")
-simpleCache("normSample", { rnorm(1e7, 0,1) }, recreate=TRUE)
-simpleCache("normSample", { rnorm(1e7, 0,1) })
+library(simpleCache) setCacheDir(tempdir()) simpleCache("normSample", {
+rnorm(1e7, 0,1) }, recreate=TRUE) simpleCache("normSample", { rnorm(1e7, 0,1) })
```
-`simpleCache` also interfaces with the `batchtools` package to let you build caches on any cluster resource manager. I have produced some [R vignettes](vignettes/) to get you started.
+`simpleCache` also interfaces with the `batchtools` package to let you build
+caches on any cluster resource manager. I have produced some [R
+vignettes](vignettes/) to get you started.
* [An introduction to simpleCache](vignettes/simpleCacheIntroduction.Rmd)
* [Sharing caches across projects](vignettes/sharingCaches.Rmd)
@@ -42,11 +59,29 @@ simpleCache("normSample", { rnorm(1e7, 0,1) })
--------------------------------------------------------------------------------
### simpleCache Philosophy
-The use case I had in mind for `simpleCache` is that you find yourself constantly recalculating the same R object in several different scripts, or repeatedly in the same script, every time you open it and want to continue that project. SimpleCache is well-suited for interactive analysis, allowing you to pick up right where you left off in a new R session, without having to recalculate everything. It is equally useful in automatic pipelines, where separate scripts may benefit from loading, instead of recalculating, the same R objects produced by other scripts.
-
-R provides some base functions (`save`, `serialize`, and `load`) to let you save and reload such objects, but these low-level functions are a bit cumbersome. `simpleCache` simply provides a convenient, user-friendly interface to these functions, streamlining the process. For example, a single `simpleCache` call will check for a cache and load it if it exists, or create it if it does not. With the base R `save` and `load` functions, you can't just write a single function call and then run the same thing every time you start the script -- even this simple use case requires additional logic to check for an existing cache. SimpleCache just does all this for you.
-
-They thing to keep in mind with simpleCache is that **the cache name is paramount**. SimpleCache assumes that your name for an object is a perfect identifier for that object; in other words, don't cache things that you plan to change.
+The use case I had in mind for `simpleCache` is that you find yourself
+constantly recalculating the same R object in several different scripts, or
+repeatedly in the same script, every time you open it and want to continue that
+project. SimpleCache is well-suited for interactive analysis, allowing you to
+pick up right where you left off in a new R session, without having to
+recalculate everything. It is equally useful in automatic pipelines, where
+separate scripts may benefit from loading, instead of recalculating, the same R
+objects produced by other scripts.
+
+R provides some base functions (`save`, `serialize`, and `load`) to let you save
+and reload such objects, but these low-level functions are a bit cumbersome.
+`simpleCache` simply provides a convenient, user-friendly interface to these
+functions, streamlining the process. For example, a single `simpleCache` call
+will check for a cache and load it if it exists, or create it if it does not.
+With the base R `save` and `load` functions, you can't just write a single
+function call and then run the same thing every time you start the script --
+even this simple use case requires additional logic to check for an existing
+cache. SimpleCache just does all this for you.
+
+They thing to keep in mind with simpleCache is that **the cache name is
+paramount**. SimpleCache assumes that your name for an object is a perfect
+identifier for that object; in other words, don't cache things that you plan to
+change.
diff --git a/man/availCaches.Rd b/man/listCaches.Rd
similarity index 71%
rename from man/availCaches.Rd
rename to man/listCaches.Rd
index 1fc6091..afc1101 100644
--- a/man/availCaches.Rd
+++ b/man/listCaches.Rd
@@ -1,20 +1,24 @@
% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/loadCaches.R
-\name{availCaches}
-\alias{availCaches}
+% Please edit documentation in R/listCaches.R
+\name{listCaches}
+\alias{listCaches}
\title{Show available caches.}
\usage{
-availCaches(cacheSubDir = "")
+listCaches(cacheSubDir = "")
}
\arguments{
\item{cacheSubDir}{Optional parameter to specify a subdirectory of the cache folder.}
}
+\value{
+\code{character} vector in which each element is the path to a file that
+ represents an available cache (within \code{getOption("RCACHE.DIR")})
+}
\description{
Lists any cache files in the cache directory.
}
\examples{
# choose location to store caches
-cacheDir = system.file("cache", package="simpleCache")
+cacheDir = tempdir()
cacheDir
setCacheDir(cacheDir)
@@ -28,7 +32,7 @@ normSample2 = rnorm(10, 0, 1)
storeCache("normSample2")
# what's available?
-availCaches()
+listCaches()
# load a cache
simpleCache("normSample")
diff --git a/man/loadCaches.Rd b/man/loadCaches.Rd
index ab57f7a..b247e1b 100644
--- a/man/loadCaches.Rd
+++ b/man/loadCaches.Rd
@@ -17,7 +17,7 @@ for stuff you already cached previously, so it won't build any caches.
}
\examples{
# choose location to store caches
-cacheDir = system.file("cache", package="simpleCache")
+cacheDir = tempdir()
cacheDir
setCacheDir(cacheDir)
@@ -31,7 +31,7 @@ normSample2 = rnorm(10, 0, 1)
storeCache("normSample2")
# what's available?
-availCaches()
+listCaches()
# load a cache
simpleCache("normSample")
diff --git a/man/setCacheDir.Rd b/man/setCacheDir.Rd
index b3989c6..de719b4 100644
--- a/man/setCacheDir.Rd
+++ b/man/setCacheDir.Rd
@@ -16,7 +16,7 @@ simpleCache() calls.
}
\examples{
# choose location to store caches
-cacheDir = system.file("cache", package="simpleCache")
+cacheDir = tempdir()
cacheDir
setCacheDir(cacheDir)
@@ -30,7 +30,7 @@ normSample2 = rnorm(10, 0, 1)
storeCache("normSample2")
# what's available?
-availCaches()
+listCaches()
# load a cache
simpleCache("normSample")
diff --git a/man/simpleCache.Rd b/man/simpleCache.Rd
index b0c2ba7..b85688b 100644
--- a/man/simpleCache.Rd
+++ b/man/simpleCache.Rd
@@ -108,7 +108,7 @@ using the parameter buildEnvir (just provide a list of named variables).
}
\examples{
# choose location to store caches
-cacheDir = system.file("cache", package="simpleCache")
+cacheDir = tempdir()
cacheDir
setCacheDir(cacheDir)
@@ -122,7 +122,7 @@ normSample2 = rnorm(10, 0, 1)
storeCache("normSample2")
# what's available?
-availCaches()
+listCaches()
# load a cache
simpleCache("normSample")
diff --git a/man/storeCache.Rd b/man/storeCache.Rd
index 5fb43fb..76b5a66 100644
--- a/man/storeCache.Rd
+++ b/man/storeCache.Rd
@@ -32,7 +32,7 @@ storeCache at the end.
}
\examples{
# choose location to store caches
-cacheDir = system.file("cache", package="simpleCache")
+cacheDir = tempdir()
cacheDir
setCacheDir(cacheDir)
@@ -46,7 +46,7 @@ normSample2 = rnorm(10, 0, 1)
storeCache("normSample2")
# what's available?
-availCaches()
+listCaches()
# load a cache
simpleCache("normSample")
diff --git a/vignettes/clusterCaches.Rmd b/vignettes/clusterCaches.Rmd
index 1475fd0..a965884 100644
--- a/vignettes/clusterCaches.Rmd
+++ b/vignettes/clusterCaches.Rmd
@@ -18,21 +18,19 @@ To do this, first, create a `batchtools` registry. You can follow more detailed
```{r Try it out, eval=FALSE}
library(simpleCache)
-setCacheDir(getwd())
+setCacheDir(tempdir())
registry = batchtools::makeRegistry(NA)
templateFile = system.file("templates/slurm-advanced.tmpl", package = "simpleCache")
registry$cluster.functions = batchtools::makeClusterFunctionsSlurm(
template = templateFile)
registry
-
```
Notice that I'm using a custom slurm template here. With a registry in hand, we next need to define the resources this cache job will require:
-```
+```{r}
resources = list(ncpus=1, memory=1000, walltime=60, partition="serial")
-
```
Then, we simply add these as arguments to `simpleCache()` like so:
diff --git a/vignettes/sharingCaches.Rmd b/vignettes/sharingCaches.Rmd
index 569d87c..8b64414 100644
--- a/vignettes/sharingCaches.Rmd
+++ b/vignettes/sharingCaches.Rmd
@@ -16,7 +16,7 @@ To solve this problem, `simpleCache` uses a second global option, `SHARE.RCACHE.
```{r Try it out}
library(simpleCache)
-cacheDir = system.file("cache", package="simpleCache")
+cacheDir = tempdir()
setSharedCacheDir(cacheDir)
simpleCacheShared("normSample", "rnorm(1e7, 0,1)", recreate=TRUE)
simpleCacheShared("normSample", "rnorm(1e7, 0,1)")
diff --git a/vignettes/simpleCacheIntroduction.Rmd b/vignettes/simpleCacheIntroduction.Rmd
index f39ec0d..1175b04 100644
--- a/vignettes/simpleCacheIntroduction.Rmd
+++ b/vignettes/simpleCacheIntroduction.Rmd
@@ -20,33 +20,33 @@ But before we start creating caches, it's important to tell `simpleCache` where
```{r Try it out}
library(simpleCache)
-cacheDir = system.file("cache", package="simpleCache")
+cacheDir = tempdir()
setCacheDir(cacheDir)
-simpleCache("normSample", { rnorm(1e7, 0,1) })
+simpleCache("normSamp", { rnorm(1e7, 0,1) })
```
Now, watch what happens when we run that same function call again:
-```
-simpleCache("normSample", { rnorm(1e7, 0,1) })
+```{r}
+simpleCache("normSamp", { rnorm(1e7, 0,1) })
```
Notice that the second call to `simpleCache()` doesn't re-run the `rnorm` calculation. In fact, it doesn't even re-load the cache, because it notices that it's already in memory. If the cache weren't already in memory, this call would load it from disk. This means you can put this code in multiple scripts and pull the same randomized data, without re-doing the compute work.
You can also force a cache to reload using the `reload` option. This could be useful, for example, if you've loaded a cache and then accidentally changed it, and want to reset. By default, a call to `simpleCache()` will not reload an object that already exists in your environment. But you can always force it with the `reload` parameter:
-```
-normSample = NA # Oops broke my object in memory.
-# Regular call won't reload because we have an object called normSample already:
-simpleCache("normSample", { rnorm(1e7, 0,1) })
+```{r}
+normSamp = NA # Oops broke my object in memory.
+# Regular call won't reload because we have an object called normSamp already:
+simpleCache("normSamp", { rnorm(1e7, 0,1) })
# But we can force reload and get it back with reload=TRUE
-simpleCache("normSample", { rnorm(1e7, 0,1) }, reload=TRUE)
+simpleCache("normSamp", { rnorm(1e7, 0,1) }, reload=TRUE)
```
What if we want to start over and blow that cache, getting a new random set? Use the `recreate` flag if you want to ensure that the cache is produced and overwritten even if it already exists:
-```
-simpleCache("normSample", { rnorm(1e7, 0,1) }, recreate=TRUE)
+```{r}
+simpleCache("normSamp", { rnorm(1e7, 0,1) }, recreate=TRUE)
```
With just those parameters (cache name, instruction, recreate, and reload), you should be able to make good use of `simpleCache`. The essence is: if the object exists in memory already: do nothing. If it does not exist in memory, but exists on disk: load it into memory. If it exists neither in memory or on disk: create it and store it to disk and memory. Now you've got the basics.
@@ -64,13 +64,13 @@ Beyond that, `simpleCache` also offers several convenient options that just make
By default, the object will be loaded into a variable with the same name as the cache. You can change this behavior with the `assignTo` parameter:
```{r}
-simpleCache("normSample", { rnorm(1e7, 0,1) }, assignTo="mySamp")
+simpleCache("normSamp", { rnorm(1e7, 0,1) }, assignTo="mySamp")
```
-After doing this command, we have both `normSample` (from the previous calls, not from this one) and `mySamp` (loaded in this call) in the workspace, and these objects are identical:
+After doing this command, we have both `normSamp` (from the previous calls, not from this one) and `mySamp` (loaded in this call) in the workspace, and these objects are identical:
```{r}
-identical(normSample, mySamp)
+identical(normSamp, mySamp)
```
This `assignTo` concept is useful if you want to create caches but not load them, or load caches one at a time. Which leads us to...
@@ -81,7 +81,7 @@ It may be that you want to create a bunch of caches that are quite memory intens
```{r}
for (i in 1:5) {
- cacheName = paste0("normSample_", i)
+ cacheName = paste0("normSamp_", i)
simpleCache(cacheName, { rnorm(1e6, 0,1) }, recreate=TRUE, noload=TRUE)
}
```
@@ -91,7 +91,7 @@ We've now produced 5 different sample data caches. They exist on disk, but not i
```{r}
overallMinimum = 1e6 # pick some high number to start
for (i in 1:5) {
- cacheName = paste0("normSample_", i)
+ cacheName = paste0("normSamp_", i)
simpleCache(cacheName, assignTo="temp")
overallMinimum = min(overallMinimum, temp)
}
@@ -105,7 +105,7 @@ In this code block, by assigning the caches to the variable `temp`, we only have
If you've got a bunch of caches and you want them all in memory, you could just load all the caches into memory with this convenience alias:
```{r}
-loadCaches(paste0("normSample_", 1:5))
+loadCaches(paste0("normSamp_", 1:5))
```
The disadvantage of doing it this way is that you've lost the advantage of using the single `simpleCache()` function for both saving and loading, but this may be desirable in some cases.
@@ -113,7 +113,7 @@ The disadvantage of doing it this way is that you've lost the advantage of using
By the way, once a cache is created, you no longer need to provide instructions:
```{r}
-simpleCache("normSample")
+simpleCache("normSamp")
```
`simpleCache` will load it if it can; if not, it will give you an error saying it requires an `instruction`.
@@ -123,7 +123,7 @@ simpleCache("normSample")
If you want to record how long it takes to create a new cache, you can set `timer=TRUE`.
```{r}
-simpleCache("normSample", { rnorm(1e6, 0,1) }, recreate=TRUE, timer=TRUE)
+simpleCache("normSamp", { rnorm(1e6, 0,1) }, recreate=TRUE, timer=TRUE)
```
## Complicated code
@@ -133,7 +133,7 @@ So far, our examples have cached the result of a very simple instruction code bl
```{r}
simpleCache("tResult", {
dat2 = rnorm(1e5, 0.05,2)
- t.test(normSample, dat2)
+ t.test(normSamp, dat2)
}, recreate=TRUE)
tResult