Merge pull request #380 from PredictiveEcology/modsForLargeArchives

Modifications for large archives; other minor changes

.github/workflows/R-CMD-check.yaml

@@ -22,16 +22,16 @@ jobs:
       matrix:
         config:
           - {os: macOS-latest,   r: 'release'}
-          - {os: windows-latest, r: 'devel'}
+          - {os: windows-latest, r: 'devel', http-user-agent: 'release'}
           - {os: windows-latest, r: 'release'}
-          - {os: windows-latest, r: 'oldrel/1'}
-          - {os: windows-latest, r: 'oldrel/2'}
-          - {os: windows-latest, r: 'oldrel/3'}
-          - {os: ubuntu-20.04,   r: 'devel'}
+          - {os: windows-latest, r: 'oldrel-1'}
+          - {os: windows-latest, r: 'oldrel-2'}
+          - {os: windows-latest, r: 'oldrel-3'}
+          - {os: ubuntu-20.04,   r: 'devel', http-user-agent: 'release'}
           - {os: ubuntu-20.04,   r: 'release'}
-          - {os: ubuntu-20.04,   r: 'oldrel/1'}
-          - {os: ubuntu-20.04,   r: 'oldrel/2'}
-          - {os: ubuntu-20.04,   r: 'oldrel/3'}
+          - {os: ubuntu-20.04,   r: 'oldrel-1'}
+          - {os: ubuntu-20.04,   r: 'oldrel-2'}
+          - {os: ubuntu-20.04,   r: 'oldrel-3'}
 
     env:
       _SP_EVOLUTION_STATUS_: 2
@@ -49,6 +49,7 @@ jobs:
 
       - uses: r-lib/actions/setup-r@v2
         with:
+          http-user-agent: ${{ matrix.config.http-user-agent }}
           Ncpus: 2
           r-version: ${{ matrix.config.r }}
           use-public-rspm: false
@@ -58,6 +59,8 @@ jobs:
           extra-packages: |
             any::rcmdcheck
             any::Rcpp
+            traversc/qs
+          pak-version: devel
 
       - uses: r-lib/actions/check-r-package@v2
         with:

.github/workflows/update-citation-cff.yaml

@@ -27,7 +27,7 @@ jobs:
       - uses: r-lib/actions/setup-r@v2
         with:
           Ncpus: 2
-          r-version: ${{ matrix.config.r }}
+          r-version: 'release'
           use-public-rspm: false
       - uses: r-lib/actions/setup-r-dependencies@v2
         with:

DESCRIPTION

@@ -19,8 +19,8 @@ SystemRequirements: 'unrar' (Linux/macOS) or '7-Zip' (Windows) to work with '.ra
 URL: 
     https://reproducible.predictiveecology.org,
     https://github.com/PredictiveEcology/reproducible
-Date: 2024-04-15
-Version: 2.0.12
+Date: 2024-05-14
+Version: 2.0.12.9011
 Authors@R: 
     c(person(given = "Eliot J B",
              family = "McIntire",
@@ -62,6 +62,7 @@ Imports:
     stats,
     utils
 Suggests:
+    archive,
     covr,
     crayon,
     DBI,
@@ -78,6 +79,7 @@ Suggests:
     rlang,
     rmarkdown,
     RSQLite,
+    R.utils,
     sf,
     sp (>= 1.4-2),
     terra (>= 1.7-20),
@@ -103,10 +105,11 @@ Collate:
     'convertPaths.R'
     'copy.R'
     'download.R'
+    'messages.R'
     'exportedMethods.R'
     'gis.R'
     'helpers.R'
-    'messages.R'
+    'listNamed.R'
     'objectSize.R'
     'options.R'
     'packages.R'

NAMESPACE

@@ -29,6 +29,7 @@ S3method(writeOutputs,default)
 export(.addChangedAttr)
 export(.addTagsToOutput)
 export(.cacheMessage)
+export(.cacheMessageObjectToRetrieve)
 export(.checkCacheRepo)
 export(.file.move)
 export(.formalsNotInCurrentDots)
@@ -75,17 +76,26 @@ export(downloadFile)
 export(fastMask)
 export(fixErrors)
 export(fixErrorsIn)
+export(gdalMask)
+export(gdalProject)
+export(gdalResample)
 export(getRelative)
 export(internetExists)
 export(isUpdated)
 export(linkOrCopy)
+export(listNamed)
 export(loadFromCache)
 export(makeMemoisable)
 export(makeRelative)
 export(maskInputs)
 export(maskTo)
 export(maxFn)
+export(messageCache)
+export(messageColoured)
 export(messageDF)
+export(messagePreProcess)
+export(messagePrepInputs)
+export(messageQuestion)
 export(minFn)
 export(movedCache)
 export(nlayers2)

NEWS.md

@@ -1,16 +1,45 @@
+# reproducible (development version)
+
+## New
+* new family of functions that are called inside `postProcessTo` that use `sf::gdal_utils` directly. These are still experimental and will only be activated with `options("reproducible.gdalwarp" = TRUE)`
+* default for `gdalMask` has changed default for "touches". Now has equivalent for `terra::mask(..., touches = TRUE)`, using `"-wo CUTLINE_ALL_TOUCHED=TRUE"`
+* `gdalProject` now uses 2 threads, setting `"-wo NUM_THREADS=2"`; can be changed by user with `options("reproducible.gdalwarpThreads" = X)`; see `?reproducibleOptions`
+* `gdal*` functions now address `datatype` issues
+* `gdal*` defaults to `FLT8S` if `datatype` not passed
+* `makeRelative`, `makeAbsolute` and similar have been created to ease many issues encountered in `preProcess`
+
+## Changes
+* `showSimilar` (e.g., `options(reproducible.showSimilar = 1)`) now preferentially shows the most recent item in cache if there are several with equivalent matching.
+* overhaul of messaging in `Cache` and `prepInputs` families; functions are highlighted with a different colour; indent level reflects nesting of both `Cache` and `prepInputs`, so it is easier to identify which message goes with which function call.
+* `preProcess` is a lot faster now for large numbers of files; uses `CHECKSUMS` more effectively and fewer times
+* `retry` now captures its `expr` so it doesn't need a `quote`; is like `try` now.
+* `showSimilar` mechanisms now returns the most recent, if there are >1 similar that are equivalently similar
+* if a user is having troubles with `googledrive` for e.g., large files on spotting connections, instructions for using `gdown` are provided
+* `showCache`, `clearCache` now have extra arguments `fun`, `cacheId`, and `...` now can take any arbitrary `tag = value` pair. The `cacheId` argument will be very fast if a user is not using `useDBI()` is `FALSE`.
+* `.wrap` and `.unwrap` can now deal with `SpatVectorCollection` (a `terra` class that does not have a `wrap`/`unwrap` method in `terra`)
+* ALTREP digesting when using `spooky` or `fastdigest` were not stable for `integers` and `factors`. There is now a work around in `.robustDigest` that stabilizes these by expanding them from their ALTREP representation  first. Since they will be saved and recovered anyway, this will have little effect. 
+* `.wrap` and `.unwrap` are becoming more mature and can handle many more classes effectively. Methods can still be written, if needed.
+
+## Testing
+* lots of testing with `cacheSaveFormat = "qs"`, which previously was not reliable especially for environments. With all recent changes to `.wrap` and `.unwrap`, these appear stable now and should be able to be used for `environments`.
+
+## Bugfixes
+* `switchDataType` can now correctly switch between `gdal` formats and `terra`
+* many messaging fixes that were imprecise or missing
+
 # reproducible 2.0.12
 
 * re-submission after removal from CRAN
 
 # reproducible 2.0.11
 
 ## Remove dependency
-* fastdigest was removed from CRAN and so is removed from here.
+* `fastdigest` was removed from CRAN and so is removed from here.
 
 # reproducible 2.0.10
 
 ## Bug fixes
-* critical bugfixes for file-backed SpatRaster objects
+* critical bugfixes for file-backed `SpatRaster` objects
 
 # reproducible 2.0.9
 
@@ -29,7 +58,7 @@
 * `Cache` can now skip calculating `objSize`, which can take a non-trivial amount of time for large, complicated objects; see `reproducibleOptions()`
 
 ## Bug fixes
-* Filenames for some classes returned ""; now returns NULL so character vectors are only pointers to files
+* `Filenames` for some classes returned ""; now returns NULL so character vectors are only pointers to files
 * Cache on a terra object that writes file to disk, when `quick` argument is specified was failing, always creating the same object; fixed with #PR368
 * `useDBI` was incorrectly used if a user had set the option prior to package loading. Now works as expected.
 * several other minor