diff --git a/README.md b/README.md
index bc0368b..872cf51 100644
--- a/README.md
+++ b/README.md
@@ -289,6 +289,14 @@ $ cd animl-base
$ npm install
```
+3. Create a directories for `~/images/queue/` and `~/images/archive/`
+
+```shell
+$ mkdir /home/animl/images
+$ mkdir /home/animl/images/queue
+$ mkdir /home/animl/images/archive
+```
+
3. Add a .env file to the project's root directory with the following items. Note, AWS creds can be found in the [TNC Cameratrap network passwords document](https://tnc.app.box.com/file/762650708780). For access, contact nathaniel.rindlaub@tnc.org:
```
@@ -301,8 +309,12 @@ AWS_SECRET_ACCESS_KEY = [secret access key]
AWS_REGION = us-west-2
# Image directory to watch
-IMG_DIR = '/home/animl/data//cameras/'
-# IMG_DIR = "c:\BuckEyeCam\X7D Base\pictures\" # Windows
+WATCH_DIR = '/home/animl/data//cameras/'
+# WATCH_DIR = "c:\BuckEyeCam\X7D Base\pictures\" # Windows
+
+# Directories for queued and archived images
+QUEUE_DIR = '/home/animl/images/queue'
+ARCHIVE_DIR = '/home/animl/images/archive'
# Log file to watch
LOG_FILE = '/home/animl/data//log.txt'
@@ -414,9 +426,17 @@ $ mbasectl -i
For adding new cameras, repeaters, and managing deployed devices, use the Multibase Server edition local web application, which can be found at `localhost:8888` from within the computer when Mulibase is running. You can remotely access it by remote-desktoping into the computer via AnyDesk/VCN and launchubg the local web app in a browser window if you're trying to manage the devices remotely. More detailed documentation on using the Buckeye MultiBase SE application can be found [here](https://tnc.app.box.com/file/794348600237?s=3x3e0onul82mxawahpo3qeffmzomm4uq).
-> NOTE: If you are having trouble adding a camera to the Base, from the Base home user interface (the page you get to after loging in and
+> [!IMPORTANT]
+> Because animl-base moves images out of the directory that Multibase SE expects them to be in (see [explaination below](https://github.com/tnc-ca-geo/animl-base?tab=local-image-file-storage-and-archive) for more detail), it will appear in the Multibase SE webapp as though there the network has never recieved any images. We reccommend using https://animl.camera for all image review, but if you need to access the image files locally, a backup of the most recent images can be found at `~/images/archive/`.
+
+> [!TIP]
+> If you are having trouble adding a camera to the Base, from the Base home user interface (the page you get to after loging in and
> clicking the "admin" button under the base entry), try "restoring the network" (hamburger menu -> Restore Network). This will search for and locate any devices that were have already been registered to the base.
+### Local image file storage and archive
+
+Typically, Buckeye's MultiBase SE program stores all image files in the `~/data/data//cameras/` directory. However, largely due to [memory issues](https://github.com/tnc-ca-geo/animl-base/issues/20) with watching an ever-growing directory of images, once animl-base detects a new image file it moves it to a "queue" directory at `~/images/queue/`, and after it's been successfully updated it gets moved to `~/images/archive/` to serve as a local backup of the image files.
+
### Sixfab power managment web app (Raspberry Pi only)
For remotely monitoring the RPi's status and configuring power, connectivity, and temperature alerts, you can access [https://power.sixfab.com](https://power.sixfab.com) from any computer. Credentials are in the Camera trap network [password document](https://tnc.app.box.com/file/762650708780).
diff --git a/src/app.js b/src/app.js
index 2d2103c..86bcc6b 100644
--- a/src/app.js
+++ b/src/app.js
@@ -1,4 +1,5 @@
const path = require('path');
+const fs = require('fs');
const chokidar = require('chokidar');
const Queue = require('./utils/queue');
const Worker = require('./utils/worker');
@@ -32,11 +33,25 @@ function validateFile(filePath) {
return true;
}
-function handleNewFile(filePath, queue, metricsLogger) {
+async function handleNewFile(filePath, queue, metricsLogger) {
console.log(`New file detected: ${filePath}`);
- if (validateFile(filePath)) {
- queue.add(filePath);
- metricsLogger.handleNewImage(filePath);
+ if (!validateFile(filePath)) return;
+ try {
+ // move file to queue directory
+ const destPath = path.join(
+ config.queueDir,
+ filePath.split('/').slice(6).join('/')
+ );
+ const destDir = path.dirname(destPath);
+ if (!fs.existsSync(destDir)) {
+ fs.mkdirSync(destDir, { recursive: true });
+ }
+ fs.renameSync(filePath, destPath);
+ console.log(`Moved file from ${filePath} to ${destPath}`);
+ await metricsLogger.handleNewImage(destPath);
+ queue.add(destPath);
+ } catch (err) {
+ console.log('Error handling new file: ', err);
}
}
@@ -57,10 +72,10 @@ async function start() {
await queue.init();
// Initialize directory watcher
- const imgWatcher = chokidar.watch(config.imgDir, config.watcher);
+ const imgWatcher = chokidar.watch(config.watchDir, config.watcher);
imgWatcher
.on('ready', async () => {
- console.log(`Watching for changes to ${config.imgDir}`);
+ console.log(`Watching for changes to ${config.watchDir}`);
// NOTE: just for testing
// const filesWatchedOnStart = imgWatcher.getWatched();
diff --git a/src/config/index.js b/src/config/index.js
index 11ec7e3..77cf169 100644
--- a/src/config/index.js
+++ b/src/config/index.js
@@ -6,7 +6,9 @@ dotenv.config();
module.exports = {
platform: process.platform,
baseName: process.env.BASE_NAME,
- imgDir: process.env.IMG_DIR,
+ watchDir: process.env.WATCH_DIR,
+ queueDir: process.env.QUEUE_DIR,
+ archiveDir: process.env.ARCHIVE_DIR,
dbFile: 'db.json',
logFile: process.env.LOG_FILE,
aws: {
@@ -17,6 +19,7 @@ module.exports = {
ignored: /(^|[\/\\])\../, // ignore dotfiles
ignoreInitial: true, // ignore files in the directory on start
persistent: true,
+ awaitWriteFinish: true,
},
supportedFileTypes: ['.jpg', '.png'],
pollInterval: 5000,
diff --git a/src/utils/worker.js b/src/utils/worker.js
index ffa4bc7..4c9480d 100644
--- a/src/utils/worker.js
+++ b/src/utils/worker.js
@@ -1,3 +1,5 @@
+const path = require('path');
+const fs = require('fs');
const Backoff = require('backo');
const S3Service = require('./s3Service');
@@ -34,7 +36,23 @@ class Worker {
await this.s3.upload(img.path);
console.log('Upload success');
await this.queue.remove(img.path);
- await this.imgWatcher.unwatch(img.path); // TODO: test whether this works
+
+ // move file from queue directory to archive directory
+ console.log('Moving file to archive directory');
+ const destPath = img.path.replace('queue', 'archive');
+ const destDir = path.dirname(destPath);
+ if (!fs.existsSync(destDir)) {
+ fs.mkdirSync(destDir, { recursive: true });
+ }
+ fs.renameSync(img.path, destPath);
+
+ // Just for testing...
+ const filesWatched = this.imgWatcher.getWatched();
+ Object.keys(filesWatched).forEach((dir) => {
+ console.log(
+ `Number of files being watched in ${dir} : ${filesWatched[dir].length}`
+ );
+ });
// Reset backoff and poll again
this.backoff.reset();