diff --git a/README.md b/README.md index bc0368b..872cf51 100644 --- a/README.md +++ b/README.md @@ -289,6 +289,14 @@ $ cd animl-base $ npm install ``` +3. Create a directories for `~/images/queue/` and `~/images/archive/` + +```shell +$ mkdir /home/animl/images +$ mkdir /home/animl/images/queue +$ mkdir /home/animl/images/archive +``` + 3. Add a .env file to the project's root directory with the following items. Note, AWS creds can be found in the [TNC Cameratrap network passwords document](https://tnc.app.box.com/file/762650708780). For access, contact nathaniel.rindlaub@tnc.org: ``` @@ -301,8 +309,12 @@ AWS_SECRET_ACCESS_KEY = [secret access key] AWS_REGION = us-west-2 # Image directory to watch -IMG_DIR = '/home/animl/data//cameras/' -# IMG_DIR = "c:\BuckEyeCam\X7D Base\pictures\" # Windows +WATCH_DIR = '/home/animl/data//cameras/' +# WATCH_DIR = "c:\BuckEyeCam\X7D Base\pictures\" # Windows + +# Directories for queued and archived images +QUEUE_DIR = '/home/animl/images/queue' +ARCHIVE_DIR = '/home/animl/images/archive' # Log file to watch LOG_FILE = '/home/animl/data//log.txt' @@ -414,9 +426,17 @@ $ mbasectl -i For adding new cameras, repeaters, and managing deployed devices, use the Multibase Server edition local web application, which can be found at `localhost:8888` from within the computer when Mulibase is running. You can remotely access it by remote-desktoping into the computer via AnyDesk/VCN and launchubg the local web app in a browser window if you're trying to manage the devices remotely. More detailed documentation on using the Buckeye MultiBase SE application can be found [here](https://tnc.app.box.com/file/794348600237?s=3x3e0onul82mxawahpo3qeffmzomm4uq). -> NOTE: If you are having trouble adding a camera to the Base, from the Base home user interface (the page you get to after loging in and +> [!IMPORTANT] +> Because animl-base moves images out of the directory that Multibase SE expects them to be in (see [explaination below](https://github.com/tnc-ca-geo/animl-base?tab=local-image-file-storage-and-archive) for more detail), it will appear in the Multibase SE webapp as though there the network has never recieved any images. We reccommend using https://animl.camera for all image review, but if you need to access the image files locally, a backup of the most recent images can be found at `~/images/archive/`. + +> [!TIP] +> If you are having trouble adding a camera to the Base, from the Base home user interface (the page you get to after loging in and > clicking the "admin" button under the base entry), try "restoring the network" (hamburger menu -> Restore Network). This will search for and locate any devices that were have already been registered to the base. +### Local image file storage and archive + +Typically, Buckeye's MultiBase SE program stores all image files in the `~/data/data//cameras/` directory. However, largely due to [memory issues](https://github.com/tnc-ca-geo/animl-base/issues/20) with watching an ever-growing directory of images, once animl-base detects a new image file it moves it to a "queue" directory at `~/images/queue/`, and after it's been successfully updated it gets moved to `~/images/archive/` to serve as a local backup of the image files. + ### Sixfab power managment web app (Raspberry Pi only) For remotely monitoring the RPi's status and configuring power, connectivity, and temperature alerts, you can access [https://power.sixfab.com](https://power.sixfab.com) from any computer. Credentials are in the Camera trap network [password document](https://tnc.app.box.com/file/762650708780). diff --git a/src/app.js b/src/app.js index 2d2103c..86bcc6b 100644 --- a/src/app.js +++ b/src/app.js @@ -1,4 +1,5 @@ const path = require('path'); +const fs = require('fs'); const chokidar = require('chokidar'); const Queue = require('./utils/queue'); const Worker = require('./utils/worker'); @@ -32,11 +33,25 @@ function validateFile(filePath) { return true; } -function handleNewFile(filePath, queue, metricsLogger) { +async function handleNewFile(filePath, queue, metricsLogger) { console.log(`New file detected: ${filePath}`); - if (validateFile(filePath)) { - queue.add(filePath); - metricsLogger.handleNewImage(filePath); + if (!validateFile(filePath)) return; + try { + // move file to queue directory + const destPath = path.join( + config.queueDir, + filePath.split('/').slice(6).join('/') + ); + const destDir = path.dirname(destPath); + if (!fs.existsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); + } + fs.renameSync(filePath, destPath); + console.log(`Moved file from ${filePath} to ${destPath}`); + await metricsLogger.handleNewImage(destPath); + queue.add(destPath); + } catch (err) { + console.log('Error handling new file: ', err); } } @@ -57,10 +72,10 @@ async function start() { await queue.init(); // Initialize directory watcher - const imgWatcher = chokidar.watch(config.imgDir, config.watcher); + const imgWatcher = chokidar.watch(config.watchDir, config.watcher); imgWatcher .on('ready', async () => { - console.log(`Watching for changes to ${config.imgDir}`); + console.log(`Watching for changes to ${config.watchDir}`); // NOTE: just for testing // const filesWatchedOnStart = imgWatcher.getWatched(); diff --git a/src/config/index.js b/src/config/index.js index 11ec7e3..77cf169 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -6,7 +6,9 @@ dotenv.config(); module.exports = { platform: process.platform, baseName: process.env.BASE_NAME, - imgDir: process.env.IMG_DIR, + watchDir: process.env.WATCH_DIR, + queueDir: process.env.QUEUE_DIR, + archiveDir: process.env.ARCHIVE_DIR, dbFile: 'db.json', logFile: process.env.LOG_FILE, aws: { @@ -17,6 +19,7 @@ module.exports = { ignored: /(^|[\/\\])\../, // ignore dotfiles ignoreInitial: true, // ignore files in the directory on start persistent: true, + awaitWriteFinish: true, }, supportedFileTypes: ['.jpg', '.png'], pollInterval: 5000, diff --git a/src/utils/worker.js b/src/utils/worker.js index ffa4bc7..4c9480d 100644 --- a/src/utils/worker.js +++ b/src/utils/worker.js @@ -1,3 +1,5 @@ +const path = require('path'); +const fs = require('fs'); const Backoff = require('backo'); const S3Service = require('./s3Service'); @@ -34,7 +36,23 @@ class Worker { await this.s3.upload(img.path); console.log('Upload success'); await this.queue.remove(img.path); - await this.imgWatcher.unwatch(img.path); // TODO: test whether this works + + // move file from queue directory to archive directory + console.log('Moving file to archive directory'); + const destPath = img.path.replace('queue', 'archive'); + const destDir = path.dirname(destPath); + if (!fs.existsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); + } + fs.renameSync(img.path, destPath); + + // Just for testing... + const filesWatched = this.imgWatcher.getWatched(); + Object.keys(filesWatched).forEach((dir) => { + console.log( + `Number of files being watched in ${dir} : ${filesWatched[dir].length}` + ); + }); // Reset backoff and poll again this.backoff.reset();