Make FileListener respect .nomedia

appinfo/routes.php

@@ -33,6 +33,7 @@
 		['name' => 'admin#nodejs', 'url' => '/admin/nodejs', 'verb' => 'GET'],
 		['name' => 'admin#libtensorflow', 'url' => '/admin/libtensorflow', 'verb' => 'GET'],
 		['name' => 'admin#wasmtensorflow', 'url' => '/admin/wasmtensorflow', 'verb' => 'GET'],
+		['name' => 'admin#cron', 'url' => '/admin/cron', 'verb' => 'GET'],
 		['name' => 'admin#get_setting', 'url' => '/admin/settings/{setting}', 'verb' => 'GET'],
 		['name' => 'admin#set_setting', 'url' => '/admin/settings/{setting}', 'verb' => 'PUT'],
 	],

lib/BackgroundJobs/StorageCrawlJob.php

@@ -16,6 +16,7 @@
 use OCA\Recognize\Classifiers\Video\MovinetClassifier;
 use OCA\Recognize\Constants;
 use OCA\Recognize\Db\QueueFile;
+use OCA\Recognize\Service\IgnoreService;
 use OCA\Recognize\Service\Logger;
 use OCA\Recognize\Service\QueueService;
 use OCA\Recognize\Service\TagManager;
@@ -36,8 +37,9 @@ class StorageCrawlJob extends QueuedJob {
 	private IDBConnection $db;
 	private SystemConfig $systemConfig;
 	private TagManager $tagManager;
+	private IgnoreService $ignoreService;
 
-	public function __construct(ITimeFactory $timeFactory, Logger $logger, IMimeTypeLoader $mimeTypes, QueueService $queue, IJobList $jobList, IDBConnection $db, SystemConfig $systemConfig, TagManager $tagManager) {
+	public function __construct(ITimeFactory $timeFactory, Logger $logger, IMimeTypeLoader $mimeTypes, QueueService $queue, IJobList $jobList, IDBConnection $db, SystemConfig $systemConfig, TagManager $tagManager, IgnoreService $ignoreService) {
 		parent::__construct($timeFactory);
 		$this->logger = $logger;
 		$this->mimeTypes = $mimeTypes;
@@ -46,39 +48,7 @@ public function __construct(ITimeFactory $timeFactory, Logger $logger, IMimeType
 		$this->db = $db;
 		$this->systemConfig = $systemConfig;
 		$this->tagManager = $tagManager;
-	}
-
-	private function getDir(int $fileid, array $directoryTypes, bool $recursive = false): array {
-		/** @var \OCP\DB\QueryBuilder\IQueryBuilder $qb */
-		$qb = new CacheQueryBuilder($this->db, $this->systemConfig, $this->logger);
-		$dir = $qb->selectFileCache()
-			->andWhere($qb->expr()->in('mimetype', $qb->createNamedParameter($directoryTypes, IQueryBuilder::PARAM_INT_ARRAY)))
-			->andWhere($qb->expr()->eq('parent', $qb->createNamedParameter($fileid)))
-			->executeQuery()->fetchAll();
-
-		if ($recursive) {
-			foreach ($dir as $item) {
-				$dir = array_merge($dir, $this->getDir($item['fileid'], $directoryTypes, $recursive));
-			}
-		}
-		return $dir;
-	}
-
-	private function getIgnoreFileids(int $storageId, array $ignore_maekers): array {
-		$directoryTypes = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), Constants::DIRECTORY_FORMATS);
-		/** @var \OCP\DB\QueryBuilder\IQueryBuilder $qb */
-		$qb = new CacheQueryBuilder($this->db, $this->systemConfig, $this->logger);
-		$ignoreFiles = $qb->selectFileCache()
-			->andWhere($qb->expr()->in('name', $qb->createNamedParameter($ignore_maekers, IQueryBuilder::PARAM_STR_ARRAY)))
-			->andWhere($qb->expr()->eq('storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
-			->executeQuery()->fetchAll();
-		$ignoreFileids = array_map(fn ($dir) => $dir['parent'], $ignoreFiles);
-		foreach ($ignoreFiles as $ignoreFile) {
-			$ignoreDir = $this->getDir($ignoreFile['parent'], $directoryTypes, true);
-			$fileids = array_map(fn ($dir) => $dir['fileid'], $ignoreDir);
-			$ignoreFileids = array_merge($ignoreFileids, $fileids);
-		}
-		return $ignoreFileids;
+		$this->ignoreService = $ignoreService;
 	}
 
 	protected function run($argument): void {
@@ -114,10 +84,10 @@ protected function run($argument): void {
 		}
 
 		try {
-			$ignoreAllFileids = $this->getIgnoreFileids($storageId, Constants::IGNORE_MARKERS_ALL);
-			$ignoreImageFileids = $this->getIgnoreFileids($storageId, Constants::IGNORE_MARKERS_IMAGE);
-			$ignoreVideoFileids = $this->getIgnoreFileids($storageId, Constants::IGNORE_MARKERS_VIDEO);
-			$ignoreAudioFileids = $this->getIgnoreFileids($storageId, Constants::IGNORE_MARKERS_AUDIO);
+			$ignoreAllFileids = $this->ignoreService->getIgnoredDirectories($storageId, Constants::IGNORE_MARKERS_ALL);
+			$ignoreImageFileids = $this->ignoreService->getIgnoredDirectories($storageId, Constants::IGNORE_MARKERS_IMAGE);
+			$ignoreVideoFileids = $this->ignoreService->getIgnoredDirectories($storageId, Constants::IGNORE_MARKERS_VIDEO);
+			$ignoreAudioFileids = $this->ignoreService->getIgnoredDirectories($storageId, Constants::IGNORE_MARKERS_AUDIO);
 		} catch (Exception $e) {
 			$this->logger->error('Could not fetch ignore files', ['exception' => $e]);
 			return;

lib/Controller/AdminController.php

@@ -13,6 +13,7 @@
 use OCP\AppFramework\Http\JSONResponse;
 use OCP\BackgroundJob\IJobList;
 use OCP\DB\Exception;
+use OCP\IConfig;
 use OCP\IRequest;
 
 class AdminController extends Controller {
@@ -22,15 +23,17 @@ class AdminController extends Controller {
 	private QueueService $queue;
 	private FaceClusterMapper $clusterMapper;
 	private FaceDetectionMapper $detectionMapper;
+	private IConfig $config;
 
-	public function __construct(string $appName, IRequest $request, TagManager $tagManager, IJobList $jobList, SettingsService $settingsService, QueueService $queue, FaceClusterMapper $clusterMapper, FaceDetectionMapper $detectionMapper) {
+	public function __construct(string $appName, IRequest $request, TagManager $tagManager, IJobList $jobList, SettingsService $settingsService, QueueService $queue, FaceClusterMapper $clusterMapper, FaceDetectionMapper $detectionMapper, IConfig $config) {
 		parent::__construct($appName, $request);
 		$this->tagManager = $tagManager;
 		$this->jobList = $jobList;
 		$this->settingsService = $settingsService;
 		$this->queue = $queue;
 		$this->clusterMapper = $clusterMapper;
 		$this->detectionMapper = $detectionMapper;
+		$this->config = $config;
 	}
 
 	public function reset(): JSONResponse {
@@ -160,6 +163,11 @@ public function wasmtensorflow(): JSONResponse {
 		return new JSONResponse(['wasmtensorflow' => true]);
 	}
 
+	public function cron(): JSONResponse {
+		$cron = $this->config->getAppValue('core', 'backgroundjobs_mode', '');
+		return new JSONResponse(['cron' => $cron]);
+	}
+
 	public function setSetting(string $setting, $value): JSONResponse {
 		try {
 			$this->settingsService->setSetting($setting, $value);

lib/Hooks/FileListener.php

@@ -9,6 +9,7 @@
 use OCA\Recognize\Constants;
 use OCA\Recognize\Db\FaceDetectionMapper;
 use OCA\Recognize\Db\QueueFile;
+use OCA\Recognize\Service\IgnoreService;
 use OCA\Recognize\Service\QueueService;
 use OCP\DB\Exception;
 use OCP\EventDispatcher\Event;
@@ -25,11 +26,13 @@ class FileListener implements IEventListener {
 	private FaceDetectionMapper $faceDetectionMapper;
 	private LoggerInterface $logger;
 	private QueueService $queue;
+	private IgnoreService $ignoreService;
 
-	public function __construct(FaceDetectionMapper $faceDetectionMapper, LoggerInterface $logger, QueueService $queue) {
+	public function __construct(FaceDetectionMapper $faceDetectionMapper, LoggerInterface $logger, QueueService $queue, IgnoreService $ignoreService) {
 		$this->faceDetectionMapper = $faceDetectionMapper;
 		$this->logger = $logger;
 		$this->queue = $queue;
+		$this->ignoreService = $ignoreService;
 	}
 
 	public function handle(Event $event): void {
@@ -80,11 +83,40 @@ public function postDelete(Node $node): void {
 		}
 	}
 
+	/**
+	 * @throws \OCP\Files\InvalidPathException
+	 */
 	public function postInsert(Node $node): void {
 		$queueFile = new QueueFile();
-		$queueFile->setStorageId($node->getMountPoint()->getStorageId());
+		$queueFile->setStorageId((string) $node->getMountPoint()->getNumericStorageId());
 		$queueFile->setRootId((string) $node->getMountPoint()->getStorageRootId());
 
+		$ignoreMarkers = [];
+		if (in_array($node->getMimetype(), Constants::IMAGE_FORMATS)) {
+			$ignoreMarkers = array_merge($ignoreMarkers, Constants::IGNORE_MARKERS_IMAGE);
+		}
+		if (in_array($node->getMimetype(), Constants::VIDEO_FORMATS)) {
+			$ignoreMarkers = array_merge($ignoreMarkers, Constants::IGNORE_MARKERS_VIDEO);
+		}
+		if (in_array($node->getMimetype(), Constants::AUDIO_FORMATS)) {
+			$ignoreMarkers = array_merge($ignoreMarkers, Constants::IGNORE_MARKERS_AUDIO);
+		}
+		if (count($ignoreMarkers) === 0) {
+			return;
+		}
+		$ignoreMarkers = array_merge($ignoreMarkers, Constants::IGNORE_MARKERS_ALL);
+		$ignoredDirectories = $this->ignoreService->getIgnoredDirectories($node->getMountPoint()->getNumericStorageId(), $ignoreMarkers);
+
+		try {
+			if (in_array($node->getParent()->getId(), $ignoredDirectories)) {
+				return;
+			}
+		} catch (InvalidPathException $e) {
+			return;
+		} catch (NotFoundException $e) {
+			return;
+		}
+
 		try {
 			$queueFile->setFileId($node->getId());
 		} catch (InvalidPathException|NotFoundException $e) {

lib/Service/IgnoreService.php

@@ -0,0 +1,75 @@
+<?php
+
+namespace OCA\Recognize\Service;
+
+use OC\Files\Cache\CacheQueryBuilder;
+use OC\SystemConfig;
+use OCA\Recognize\Constants;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\Files\IMimeTypeLoader;
+use OCP\IDBConnection;
+use Psr\Log\LoggerInterface;
+
+class IgnoreService {
+	private IDBConnection $db;
+	private SystemConfig $systemConfig;
+	private LoggerInterface $logger;
+	private IMimeTypeLoader $mimeTypes;
+
+	public function __construct(IDBConnection $db, SystemConfig $systemConfig, LoggerInterface $logger, IMimeTypeLoader $mimeTypes) {
+		$this->db = $db;
+		$this->systemConfig = $systemConfig;
+		$this->logger = $logger;
+		$this->mimeTypes = $mimeTypes;
+	}
+
+	/**
+	 * @param int $fileid
+	 * @param array $directoryTypes
+	 * @param bool $recursive
+	 * @return list<array>
+	 */
+	public function getDir(int $fileid, array $directoryTypes, bool $recursive = false): array {
+		$qb = new CacheQueryBuilder($this->db, $this->systemConfig, $this->logger);
+		$result = $qb->selectFileCache()
+			->andWhere($qb->expr()->in('mimetype', $qb->createNamedParameter($directoryTypes, IQueryBuilder::PARAM_INT_ARRAY)))
+			->andWhere($qb->expr()->eq('parent', $qb->createNamedParameter($fileid)))
+			->executeQuery();
+		/**
+		 * @var list<array> $dir
+		 */
+		$dir = $result->fetchAll();
+
+		if ($recursive) {
+			foreach ($dir as $item) {
+				$dir = array_merge($dir, $this->getDir((int) $item['fileid'], $directoryTypes, $recursive));
+			}
+		}
+		return $dir;
+	}
+
+	/**
+	 * @param int $storageId
+	 * @param array $ignoreMarkers
+	 * @return array
+	 */
+	public function getIgnoredDirectories(int $storageId, array $ignoreMarkers): array {
+		$directoryTypes = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), Constants::DIRECTORY_FORMATS);
+		$qb = new CacheQueryBuilder($this->db, $this->systemConfig, $this->logger);
+		$result = $qb->selectFileCache()
+			->andWhere($qb->expr()->in('name', $qb->createNamedParameter($ignoreMarkers, IQueryBuilder::PARAM_STR_ARRAY)))
+			->andWhere($qb->expr()->eq('storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
+			->executeQuery();
+		/**
+		 * @var list<array> $ignoreFiles
+		 */
+		$ignoreFiles = $result->fetchAll();
+		$ignoreFileIds = array_map(fn ($dir): int => (int)$dir['parent'], $ignoreFiles);
+		foreach ($ignoreFiles as $ignoreFile) {
+			$ignoreDir = $this->getDir((int) $ignoreFile['parent'], $directoryTypes, true);
+			$fileIds = array_map(fn ($dir): int => (int) $dir['fileid'], $ignoreDir);
+			$ignoreFileIds = array_merge($ignoreFileIds, $fileIds);
+		}
+		return $ignoreFileIds;
+	}
+}

src/components/ViewAdmin.vue

@@ -18,14 +18,19 @@
 			<NcNoteCard v-if="nodejs === false">
 				{{ t('recognize', 'Could not execute the Node.js binary. You may need to set the path to a working binary manually.') }}
 			</NcNoteCard>
-			<template v-else-if="settings['faces.enabled'] || settings['imagenet.enabled'] || settings['musicnn.enabled'] || settings['movinet.enabled']">
-				<NcNoteCard show-alert type="success">
-					{{ t('recognize', 'The app is installed and will automatically classify files in background processes.') }}
-				</NcNoteCard>
+			<NcNoteCard v-if="cron !== undefined && cron !== 'cron'">
+				{{ t('recognize', 'Background Jobs are not executed via cron. Recognize requires background jobs to be executed via cron.') }}
+			</NcNoteCard>
+			<template v-if="nodejs && (libtensorflow || wasmtensorflow) && cron === 'cron'">
+				<template v-if="settings['faces.enabled'] || settings['imagenet.enabled'] || settings['musicnn.enabled'] || settings['movinet.enabled']">
+					<NcNoteCard show-alert type="success">
+						{{ t('recognize', 'The app is installed and will automatically classify files in background processes.') }}
+					</NcNoteCard>
+				</template>
+				<p v-else>
+					{{ t('recognize', 'None of the tagging options below are currently selected. The app will currently do nothing.') }}
+				</p>
 			</template>
-			<p v-else>
-				{{ t('recognize', 'None of the tagging options below are currently selected. The app will currently do nothing.') }}
-			</p>
 		</NcSettingsSection>
 		<NcSettingsSection :title="t('recognize', 'Image tagging')">
 			<template v-if="settings['faces.enabled']">
@@ -287,6 +292,7 @@ export default {
 			nodejs: undefined,
 			libtensorflow: undefined,
 			wasmtensorflow: undefined,
+			cron: undefined,
 			modelsDownloaded: null,
 		}
 	},
@@ -322,6 +328,7 @@ export default {
 		this.getNodejsStatus()
 		this.getLibtensorflowStatus()
 		this.getWasmtensorflowStatus()
+		this.getCronStatus()
 
 		setInterval(async () => {
 			this.getCount()
@@ -415,6 +422,11 @@ export default {
 			const { wasmtensorflow } = resp.data
 			this.wasmtensorflow = wasmtensorflow
 		},
+		async getCronStatus() {
+			const resp = await axios.get(generateUrl('/apps/recognize/admin/cron'))
+			const { cron } = resp.data
+			this.cron = cron
+		},
 		onChange() {
 			if (this.timeout) {
 				clearTimeout(this.timeout)

src/test_wasmtensorflow.js

@@ -1,4 +1,4 @@
-let tf = require('@tensorflow/tfjs')
+const tf = require('@tensorflow/tfjs')
 require('@tensorflow/tfjs-backend-wasm')
 
 tf.setBackend('wasm')

test/ClassifierTest.php

@@ -76,6 +76,10 @@ public function setUp(): void {
 		$this->queue->clearQueue(LandmarksClassifier::MODEL_NAME);
 		$this->queue->clearQueue(MovinetClassifier::MODEL_NAME);
 		$this->queue->clearQueue(MusicnnClassifier::MODEL_NAME);
+		$this->config->setAppValue('recognize', 'imagenet.enabled', 'false');
+		$this->config->setAppValue('recognize', 'faces.enabled', 'false');
+		$this->config->setAppValue('recognize', 'movinet.enabled', 'false');
+		$this->config->setAppValue('recognize', 'musicnn.enabled', 'false');
 	}
 
 	public function testSchedulerJob() : void {
@@ -108,6 +112,33 @@ public function testSchedulerJob() : void {
 		]);
 	}
 
+	/**
+	 * @dataProvider ignoreImageFilesProvider
+	 * @return void
+	 * @throws \OCP\DB\Exception
+	 * @throws \OCP\Files\InvalidPathException
+	 * @throws \OCP\Files\NotFoundException
+	 * @throws \OCP\Files\NotPermittedException
+	 */
+	public function testFileListener(string $ignoreFileName) : void {
+		$this->config->setAppValue('recognize', 'imagenet.enabled', 'true');
+		$this->queue->clearQueue(ImagenetClassifier::MODEL_NAME);
+
+		$this->testFile = $this->userFolder->newFile('/alpine.jpg', file_get_contents(__DIR__.'/res/alpine.JPG'));
+		$this->userFolder->newFolder('/test/ignore/');
+		$this->userFolder->newFile('/test/' . $ignoreFileName, '');
+		$this->ignoredFile = $this->userFolder->newFile('/test/ignore/alpine.jpg', file_get_contents(__DIR__.'/res/alpine.JPG'));
+
+		$storageId = $this->testFile->getMountPoint()->getNumericStorageId();
+		$rootId = $this->testFile->getMountPoint()->getStorageRootId();
+
+		self::assertCount(1, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'one element should have been added to imagenet queue');
+
+		$this->testFile->delete();
+
+		self::assertCount(0, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'entry should have been removed from imagenet queue');
+	}
+
 	/**
 	 * @dataProvider ignoreImageFilesProvider
 	 * @return void
@@ -123,7 +154,9 @@ public function testImagenetPipeline(string $ignoreFileName) : void {
 		$this->userFolder->newFolder('/test/ignore/');
 		$this->ignoredFile = $this->userFolder->newFile('/test/ignore/alpine.jpg', file_get_contents(__DIR__.'/res/alpine.JPG'));
 		$this->userFolder->newFile('/test/' . $ignoreFileName, '');
+
 		$this->config->setAppValue('recognize', 'imagenet.enabled', 'true');
+
 		/** @var StorageCrawlJob $scheduler */
 		$crawler = \OC::$server->get(StorageCrawlJob::class);
 		/** @var IJobList $this->jobList */

test/bootstrap.php

@@ -9,5 +9,3 @@
 
 // load all enabled apps
 \OC_App::loadApps();
-
-OC_Hook::clear();