diff --git a/appinfo/routes.php b/appinfo/routes.php index be7efa0f..c9cdbcf4 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -33,6 +33,7 @@ ['name' => 'admin#nodejs', 'url' => '/admin/nodejs', 'verb' => 'GET'], ['name' => 'admin#libtensorflow', 'url' => '/admin/libtensorflow', 'verb' => 'GET'], ['name' => 'admin#wasmtensorflow', 'url' => '/admin/wasmtensorflow', 'verb' => 'GET'], + ['name' => 'admin#cron', 'url' => '/admin/cron', 'verb' => 'GET'], ['name' => 'admin#get_setting', 'url' => '/admin/settings/{setting}', 'verb' => 'GET'], ['name' => 'admin#set_setting', 'url' => '/admin/settings/{setting}', 'verb' => 'PUT'], ], diff --git a/lib/BackgroundJobs/StorageCrawlJob.php b/lib/BackgroundJobs/StorageCrawlJob.php index 0666a585..4ca27865 100644 --- a/lib/BackgroundJobs/StorageCrawlJob.php +++ b/lib/BackgroundJobs/StorageCrawlJob.php @@ -16,6 +16,7 @@ use OCA\Recognize\Classifiers\Video\MovinetClassifier; use OCA\Recognize\Constants; use OCA\Recognize\Db\QueueFile; +use OCA\Recognize\Service\IgnoreService; use OCA\Recognize\Service\Logger; use OCA\Recognize\Service\QueueService; use OCA\Recognize\Service\TagManager; @@ -36,8 +37,9 @@ class StorageCrawlJob extends QueuedJob { private IDBConnection $db; private SystemConfig $systemConfig; private TagManager $tagManager; + private IgnoreService $ignoreService; - public function __construct(ITimeFactory $timeFactory, Logger $logger, IMimeTypeLoader $mimeTypes, QueueService $queue, IJobList $jobList, IDBConnection $db, SystemConfig $systemConfig, TagManager $tagManager) { + public function __construct(ITimeFactory $timeFactory, Logger $logger, IMimeTypeLoader $mimeTypes, QueueService $queue, IJobList $jobList, IDBConnection $db, SystemConfig $systemConfig, TagManager $tagManager, IgnoreService $ignoreService) { parent::__construct($timeFactory); $this->logger = $logger; $this->mimeTypes = $mimeTypes; @@ -46,39 +48,7 @@ public function __construct(ITimeFactory $timeFactory, Logger $logger, IMimeType $this->db = $db; $this->systemConfig = $systemConfig; $this->tagManager = $tagManager; - } - - private function getDir(int $fileid, array $directoryTypes, bool $recursive = false): array { - /** @var \OCP\DB\QueryBuilder\IQueryBuilder $qb */ - $qb = new CacheQueryBuilder($this->db, $this->systemConfig, $this->logger); - $dir = $qb->selectFileCache() - ->andWhere($qb->expr()->in('mimetype', $qb->createNamedParameter($directoryTypes, IQueryBuilder::PARAM_INT_ARRAY))) - ->andWhere($qb->expr()->eq('parent', $qb->createNamedParameter($fileid))) - ->executeQuery()->fetchAll(); - - if ($recursive) { - foreach ($dir as $item) { - $dir = array_merge($dir, $this->getDir($item['fileid'], $directoryTypes, $recursive)); - } - } - return $dir; - } - - private function getIgnoreFileids(int $storageId, array $ignore_maekers): array { - $directoryTypes = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), Constants::DIRECTORY_FORMATS); - /** @var \OCP\DB\QueryBuilder\IQueryBuilder $qb */ - $qb = new CacheQueryBuilder($this->db, $this->systemConfig, $this->logger); - $ignoreFiles = $qb->selectFileCache() - ->andWhere($qb->expr()->in('name', $qb->createNamedParameter($ignore_maekers, IQueryBuilder::PARAM_STR_ARRAY))) - ->andWhere($qb->expr()->eq('storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) - ->executeQuery()->fetchAll(); - $ignoreFileids = array_map(fn ($dir) => $dir['parent'], $ignoreFiles); - foreach ($ignoreFiles as $ignoreFile) { - $ignoreDir = $this->getDir($ignoreFile['parent'], $directoryTypes, true); - $fileids = array_map(fn ($dir) => $dir['fileid'], $ignoreDir); - $ignoreFileids = array_merge($ignoreFileids, $fileids); - } - return $ignoreFileids; + $this->ignoreService = $ignoreService; } protected function run($argument): void { @@ -114,10 +84,10 @@ protected function run($argument): void { } try { - $ignoreAllFileids = $this->getIgnoreFileids($storageId, Constants::IGNORE_MARKERS_ALL); - $ignoreImageFileids = $this->getIgnoreFileids($storageId, Constants::IGNORE_MARKERS_IMAGE); - $ignoreVideoFileids = $this->getIgnoreFileids($storageId, Constants::IGNORE_MARKERS_VIDEO); - $ignoreAudioFileids = $this->getIgnoreFileids($storageId, Constants::IGNORE_MARKERS_AUDIO); + $ignoreAllFileids = $this->ignoreService->getIgnoredDirectories($storageId, Constants::IGNORE_MARKERS_ALL); + $ignoreImageFileids = $this->ignoreService->getIgnoredDirectories($storageId, Constants::IGNORE_MARKERS_IMAGE); + $ignoreVideoFileids = $this->ignoreService->getIgnoredDirectories($storageId, Constants::IGNORE_MARKERS_VIDEO); + $ignoreAudioFileids = $this->ignoreService->getIgnoredDirectories($storageId, Constants::IGNORE_MARKERS_AUDIO); } catch (Exception $e) { $this->logger->error('Could not fetch ignore files', ['exception' => $e]); return; diff --git a/lib/Controller/AdminController.php b/lib/Controller/AdminController.php index c4855077..c9d23335 100644 --- a/lib/Controller/AdminController.php +++ b/lib/Controller/AdminController.php @@ -13,6 +13,7 @@ use OCP\AppFramework\Http\JSONResponse; use OCP\BackgroundJob\IJobList; use OCP\DB\Exception; +use OCP\IConfig; use OCP\IRequest; class AdminController extends Controller { @@ -22,8 +23,9 @@ class AdminController extends Controller { private QueueService $queue; private FaceClusterMapper $clusterMapper; private FaceDetectionMapper $detectionMapper; + private IConfig $config; - public function __construct(string $appName, IRequest $request, TagManager $tagManager, IJobList $jobList, SettingsService $settingsService, QueueService $queue, FaceClusterMapper $clusterMapper, FaceDetectionMapper $detectionMapper) { + public function __construct(string $appName, IRequest $request, TagManager $tagManager, IJobList $jobList, SettingsService $settingsService, QueueService $queue, FaceClusterMapper $clusterMapper, FaceDetectionMapper $detectionMapper, IConfig $config) { parent::__construct($appName, $request); $this->tagManager = $tagManager; $this->jobList = $jobList; @@ -31,6 +33,7 @@ public function __construct(string $appName, IRequest $request, TagManager $tagM $this->queue = $queue; $this->clusterMapper = $clusterMapper; $this->detectionMapper = $detectionMapper; + $this->config = $config; } public function reset(): JSONResponse { @@ -160,6 +163,11 @@ public function wasmtensorflow(): JSONResponse { return new JSONResponse(['wasmtensorflow' => true]); } + public function cron(): JSONResponse { + $cron = $this->config->getAppValue('core', 'backgroundjobs_mode', ''); + return new JSONResponse(['cron' => $cron]); + } + public function setSetting(string $setting, $value): JSONResponse { try { $this->settingsService->setSetting($setting, $value); diff --git a/lib/Hooks/FileListener.php b/lib/Hooks/FileListener.php index 39569660..0ad86ab6 100644 --- a/lib/Hooks/FileListener.php +++ b/lib/Hooks/FileListener.php @@ -9,6 +9,7 @@ use OCA\Recognize\Constants; use OCA\Recognize\Db\FaceDetectionMapper; use OCA\Recognize\Db\QueueFile; +use OCA\Recognize\Service\IgnoreService; use OCA\Recognize\Service\QueueService; use OCP\DB\Exception; use OCP\EventDispatcher\Event; @@ -25,11 +26,13 @@ class FileListener implements IEventListener { private FaceDetectionMapper $faceDetectionMapper; private LoggerInterface $logger; private QueueService $queue; + private IgnoreService $ignoreService; - public function __construct(FaceDetectionMapper $faceDetectionMapper, LoggerInterface $logger, QueueService $queue) { + public function __construct(FaceDetectionMapper $faceDetectionMapper, LoggerInterface $logger, QueueService $queue, IgnoreService $ignoreService) { $this->faceDetectionMapper = $faceDetectionMapper; $this->logger = $logger; $this->queue = $queue; + $this->ignoreService = $ignoreService; } public function handle(Event $event): void { @@ -80,11 +83,40 @@ public function postDelete(Node $node): void { } } + /** + * @throws \OCP\Files\InvalidPathException + */ public function postInsert(Node $node): void { $queueFile = new QueueFile(); - $queueFile->setStorageId($node->getMountPoint()->getStorageId()); + $queueFile->setStorageId((string) $node->getMountPoint()->getNumericStorageId()); $queueFile->setRootId((string) $node->getMountPoint()->getStorageRootId()); + $ignoreMarkers = []; + if (in_array($node->getMimetype(), Constants::IMAGE_FORMATS)) { + $ignoreMarkers = array_merge($ignoreMarkers, Constants::IGNORE_MARKERS_IMAGE); + } + if (in_array($node->getMimetype(), Constants::VIDEO_FORMATS)) { + $ignoreMarkers = array_merge($ignoreMarkers, Constants::IGNORE_MARKERS_VIDEO); + } + if (in_array($node->getMimetype(), Constants::AUDIO_FORMATS)) { + $ignoreMarkers = array_merge($ignoreMarkers, Constants::IGNORE_MARKERS_AUDIO); + } + if (count($ignoreMarkers) === 0) { + return; + } + $ignoreMarkers = array_merge($ignoreMarkers, Constants::IGNORE_MARKERS_ALL); + $ignoredDirectories = $this->ignoreService->getIgnoredDirectories($node->getMountPoint()->getNumericStorageId(), $ignoreMarkers); + + try { + if (in_array($node->getParent()->getId(), $ignoredDirectories)) { + return; + } + } catch (InvalidPathException $e) { + return; + } catch (NotFoundException $e) { + return; + } + try { $queueFile->setFileId($node->getId()); } catch (InvalidPathException|NotFoundException $e) { diff --git a/lib/Service/IgnoreService.php b/lib/Service/IgnoreService.php new file mode 100644 index 00000000..1616ea24 --- /dev/null +++ b/lib/Service/IgnoreService.php @@ -0,0 +1,75 @@ +db = $db; + $this->systemConfig = $systemConfig; + $this->logger = $logger; + $this->mimeTypes = $mimeTypes; + } + + /** + * @param int $fileid + * @param array $directoryTypes + * @param bool $recursive + * @return list + */ + public function getDir(int $fileid, array $directoryTypes, bool $recursive = false): array { + $qb = new CacheQueryBuilder($this->db, $this->systemConfig, $this->logger); + $result = $qb->selectFileCache() + ->andWhere($qb->expr()->in('mimetype', $qb->createNamedParameter($directoryTypes, IQueryBuilder::PARAM_INT_ARRAY))) + ->andWhere($qb->expr()->eq('parent', $qb->createNamedParameter($fileid))) + ->executeQuery(); + /** + * @var list $dir + */ + $dir = $result->fetchAll(); + + if ($recursive) { + foreach ($dir as $item) { + $dir = array_merge($dir, $this->getDir((int) $item['fileid'], $directoryTypes, $recursive)); + } + } + return $dir; + } + + /** + * @param int $storageId + * @param array $ignoreMarkers + * @return array + */ + public function getIgnoredDirectories(int $storageId, array $ignoreMarkers): array { + $directoryTypes = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), Constants::DIRECTORY_FORMATS); + $qb = new CacheQueryBuilder($this->db, $this->systemConfig, $this->logger); + $result = $qb->selectFileCache() + ->andWhere($qb->expr()->in('name', $qb->createNamedParameter($ignoreMarkers, IQueryBuilder::PARAM_STR_ARRAY))) + ->andWhere($qb->expr()->eq('storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT))) + ->executeQuery(); + /** + * @var list $ignoreFiles + */ + $ignoreFiles = $result->fetchAll(); + $ignoreFileIds = array_map(fn ($dir): int => (int)$dir['parent'], $ignoreFiles); + foreach ($ignoreFiles as $ignoreFile) { + $ignoreDir = $this->getDir((int) $ignoreFile['parent'], $directoryTypes, true); + $fileIds = array_map(fn ($dir): int => (int) $dir['fileid'], $ignoreDir); + $ignoreFileIds = array_merge($ignoreFileIds, $fileIds); + } + return $ignoreFileIds; + } +} diff --git a/src/components/ViewAdmin.vue b/src/components/ViewAdmin.vue index 072f50ca..a60a904f 100644 --- a/src/components/ViewAdmin.vue +++ b/src/components/ViewAdmin.vue @@ -18,14 +18,19 @@ {{ t('recognize', 'Could not execute the Node.js binary. You may need to set the path to a working binary manually.') }} -