From 37b41cb8593d23827c902d075071fd8b0f7af0b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 21 Jan 2014 16:04:38 +0100 Subject: [PATCH 01/51] move getUnindexed from Indexer to Status class --- ajax/lucene.php | 2 +- lib/indexer.php | 57 ------------------------------------------------- lib/status.php | 57 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 58 deletions(-) diff --git a/ajax/lucene.php b/ajax/lucene.php index 9d33bd044..7427abb5d 100644 --- a/ajax/lucene.php +++ b/ajax/lucene.php @@ -11,7 +11,7 @@ function index() { if ( isset($_GET['fileid']) ){ $fileIds = array($_GET['fileid']); } else { - $fileIds = OCA\Search_Lucene\Indexer::getUnindexed(); + $fileIds = OCA\Search_Lucene\Status::getUnindexed(); } $eventSource = new OC_EventSource(); diff --git a/lib/indexer.php b/lib/indexer.php index db8bb03cb..847ca59d2 100644 --- a/lib/indexer.php +++ b/lib/indexer.php @@ -244,61 +244,4 @@ private static function extractMetadata( return; } } - - /** - * get the list of all unindexed files for the user - * - * @return array - */ - static public function getUnindexed() { - $files = array(); - $absoluteRoot = Filesystem::getView()->getAbsolutePath('/'); - $mounts = Filesystem::getMountPoints($absoluteRoot); - $mount = Filesystem::getMountPoint($absoluteRoot); - if (!in_array($mount, $mounts)) { - $mounts[] = $mount; - } - - $query = \OCP\DB::prepare(' - SELECT `*PREFIX*filecache`.`fileid` - FROM `*PREFIX*filecache` - LEFT JOIN `*PREFIX*lucene_status` - ON `*PREFIX*filecache`.`fileid` = `*PREFIX*lucene_status`.`fileid` - WHERE `storage` = ? - AND `status` IS NULL OR `status` = ? - '); - - foreach ($mounts as $mount) { - if (is_string($mount)) { - $storage = Filesystem::getStorage($mount); - } else if ($mount instanceof \OC\Files\Mount\Mount) { - $storage = $mount->getStorage(); - } else { - $storage = null; - Util::writeLog('search_lucene', - 'expected string or instance of \OC\Files\Mount\Mount got ' . json_encode($mount), - Util::DEBUG); - } - //only index local files for now - if ($storage instanceof \OC\Files\Storage\Local) { - $cache = $storage->getCache(); - $numericId = $cache->getNumericStorageId(); - - $result = $query->execute(array($numericId, 'N')); - if (\OCP\DB::isError($result)) { - Util::writeLog( - 'search_lucene', - 'failed to find unindexed files: '.\OCP\DB::getErrorMessage($result), - Util::WARN - ); - return false; - } - while ($row = $result->fetchRow()) { - $files[] = $row['fileid']; - } - } - } - return $files; - } - } diff --git a/lib/status.php b/lib/status.php index 5b243f271..f0c27c179 100644 --- a/lib/status.php +++ b/lib/status.php @@ -82,4 +82,61 @@ private static function update($fileId, $status) { '); return $query->execute(array($status, $fileId)); } + + /** + * get the list of all unindexed files for the user + * + * @return array + */ + static public function getUnindexed() { + $files = array(); + $absoluteRoot = Filesystem::getView()->getAbsolutePath('/'); + $mounts = Filesystem::getMountPoints($absoluteRoot); + $mount = Filesystem::getMountPoint($absoluteRoot); + if (!in_array($mount, $mounts)) { + $mounts[] = $mount; + } + + $query = \OCP\DB::prepare(' + SELECT `*PREFIX*filecache`.`fileid` + FROM `*PREFIX*filecache` + LEFT JOIN `*PREFIX*lucene_status` + ON `*PREFIX*filecache`.`fileid` = `*PREFIX*lucene_status`.`fileid` + WHERE `storage` = ? + AND `status` IS NULL OR `status` = ? + '); + + foreach ($mounts as $mount) { + if (is_string($mount)) { + $storage = Filesystem::getStorage($mount); + } else if ($mount instanceof \OC\Files\Mount\Mount) { + $storage = $mount->getStorage(); + } else { + $storage = null; + Util::writeLog('search_lucene', + 'expected string or instance of \OC\Files\Mount\Mount got ' . json_encode($mount), + Util::DEBUG); + } + //only index local files for now + if ($storage instanceof \OC\Files\Storage\Local) { + $cache = $storage->getCache(); + $numericId = $cache->getNumericStorageId(); + + $result = $query->execute(array($numericId, self::STATUS_NEW)); + if (\OCP\DB::isError($result)) { + Util::writeLog( + 'search_lucene', + 'failed to find unindexed files: '.\OCP\DB::getErrorMessage($result), + Util::WARN + ); + return false; + } + while ($row = $result->fetchRow()) { + $files[] = $row['fileid']; + } + } + } + return $files; + } + } From 47a75c0b325ba49d2bea210294b44806ef14ce11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 22 Jan 2014 18:52:33 +0100 Subject: [PATCH 02/51] update search_lucene to use new jobs api - uses batch updates because filesystem hooks don't provide file ids - also refactored most of the static access code to new & instance method calls --- ajax/lucene.php | 56 +++--------- appinfo/app.php | 8 +- lib/hooks.php | 93 ++++++++----------- lib/indexer.php | 192 +++++++++++++++------------------------ lib/indexjob.php | 30 +++++++ lib/lucene.php | 199 ++++++++--------------------------------- lib/searchprovider.php | 136 ++++++++++++++++++++++++++++ lib/status.php | 27 ++++++ 8 files changed, 351 insertions(+), 390 deletions(-) create mode 100644 lib/indexjob.php create mode 100644 lib/searchprovider.php diff --git a/ajax/lucene.php b/ajax/lucene.php index 7427abb5d..5f7aa5911 100644 --- a/ajax/lucene.php +++ b/ajax/lucene.php @@ -5,8 +5,6 @@ OCP\JSON::checkAppEnabled('search_lucene'); session_write_close(); -//FIXME refactor db queries and logic into the lib folder - function index() { if ( isset($_GET['fileid']) ){ $fileIds = array($_GET['fileid']); @@ -17,50 +15,16 @@ function index() { $eventSource = new OC_EventSource(); $eventSource->send('count', count($fileIds)); - $skippedDirs = explode(';', OCP\Config::getUserValue(OCP\User::getUser(), 'search_lucene', 'skipped_dirs', '.git;.svn;.CVS;.bzr')); - - foreach ($fileIds as $id) { - $skipped = false; - - $fileStatus = OCA\Search_Lucene\Status::fromFileId($id); - - try{ - //before we start mark the file as error so we know there was a problem when the php execution dies - $fileStatus->markError(); - - $path = OC\Files\Filesystem::getPath($id); - $eventSource->send('indexing', $path); - - foreach ($skippedDirs as $skippedDir) { - if (strpos($path, '/' . $skippedDir . '/') !== false //contains dir - || strrpos($path, '/' . $skippedDir) === strlen($path) - (strlen($skippedDir) + 1) // ends with dir - ) { - $result = $fileStatus->markSkipped(); - $skipped = true; - break; - } - } - if (!$skipped) { - if (OCA\Search_Lucene\Indexer::indexFile($path, OCP\User::getUser())) { - $result = $fileStatus->markIndexed(); - } - } - - if (!$result) { - OCP\JSON::error(array('message' => 'Could not index file.')); - $eventSource->send('error', $path); - } - } catch (Exception $e) { //sqlite might report database locked errors when stock filescan is in progress - //this also catches db locked exception that might come up when using sqlite - \OCP\Util::writeLog('search_lucene', - $e->getMessage() . ' Trace:\n' . $e->getTraceAsString(), - \OCP\Util::ERROR); - OCP\JSON::error(array('message' => 'Could not index file.')); - $eventSource->send('error', $e->getMessage()); - //try to mark the file as new to let it reindex - $fileStatus->markNew(); // Add UI to trigger rescan of files with status 'E'rror? - } - } + $user = OCP\User::getUser(); + + OC_Util::tearDownFS(); + OC_Util::setupFS($user); + + $view = new OC\Files\View('/' . $user . '/files'); + $lucene = new OCA\Search_Lucene\Lucene($user); + + $indexer = new OCA\Search_Lucene\Indexer($view, $lucene); + $indexer->indexFiles($fileIds, $eventSource); $eventSource->send('done', ''); $eventSource->close(); diff --git a/appinfo/app.php b/appinfo/app.php index b186f6146..61845c35b 100644 --- a/appinfo/app.php +++ b/appinfo/app.php @@ -27,11 +27,11 @@ //add 3rdparty folder to include path $dir = dirname(dirname(__FILE__)).'/3rdparty'; set_include_path(get_include_path() . PATH_SEPARATOR . $dir); - +/* OC::$CLASSPATH['OCA\Search_Lucene\Lucene'] = 'search_lucene/lib/lucene.php'; OC::$CLASSPATH['OCA\Search_Lucene\Indexer'] = 'search_lucene/lib/indexer.php'; OC::$CLASSPATH['OCA\Search_Lucene\Hooks'] = 'search_lucene/lib/hooks.php'; - +*/ OC::$CLASSPATH['OCA\Search_Lucene\Document\Pdf'] = 'search_lucene/document/Pdf.php'; OC::$CLASSPATH['OCA\Search_Lucene\Document\OpenDocument'] = 'search_lucene/document/OpenDocument.php'; OC::$CLASSPATH['OCA\Search_Lucene\Document\Odt'] = 'search_lucene/document/Odt.php'; @@ -63,7 +63,7 @@ //remove other providers OC_Search::removeProvider('OC_Search_Provider_File'); -OC_Search::registerProvider('OCA\Search_Lucene\Lucene'); +OC_Search::registerProvider('OCA\Search_Lucene\SearchProvider'); // --- add hooks ----------------------------------------------- @@ -86,6 +86,6 @@ //listen for file deletions to clean the database OCP\Util::connectHook( OC\Files\Filesystem::CLASSNAME, - OC\Files\Filesystem::signal_delete, + 'post_delete', //FIXME add referenceable constant in core 'OCA\Search_Lucene\Hooks', OCA\Search_Lucene\Hooks::handle_delete); diff --git a/lib/hooks.php b/lib/hooks.php index 194cb4092..e6f93c159 100644 --- a/lib/hooks.php +++ b/lib/hooks.php @@ -48,35 +48,9 @@ class Hooks { * @param $param array from postWriteFile-Hook */ public static function indexFile(array $param) { - if (isset($param['path'])) { - $param['user'] = \OCP\User::getUser(); + $arguments = array('user' => \OCP\User::getUser()); //Add Background Job: - BackgroundJob::addQueuedTask( - 'search_lucene', - 'OCA\Search_Lucene\Hooks', - 'doIndexFile', - json_encode($param) ); - } else { - Util::writeLog('search_lucene', - 'missing path parameter', - Util::WARN); - } - } - static public function doIndexFile($param) { - $data = json_decode($param); - if ( ! isset($data->path) ) { - Util::writeLog('search_lucene', - 'missing path parameter', - Util::WARN); - return false; - } - if ( ! isset($data->user) ) { - Util::writeLog('search_lucene', - 'missing user parameter', - Util::WARN); - return false; - } - Indexer::indexFile($data->path, $data->user); + BackgroundJob::registerJob( '\OCA\Search_Lucene\IndexJob', $arguments ); } /** @@ -87,12 +61,18 @@ static public function doIndexFile($param) { * @param $param array from postRenameFile-Hook */ public static function renameFile(array $param) { + $user = \OCP\User::getUser(); + if (isset($param['oldpath'])) { + //delete from lucene index + $lucene = new Lucene($user); + $lucene->deleteFile($param['oldpath']); + } if (isset($param['newpath'])) { + $view = new \OC\Files\View('/' . $user . '/files'); + $info = $view->getFileInfo($param['newpath']); + Status::fromFileId($info['fileid'])->markNew(); self::indexFile(array('path'=>$param['newpath'])); } - if (isset($param['oldpath'])) { - self::deleteFile(array('path'=>$param['oldpath'])); - } } /** @@ -102,41 +82,38 @@ public static function renameFile(array $param) { * * @author Jörn Dreyer * - * @param $param array from postDeleteFile-Hook + * @param $param array from deleteFile-Hook */ static public function deleteFile(array $param) { + Util::writeLog( + 'search_lucene', + 'deleting status for ' . json_encode($param), + Util::DEBUG + ); // we cannot use post_delete as $param would not contain the id // of the deleted file and we could not fetch it with getId if (isset($param['path'])) { - $param['user'] = \OCP\User::getUser(); - //Add Background Job: - BackgroundJob::addQueuedTask( + $user = \OCP\User::getUser(); + //delete status + $deletedIds = Status::getDeleted(); + foreach ($deletedIds as $fileid) { + Util::writeLog( 'search_lucene', - 'OCA\Search_Lucene\Hooks', - 'doDeleteFile', - json_encode($param) ); + 'deleting status for ('.$fileid.') ', + Util::DEBUG + ); + \OCA\Search_Lucene\Status::delete($fileid); + } + //delete from lucene + $lucene = new Lucene($user); + $lucene->deleteFile($param['path']); } else { - Util::writeLog('search_lucene', - 'missing path parameter', - Util::WARN); - } - - } - static public function doDeleteFile($param) { - $data = json_decode($param); - if ( ! isset($data->path) ) { - Util::writeLog('search_lucene', + Util::writeLog( + 'search_lucene', 'missing path parameter', - Util::WARN); - return false; - } - if ( ! isset($data->user) ) { - Util::writeLog('search_lucene', - 'missing user parameter', - Util::WARN); - return false; + Util::ERROR + ); } - Lucene::deleteFile($data->path, $data->user); - } + } } diff --git a/lib/indexer.php b/lib/indexer.php index 847ca59d2..1ac3164b9 100644 --- a/lib/indexer.php +++ b/lib/indexer.php @@ -18,6 +18,68 @@ class Indexer { * used as signalclass in OC_Hooks::emit() */ const CLASSNAME = 'Indexer'; + + private $view; + private $lucene; + + public function __construct(\OC\Files\View $view, Lucene $lucene) { + $this->view = $view; + $this->lucene = $lucene; + } + + public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) { + + $skippedDirs = explode(';', \OCP\Config::getUserValue(\OCP\User::getUser(), 'search_lucene', 'skipped_dirs', '.git;.svn;.CVS;.bzr')); + + foreach ($fileIds as $id) { + $skipped = false; + + $fileStatus = \OCA\Search_Lucene\Status::fromFileId($id); + + try{ + //before we start mark the file as error so we know there was a problem when the php execution dies + $fileStatus->markError(); + + $path = \OC\Files\Filesystem::getPath($id); + if ($eventSource) { + $eventSource->send('indexing', $path); + } + + foreach ($skippedDirs as $skippedDir) { + if (strpos($path, '/' . $skippedDir . '/') !== false //contains dir + || strrpos($path, '/' . $skippedDir) === strlen($path) - (strlen($skippedDir) + 1) // ends with dir + ) { + $result = $fileStatus->markSkipped(); + $skipped = true; + break; + } + } + if (!$skipped) { + if ($this->indexFile($path)) { + $result = $fileStatus->markIndexed(); + } + } + + if (!$result) { + \OCP\JSON::error(array('message' => 'Could not index file.')); + if ($eventSource) { + $eventSource->send('error', $path); + } + } + } catch (Exception $e) { //sqlite might report database locked errors when stock filescan is in progress + //this also catches db locked exception that might come up when using sqlite + \OCP\Util::writeLog('search_lucene', + $e->getMessage() . ' Trace:\n' . $e->getTraceAsString(), + \OCP\Util::ERROR); + \OCP\JSON::error(array('message' => 'Could not index file.')); + if ($eventSource) { + $eventSource->send('error', $e->getMessage()); + } + //try to mark the file as new to let it reindex + $fileStatus->markNew(); // Add UI to trigger rescan of files with status 'E'rror? + } + } + } /** * index a file @@ -28,42 +90,28 @@ class Indexer { * * @return bool */ - static public function indexFile($path = '', $user = null) { + public function indexFile($path = '') { if (!Filesystem::isValidPath($path)) { return; } - if ($path === '') { + if (empty($path)) { //ignore the empty path element return false; } - - if (is_null($user)) { - $view = Filesystem::getView(); - $user = \OCP\User::getUser(); - } else { - $view = new \OC\Files\View('/' . $user . '/files'); - } - - if ( ! $view ) { - Util::writeLog('search_lucene', - 'could not resolve filesystem view', - Util::WARN); - return false; - } - if(!$view->file_exists($path)) { + if(!$this->view->file_exists($path)) { Util::writeLog('search_lucene', 'file vanished, ignoring', Util::DEBUG); return true; } - $root = $view->getRoot(); + $root = $this->view->getRoot(); $pk = md5($root . $path); // the cache already knows mime and other basic stuff - $data = $view->getFileInfo($path); + $data = $this->view->getFileInfo($path); if (isset($data['mimetype'])) { $mimeType = $data['mimetype']; @@ -71,27 +119,29 @@ static public function indexFile($path = '', $user = null) { $doc = new \Zend_Search_Lucene_Document(); // index content for local files only - $localFile = $view->getLocalFile($path); + $localFile = $this->view->getLocalFile($path); if ( $localFile ) { //try to use special lucene document types if ('text/plain' === $mimeType) { - $body = $view->file_get_contents($path); + $body = $this->view->file_get_contents($path); if ($body != '') { $doc->addField(\Zend_Search_Lucene_Field::UnStored('body', $body)); } + // FIXME uther text files? c, php, java ... + } else if ('text/html' === $mimeType) { //TODO could be indexed, even if not local - $doc = \Zend_Search_Lucene_Document_Html::loadHTML($view->file_get_contents($path)); + $doc = \Zend_Search_Lucene_Document_Html::loadHTML($this->view->file_get_contents($path)); } else if ('application/pdf' === $mimeType) { - $doc = Pdf::loadPdf($view->file_get_contents($path)); + $doc = Pdf::loadPdf($this->view->file_get_contents($path)); // commented the mimetype checks, as the zend classes only understand docx and not doc files. // FIXME distinguish doc and docx, xls and xlsx, ppt and pptx, in oc core mimetype helper ... @@ -135,9 +185,8 @@ static public function indexFile($path = '', $user = null) { $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', $mimeType)); - //self::extractMetadata($doc, $path, $view, $mimeType); - Lucene::updateFile($doc, $path, $user); + $this->lucene->updateFile($doc, $path); return true; @@ -151,97 +200,4 @@ static public function indexFile($path = '', $user = null) { } } - - /** - * extract the metadata from a file - * - * uses getid3 to extract metadata. - * if possible also adds content (currently only for plain text files) - * hint: use OC\Files\Filesystem::getFileInfo($path) to get metadata for the last param - * - * @author Jörn Dreyer - * - * @param Zend_Search_Lucene_Document $doc to add the metadata to - * @param string $path path of the file to extract metadata from - * @param string $mimetype depending on the mimetype different extractions are performed - * - * @return void - */ - private static function extractMetadata( - \Zend_Search_Lucene_Document $doc, - $path, - \OC\Files\View $view, - $mimetype - ) { - - $file = $view->getLocalFile($path); - if (is_dir($file)) { - // Don't lose time analizing a directory for file-specific metadata - return; - } - $getID3 = new \getID3(); - $getID3->encoding = 'UTF-8'; - $data = $getID3->analyze($file); - - // TODO index meta information from media files? - - //show me what you got - /*foreach ($data as $key => $value) { - Util::writeLog('search_lucene', - 'getid3 extracted '.$key.': '.$value, - Util::DEBUG); - if (is_array($value)) { - foreach ($value as $k => $v) { - Util::writeLog('search_lucene', - ' ' . $value .'-' .$k.': '.$v, - Util::DEBUG); - } - } - }*/ - - if ('application/pdf' === $mimetype) { - try { - $zendpdf = \Zend_Pdf::parse($view->file_get_contents($path)); - - //we currently only display the filename, so we only index metadata here - if (isset($zendpdf->properties['Title'])) { - $doc->addField(\Zend_Search_Lucene_Field::UnStored('title', $zendpdf->properties['Title'])); - } - if (isset($zendpdf->properties['Author'])) { - $doc->addField(\Zend_Search_Lucene_Field::UnStored('author', $zendpdf->properties['Author'])); - } - if (isset($zendpdf->properties['Subject'])) { - $doc->addField(\Zend_Search_Lucene_Field::UnStored('subject', $zendpdf->properties['Subject'])); - } - if (isset($zendpdf->properties['Keywords'])) { - $doc->addField(\Zend_Search_Lucene_Field::UnStored('keywords', $zendpdf->properties['Keywords'])); - } - //TODO handle PDF 1.6 metadata Zend_Pdf::getMetadata() - - //do the content extraction - $pdfParse = new \App_Search_Helper_PdfParser(); - $body = $pdfParse->pdf2txt($zendpdf->render()); - - } catch (Exception $e) { - Util::writeLog('search_lucene', - $e->getMessage() . ' Trace:\n' . $e->getTraceAsString(), - Util::ERROR); - } - - } - - if ($body != '') { - $doc->addField(\Zend_Search_Lucene_Field::UnStored('body', $body)); - } - - if (isset($data['error'])) { - Util::writeLog( - 'search_lucene', - 'failed to extract meta information for ' . $view->getAbsolutePath($path) . ': ' . $data['error']['0'], - Util::WARN - ); - - return; - } - } } diff --git a/lib/indexjob.php b/lib/indexjob.php new file mode 100644 index 000000000..9c1e297b4 --- /dev/null +++ b/lib/indexjob.php @@ -0,0 +1,30 @@ +indexFiles($fileIds); + } else { + \OCP\Util::writeLog( + 'search_lucene', + 'indexer job did not receive user in arguments', + \OCP\Util::DEBUG + ); + } + } +} diff --git a/lib/lucene.php b/lib/lucene.php index b225d33a5..39ad815d9 100644 --- a/lib/lucene.php +++ b/lib/lucene.php @@ -9,14 +9,22 @@ /** * @author Jörn Dreyer */ -class Lucene extends \OC_Search_Provider { +class Lucene { /** * classname which used for hooks handling * used as signalclass in OC_Hooks::emit() */ const CLASSNAME = 'Lucene'; + + public $user; + public $index; + public function __construct($user) { + $this->user = $user; + $this->index = self::openOrCreate(); + } + /** * opens or creates the users lucene index * @@ -26,22 +34,20 @@ class Lucene extends \OC_Search_Provider { * * @return Zend_Search_Lucene_Interface */ - public static function openOrCreate($user = null) { - - if ($user == null) { - $user = User::getUser(); - } + private function openOrCreate() { try { + //let lucene search for numbers as well as words \Zend_Search_Lucene_Analysis_Analyzer::setDefault( new \Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive() - ); //let lucene search for numbers as well as words + ); // Create index - //$ocFilesystemView = OCP\Files::getStorage('search_lucene'); // encrypt the index on logout, decrypt on login + // TODO profile: encrypt the index on logout, decrypt on login + //$ocFilesystemView = OCP\Files::getStorage('search_lucene'); - $indexUrl = \OC_User::getHome($user) . '/lucene_index'; + $indexUrl = \OC_User::getHome($this->user) . '/lucene_index'; if (file_exists($indexUrl)) { $index = \Zend_Search_Lucene::open($indexUrl); } else { @@ -67,17 +73,9 @@ public static function openOrCreate($user = null) { * * @author Jörn Dreyer * - * @param Zend_Search_Lucene_Interface $index an optional index - * * @return void */ - static public function optimizeIndex( - \Zend_Search_Lucene_Interface $index = null - ) { - - if ($index === null) { - $index = self::openOrCreate(); - } + public function optimizeIndex() { Util::writeLog( 'search_lucene', @@ -85,7 +83,7 @@ static public function optimizeIndex( Util::DEBUG ); - $index->optimize(); + $this->index->optimize(); } @@ -103,19 +101,14 @@ static public function optimizeIndex( * * @return void */ - static public function updateFile( + public function updateFile( \Zend_Search_Lucene_Document $doc, - $path = '', - $user = null, - \Zend_Search_Lucene_Interface $index = null + $path = '' ) { - if ($index === null) { - $index = self::openOrCreate($user); - } // TODO profile perfomance for searching before adding to index - self::deleteFile($path, $user, $index); + $this->deleteFile($path); Util::writeLog( 'search_lucene', @@ -124,9 +117,9 @@ static public function updateFile( ); // Add document to the index - $index->addDocument($doc); + $this->index->addDocument($doc); - $index->commit(); + $this->index->commit(); } @@ -136,27 +129,22 @@ static public function updateFile( * @author Jörn Dreyer * * @param string $path path to the document to remove from the index - * @param Zend_Search_Lucene_Interface $index optional can be passed ro reuse an existing instance * * @return void */ - static public function deleteFile( - $path, - $user = null, - \Zend_Search_Lucene_Interface $index = null - ) { - + public function deleteFile($path) { + Util::writeLog( + 'search_lucene', + 'Lucene::deleteFile('.$path.')', + Util::DEBUG + ); if ( $path === '' ) { //ignore the empty path element return; } - - if (is_null($user)) { - $view = Filesystem::getView(); - $user = \OCP\User::getUser(); - } else { - $view = new \OC\Files\View('/' . $user . '/files'); - } + + //TODO remember view as instance member? + $view = new \OC\Files\View('/' . $this->user . '/files'); if ( ! $view ) { Util::writeLog( @@ -167,10 +155,6 @@ static public function deleteFile( return false; } - if ($index === null) { - $index = self::openOrCreate($user); - } - $root= $view->getRoot(); $pk = md5($root.$path); @@ -180,8 +164,7 @@ static public function deleteFile( Util::DEBUG ); - - $hits = $index->find( 'pk:' . $pk ); //id would be internal to lucene + $hits = $this->index->find( 'pk:' . $pk ); //id would be internal to lucene Util::writeLog( 'search_lucene', @@ -195,123 +178,11 @@ static public function deleteFile( 'removing ' . $hit->id . ':' . $hit->path . ' from index', Util::DEBUG ); - $index->delete($hit); - } - } - - /** - * performs a search on the users index - * - * @author Jörn Dreyer - * - * @param string $query lucene search query - * @return array of OC_Search_Result - */ - public function search($query){ - $results=array(); - if ( $query !== null ) { - // * query * kills performance for bigger indexes - // query * works ok - // query is still best - //FIXME emulates the old search but breaks all the nice lucene search query options - //$query = '*' . $query . '*'; - //if (strpos($query, '*')===false) { - // $query = $query.='*'; // append query *, works ok - // TODO add end user guide for search terms ... - //} - try { - $index = self::openOrCreate(); - //default is 3, 0 needed to keep current search behaviour - //Zend_Search_Lucene_Search_Query_Wildcard::setMinPrefixLength(0); - - //$term = new Zend_Search_Lucene_Index_Term($query); - //$query = new Zend_Search_Lucene_Search_Query_Term($term); - - $hits = $index->find($query); - - //limit results. we cant show more than ~30 anyway. TODO use paging later - for ($i = 0; $i < 30 && $i < count($hits); $i++) { - $results[] = self::asOCSearchResult($hits[$i]); - } - - } catch ( Exception $e ) { - Util::writeLog( - 'search_lucene', - $e->getMessage().' Trace:\n'.$e->getTraceAsString(), - Util::ERROR - ); - } - - } - return $results; - } - - /** - * converts a zend lucene search object to a OC_SearchResult - * - * Example: - * - * Text | Some Document.txt - * | /path/to/file, 148kb, Score: 0.55 - * - * @author Jörn Dreyer - * - * @param Zend_Search_Lucene_Search_QueryHit $hit The Lucene Search Result - * @return OC_Search_Result an OC_Search_Result - */ - private static function asOCSearchResult(\Zend_Search_Lucene_Search_QueryHit $hit) { - - $mimeBase = self::baseTypeOf($hit->mimetype); - - switch($mimeBase){ - case 'audio': - $type='Music'; - break; - case 'text': - $type='Text'; - break; - case 'image': - $type='Images'; - break; - default: - if ($hit->mimetype=='application/xml') { - $type='Text'; - } else { - $type='Files'; - } + $this->index->delete($hit); } - - switch ($hit->mimetype) { - case 'httpd/unix-directory': - $url = Util::linkTo('files', 'index.php') . '?dir='.$hit->path; - break; - default: - $url = \OC::$server->getRouter()->generate('download', array('file'=>$hit->path)); - } - - return new \OC_Search_Result( - basename($hit->path), - dirname($hit->path) - . ', ' . \OCP\Util::humanFileSize($hit->size) - . ', Score: ' . number_format($hit->score, 2), - $url, - $type, - dirname($hit->path) - ); } - /** - * get the base type of a mimetype string - * - * returns 'text' for 'text/plain' - * - * @author Jörn Dreyer - * - * @param string $mimetype mimetype - * @return string basetype - */ - public static function baseTypeOf($mimetype) { - return substr($mimetype, 0, strpos($mimetype, '/')); + public function find ($query) { + return $this->index->find($query); } - } diff --git a/lib/searchprovider.php b/lib/searchprovider.php new file mode 100644 index 000000000..3f95a96a0 --- /dev/null +++ b/lib/searchprovider.php @@ -0,0 +1,136 @@ + + */ +class SearchProvider extends \OC_Search_Provider { + + /** + * performs a search on the users index + * + * @author Jörn Dreyer + * + * @param string $query lucene search query + * @return array of OC_Search_Result + */ + public function search($query){ + $results=array(); + if ( $query !== null ) { + // * query * kills performance for bigger indexes + // query * works ok + // query is still best + //FIXME emulates the old search but breaks all the nice lucene search query options + //$query = '*' . $query . '*'; + //if (strpos($query, '*')===false) { + // $query = $query.='*'; // append query *, works ok + // TODO add end user guide for search terms ... + //} + try { + $lucene = new Lucene(\OCP\User::getUser()); + //default is 3, 0 needed to keep current search behaviour + //Zend_Search_Lucene_Search_Query_Wildcard::setMinPrefixLength(0); + + //$term = new Zend_Search_Lucene_Index_Term($query); + //$query = new Zend_Search_Lucene_Search_Query_Term($term); + + $hits = $lucene->find($query); + + //limit results. we cant show more than ~30 anyway. TODO use paging later + for ($i = 0; $i < 30 && $i < count($hits); $i++) { + $results[] = self::asOCSearchResult($hits[$i]); + } + + } catch ( Exception $e ) { + Util::writeLog( + 'search_lucene', + $e->getMessage().' Trace:\n'.$e->getTraceAsString(), + Util::ERROR + ); + } + + } + return $results; + } + + /** + * converts a zend lucene search object to a OC_SearchResult + * + * Example: + * + * Text | Some Document.txt + * | /path/to/file, 148kb, Score: 0.55 + * + * @author Jörn Dreyer + * + * @param Zend_Search_Lucene_Search_QueryHit $hit The Lucene Search Result + * @return OC_Search_Result an OC_Search_Result + */ + private static function asOCSearchResult(\Zend_Search_Lucene_Search_QueryHit $hit) { + + $mimeBase = self::baseTypeOf($hit->mimetype); + + switch($mimeBase){ + case 'audio': + $type='Music'; + break; + case 'text': + $type='Text'; + break; + case 'image': + $type='Images'; + break; + default: + if ($hit->mimetype=='application/xml') { + $type='Text'; + } else { + $type='Files'; + } + } + + switch ($hit->mimetype) { + case 'httpd/unix-directory': + $url = Util::linkTo('files', 'index.php') . '?dir='.$hit->path; + break; + default: + $url = \OC::getRouter()->generate('download', array('file'=>$hit->path)); + } + + return new \OC_Search_Result( + basename($hit->path), + dirname($hit->path) + . ', ' . \OCP\Util::humanFileSize($hit->size) + . ', Score: ' . number_format($hit->score, 2), + $url, + $type, + dirname($hit->path) + ); + } + + /** + * get the base type of a mimetype string + * + * returns 'text' for 'text/plain' + * + * @author Jörn Dreyer + * + * @param string $mimetype mimetype + * @return string basetype + */ + public static function baseTypeOf($mimetype) { + return substr($mimetype, 0, strpos($mimetype, '/')); + } + +} \ No newline at end of file diff --git a/lib/status.php b/lib/status.php index f0c27c179..6b9f55de2 100644 --- a/lib/status.php +++ b/lib/status.php @@ -3,6 +3,8 @@ namespace OCA\Search_Lucene; +use \OC\Files\Filesystem; + /** * @author Jörn Dreyer */ @@ -53,6 +55,12 @@ private function store() { return self::insert($this->fileId, $this->status); } } + public static function delete($fileId) { + $query = \OC_DB::prepare(' + DELETE FROM `*PREFIX*lucene_status` WHERE `fileid` = ? + '); + return $query->execute(array($fileId)); + } private static function get($fileId) { $query = \OC_DB::prepare(' @@ -139,4 +147,23 @@ static public function getUnindexed() { return $files; } + static public function getDeleted() { + $files = array(); + + $query = \OCP\DB::prepare(' + SELECT `*PREFIX*lucene_status`.`fileid` + FROM `*PREFIX*lucene_status` + LEFT JOIN `*PREFIX*filecache` + ON `*PREFIX*filecache`.`fileid` = `*PREFIX*lucene_status`.`fileid` + WHERE `*PREFIX*filecache`.`fileid` IS NULL + '); + + $result = $query->execute(); + + while ($row = $result->fetchRow()) { + $files[] = $row['fileid']; + } + + return $files; + } } From 84781c24aee8ad32fede7b0d1e06cab464d25234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 23 Jan 2014 15:28:26 +0100 Subject: [PATCH 03/51] fix indexing via cron --- ajax/lucene.php | 9 +++++---- lib/indexjob.php | 7 ++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ajax/lucene.php b/ajax/lucene.php index 5f7aa5911..1adbe3b4c 100644 --- a/ajax/lucene.php +++ b/ajax/lucene.php @@ -6,6 +6,11 @@ session_write_close(); function index() { + $user = OCP\User::getUser(); + + OC_Util::tearDownFS(); + OC_Util::setupFS($user); + if ( isset($_GET['fileid']) ){ $fileIds = array($_GET['fileid']); } else { @@ -15,10 +20,6 @@ function index() { $eventSource = new OC_EventSource(); $eventSource->send('count', count($fileIds)); - $user = OCP\User::getUser(); - - OC_Util::tearDownFS(); - OC_Util::setupFS($user); $view = new OC\Files\View('/' . $user . '/files'); $lucene = new OCA\Search_Lucene\Lucene($user); diff --git a/lib/indexjob.php b/lib/indexjob.php index 9c1e297b4..4c42a16f8 100644 --- a/lib/indexjob.php +++ b/lib/indexjob.php @@ -6,6 +6,8 @@ class IndexJob extends \OC\BackgroundJob\Job { public function run($arguments){ if (isset($arguments['user'])) { $user = $arguments['user']; + \OC_Util::tearDownFS(); + \OC_Util::setupFS($user); $fileIds = Status::getUnindexed(); \OCP\Util::writeLog( 'search_lucene', @@ -13,11 +15,10 @@ public function run($arguments){ \OCP\Util::DEBUG ); - \OC_Util::tearDownFS(); - \OC_Util::setupFS($user); $view = new \OC\Files\View('/' . $user . '/files'); + $lucene = new \OCA\Search_Lucene\Lucene($user); - $indexer = new Indexer($view); + $indexer = new Indexer($view, $lucene); $indexer->indexFiles($fileIds); } else { \OCP\Util::writeLog( From d84026cfa22c38416d50a481e0a4fb77652f04dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 23 Jan 2014 15:28:56 +0100 Subject: [PATCH 04/51] increase version to 0.6.0 to indicate new background scanning approach --- appinfo/version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appinfo/version b/appinfo/version index be14282b7..09a3acfa1 100644 --- a/appinfo/version +++ b/appinfo/version @@ -1 +1 @@ -0.5.3 +0.6.0 \ No newline at end of file From fa2e0f8dab5534f1ea15be85738f3e2fce8f5b41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 23 Jan 2014 15:30:06 +0100 Subject: [PATCH 05/51] readd doIndexFile as a deprecated method to prevent errors in case old jobs are still in the jobqueue --- lib/hooks.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/hooks.php b/lib/hooks.php index e6f93c159..df1819554 100644 --- a/lib/hooks.php +++ b/lib/hooks.php @@ -116,4 +116,16 @@ static public function deleteFile(array $param) { } } + + /** + * was used by backgroundjobs to index individual files + * + * @deprecated since version 0.6.0 + * + * @author Jörn Dreyer + * + * @param $param array from deleteFile-Hook + */ + static public function doIndexFile(array $param) {/* ignore */} + } From 0bac4cbb4b6e6ba9d2d2af579d238b8800f5a87c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 24 Jan 2014 16:16:54 +0100 Subject: [PATCH 06/51] add standalone optimize job, use fileid as unique id in lucene index, prune old index documents in optimize job --- appinfo/app.php | 6 +++++ appinfo/update.php | 11 +++++--- lib/hooks.php | 54 ++++++++++++++++++-------------------- lib/indexer.php | 60 ++++++++++++++++++++++++------------------ lib/indexjob.php | 4 +-- lib/lucene.php | 64 ++++++++++++++------------------------------- lib/optimizejob.php | 45 +++++++++++++++++++++++++++++++ 7 files changed, 140 insertions(+), 104 deletions(-) create mode 100644 lib/optimizejob.php diff --git a/appinfo/app.php b/appinfo/app.php index 61845c35b..fdf462664 100644 --- a/appinfo/app.php +++ b/appinfo/app.php @@ -65,6 +65,12 @@ OC_Search::removeProvider('OC_Search_Provider_File'); OC_Search::registerProvider('OCA\Search_Lucene\SearchProvider'); +// add background job for index optimization: + +$arguments = array('user' => \OCP\User::getUser()); +//Add Background Job: +\OCP\BackgroundJob::registerJob( '\OCA\Search_Lucene\OptimizeJob', $arguments ); + // --- add hooks ----------------------------------------------- //post_create is ignored, as write will be triggered afterwards anyway diff --git a/appinfo/update.php b/appinfo/update.php index 303355ca2..ccc5a218f 100644 --- a/appinfo/update.php +++ b/appinfo/update.php @@ -3,10 +3,15 @@ $currentVersion=OCP\Config::getAppValue('search_lucene', 'installed_version'); if (version_compare($currentVersion, '0.5.0', '<')) { - //force reindexing of files - $stmt = OCP\DB::prepare('DELETE FROM `*PREFIX*lucene_status` WHERE 1=1'); - $stmt->execute(); //clear old background jobs $stmt = OCP\DB::prepare('DELETE FROM `*PREFIX*queuedtasks` WHERE `app`=?'); $stmt->execute(array('search_lucene')); } + +if (version_compare($currentVersion, '0.6.0', '<')) { + //force reindexing of files + $stmt = OCP\DB::prepare('DELETE FROM `*PREFIX*lucene_status`'); + $stmt->execute(); + //FIXME wipe index on disk because primary key changed + +} \ No newline at end of file diff --git a/lib/hooks.php b/lib/hooks.php index df1819554..bc0c4fa02 100644 --- a/lib/hooks.php +++ b/lib/hooks.php @@ -48,9 +48,18 @@ class Hooks { * @param $param array from postWriteFile-Hook */ public static function indexFile(array $param) { - $arguments = array('user' => \OCP\User::getUser()); + $user = \OCP\User::getUser(); + if (!empty($user)) { + $arguments = array('user' => $user); //Add Background Job: BackgroundJob::registerJob( '\OCA\Search_Lucene\IndexJob', $arguments ); + } else { + \OCP\Util::writeLog( + 'search_lucene', + 'Hook indexFile could not determine user when called with param '.json_encode($param), + \OCP\Util::DEBUG + ); + } } /** @@ -62,12 +71,12 @@ public static function indexFile(array $param) { */ public static function renameFile(array $param) { $user = \OCP\User::getUser(); - if (isset($param['oldpath'])) { + if (!empty($param['oldpath'])) { //delete from lucene index $lucene = new Lucene($user); $lucene->deleteFile($param['oldpath']); } - if (isset($param['newpath'])) { + if (!empty($param['newpath'])) { $view = new \OC\Files\View('/' . $user . '/files'); $info = $view->getFileInfo($param['newpath']); Status::fromFileId($info['fileid'])->markNew(); @@ -76,43 +85,30 @@ public static function renameFile(array $param) { } /** - * remove a file from the lucene index when deleting a file - * - * file deletion from the index is queued as a background job + * deleteFile triggers the removal of any deleted files from the index * * @author Jörn Dreyer * * @param $param array from deleteFile-Hook */ static public function deleteFile(array $param) { - Util::writeLog( - 'search_lucene', - 'deleting status for ' . json_encode($param), - Util::DEBUG - ); // we cannot use post_delete as $param would not contain the id // of the deleted file and we could not fetch it with getId - if (isset($param['path'])) { - $user = \OCP\User::getUser(); - //delete status - $deletedIds = Status::getDeleted(); - foreach ($deletedIds as $fileid) { - Util::writeLog( - 'search_lucene', - 'deleting status for ('.$fileid.') ', - Util::DEBUG - ); - \OCA\Search_Lucene\Status::delete($fileid); - } - //delete from lucene - $lucene = new Lucene($user); - $lucene->deleteFile($param['path']); - } else { + $user = \OCP\User::getUser(); + $lucene = new Lucene($user); + $deletedIds = Status::getDeleted(); + $count = 0; + foreach ($deletedIds as $fileid) { Util::writeLog( 'search_lucene', - 'missing path parameter', - Util::ERROR + 'deleting status for ('.$fileid.') ', + Util::DEBUG ); + //delete status + \OCA\Search_Lucene\Status::delete($fileid); + //delete from lucene + $count += $lucene->deleteFile($fileid); + } } diff --git a/lib/indexer.php b/lib/indexer.php index 1ac3164b9..a35d04ae0 100644 --- a/lib/indexer.php +++ b/lib/indexer.php @@ -37,31 +37,42 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) $fileStatus = \OCA\Search_Lucene\Status::fromFileId($id); try{ - //before we start mark the file as error so we know there was a problem when the php execution dies + // before we start mark the file as error so we know there + // was a problem in case the php execution dies and we don't try + // the file again $fileStatus->markError(); $path = \OC\Files\Filesystem::getPath($id); - if ($eventSource) { - $eventSource->send('indexing', $path); - } - - foreach ($skippedDirs as $skippedDir) { - if (strpos($path, '/' . $skippedDir . '/') !== false //contains dir - || strrpos($path, '/' . $skippedDir) === strlen($path) - (strlen($skippedDir) + 1) // ends with dir - ) { - $result = $fileStatus->markSkipped(); - $skipped = true; - break; + + if (empty($path)) { + $skip = true; + } else { + foreach ($skippedDirs as $skippedDir) { + if (strpos($path, '/' . $skippedDir . '/') !== false //contains dir + || strrpos($path, '/' . $skippedDir) === strlen($path) - (strlen($skippedDir) + 1) // ends with dir + ) { + $skip = true; + break; + } } + $skip = false; } - if (!$skipped) { - if ($this->indexFile($path)) { - $result = $fileStatus->markIndexed(); - } + + if ($skip) { + $fileStatus->markSkipped(); + \OCP\Util::writeLog('search_lucene', + 'skipping file '.$id.':'.$path, + \OCP\Util::ERROR); + continue; } - - if (!$result) { - \OCP\JSON::error(array('message' => 'Could not index file.')); + if ($eventSource) { + $eventSource->send('indexing', $path); + } + + if ($this->indexFile($path)) { + $fileStatus->markIndexed(); + } else { + \OCP\JSON::error(array('message' => 'Could not index file '.$id.':'.$path)); if ($eventSource) { $eventSource->send('error', $path); } @@ -102,14 +113,11 @@ public function indexFile($path = '') { if(!$this->view->file_exists($path)) { Util::writeLog('search_lucene', - 'file vanished, ignoring', + 'file '.$path.' vanished, ignoring', Util::DEBUG); return true; } - $root = $this->view->getRoot(); - $pk = md5($root . $path); - // the cache already knows mime and other basic stuff $data = $this->view->getFileInfo($path); if (isset($data['mimetype'])) { @@ -132,7 +140,7 @@ public function indexFile($path = '') { $doc->addField(\Zend_Search_Lucene_Field::UnStored('body', $body)); } - // FIXME uther text files? c, php, java ... + // FIXME other text files? c, php, java ... } else if ('text/html' === $mimeType) { @@ -173,7 +181,7 @@ public function indexFile($path = '') { // Store filecache id as unique id to lookup by when deleting - $doc->addField(\Zend_Search_Lucene_Field::Keyword('pk', $pk)); + $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $data['fileid'])); // Store filename $doc->addField(\Zend_Search_Lucene_Field::Text('filename', $data['name'], 'UTF-8')); @@ -186,7 +194,7 @@ public function indexFile($path = '') { $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', $mimeType)); - $this->lucene->updateFile($doc, $path); + $this->lucene->updateFile($doc, $data['fileid']); return true; diff --git a/lib/indexjob.php b/lib/indexjob.php index 4c42a16f8..3ebe95262 100644 --- a/lib/indexjob.php +++ b/lib/indexjob.php @@ -4,7 +4,7 @@ class IndexJob extends \OC\BackgroundJob\Job { public function run($arguments){ - if (isset($arguments['user'])) { + if (!empty($arguments['user'])) { $user = $arguments['user']; \OC_Util::tearDownFS(); \OC_Util::setupFS($user); @@ -23,7 +23,7 @@ public function run($arguments){ } else { \OCP\Util::writeLog( 'search_lucene', - 'indexer job did not receive user in arguments', + 'indexer job did not receive user in arguments: '.json_encode($arguments), \OCP\Util::DEBUG ); } diff --git a/lib/lucene.php b/lib/lucene.php index 39ad815d9..ba6bb5b89 100644 --- a/lib/lucene.php +++ b/lib/lucene.php @@ -25,6 +25,12 @@ public function __construct($user) { $this->index = self::openOrCreate(); } + private function getIndexURL () { + // TODO profile: encrypt the index on logout, decrypt on login + //return OCP\Files::getStorage('search_lucene'); + return \OC_User::getHome($this->user) . '/lucene_index'; + } + /** * opens or creates the users lucene index * @@ -44,10 +50,8 @@ private function openOrCreate() { ); // Create index - // TODO profile: encrypt the index on logout, decrypt on login - //$ocFilesystemView = OCP\Files::getStorage('search_lucene'); - $indexUrl = \OC_User::getHome($this->user) . '/lucene_index'; + $indexUrl = $this->getIndexURL(); if (file_exists($indexUrl)) { $index = \Zend_Search_Lucene::open($indexUrl); } else { @@ -79,7 +83,7 @@ public function optimizeIndex() { Util::writeLog( 'search_lucene', - 'optimizing index ', + 'optimizing index', Util::DEBUG ); @@ -97,22 +101,21 @@ public function optimizeIndex() { * @author Jörn Dreyer * * @param Zend_Search_Lucene_Document $doc the document to store for the path - * @param string $path path to the document to update + * @param int $fileid fileid to update * * @return void */ public function updateFile( \Zend_Search_Lucene_Document $doc, - $path = '' + $fileid ) { - // TODO profile perfomance for searching before adding to index - $this->deleteFile($path); + $this->deleteFile($fileid); Util::writeLog( 'search_lucene', - 'adding ' . $path , + 'adding ' . $fileid .' '.json_encode($doc), Util::DEBUG ); @@ -128,47 +131,17 @@ public function updateFile( * * @author Jörn Dreyer * - * @param string $path path to the document to remove from the index + * @param int $fileid fileid to remove from the index * - * @return void + * @return int count of deleted documents in the index */ - public function deleteFile($path) { - Util::writeLog( - 'search_lucene', - 'Lucene::deleteFile('.$path.')', - Util::DEBUG - ); - if ( $path === '' ) { - //ignore the empty path element - return; - } - - //TODO remember view as instance member? - $view = new \OC\Files\View('/' . $this->user . '/files'); + public function deleteFile($fileid) { - if ( ! $view ) { - Util::writeLog( - 'search_lucene', - 'could not resolve filesystem view', - Util::WARN - ); - return false; - } - - $root= $view->getRoot(); - $pk = md5($root.$path); + $hits = $this->index->find( 'fileid:' . $fileid ); Util::writeLog( 'search_lucene', - 'searching hits for pk:' . $pk, - Util::DEBUG - ); - - $hits = $this->index->find( 'pk:' . $pk ); //id would be internal to lucene - - Util::writeLog( - 'search_lucene', - 'found ' . count($hits) . ' hits ', + 'found ' . count($hits) . ' hits for fileid ' . $fileid, Util::DEBUG ); @@ -180,9 +153,12 @@ public function deleteFile($path) { ); $this->index->delete($hit); } + + return count($hits); } public function find ($query) { return $this->index->find($query); } + } diff --git a/lib/optimizejob.php b/lib/optimizejob.php new file mode 100644 index 000000000..602880340 --- /dev/null +++ b/lib/optimizejob.php @@ -0,0 +1,45 @@ +setInterval(86400); //execute at most once a day + } + + public function run($arguments){ + if (!empty($arguments['user'])) { + $user = $arguments['user']; + \OCP\Util::writeLog( + 'search_lucene', + 'background job optimizing index for '.$user, + \OCP\Util::DEBUG + ); + + $lucene = new \OCA\Search_Lucene\Lucene($user); + // check if we have to rebuild the index because old pk: entries + // from pre 0.6.0 are still in the cache + + \Zend_Search_Lucene_Search_Query_Wildcard::setMinPrefixLength(0); + $hits = $lucene->find('pk:*'); + foreach ($hits as $hit) { + \OCP\Util::writeLog( + 'search_lucene', + 'deleting deprecated index document for ' . $hit->id . ':' . $hit->path , + \OCP\Util::DEBUG + ); + $lucene->index->delete($hit); + } + + $lucene->optimizeIndex(); + } else { + \OCP\Util::writeLog( + 'search_lucene', + 'optimize job did not receive user in arguments: '.json_encode($arguments), + \OCP\Util::DEBUG + ); + } + } +} From 45ebfca992f33c9a0d14c76142ef18516a8df98e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 29 Jan 2014 19:10:29 +0100 Subject: [PATCH 07/51] change code to be accessible from tests --- lib/searchprovider.php | 2 +- lib/status.php | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/searchprovider.php b/lib/searchprovider.php index 3f95a96a0..55fe34c1b 100644 --- a/lib/searchprovider.php +++ b/lib/searchprovider.php @@ -78,7 +78,7 @@ public function search($query){ * @param Zend_Search_Lucene_Search_QueryHit $hit The Lucene Search Result * @return OC_Search_Result an OC_Search_Result */ - private static function asOCSearchResult(\Zend_Search_Lucene_Search_QueryHit $hit) { + public static function asOCSearchResult(\Zend_Search_Lucene_Search_QueryHit $hit) { $mimeBase = self::baseTypeOf($hit->mimetype); diff --git a/lib/status.php b/lib/status.php index 6b9f55de2..3ac12f036 100644 --- a/lib/status.php +++ b/lib/status.php @@ -30,6 +30,12 @@ public static function fromFileId($fileId) { return new Status($fileId, null); } } + public function getFileId() { + return $this->fileId; + } + public function getStatus() { + return $this->status; + } // always write status to db immediately public function markNew() { $this->status = self::STATUS_NEW; From c3440a65925e1a9b02d727c0951ae80f03a604f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 29 Jan 2014 19:12:07 +0100 Subject: [PATCH 08/51] add status and searchresult tests --- tests/unit/README.md | 56 +++++++ tests/unit/bootstrap.php | 22 +++ tests/unit/data/document.docx | Bin 0 -> 4629 bytes tests/unit/data/document.odt | Bin 0 -> 24265 bytes tests/unit/data/document.pdf | Bin 0 -> 16484 bytes tests/unit/data/document.txt | 1 + tests/unit/phpunit.xml | 21 +++ tests/unit/testcase.php | 140 ++++++++++++++++ tests/unit/testsearchprovider.php | 258 ++++++++++++++++++++++++++++++ tests/unit/teststatus.php | 160 ++++++++++++++++++ 10 files changed, 658 insertions(+) create mode 100644 tests/unit/README.md create mode 100644 tests/unit/bootstrap.php create mode 100644 tests/unit/data/document.docx create mode 100644 tests/unit/data/document.odt create mode 100644 tests/unit/data/document.pdf create mode 100644 tests/unit/data/document.txt create mode 100644 tests/unit/phpunit.xml create mode 100644 tests/unit/testcase.php create mode 100644 tests/unit/testsearchprovider.php create mode 100644 tests/unit/teststatus.php diff --git a/tests/unit/README.md b/tests/unit/README.md new file mode 100644 index 000000000..2bbd13ec3 --- /dev/null +++ b/tests/unit/README.md @@ -0,0 +1,56 @@ +Testing search lucene +===================== + +hooks.php +--------- + +make sure +- indexFile() registers an IndexJob +- renameFile() deletes the file from lucene index and adds an IndexJob +- deleteFile() deletes the file from lucene index + +indexer.php +----------- + +indexFile() adds file to the index if possible (different filetypes) +indexFiles() indexes files and changes status accordingly + + +indexjob.php +------------ + +- run() sets up correct FS and indexes all unindexed files + +lucene.php +---------- + +- openOrCreate() creates the index on the fly, opens existing index, check readonly index? +- optimizeIndex() optimizes the index? +- updateFile() deletes the old entry and adds a new one +- deleteFile() deletes the old entry +- find() finds an entry, try various queries + +optimizejob.php +--------------- + +- run() cleans up entries without a fileid (removing old pk entries), optimizes the index + +searchprovider.php +------------------ + +- creates valid oc search result objects + +status.php +---------- + +- setUp() create folder and two files + +- fromFileId() loads a status +- markNew() sets status to N +- markIndexed() sets status to I +- markSkipped() sets status to S +- markError() sets status to E +- delete() deletes a status +- getUnindexed() return list of unindexed file ids +- getDeleted() returns list of deleted file ids + diff --git a/tests/unit/bootstrap.php b/tests/unit/bootstrap.php new file mode 100644 index 000000000..a25813a26 --- /dev/null +++ b/tests/unit/bootstrap.php @@ -0,0 +1,22 @@ +^1!3rv6sb{|L7JgMx+Mi9L>i=HU}yvxT0lyat|6pBQb3SK z>3HGZ_kMC+-|I7L&6+=EpL3pl_7nRD0b^hS0l2uh0QoeZ2Y_pVfBD`plo9@rRb2hN9%`ci$6?8ldh;2jyY?$oPCe|MsIDlFthS-~1~+rl6pR$3{-O`+mx zHfWUNS?F|REs#Lqp>KKSoh3qm+I}LS&@WTsxuYYZU0suKCr?k_{lZGX;dxI(hwY%j zLU>0gR>dvZ(7V8OrP1A+q-gm*xCL~ zQA{7+CmxcZEuXM5_q1vpQme^u+(u=e7s>X1JbtXu^(i{YZ~zW5<5TzjG~=G*eAf5b zdvSvVy8;*iy2)fnn{ETPx4bgDAQAJq=q^nXv4+qy%vGQ z&^muzZq##WU9*YQI#q0S{M%My*d?pZw>E5dsH#^_l;%(&;j#=(a*&%Ap^{|p>3DcN zz6m(L@QntY)6CK_==#PU+HccJfLYn}pI6smTrm)G7H?8}nO_XrpBNzfhXD%*dslU1 z6I=7E!cY8yQ8# z9*7~x4VgLxoT?ib$Jx)A#1xlkLCmkliFhqpIaX0&$oT$G{yPm#E9EdxM=Td1qEH6j z%7eK%ivbC_Pj8=MzO9Xi_4>?3q$gU#H!69Ob5HUlWd#Pb8UhLTUOAaP{@$ps`)RQzrO`y>L!E&}p z%~t=)AM}mV+?rkEk8cg6Zky=59a%fkSv_993l3BeV2zK^{$A9D9$&s7Tf)4*=sWbS z{KY**qQY3}aze6t8sAzJj+yXXwYhyT?N?H8@kk&~uEbiMcOl;HUI@%D?x)Gp%PH3|0x&#^Wj>#7bgogMN znC=JO=Mzdc$Ve7ZUA>>ra>(P=&GjnV`Re6NyNwqBM*^AJAt8_3&A0-ZH6?zrm=!W5&9w9U5RztY*$3GDoucsS0Vb5v^hN=v6uJYj+!E zK>cy3<#gY^lWknP$G#SAROA$hZ7RS*JJow~W3)ibP3pqA7nUQc=7wX77Id0+fB87g z-tyW09UF;JZdf7=$cO>?f4K<}wULT?^e?^JY>^+}A>U zwyHcm&g}$J!{41F8ai4f0#$2>5QVqG3sTfWX4O`kvYqSD_H8BmqYM}h5#_e0?_iBB z(KAgqm%xG)WKX~M&KBv8HLG5D5)x^2;+PyTJlR-TCRjRt7Jm|I5zefVo9T`_R?0NZ zrVl+E+$O#f8&Wj*jOS8l@i>1fQPO|JcD+4~XxrG(DFDxA40+d|K8#H>Q6m)(s`Zy8 zSKB?5Ol44dvRt0>3V1MfFzty1%GxONa8Z|P?bW$#imn(O9fgDBD|3Fxh8rSjJG@Y4Zlb^KBc%{iHR<(o6JKyG)R){7OugR z&~Ick5DeX1ay&`f8{(0M6VRo*WnfXz2=j3#1CG_?XyTqeSC7L(_vbCtfK?cBQB!13 z-j2-UM?={QqGGVnpH0V)l1#)uj(j~5&ngz3$%KF@+R%On>`s4upDev#S=z_J=Fg0q72DrB`whQ9KU#rvr=5aDkq4dkm0@7U9(VQ zznC-~W`^28L8x)C#uNr$KR?JAmDQDfp96f5adSj*dLR`@iH}uYiq;2XKZtD%TBv;~ z=12)mvZ}$dQc~BXsU9iwb-=7nSIfvO%Hxm-myZ>mYYa@E)8dWnDt!(yk<;$%$oqAViejrVnZ?~q5TUA+i zKtS&Iu*iVZ)L!nYS2j+_PN6mR@f#T;D#86q2CWDN3*`)F9KtaXyk$ZuiJZr8I~x3 z9nN{I}Mo_EumGIg3B11&It-c1iyG3Ga8vqs_wJSzw?JbO-Kg#%0 z5=SvM1(d8h__;c}`GUiFZTw-qa(TC*t0&omw(xf5k%xX5}VjH+VDF-wOXJg7+68gCs5 zgpc)fyU)oRC#Yw&awV%A^*>VO3q&7YV5Y%QoC2#R72M7eBPuH%k8v1ApS7_KETryW zaXI`v9SwNd+YbBu77epWOBC9d{>BUJFUcy94TsA1#2j|gEpgWo@!5;P~9%a-F(Cp(o?oNxC&%>RHOK2|-#2{ba$YT)&fR4ajStqJ^_3c$vi- zz6H3-LIs&O-2~~dHe-9BvoA_!j+@uX_8@k|`MJ?}toOFLEFR$&zcJA%Tv2=Ay^mct zK*d_OE>}pi8k*)JbW;WCpO~=f-r0WdKpEIA(Y1scE1vCf1md8h|b2#QB@Tny!>Z{t}*0Z7LSDTy_ktAM9F(>z*7f`uHu#Vr95lv3%ER=HOo<%dMh(aOVM>oa=YdmhTL$BA|=fF z+YMe@YVaCOO)v!DTK5CN-Wq)DyH#^N#n{0Dex&Lt?zM8iG$BwCk^I!XLWUQfT%PnBr z>=}@7i!0BDlFcsZTLpVkvPjNO31{ zB6-#xD0yd#x&CAD4skUa?CTK_yj^UbF^`s<`Z`3VGcj6m&#E5RQZ$%| z_0ASg7w-AK@;oti$)-rS)HGwUt5zSBCf%D|UTgM1M=G=_1~FH&wCK`c%`v>KP7isK zJ`k^G#5l4ekx=8 zt~t7?7HzTRQ$7MmuFOU$n8-zusfy>?nbf!V#3l0qewOyKGV+!JYsLHS_e`H1Jxc{$ zNi(@wYf18w_~1Y8)#R7t>pi%1B9|u2P~Fqf+(rLNosDr)s+a0)JMazbMXw!BJG z+g{UMqsPxwYcna#I7t`Mc_94f)JAd!xe4YqCjqU+DciooYI5BjW12qMOE_vp(^ifH zj3CA4!-HHcs$sMO_;X%N4#jhaZ&A2N4_YRuAGWK00o$X(xO8F_bmm4%ed7vA8uUds z%=>-;jay2m9qkC=W>=G->JH}^M+&%tO0!hQ3qlPJh^YAvNw(xnU?`XVVHNfliw66~BXM=N30F`JKYJDt$C`AmG;-z;!Qtbq4V7xOO7&JMg-Nyt2x_g6#5E ze=^R$U+KE7y0V!>{*^tKRu5a0ssTivM=f-_h5F|L>c#5ne<8(`kN(Uw^Mx)$uE`iT>YOfq=0t TCjbC|d-*xLd`swmJ@S75m@Ut` literal 0 HcmV?d00001 diff --git a/tests/unit/data/document.odt b/tests/unit/data/document.odt new file mode 100644 index 0000000000000000000000000000000000000000..7c6ed6745e95b554be657378fa12b2981d85c51e GIT binary patch literal 24265 zcmeFZV{l|&^fnqN6WbHpHYYYGnb@|Sj&0kvCbn&JVjC0Z_Wb_!zE$_bt$OdL`{j03 zSD%yYK5L(ib)NOCXSJdXI0QNf2n+}aW`>W5?l22HB?t(}zc=ttAT|~@Ce9xACIi~2 z$>UOScA@oF&OeO@4^^&I$joM^f|NH0U}OI@S1lYSjJ8~SE>w!f1+Y~p?Z=9^?ockbLstjVJ7$% zM9D|o@t47^ZNkRRYvZm)@Z(e=du4?E+HfjdhYuHC7-0*)-IwO*J)AW zwBXl!c=)$%w1v;L8DH^z!Oz=Rvn+?<^3oj7G4j-#OIqFA2u8rmgj?6SPFpY(946hL zjbfS9?Mqv?x9z{(Ztu4v_vlyiH%BwMUqHLr!dHzm0#(nP-M0uOt{od&S4Ww4+n(pu zl$4YfFI(5Xf1PsQ@b`Dyw7;H51m!c$`1Wy1vt4oDn_Mr~O(s&hu6?d~kF%VveE)Xb zUq@ij>JuGJgaSU!^>QA8-`sSX;ZGH4%eM0<`!vDv7#D{AUW5p^&)m6b{rJ1{6@(`L ztI0gRM$hMQDb{_Ebh%n<#LyIJZ$~Qxop6wpq38XMG=^($l6A%y!2@&4@jCm;xior^ zkng&pvc~Wdo0c*2cI)5d-Soq0JoM?`Oz3?kAOY`Q@0r^BTx%y_2d7{)-R|W*7VsW8 zR_D!xX*CGVvy0IWQJUolD+t3AWas{~X+@{=qU-N=X7akv->uUdP$YLhPrvl$RYSO( zyZhX)_ZYWM(w1Jpzr~}Wp?fUHxi!d`kb}eeeLh6OiI;ZbMqj%7b#+AGDQ$!=Br*zu z;u$!wQ(~F|z`6MvSMTnRb{>^=+j3DChm&C@V6bi3a&`XE7|MTPz?aRz7uoYLR+dKk z?z11w>A4*DdcfJ+(kUrE-D9;}mEXl{jO`2-c@q%J|MW}eYY(eu^iCIesl!0*U(KB> z+D-`3E$fz*gztZ#E7fbaecvuM5^Y+xQoMef*};FhAJXcL>FHtsG*qSpuV&1$U6vPe zKOg=rQ#uI1V+YEda?7)>&mYF?aPmNITbD|$u7_gAsjC9Q)}Wbp(dWNX3Z4nJeD5~{ z9VJHTz^0WHt%!(8fUBp<^XrI*~CQ8yU+SU&T#K4*CYMH!8QRS&lFUyW%XlUuGmHI zSIhZs8nzz#K~vNy(RSI>q@J79a>S*%IA!OAo$r0--?kZ_f}mv`wm+ByVb*+SOwsls zAz0UPq@De5Vc?Y@U>r;}jj?w}SKi0GPKP6cpUfLjtcNHLOlvgEkXs1|%3uB2o>@Ih z0nfF}p{9e&c%dUh$+YVI4RBZti#-edeLRo5NXQ_2-l;mS26A?-yHLKfU^sc89LyZ9 zg9KS{()wL2!qZ)pIqoD}w+{;nsW7FWoF%*R-QF*bqTqXh1ilZ2gv335FbUZ`Z(G-C z9FQVm0#CIq{P$F#i zb?KwL+qa1Je!a+-Ym=?MLm28zMJ?Tr)e_%b(1}7e+Qf`}x6XC{0v%H>wAjJzAf-UO zmbu5<#!e3O2XIvhVYVtP`)ldmXROqD$VTe8Gtk0sgPGQ+5kB`q2QkH6*P&F3QtAtT z!fL*#*Y~{y=CWJIc#9cdHE-BL;k?s5Pb$Om++?r{Pd}K46CDSNaUpcSF4eMV=_9;V z3suHJG+-XUO&i)B>)9Uy9ZZdH+)f?bp5IQTK0=p8K2BiwA&*@)XTFIGxY~W1QOumm z14i^3r91zoAybjZsRNs-7uQWMIAq8(hUKcaIQw*?)M#Wjujg=9x+4qF?U@6QF_61=$VR@l0JVvsi-#Ttx)bM|C{H`y78Hg*P?Z z;bh&9M#Y#ouD8+RM32EYx{H0i*TBfNTVT}f9N3q(ktwbGZjS7db_i@j{=75F_Kx8g zSXPS+YAqio3DLxe?%at7Lp`S!-jNc%NSeO>ffnR?8e+^6!383Msxcwwc1iLo`; zb^-ZVOb+Q1iTeOqAn>V^Xf0`mYac$=_-S%3QLxC%&(A1Na@zzK%mO>5)*ZU4`#09k z%e31w5i_+PH6&CdgAK%`2cdRqQLY*sNx2@%?io$uUMEy!&MN&!??MKO%Bt&zP5h2_ zQm2Ib03jP1a)^;T4wS5QK?Yc4oB?E72$t*4+qOE>kb9UxZ+0F8fx}H7wBUXE%z;_q zsg=evFW}R=T}~tmi=E$lcK?qIqa?>6SAtwKK=N-~2ro9YkB>OA@0m*OUZw0dt6@30 z^A8!)+i#tFp4QY-g4LKov@R$);?j|_%zps;W@C0%Dn+oY|<_YBJHt5|Xh+HO+Ynbq*f?zhU)a|wML`y2)OqF8L;0M#RVOTCRK7@Ra0VfaeJ5qvBF9ZnSMzL z51a7>O5O_JHeNL(fUxD|L}M4(g{2^-iAO&Y4r5#C>1OAj0<-J?qT6%Tgr#6hiI?h56CUYXwPmL$@PPTLLigpM6Qu`XoQqDLP< zIGvvM3qUB~?^o$jL8!uh|Pp%))-$L zk_=7H|1ze~><=H+8viUZ3NHM-Pb@_XGQH}uqEp=t!YbYsHih5db7Kh47(`hlm{&i@ z`aznnL8>O2CX#|qi{T!$gu#2=>7=S7x-<%0ToPVPQ@108UrD8Wkn)Dd$2+k=cWVvCk{ zR$?cSr6Gs?vW#O2`lN?0*)Gb7-xbkB_|#M(A&(9*qY}wIgN<*x?tUxzVbRy9kGRE- zeRweESB*BF)Bi{=lk%&6m%ZLl1Wldtooux}DAXD5oXELg+q3gz5f5+zuEh*6ahToJ z^@J-KkDbZE^ejY8nZK)mmN8Vg%W|3y*95Jkt$GQw{1#iP@=)Qli&04e5wiJ>5Q2_N zxdF}+{%VTw;Y79;EjOVg!|@@h%4D(Dn5l74d_N~)i-0l*C%K@8@!Y1X3<*GFEsk290S4_?We&woUurSA&50 z1mp?4EAeFh>X-9^jrBI-l}Ar6xNG!`X>r4?D%=W1Fq$h6`=r6sda|P~qD>qJ>H9LY z{5%InYoUT_HQFA>ygN8-Za|DuyRBRrnBKuuo!Xcv&&k6-b(11FCs^%7#Vq7mq=e^oHGy8OJv8LzKlYr)!A|;hQ z6^*YEJa%FA785U75WBCX=feikAAmPMIQ#EL=9pzm!R4v;H`jphUJF-3tm0?Q-QBo# zS|@x9r({x3ig3zO>l{odbgFFSd1KIRzc2HPZgcM)xkMpK0M z-9`-S42|S9QV$PD?-mJO2_JXObzP{*h)W>c+9WBhExR!Z+V3_2SxaY-m*X+vN?T(t zf`r2y{pS1m{_v{K1eAKKswhM8M3qu&Z!|5#f(JJDSb4iJI{R*l6D0 zaVBp`j)aYlsuo2H5AkM(2TW=jqJ&xv9K{@H4f`En#tozj<ry zz*P^r7%xHa;*aF5zuD4AMoLF43O+o|LGI6*^SW3%eIUFLV2JlO`lnbn4ClFAi?M&) zT}&O4$+C@4+@X0R*KAv{z`Mg)d8zagf;gHvOBO^}7Z8&k74V+{;Y}`Z{w{l)%#CBq zR9^26CsGMrR}IoHwPK<=!T$uPX3Y&Ke7fQamArIcwV(UN`DuU@qOe^AM?q_s?ur|R zjsgQ)EV|QMa2mqTn@|ZrAXfrb*NIrC1+j!JgyYyJgO$LH=5Na-V8DCO_YOqZ`jZc& zsuK&B6jTymm=z0mach5tv02oZ^tI?8X6X=}5z|Q@_mL{wQO=kfgoWT+(fQBx;riI{ z)>L-6C1lnm)iG^$EPGoT4B?p82WCE9P5g-8`)=HSA7{Je1X9|z?uOVjuTeyE81Rbf z0-qhB1fFFhGe!DXZ+Jz!9&&ME5>#nItqCqnU@m!3ve#{J1C}@aixwwqMFtb``dNp> z>wiyohG8?-Eaz~bdCaPqiv-?xvBhH=G$07M&dUm8A_!dA{ur_~l!Ce3xs4@|TN2vr zywU+W!u3Z|*R?)fvq3{Zd_Q%8t7;BwmKi4}7@wri&EZ{w(8u|KQ&Qx6r$hq~(iB2? zx-U28U<6{jtC_YtdBrZ+1~`&on|XvieXuov8!rNlw*cKuQ`JS{KRU+9SmZ3gD;<&# zl5wD_xOZ@ns!qwq8gPt7(G$Uj@zws7*3mchj(&PqcoTydzxK>4>_DUVQXPVxi*c(M zRV)b^B&!4aIUS#RG6=)S(eZj#dDy`H&%kOiia+O$cVh$Uua2SR5o3Li0ThTLZ<@8U zOFJ`TmF$ebY-dr8m%B<)^C?BEvitiTkiYO7G!jcKq6_vYvNQ8L2BAn zO+o71<+L~zOhcbYH+fSm*u~w$4?a&E)3R5`n&SeEGRyu4ZaWFgL$8n{p{jj>aCk_O z;vn6FGNwK_iU>Yatv`CUzWDH8=UF%w?Ldh8i9H86?)Uqv^}5$%?Qw zPK>kWl5J!9c(&w?A%ft%VnlF{eVU<%x=R~HP2^%^l; z2!SS!&2LCOPxTG*Yn%a8#$s|4`#UoR>uwH4xhK$fS@|rM>{!v4BLTQNl_>3G9?Yd0 z9@xpE`V$DP7V)1e!-&m|YP+c_hJMYp64EPM%QJfgv?*R1H5qwd4N?t)I@AG28#mb5 zGQ9zL8pv{)7(id(Ao8Z+oBp!5jf7z-Xpkp-*S5MZuqy0URun5SjzGc4rfJ;wdY50e3f8-Vi5J8~fW2wR0NVx3zg0{$MT?K5N zW2mTqVAJ3Yk0;Y=G(iIimd%ZF=ykSt13NzXZCI{2aNi~}=4c7NJrsclnWVcFRMqBs zY^0OAZyvNMpn>8{IND*%;ro1$$=n4|9ZkaWXeb9wufkAu*hs`>eN1Tq4+xKS;Ju() za2?iAi1=3a*ZllRscmr^0Pn_Aa z67!f@<_yp{R1iv_;5A3$`hu|&f8C<1nwAL1^9u{ITd~Yb@((Psb^bHPjD1)uy%78z z1QDGu$sc?~#69b^VqqE!A!XjxUQ*RBELo&D-$A~t=OdVT?Q<`l+bZz$@sY}^ur)d) z92N5>GQ1!=5;h?Yz8PWWFD_E@dSrO{S|-b(D6TEv^yJ_ex9@~HbGu=d-(MY!88JZ2 zEi&^jm#l0?`DL^*%trkA3yYXbMmz6)Jbn=a#34dI46^=5EBo2pS@E8*e(c_|C{%B9 zIu>kPwP$Z2b0#Rmnje3!Ej}GPPpVKjDT~k2M<@i5sXDJXG#w4&#RS?b`9bR!pOW4` zz+BpUU~bybALzRc&8$pb~2p;DGx|f5#C;eP%geZF2bW-@hL-ScPZXA0qGz2IuL-Fo&efpWA?kXJ; zK`ihVFhMbydXN}{jy_;I9Pp*;Wl9jRvX*OcB!n3~Xd8h&)`ASHQt>s&s%FE+va!pF z!-h~+XU?lvI8}$M{hV=Z3MIS@Uo~c>Tr4fbwv1cTqxj)R46P$GvMLnn@&Vz}`gK4l zIA%zUwTkg5V2Y|lHPt0B@-$ax7#NmFG}@M-$E4tPVp6IeFK{ya59U1#WG?hv2{Hso z7#;@ACiFz`q;XOA&i-RO7-bo&sxo6Ew-{>l?Xq%?k8DkzoA<@rDIdD*JTIO zXQM?V;;yO$nX;VJ0SW)drBrRlZ*TN(6%K*QBHyU$*2IsG(+U&F_a~*)Lm-vQB=zsn ztGT=ckOD~8YLc^w1$a=(c zGyJL(imzxu(->iGvGX|sItrR^=<~>!qelfrP|b~XzdGqD+2cyXRAE?7I4^u4)`xzY`$8aHH67n;N$b{nvmQa$^okHg}bCy)zbQ)-oPK7CWHbEM;?(t4x8jr z(e1Ab1q^9LD~r5c-!i$60<3i2yysqR(^p$G8xL|%MB9&yk21ly@KI73UH!}n_(a?y z2Bq~EqL!$D9n*@c7I=JP>|*=@_${Bp4i|AK=sCW{Q(t_E9I%@4*%_#DTy`hR+vNYE z51z}W&*q2y@llX%SA8AmCf**OQU#CVI>jOfg6;N`P}4_$cjqigZ_!eT?)QCGxF@9$X^U5|@)U=e>8;K!#UIv$Jx!T_Lf7t?_W~w%RKrvtR%jMy>BkQj#ZcI!bAW;6d zOZ}t@Q#EnLwdZ5;>x*Ro+Z!_UuK zSu@RO8;er0=kfa~0i!!B1yXu?8z|!PZW$`-NWw}8fFw#Mkn06$P&pjk<+cpsL;$E> zG-a2c7n~+P;UlY_nr$<+eJpF)*&LR?9w*=T|I#*lbDOEJL#`}e7ZhNYva2lRHS1>2 zu@gBbWhpgUqV(I#NUrO;1xDNnwHQQWH@>H%AN~x{$NQ&%s3H<_RB#)#2IJoV zO!4g)BI;X-6}Gf5)jA|s!C?9=#Y-bvHs6R_G7a^Z>Odhph(=PfyqYmf>kY#6+F} z#RSb@sQ~0F#1&1B$Nmpy7Bqjk%Zf;LoPL7K?^NJF%!bi1Hf2@L4ct+Z@K{2U*6Uf? z6-G^*v>6iR1xpa72{|_5c*al^^_J-G_D=d1g?{{=`7AD0o$k(&`*}}z%wj&N9GQ5JlevPx-`uAnO3Zt5| zih38?m9nGEYC#Yxd0-#w7EGfqdg-clw1xg!_m57@H*J&UP)rFGcxPtFn-R-nY&$?T zp;U|O%kjD@m+YNhmzKT$m+^z!vhBExJJhmmBTqbb3ZcYz|D}2PlYu811eCKumhVTa z1a&r~qIMcfh)7B+gd*f1gO$iwy@4)OWctr!NnsD{?+9%xVdD`j#-Rf;jY*hbgnzzy z&`A!NZDpVH58~$GQB8;teqv+I9+S*oOpeP>P5@KnsC8ARr{;dSNqv7oRkI%zC$p_BICrrwjV^_5bFqJEM$5Fss7 zTEdVPC{s?Z?-pxF!}%jYrfUwFDK2S%vtfSA7NK4U6&!|qXoF$PP0s$+QFCahp+&8d zB$eU@26oypuUhFthczW#-?5rBV@k>#4~mXa615vg&JeB;j)Ki>F=VXtvjF}s47VR% zNemaAntpMZ04`_ofXcVA3KTao=*hiwAGqL0U6;yRnEk>Ev+#d9g!oD ze$ECuPkhNro7dBjt}-(f;V~j6Dbg}TM-|jF%6MQZYtKRnTq~+aCkKfNYGVRzQ{f$u z)Nu<7nC6Dkyg|4z%y?iVPgRZ6hP;Q?)HeP!LcgARfWf$KtNpGFkdpXS$fLsTToQZ6 zW;Fzh+>lZ~P^;Z`myhIAX57J^UGtrN4AUcvo(*q8!pVd%PtY4ngP$t`4&!*ff8hAK- zgp|zv8vX z+bN%eGNEp~FU_<-l5ojlIRb;on7+{Lro4e?OeU%PR`(^M=O&REwB=?XSmS4_3jIq8 zC#7QXEvpLgHwrMf@hU3FkUL&Kg5@fWd!WvMJw)YZueRf=y#atb;W|pM9lt}?qwMPy zQ4wLm&H)No1xX3ml3e{U?F1Z0+@R6HUjuU_YRTBjTyJ7)wG#J4sfiIZ&;}XP4@%jl z^e(*2cb@2VevR~R(4>3pb;A9Gq4~m#qg#?~*0aGReE)#m~Uxm*fnTh!X+)T%)gQdXv<092>jvB6VpN zn7Dz~kqHRbpZs6b%Fusdv>RCBV*pY4NfzxOk?BPydrvH>LD5{D+_tsojkrOpk!fX4 zk@U?s=8WwpB+nt#R1rTi!cS0#hz^s;M##xWl+UU>wJVyD*PIIiL|bO6d%R{|m9nKW z$03wtsY4i~v+$B&8btq~4$vD}P#3GN`l`Y;5a=@}HpvDm!h11|lcN|Q@3>NEU(0se zV&x1V3Gz|+`RU|5HI<4j$seH>W3ci${%1lqt>N^deuRKve$x{w{MC8HwC1-HSdsFT zCce02e*5~I$uQs&$@-@R9ewd9tJhR32pkSY0izgx&Tt&HT?2l?13o_=g-A>pSGFoh z43Hxo)TC#UfHlSao8rMfgO@y)#mVlX21N~Y{}q#y=*esxp(boH3P(AwNb;+WdUXew ziuU8uF&7z`HsqQQCnuSZG9^lmLH`J1eCc4pAda);bhb$tc02&eAcT;??cUI>YYwf! zS12(vBrzE;{M&{@T<|$OJT?U9ARFeL>_0qKA;=auHaHO!C;q_>-HQ~@@@yI?)Z6LQ~DAtsJ1+&MwLov-iZ+_ zf0?Pac%^)jAkR!SnDF3XDg(CTg5t;)tDfWc9Vm_0fVv^~8a5v0=wN+`QV+3VI)uO( z+#aB5sS7cqy5>48fRWUlaG6};*?t#23;!cwVEkJExf}I+K}#K(x$^^kAo&7SmkVwH zcJ80^WgLDJpe+XPbd!mwSNJy%_<-1^p6}2ai#D*_to&>8_tF;&MbkE>2~LWGy6+}N zG2~gM*%w}BD(;|8#VTBB`UT8?d~fk-)ZY-XwhCBqU!~DxGB=RzMcbfop^OmZjoullR}rl)n!HZgc}}CZCU3 zp@#Cr7f_Yu^E_}0MR*q%TBKyJ7X^Xu)~lhXQ@cLGszj4@Pom?p^pp3pX`$78BwZ1Q zuw7SViosRfx#To}XWibd*jVt36xs)hlHT$H5hHv<5u^0u#%C^+=F;NrSN`RdxwO=f zMnX#rh8Qe5;*Urc=)(ewre1nF?s7v>| ziS1Qm_(%kae{uclyW1wSsbVp?R?=%O2aAX?Kp5wQayV6MG~qi9AiuLLZFF$4H^wvn z&DV~jP-7`J1LoUP)eb!={Jbh8HzUhLiUBu0VTbIkw$zHXm#1qao@@vzN_qwZob(BP z9g-oeN!+v9ahSyd%NBqVd9EcRn{UD*T}-BRJIQvguuZ8O+93-iSQb7|wz`yICZVzD zp9zoXk`o7{9DDF&wlqONx6efuc|NOCDj1mR7l=PI{E3G=lfOO?CWu?n{LZd{UD+3Doh*n6B4gNP}h!|OUB8A>4 zoI+&jMt%o8qZkEZcTPhPFIMoV8t~6FAvnR$d7%?w0oTMrx^LO(9Yjllv-Heq%P>YIzg1o5Bm=jFxbW`nu2wIsGy3+^3+0YXwk5z-vA@gqp`72o%q4vsA z!ezKQ?cPX+GRtJ=rG$?y*9s7+06nE|6i`XY`iHdoc@c6r4MbcoA94^QeXIGaCP4bM zCi&DJf}e`uL#7G~(g}SyeL?r#($SK@^mo#iPs!Sn2oDn_6JMM|Z4gzm)xH(5VGP_;$K6YS+pJwi?lK?1kfLJ_E;SwI2V zw7T|1_A`WAZCL~HejrrmN=U>yHgL6mZ3D_03sCzp$S}l+v4o-zpaq}%1l}$NhHHRP z+h2T=SHhG$mLWLF)zrTHNtDy=tdQn1Z zzB8pI2jk+n?0_PO3DIx29wRvS;Jc@VIe_-ABvV`kBg~a3c@R@jAr&?rwjycU)4J6^ zm7UGv<3D3ujzI*id#eh0-!3t##~i2DtTy$55S1>|=xzafiuW>WqHy_HtKq=(PQvD-pu7oHKS%znYTH< zLFo^2Jlzc3w{+OqdqVjs-;FZykINRbT%Tsg^_eW`H_I+m5veXK!O1LdFZ`mjfnsJH zCnNdxM<4BSyYK3Ne(8Dnq4QS(w~)mhU$6TqPFmvcbvx}7_cMY3U|NQ?rFJ}-JqOac z7VH>9E~t!w7T?c?vZV4OPtxRjExNQWlBcU`LUP-rJUh*a!32HF0oB75@R1D^T`3v2 zMGizyNO7WJ)HRJYX}3F5V`tgDyjVE=(k*_OaGY7fsAbNd0!FoKcDtyMTtjL2)3}}7 zMBDI}m5wISW}x4-RGST$qA1R)?1lC%tOU#St3pZ@@1$oNryA!S&zuuON-?$(qFsEZ zGT0wV(9-|2rmhZ3D-(stKJHXHYR!l^QOO3F=BG5rZJ2T{nc z@*Zk|jVd{Qxj$y5rVjQXw5aI8tn+(;1?T41qw_Cr)hOLO4)?C~eq&}v)ag2;w-eT{ z7pR!UQ-NENfhx+U1@j7DA2|p5$ngUQ69>u-8T!EAmKJIJaw)nagjRwVvTr?0YFZ9k zQ!Fpl$2GcLuRtwpm&(2zUX^yrH%TKJJsqbh_AmgX_&xeIgubNo?!2DNJ^yv*{qy;Q zcRfT3QM1YwD`cB*v65swaVk7mR9yVvV*g_T==@kfyL-X>w#O7F4IpW3e|%5=bD0G@ zCk^}14=+l&Q1ndpEw{YxpV_;8f}lnB1t{876R`QTadqY|hGRH{?P(+6&89Ss<$+RG zj!Em|h-0^#@MwrYZ&yMy;VA#RKl3Ief`sjWl}^qnSOB&q)ooVj9$i~`rQbXkq+u1W z!i+TPc=8F(^(JFNISN15P()&vYz)iJm?aN0yc&VJH|xT-X>Mch30JE175Qgai}CXQ zooA^OcZYmSwyPt%=__^7dN%pvrx-WCjVq(|stwsuuzik~D1Awi0#4*7IIq(ba9r z4nOAdpx11Yv{)2>vBtv7jTP_Q@6qgk z4@34wx3KiDKN%!n?!gVe+xI%*W@X+3-`zHBIK?kSsdxa=3&_E44|M_RaU%%};|+Vb z+^%^vk*{1%aO$@0rxZNsL~#M2O}VEuw$u@J!*uS>^Xcx0PpB+;aB-G6AGADPOY0RPSwSI6vqH?JZ7zpQI?jFEDr(YMw0?M zIx}xZ&A-uUrp=}|Ezz&AegFNnO$-8==+|N7nQguyJyCK?BvDLagJ`1vgqXM!a~H7Ol-0?gwPGFuYURGhm}5HPr>j2CXBa_)tSc@pu31m;z?Msf(0%=2It_q?1L|3v z13Mh#6D1K!2)l@f(CWF8btntN5l4et9XiEmq5^L7$H8>^XC_m^y&V!H^BB-nzZ&kW zRX-I!fB2@(UnX(Gnb>Hy*na;i+1;JJ1+D)fa)gp*A{LTc;7Gk-ek#0xndA7{+z|7F zCV_aOC+-ll(?zNbXb^sW4F{5bVo#NWN*wKbmECg|sMBzYhb0Z1^EW|Ot&*yoSGj)9 zK!eF6y7WRy02d%%w3e2(2!&*SikW|iAfN|opg=jW+#y&V$2_)|j<$hKM~$Jl9G9No zeEnwlmngAZbz$5vnFv(-U25MXUUEAd9D*nvd+;cHIh*27>Nu~+ozT0FlrTp}%di3$Z3}bDmRMQYm|sVh z4E&xCEvzPaN^=!IE1y58v$&%DRa{D?IKWTGot1v|- zZ?c;#O{Tq6Rsj`Cam3Nt_dpGrFhhlD@g7NWuW$kB3kVgg%t#!u7T^ywkL zL2qE;LzJR^M#Pw;h*3j@C=(RM0>nt&$j#EZ-q&j&k^mQGR~$=RjrsjUsbF?7=C8Rr z#*Pb?A(`PHhzyK&KiL?oHH+5hz-L5@Xgm|!u1d9>et5JlpIc4L<>Uh7Aj z1O4yb2yhm3MvL4Mwu*@OAGhOuR2{!5tg1EtC9|}w>mlEYVvllUJ=kKeS=3w$Lg2ax5)qfCd&DrJsX0v@0YmOr$a%>z#w2rK3n=>3u93l9aV{UD{+l z7fTp~re@1ZN@5`5-(XSv{7d)(_Neta4Bvdx0^M$MODUcSgx(w3S zhHKUX`Ae*GVQKL`A%)@bGhpODKpd{>t6Vh7c-Oi1RkXn-rZ+NDI~&&Iq3~$=oSI+g zi)+GO=z+f3IbtL4b4p*@Z0z!jKyOz!iT=WcOf{)1>HK#= zFwI42qBsY72Dj6ED-rF|QV#x41g`U{#*102si2RLVGp^9cxZ009OOGiO==#cdK=*^ z+S<;$MHK~pU@bwgzHx!FCUiNWDS@gQC(T@MGyQV9$;wTEhs-bOT7APLHM{*=iP-m) zC)#F7*RcuDYV-UEbB}t2q4Y$az$!oY5qzhp!Dj8$LJ>yVvbNTveXd#y3x^X-YU%?} zm_L{8tIb4+TX(S0`bl}p7T=3}(>xS4w#o);QDEp`ejowsvZ$uCzSt|K%iV~zmWKCZ zn?7IY3-#&;>(q8Q5r-EeFe}2&puO<8os`Yuk_}aimO$3r4rb50wm?j>R+^f)ly5{2 z;>F|D&-^*?XA2O_nWWG49|zzA$Iryc<#uf5CBKq(OJS^xE6*f8E*V+!M5ja_;FNxh zvLygu7G=q_9A}EuoV}foY>JT5#Ug@Tb;>WEE_SBrP!EmyN8I@vql}+{qp(b@dmH*Z zE_0k0@kG*SQ_SF;5IxvjL9nQDm zRtjLbzT1mrB#@QcLU-_+c`YBsxxB1l_4WY9xE5sl6GA0xX{fl5c^Zc;bb@Z4CC)q` zr3_o*rO#zlg19#3mM_KsZwd+jkE#v-|3?33Rhj>vj?*WSfC@=iZA?2Au;LX&N=#m~ z=7)a3|5Ve$z`*=hn)~lR|LX+@JTS7eb^cd(>uzIRtYKqE+Jf@AttU(+Bq*iX(2Rc+t)~xL~fVKAd*C*t_QQd5!CEP#I#KCNImvx4Z_1x?0!>Dvk ziKn)zO+&ke5mOe|K+EKbGg40gpzYJ~{Q1OqeQ8@7wArLKD4$V>x(wmX;*QrdreKFT zmmYnn`x3pI{g6q-(VS{}vo2fBd0N%EH;|ZfnG;pIEi4eyHf`*+H_>vhAU~{fy?h%A zJcm;(z2fCvd6_$3ng)BUITrRo87fpe0&qr(z2Hkn9uG@2FT}}SG|x|kgg}4NRerYSPKE7%!n5u5Zdu!s z?Jqp&w%jjxZOQxa5+Is6-q|>XhMtaWmBz!>WO`w)_KESxOv$@;MY=;@jFRQq^Q2Jbl`pH(q`>0hvR+d zS&D&cvn-zdeDYbYZJ)ezi+D4=os;sIiSE&6l3dbXr2b$N(5g%PezI%Hh;S8oc8fIr zCU*Zyor_L}g=&t_5xQ{|Jn%X0b>8zf&s5uu>(}_5lM=HF77WboK%SiA-T%*>O%DOy zg=tVqy@!Z7Vb;&fbERksCos=DVOP6w^Zsx_iBt^~w&ODGWe@mKPesGzr*eD+L501p zR1)pwed!{CNXh+z_QUH+f?^sS#rei;u}o(#6E)2%#X>AF;n+y|OQZqIZIR{*cx&F8 zt|~9MN+YxRkQenR@VVZnE<-Wx9Z9cgnvW$S( zv?AJy87vJ=%j9czfaV%DQ@NCoL(;7EiMnSN%BR%%i3kHc?3Y}o4*~W~v(C+VCg)3B z=w4F%ec9t0IcN{A!q?^59_J5$!`m3o8uQZIYW=e1d78R^9}^wUwkp*Gi|B}ZrRO1} zys>g+ov9#B=bL*}Gzz={3k#c|E5z7=vCovR^Cl}wc5IGAytf8C6Z1R~rSFvQx2pX) zYCVCJC9#F@%4uXZZ-=4sZGjXGe^J=Yh)TTR`#od;!z-8Q|t z@onNbOHjI$vOLxpP#W&CZN7n3sT{a1+h7E;iJ#^ns9BM#hPg)?U+K#=yrhGPQ}DtI zPfGTE!bxv5d37yjU=Tbc|0);CMawgv4wB(H1Td$v64DI`m6~nGv5DtCJ067Xu!_Kd zg>8@J=^HFQ`B~jSuuupJ9&iEBNB_nU{*=Kg&LFQt%UNv!H*Gcrb{WK{Rk*?d^U&X- zLkud6Slq?BlA8q!3IwNauxD~ZZz_@rek`PIE=l1War6 zRyO;iAjwwDl?cncv;vNs@cp7tfvmyYaY)9-Bwpl(!FXoEx?QwUs(rqzdKo+=iZ7V5 z@Qfx|$b%{pR2LxyF(Y^Jsah7geF^pv^AZhtGC!DE{YseGp`o z5__KlKbIfHJpDE&!2w8hI9iKFn)gGMM1s^hB#F>WZO93^C8rZHQH;`MOz7aS2`S`G z%v|eg#_zsSF?JZ?v?!H1i9eatCwjfBpHNz4`lVnixQ(@94B41sYW&qQVf*$6Rf)tn zm5-J$^nQ_$M5R5d_zDdrHU~-+!}KaayrED^*|Rl96PV>JTAL-L3bk+Y#s!pnV)cV@ z!qS$)26Pznl<1-Llx46gP9z%RD`p~!;F9E}{3HYag=t-OBhyhHA=sNl`xTpTvr^x+ ziyJOqE|!G);9ZM4A1I*68V6=Gg_NFifbOJlKu^f!G(RW=`k-d`v&3qeKi?o5w9|eE z?=!$TY7KvmnGH(FFCH@K)BjDFDWo-06az(pZR&TVq@y0rctv9dQ5*oQy#O1da1z&; zc1g7uV(|Ak_HC7U2;BqYlGY9P5gVd9MNMirHbTD;k%&^Dr_)8{hhRe`fYut*l~2^2Ii+OjzQ$Fk)nv+rdrU%def zV?nPTa+_(5xAtc>FN}rE#SY4NzcC%B&&blu1~ggH{2(m^*Jpl(QYu3hQPYNN$|RIb zm+X?-X(g>}q)g*w{_1lU!k6x(`^lu}m!O;$N+&zEBMV)5|J$CB$=7~GUJv8`T+Zq4 zXd&yR)_>y*Tu}xZhTKmsbrBi_L<<`PIaII%h zcG5N1aN**5IZ=Xii9sLt;3sp{mPbBrC=w1PwRZmg$Vny^oiwV%TdUg zQtpr=iQE+(&XnrEE22{W|9^aT?K|^4-+AVlneRLA%;$Z^LP<4zKb}xoRTbh=?LKC? zbSmt7&=!>(p?vVCS8{qr%*k$TGkj+vlnk>^B#LHOJ+9;d6Q}alvS{($CF+fSQGUKl z%kZAYxg392<1tz;#2d_mr9RVH4V~-?Um5gAWW6KyT9**4i@nA-3ij(P(G5zXEMj>m z3%s2#$A9evpb1QzX*efA8Q(FpeJ=}ro2H=0@ffoU)Z@j6)Kt4j{%Ga4i>9X^II~5> zNuD9stI=Eie44?eiOXI@%>0$;&CUlwC>53suKso=#$@ONA)ko`&Ef{1)K6%1#U0ul z+3=hY=Wk!uZpRWO>OCv+mC=8AAuTx_UiWqD>gpq7xz^LA!ajB8FCsR^XTUZN^5o?g zKa9@AED1Y$hDr5BDG)UA5>l&BgS1)i_Q$ZL-lrz|uTDRfeJ~A$J@RU@Mdz|X#pMeOZ=wp z=dR8^FKujrot&8taY zFs$8%O>!4_+qGaKx2DvCM~oaIj|HjNRdSQd$q~nF@%55Km#`Tq<|y?KlO^%QL5xC0 z1Q#Xoj)?Zu2i$uc&>wTD&mxanH`VVEh`ow9vlSF)410_mH0&-suE@hUIyIpkF+#926sQrQvPvyo_&TDi@8EMT|QPnQYP}FD+EqZ##*zxi!d)rrX zlssww&70&AhCjIZ-6c(rUhLdj>czGUB$40sArabY*$V#_`zFDY0awqfeNB_rtn8o- z+FG^Bf9Wo~GF!2%fhc8K?n0;sZ!=Ygc(0%}2PD`?1taM~(y}`*(--q{MJ5yWi_~U> zJBXFUZXM&gQE|!K#57me*_@~1WPdQ2+nP5oT*;o%l5B0xYiM3e(pd|yN+<2OO4>s0 zZ3+LLsX>r6y%+N{@tHwhw>HGBx%<|nUue%-%`HY{vr88f4UkYC_wN!dzLR7TKgX~I z*}QB+25qNd^-r1b%o|kfU^YgQvP5c?5p305N}TGrhC1<@XWmRnv+&ckQtccDuW7lG zQ8VY6Ezv@Q3#xRNhe;M>BZ~-E*`uCc2BmMxv__|Q7EIga))6Z{D5eKl2Sx&gK8&C~ zj>MfQ;i$TH-9egB-x2SQ;oY%@B6!vEr#SwiKE3|X0+^1-i7OlDGB#qyXme)KJg`EW z41qVPF%1%4HYp#&R-5^KYr-WE*T~YbC$j)SCa|w=*4FGucuP#9|5#OW`WM>g-CDz* z9EO>U(i{OG8ynpflvp67*{-TANcmK&CgHbUdc-P1fT$<1ppi<{Uwf>YR9RSt$j74g zgkD!7n_I^k1 z=eH^*^xU^-NqCO3Ecl#eKh$AX!Mq2#JVIcnHken=^14ffGCL3dEH(l}P5foO>7*<- z3I#W6-S+IN57dvRUI*sg9Exn;Fn1Yq6pD}|l}{pE)~y|CndkVBB6-re#CGBNSDEz4 zN3=sMuh|G@jouK#xJ{N*7#CVq3A`x#qSb-!nRTG zWKt?&9LUcNJU8e}OR*{=(73f~b1$}f19EOMIZT_GOU>grv;PMPaHx+=IzdjSEIDz8 z-=Gd{(m(OpN69o4!LA0Ikag6+AVianU%Qg9^CPsJ|gna0blsE%8m% zcYNGhfa;DxKW;5B%lEIf03~iM0K;J5NT;3EK#ipidRppe0#yL^*Jxu*_K~lI^L}TK!P$_C{qo-lLvX`H;T+uU$>|@0X9nRe>*g2!|&K5iehS!n6pWIi`ZI?Wa>P0d1r$_z&}b;xsqDTSMr5rxx}S$hjRisw^AcU8=7ud zv%1%o8oJoJ1P{lHU*A$u*m{_h323Yqz5bw$`8LD(tYt>#Fi!|R8+=>RUiP%nxO0qt zW-@76hiOKZ#3iw`aaHTIb;jxY)#WdHw4#WJpM9$t5sb46G>cbIuw!51t@mgj*t81h z5>&lA5?=Ei-z6^U?%=jkc}Cv+c48BVot#U(A7miK{<%9q>z3V1j#ClTx(cfEt?6_+ zj9=K6vWb_wzlHSVsw#HV7o!@eQc{bB1$sgZiD)k!tyQ$Kyb<(4h!CD#xAFm?+ud2* z&5%n>?xjTZmM^5jNs20jCl^FMLz{tVc$>;dT-rFF8QEtB5X`jneMT_qdD=iRir>wz zkgTEIVgNG-W!7UKl%KnK%`(LJ3U^YZ|jvA()Y27cITJJdUks+dR8XUC@=$t`z`;I z8_z1g-$^XfqQnuAQ~RV)!qbyxDnGwW%~jW}m2Oj%qXhURxtU}!zsQi=kMzRl*A&q! zELC%6ldRx;cvX8L`8)b^`cm~kz*`V$S(E^aVPFA0Kgs;a>I!oUxAHqkM-K^nQ$yJ{ z+?6dDk1xOFq0Pb)(&t4w2bs9^h;eY*;_;s0P05t+QNA{f;HGZ2XAf*RY#xX5nv~Qc9-^SK+OC|Q(#Q~ zRvO01G)Y8G09uWzOGk8+D^%wbKayRrp8s@4_qjGPdnZi=;QaJkZdK!-@&A);or8wev3zudF)lPJa4CUyS#bxW~a32u?7!R3@ z==)ey4NaCx>UYdD-iyn&IO21BFzsDcuv$jBozn`~{B!ImMZj?>$Tae6>jln?hSBvP z7sT~`O34M z5R2`xf+;h+2-~aWGV-rW8NZGeZPDNe`ozXVkP{Ieo*>sALGJ}stod#sOHYW0hx_aV zRfJ(6xJNZvdF&UvHw=ZwhETt8#$;l=JXHA$Fu3pgZDAz@K~urNSG+E_#gBrFCJ6W-PSRZi9c3OYI=ngA%OR++>_qT{vrEw2%<4qbTkGIW#4fK z24atZ0if<)NQ@jOb`<~ck{iV7k4$%a7Z?=tyQs$jN#R{}4+sk4go1cD?+J_V7WlZM z9QN`=rA2mgq0U%&C)AW*DHe~5|*3vmK>d*2C$gkgP$ z?RRasJqo6`Gx>m`g5uoTaHQ9HTXQK1TM04t1BJF8D0f_|fTH3;Zo=G3UT}m1KxEhJ z`*56qdiq8Jf9qR&+A1`pPnd<#EiN(pFdrN< z`9TeWbw%T#=cBd!r?OZ-x`Y+w@YVYu`J`>rA^?(M9Tc1gJs#89y zjw2;wi&cWJ&jv6{_fC^Jni*i;<|g=VvdhIxfXZ_3kV9$>8>4F36GIhZ$+SDeF<~tA zx>^rK(^d_nTl1EuJM0<H3L4;O$Imwt_F`y=ni zH#7F%@30tmcZ3H72}1z?e|oq{<%PC~p#Gu9a!7sH;pc>cq5l_6>>lEwmj1iGQ-^55 zaT6O2RDxsNARg#{7<-s=Jls9JJg|2RuYc$rW~w8?9ZReKa?N2{xQg~0c-XjLz&21v z)mRarp?enibADm(j+Mq;$Or64?=SER+(^yqDO2)9l@0;bn-=uj&F@Yu!d!q8Tu@0x+hxtsnXgb z)e!0YHNH%418Mc7>NWgZ>mV70B>9gk6_}oPloey%CKGEQXrADz=e@iJgc~b8EA#1M zbtH_W(m(c&ANb+6jW|M*4bH;xA>ri~Bxm zCk`vYsTeii6!c3!Y6flwThf4|Qar0$s$PA*2BDw4XdB$JL4lhcF;*p-%-Bf>#;$4f zweXM79&%*JtrP&A%-R_ccunuvCw|N#GWVds=%9Jjiyx z))#N`>kZ;}O&tCnq2btYKlPgaNezGD#y!xzM(l&&;o+wHe&R9v6Z9Z6{yJv%^?t{( zJ_P3=Q~n(X$Cdjj^$?t2ne*>RIBMBXeTN|7TK0oz{*Hp<()~na{-=NbVAj8b{pxI` zL%C#y>}InjG!>eCNw=REv}WGDKsZ1jRe5b$le&2;R7N^BWz-B z53r>XHrKTW2mgD-vVdd$TS!!#V6JGaX;k*zh^KAF!=y*b#hu!HGC;j}%vZ+HFXRXO2 zy~X8OrDf!n%YD7#oIzW)cavj{VJn#R1qqzf8hgQGGwrgcHdVvPVT=3Vq5{tyw)67W zDaVV$@ze41m)d|0B{k`nV3uBu6{mx=sIO@{zD-S!Pux8k)MeAKTgkECb%8ivu0xL< zkt{FZqz60(vRTp|#z(GguvgFT!kfCBk2XOjle#%8f{xa(Bp$svyq=$qPA_loYHzi9 z(t~_Kde;^zJHwm=qomiGS+F5Sz`n*hT_TyoDlf&Yz|9%s>4XoE5+hW~qm|Oz8F;GzmDJrRg+;@jK zey(S6(>64u7hR@WuL;|zR>lrwWev{&GX0y7Z!>C&IpjN!&3B|dtbx6haSX3pXj583 z0-ZkVb^#?&m2QQu9~2L22-?Dq7P^m$B*KX|+|JWYpg-`{)Y8GhKGGCHB_{V{Wf!Nd-Mo6f?bVSLBD;o z0d&8U4sq?MJsqGM(S#`6-@uzr(8(VyZkF*x%OGG(1R4<+L~rxPpU6E3+{YTCE82|| z9%{6}JMTnN6k_!E>;d9F1^u>zhQG?$LUa`+xbHP7lSp3~?NFq80$nTudmiDs99?%+ zelnK;;+0PlY@ij{sgtZ`kkVqa6FwE5j!vsowZ<1WC>ccChYLb){_hZswfNo^4quET zb<*y%hLvCzlg6N5c&{VU1=v$u3z*Yol5g!n>rIzK_=fQ}i zEL(Mk+(61R2!Nus1S)~4>2oc6g(v&%Cr;O8|9s=LiMkSIWhCk+b1;=(%2%-HHM)(| zmZKu06*TOo2ll~S{pZjpy=I}TdhLi@*p}IZ!$S;f zw_S|LD1(5VXDyeOe4P{s zna0e0y$4HE+rngy%F$-17H_;drn!1_!yAp?&NTq_EB2mP6UmJ6C;|ZgQN@ziOvAD} z%|uc`DrvnvPF8g=sZkJb>MlOhuG$(DqqA`g;Wu^e0Pdh>BCe?GO#*i7kWw@vJzhe_ zH$vMI6+xmOusz+FA~+`oahP}0FS|%AdlQ(2=f}i+1+9Dr{6S2z9srA3`xWbDv?uE18v`31<+Zw5vl90Q`lpdv#p~o6A1}Jz%e!IsqFpXn>6{d?} zrcU?*9E?8!NZ%A(4HvpFg6ZTG+NZZ zg)Li`o~msVOi80pugPWWJmlH=l>_j>rIfxZJQp%+Il+c;pSte zEwXLU#5MfPoM-p8d@iLx-ugN~QD07|)bL17iWQO3l`z~=rZ=@FCe%}7ni_timx_;# zpFWo1n?UrW5Nr;HHY0R9kh{1m;gx+_xuyZ^5}4yzbnIHHa$^1xf>}WKv>I|=gIvFA4|U{ z0S&M;`1?Zi(fYhYL4RJIJ{$B5bhMwte*u~P*A0vL|8c|m`vyjbOUJ@U&-kDBtu;3& z1&zUYW5xD`waWVi3RuP)Gp>~U@5X|DK)_KDdL}-U_1}I|e!;AUtn&d9)gmw#O^7H{ z5k*$j(xbqR7}TfOnMaJmGSa(>O4}o8h{qv9iaJF%iW}8KGDGa&(NCW5ylt7rvETrSdDOltUJP$>Mt+b<3-;0n)|tTq^sIrQRc^;`Y&#OI`t4rHt}f8 z&o`LsOb;IreuGF`&OI#d&f*TY1j?Lp)w#Rt8Na}5M%n)ILVRKYV}7D!j*biR$w{CLRaz_Y*(cck`!#2g&olJM*eUAVO%Es%;%kW~w> zP!zguGZo+5aIXK-iE$(uIyvWZ8{iD?jOE>M2E zLs|Gr^8{AM3P@0uD63wm`tJsgN8U9!l$#NlDQz^AE{24Z1D$M5kq;Bso0%Jj6nEtyHu_bus03Hr_*=l@@okPZoKyKJ(1T3- zB3a))$^$c)VQqXk0=*W^>^Dpef?Ix=p8Lf4w|?$)@Rf~M}~N2vF}UrEaK>893909R{c`VSr2oVkJz6LLvV>YHgZJh8a3P<0F!8%nM3c9V%rBg; z5Qy>{^9ZrWz`vej?Pje@NFkY?a%qKfZ-;OCeYc%QspmB_rI*VCryPEW#T(Qoc?iJ> zLB!PSbqBgG*G$qPl}p?ETp=zvwuSRA>`TQ%9vSf$fjZ2~>m7Wka}t;w*}auSlEMv1 zVWDy;azC&um9!~UtzQ&NjwwQF1($M7PBrj5v>`IVL2qQaBJ#@MCn8wfk7Kh3tnbwX zT;1Rb)B}&K*?xHp>Ab%|_>Dn%%SwHnBzAT-d;PjlC>7^g$g@ETbA?O~g(;<@;}1Y209k`Bl9G~(qs_Tw z3t%B~^dsOA|71U0EOJIBBMHd<&B>;0(YR}gh#Cy#lI=_<5P)F@lea;+suCO1nE~rn z;k>6;P1_qnb!q71Z$f$Co1$*>YD`6$XG=!^fTv1dRZjdZr=3{Bks5aPh@-Fu z8!JJ@agkh6x9@7ICz{1bsG7C7x+WhtdTon5zNhsm83s>NMd=RUqS+r}g1O+-lhEixcoR=X^p2C1Nfcq7Ik!_M6r}$0Ga}!G#I#Rt)m(e3!P8_Cvw=xcSS4Xc z{p_KY*$*k8T9TKTlQ1WJV2@5?7qHtn8eqfo5r4LricBHkRs5i(b&<>1%XOO{x8+Ue zI1~#pzTmR=%hZMDNSn%6pHdb5K5zVS>=#;t9mRxf3B)% zp(HUfG*PZntRTDct)M?|ec6zxUnv}nqCuUmW*fS-zK`9QORwFy-Jo4>4k$ESz1loN zWw0D(HiK(t1eU@?Eih)-Qe7;`UI@kxT(4_@n?e6@Mu>IPMnb_zi0S&{O9+oPgLb$d z*R>CCmeJ3RK0zyKX=xmH+(J$y;J1i;A&+)075z(88j9KgbSsl&qQY5m$Ig&ux6I)< z({g1mk1N7rIU5t@pAx2Nh#T@L@+?yopu5sZ?~5jU*5l%(c(PouVML< zeusTuBTJ(cD$&tX#ZcBDSm*BQZ_@*5wr^vN9Y{}ya2UYyKJu6^WbZ%hcPgk6qp4Oy zsc z+IeO?Br8OT+JdhIY_TZ}F4Irb!D)dDD}fe$15J=WU@MvYp=>VqgXhYagka#%rRiYE zkYO%(U~V^mAa$>2JZ|eqUry)n!aD}DO?pKT^+%^=B8Nal>-+3ks#e&0a%nZXu{4;< z<8yJTj+$MbrcITo7l~7miU9m)ebXEAACqa75t2%U4_hG~r{Xj&e*M6y)onk`ieR0o zoe7zb|1)gIwXfHFQuVX)emB>hQt|+2qBCt`SAI-=3ag+vZJP$>+4_Nx#4#s$q7W6o zxXrbkh>mDJbdyuDd7D?~rxpCWu+6VdZYpP+FT+^G!WwcSaYsXh80a%gmpzQS^Rf(6 zv8u`>`T4UIXQYddx7{x+S{6&K)%b75kIT~vF931e@Qs=ZcOCiq(3#?qu_LM6mjJ7^ z$IEkDx6I|a&`g+$PPe&|)&&TpYZP9QbZGqwJ+0{b+WZH(qjBw(-B$lO#JAlT|Mx`y39$pB zt8o7)kq1vGX;J4~{}oan`Qbp!lXF6Mtrk+vn4}p*ZoDnPw5tg^`j2=8RYXUzr1%RI zsCSL02*GjjhoYMp!8=jsM1Kw<4g5KALiHY{fD;HDl9X0bx(RYvZrm*>D0nee{4I7U zBT*tm(O+NoDhb_Xdp2UL1jjUo!wP%Lr-egiXQ+XSrv-B2G&wF=wv6&-5Sx?to(MYy z;iIPBJSgc%2$zBPi~R3pV>F>V%}tU8gAL*#VRl(UE@Ki&hY&(0FVA9<`oVb$+FJ;t z&&hgNOM=53aTbxM5>Dh2`ocA&`4eL=q*r*fD@kUNAi}D- z1$E6yq1z*lV4{(dpcgYt$!GlC6q9DK2rE%puw;~3SxHMo5qrpn+Rj?5vUTov?)J8? z5{k4W)O&9q1#*fsSwUcp(V)9crQWL(p#_geMe2yqiR57ikF2$Lytgm>s)jQ~H;%GY z!WCL~lWLb7GDWL2&@HGnP|*>|rI#6;UpD~rOm=A}jhpJv2xp_V*R6!Ry1+oppgozkGaa|q)|Q%CSJDn_n4*Y~E7jzN7-+Ld zor$uQ)f(%#6q)Jjir5*9B|tsPm8sMObsjaQN>#azQLth?^pXs?tDII)%54jYJT#6i&<+5r@X!Ue^7O_;kP?MXV)LJ|u{;lyfD zL`CZb(rpX#e=UaF=_B8Uua**}R4VhI!z z?qJ4TM#$lK3TbLfg`qA!EvH;gIK_cGZS}*Dq9Uy|WQCbt+gN3KyjmA-U*EiJskc$4 zJ=9Zz$2q%Ge*pVh=#hHHXR*Ld$OHBmms2^SI@Zt7zl%NC3skOEduk`*8-OP#k;9-Ay4dKFe(wXmC^7G1fdKmJPhx5m4 z4PWAUwyB0HCWx1ov@)WK#BOe%kX(f)_1m?;I2fG$W`8K8@tX;O!3FAQ2cp3Ad`Sdx z59fs!Ym$Vz{Y?g&Pbx=7MW#Sv-=F+NO9T(xr<=Kgm}ksnW#jvaR{N?-)og8bh0$+q zFSq;C+qWKvSlHWpWw2Kh&FuQqx4=l)AF!UIbKwoUM`LY&wC`H%78=kN_Kge-l^x1| zK^zOEZrxgtRfu@#0<;Q#kyvHXi$iGdY3>u273kXQ_{aebLyEXba=8 zT#2y>O=s>)7;7EJn>satt=mu3dTysp33HVNll`I2JXH5PXVY$S&ybZARhDJ`z#}=| z`dOHrH?%ptIPi>pF54sPvkDE{@LshuyhzTWZ2?N=s-#-{YBD14FHKO~wmR&z-*xKPp&&IQVs_+uqG^mVIUbcL4Z3Tb2 zF+-ZLb??+s)7YK0;8&4c*#yn)!B!8{$?e+2KmwN6ZQ_-2Q|^M^iu&W1ouVYL|1BiV zxv=`=+96b5L9XQu^b>-f$r)Ji-OT}CYcX4y3CwXkU!rm#{4pJ$s}{}_&cfhLt?R8W z5cM&J$FwK8KX_H@pI}rxMSsy{i8R_IpDxgMDZPsB&<9KOK|0f~Q9A>|rjH7Q2yS`? zLG_wl>p^l;r?y~Z2b8)>5!w=iwMR|F9+K=+$tg*}T&5(Bt>AQ=1hg=NRe`}u^C>-Q z5$h*aWkSRLv;OOlgyyOus`)zO#0jXkpbFpmRr;neVU&9C)DXoW-WargoVGr%XQz0* z3=ZROh()=W0Mb45kRHYO#wR@I8>$K$qzWtMmd08H(@??Nm%H83INt$>vP-Q*-wO*t z<${OE*g_O1Q+qMyzfjPrR8RTRg(cf9?C-*q6T#d?*&Qb^6^|5-*l4pQjL-pzr=GAW zL}I6trN!0`M4m1!iGM+8{7&_?d@ea1SA8%BayvfQn;O4mLBIfLSQrGWC5gDARGs&d zsOVI6brmg9E~IaMbJ{l!+{FNSamT|ZPIKUMe{8|Qc?-EAgbi>S37M#d7nPF7*+6-S;@gb zom*X$F^0t~vt?=n9W$ukNi%(vn=jV^$XA81t^(7nYg$BRjLS`~^?3Q?mX*_Fy@K|R zSC{_y3pBPDnyJD)|Sb7fW4IR{)>Sw$bSQ-+`nszE#D!7Q_ z!Q69E>Oo-j_Di0@6Bx^jB#Ns5i}n-++UsnuS7)}VcyC9wk#(D!JL)YRj2acwn2faC zkaJ1*y6*U{M}R(`?Z%H>x0 z9eeBbr%ZH;h^s)EvZdTbq?&7iE4A6% z;wVvA1z6ka^pAc|LnPLSAIa2v<54fphZYm)`nEGrpvkZy1Mo_N!}qQf6bGf{Z#S(&EfdeRSudF=?!DFypkI_nJiZ zQJso?x40DGD)sp|Xx9`35l%>C-$1IxiUP}g@E0Ph>KU6@KTW0IaR~sLmSr*k8E>z* zt8+1mI3<2rlgdI=Vba$7rQABZ6D*QeVESN(T#MV?7S&I;*J3ca1g1kLo?8XcceK{k z5rL&;r+z>2HkSy=O9D8$xj4D%>SnX^r?!V=i?ztj5&)o)%7HyhDY|`vvsCfC-ry{? zq`k3IE>111EgnE*^2gq5DQ0HOs=V3GB10_~Fz_X)mW1b7DZm^yA+Eqs@kvu}%vH|j zJR>EeWOTG7BPFE+6TV&uqT-dI1L+lJI}M8s<%bY^xP9AMO{6Gs+W3qAaA1s?A->70 zB^>KTI&(HI6P0$=lK)HH?3eBaJf;n^T9<2i6L4q~jD{}@oOpJ9>bSSevyMYm5+j-}8=c(W5#rSKe+2lMF8(N+8 zSWMGmVkmM3ePvUYqxAIgWz~Hu@Dc^#&vBuN;mcnK*a-0AOrq)Z!ZD-1>kp_Bu#^*` zg_miq4bq9s4XPlG3(zT1`faoO5R+|LUR+cA3;1cXH(ykeNFy!`iMX0PGp-teE<8>L<|Tnmpp;!i~|IS6E(9*3dxm#vjbFlMUqo#o%c@B zo>)jCCr~vn}Bp_$(>np<0b>=8HEAtdXq)SeVj2}s-D@eQSd~hnr)a3CIYuBT^ziy z%AyPVdzKsJnn|n~4~8GP7PA8y;crbAA~wP^v%^aJcq)p9r^Z0TGga1yZ*Y1SW6)>n zT+gmU^PTImjNeX&2NHweEAFe+KpKDb*mUZA)dagtvM@0z7KeFR32N2$z%6SmNRg!{ z26&aLf!Xoka|Es~MsJOv59?r2XPW))e~`1DZ#H{(1=o|I$g_n5;>^^HB;uc9qpbYpHa4 z|0>IJ9dRl3+ua73?^WV{r_IT0CG)jq<}1YI0t}gw<`ve%a!?6LcZTz%zO4@Ej+}*G zLYyruma3&Wxb|?wZMYFgX+voU-zlKIsi05&oaknME1U;i$r5)spsZ@Mzii=*D6u{3 zdfmiol;b%4x&%v;IIo6bpIc8iW!h4jz(iImijB*t#{U)*BWQH=AU)zJwQjua|$@MG@M5zv6UhbdQUDJoG zVDUaz8ySPU5VU^rZItLAXw^StEW8bl%)~fLk&ABiT0}hERr?N5Ph0uwic1LNCY{F&X7WURh}lZFTUaz%-w}Zv;?anXd_%w;1{?fEASF} zuku4h28YWXUrdTL0iy4~E>WVf5Nvo)3gR52Wk#a}1AFYz<#S9Q6%jH6xMOzk#}SXb z3y~d7wfvK0`+QXhh5Jk{tI&}S`bO{v!@d|83C#l-TT+wyLS`AcqHWy>upz|4Xp0Ta z1Xm!iV?@Jb`LB@<o})Kcp*YVChvTr13cBOS*!U{BbewRn4Ku;sdcT4Qf7 zbga4@YM9`@Lq&E*y2Nc?x#@AG#ie1#+i>f#o#$7;ZsR#EH4%(&;fc3qj}ZCa*%K10AkP~@u?)&NN^>{sr;7mQFn@f%GA(n6ZsLlz>v#3lud;4_==!Jnon0 zDiGITSZlS(bDFT=cfXxL^}M+eeL2eZN+N}h^e_ty5h_rtLA^t(TSM`}(`szGRkC7h ztGHcM(BEDq>Iij7b@{sO%(G14bTkGrCY_X=-Ine`7}4#@F<(lT7$3^GSZadL;utK# zEp-Avc)j$UL*M~!#1FLwTteRxl*7eCDZ!|GNR$yl(RErk>*uw79oO{&JQbfNYLvDN zJRB7`(#WucO7oCJ5wI3a2O(l(?*|aln*>R%A9-vCo z9R4)baQ*2<{l);Xw8rW_Oh}n0&Hb?h+>iV0`AI&;n-Px2UPs3g}3mrB*1!6M`=K$De^L%E#8T7KyuwPA%S-NEP@g7{E+{h#o zg^d5Lbau(e7(hW@0kC&1z1z#bn|fw8h<<2r~-Z z%iinlLpKR)i*XVN&YuJ;Ynei@`Z_@KJ*4}P88J6SMFU0!(l0}|j0|QWs0zc4b_+|T zJiNJaFlTj&l%PTBhHwCjvcS#I+bbw?1G+<&R1n_p%X(^1X}gu>o!W;V zoE}%ZJ*%pB9i>)UILssN;5my$yNSdTmAFkWOb^1%q<8f6J?J2`J}!6A95zeU32z)J zqnw9T?@c%aJm;CW6OT)wJ8vVNIZfP-kR5)Wqt7;dQG>5DZ@r!anJ{fAzCf^TT#_@Y z=zvRA{(1BHfU;&vro_FlUHmA&iO|IaR(i#fS*5j1_*N^ZmLo*qUT+khBO1Z=h)p5{ zr`@|1tH=(sJis>(_%ly*GzOehTae~~iGa=aq3_r$l0hE3%Vw8~4|sc#BxnaiR(X)K zCg$rot?EDR=|M{@*vssUm@KO0-E?c8Z{O^BLbPwRuVSg*T3;bvB@%EtH@#M*wJtih zJO3C(K2)zqZ9qOWdeiRBY`{G4b-HkKic}xHd1t!(0m;HYctv|9T-)Hsu$P}k^lHzP z0XN9((Mjc>-#%Bubm^(Q8d>w$LR52X%w)c8cn9fhK_vbKX^#lz70m;VvYQ%FG?m(NYVA9)nfCs8#USg$sIx5Y z_PWO@5P&Wi0ur4M|DZhTjFqlj32f z7>;133IsSo-yu_FSZueEji@EeBhtbGG&nQdIGv~yr|gBgP;m8>GTBqcf;|F_+1Dy~ zu|o7~g<2V7fjHYpwnOacJBWPQ>Ah7x2e0*2?|2d?uDmRsczY9kId7ru9;)2@ z*5avnprC6YtaM{#s{*SYIW(GDuVl`IIomYX^}U7Bv{g7)X-kPTxQ3hqbn58a7?jS1 zbB2eSA3#!2fFEhcSN4r{@BU1Kr{o>>0q38|On(mS>U-m14e* zdRDswpcI@dD4LV-`%-qGj!V2dM?RmWJLbVIahhIJS3Fm%L<&J`51*G9?=ee|vFRnT z!%2j?YGa1`dt!VW`whrLK&E_kJpi%MIXtmD9IX@EiapTl;m6C}^;aIwRej*%9A`tn!V-c67<6msFAB$NmtII)alb1x(LdSuyh~NRMVmK zQ2`hXy$k3~_?X%6%-&Z$=7$c;N}7qG^`#1wrG7nRTGhvO)T%`-8Fw#iw zBrd_fNV}=!q=F;=d_UiMcGlZBdtiFOeiM2l(0ax8iXaJ2#Pg55G~csvp12C{lD-ad z0l!3hV0)vVm_B@#@T4uvYts}3)UG4XM-*3y?Rh7SmTA};?2#v8(XcV}41Gmh`ZCp# zfyWM#`-i~v(axzE`Lb0E3hF~p*rB6b|Gi&;gyx^fqZ>n;^&V7eI>Z{9U>wrE$ew9_x#PjIku!Xa4A|o`l}Z*}KedBd1iJ!~ z*o)-kz>562BFw}IMePdWq9=YPvXTTmUE5YH9ds-qxFrOesx*C8w8p*d_D5%BZp#sz zQIZb#lhHx=%ckVcFcNTXo(LCWXTh80qm2@yy^wb=PX-D=htW0VF$@y+Kiyz zUgZS5U6BhNrZ7ntCbQ%G*_s}?Z}*WLZf|wob;^xPwHxug<{pO2{`kovja(7(&#DCT5-cC|fa)$&D4UCVjr%G%0;PxtYOi>vhB z9yMQe``wZM;wym4XZd#Sv3vNeeJUeYogC(lk)Y-6$Do_;59g`6HmUfn-?+18D?@Z1 zf9P(pne-La=g6>S1G|hhIN=&(08&Sxn`~8lO!mE4XpXhXk|C}P3^thyrlB>P zBxa0xQ2KxjhDVPCEBgMn+q$IIC)i5PO~c8cxZavE@#&fejrz>_1z)w#;aY? z{YD4h$+eY)E9Ui+Xn^z!>3-hn52pQ4_LaFzLUX;BxkeQxKeP(0D0uGL3u=&L>1}B; zHv#pRvOoScUJE$~9nIq`S*tiNrEOsRX~w9LPgRFsALk#@Gg%1{HC62}rZB3d{1j2k zd~ta-GW6lW9o9&Z3JofFDXb%^__g?CoT4=wkZRZT;eB03V{yp3iDq;WXwVcroD_Ie zBugyzJJyOJqH@d&C8!bYOP2BO49j#oKa zk|A!83z#CTH@*Doz$f7sz@PLZ)9YdP^*(cT_ueY!sOp`hk;iSXc+m6RUAf9l+s!7c zxq)1M$Cgd;>7}ORCiHBjmEx+*r8w&K?2-Cvwzt#O_4TRH>-M7izH@&4t^OW9V_?+I zY=Ik1o0aSJX<n8F~hLe0LXM7KSPQ+w>Esq1B-R_Mrl%tG1;hy2pSV!4B;M zOg=dA+|;JA)3<@>m@i@IcCPnvZkf-)sUa;k?X6>y)XkS*ES4ZQs`oA|6sKDSH{ z7{kGe2BG@j-mu`3nF-y0xS26Li90d)Xh^N(CVx~>N#0OnZS4qoxyveg@hd7mu6=KCzesJCc6}OE^y1Cvv#RiX zp?_Bn>-_o!z77FD!RXs7AF3#RcE-SWE8a%!cMF8K)yi1g-$6wguSN<`*R?5}QCCR< zq&XG5z{9@0(9SfmKFhe0LDCGHSDX3}Ewqzmf%Z(BeiujHFTk5q5(g=s3dgdkf1bOJ zT@-qy$SoKJI^<%ryBy_gPMK1IqJD6b@m_yB{n}*_hpn=kd%g*HfTXK-xL_#_Eqc@B zD(lFLX4KxlYPPYI;>dOry+417*c|1wtiyy)#b%+Q;~8mvETkmT$?DJ%uYHYuB@0@+ z1Ubl}f5Ms#K#yZR+LZnEHV{778;l`jYT5SI5}vcL7vMvm;tw^QbxAmyWnaeuRm*$5 zM_9&Bd?SIVu`)ShSH8C>4Il2=eVyjcChnNt7 zqb^dFYS&QI3~@k2r%W^n7fC+NAEPcjI+pe;V{1$mbaQg9s7eHV@{Fj8Uj;2%3~i9@ z(zU2c*y>4;s7lnZ;>Z7PS+FHSZ^Ad?bFrd}{%5iq^?K$n;~`U$1oHf6ycseY3JSz^ zaInx@Z^iDqdv`wk*^b9be0WL7#DT;?#E!7AsL-WK^GtQ~4doiOYUt_6PXWatZ{67z zuSn|t)m!Svm6&U+j8i;J>%Tx=~9Uo4FT94u7aE8ztWHl}|?gmkzCHf9nzR_Y@1Dpw1DF=>;f#Y6rA{QEG` z(zsV8vzpI<4(xge@g(W8zwjh7q6{@AugfpgT>J+LkRlY=Y(A0RA{^2yV6~725Jf!R z8?;1qy!LKgxgl6ScF-g0eCm7+u=+nTAl9Wq7C@DJrNA5|llQ$w&_&Naj*u39_FpW3 zSJ5%wXjZhrZdu4rCu3x@HsJD)kw-~q^O;z((U*D^*!Nes zf(`2u+4V|2rL#V}uMcBjl$4uX$~L>?8EbN=vk-WI%{ll@=U&dz4x<2 zVgM(}bMpdZ^yaPic_^$^YfXrlz$K&l6nNr}xRZow*L1-@$&txU@-fHtRao0nzfSt@ z23XcfIl8cvY(oF5>Wbo|Dk|hycu=wKJgxpH!DwlCpjm{0>wJ$9wJjU}s2Qi3uUXtV z;a#J%kL^4c!EiCHZ|)DyL+30ZZz-*;A3|+Jb*(;QIjDcWn!U-GMfp+|{Hb z3;Up%m2|wH960oiNa4#iko6E_3!{DJXAI|bGc<7o^rwB~?r=EVa}CtrjM)mC=a&(8 z@Ym>GbVeH^JND`|>2Ce#X4mJ|Y@l~&9M5>{A?E;feW42ueK{EERMmVy2G9~9T}ElP zIW6NuLu7l;{H))AtV!Brbx5*YNsvU4RVamhL^;_)h)Ha`6UcCNCJnrE$O7lPWon%e zr_iWiNfnS&((G^FtmJ4ND*i7K>z|?MpHb>`bgYc5{|Z@W`FF^=f{QhPM%c>IUJziX zZ);+0Z}pKB_|cWpwE)oY^L;)P5+-^8TU~n-D@%ERt%)J@XV##R9WLWX;{%hQ-^y8? zijIXDmx_Uw0hf-JmI;@Ql}?jJ)Lz%zM4!*n$Q*!63r)jkrw_2S$7N-sgQocu_SvDL zr(=eu5zw_30hkyW+ketQ)5zNcER=9rKS(~=eW3*&vxDHOSu?VAUz28HKPCxUgC>)-~2DG|i@1^T9Ig{EPN_;m75 zm_CPgaBmK~Vnb@LGE36e(VSEB9UaWGaqp&{@>(B-oTJgUio=bYTXg@9yBZ0Gn|O;L z%C9{voh)OP2&wuE8e4df+8Q7gwJg<-q|}}s%~3mOH#$ZLU@p{{#vYl4uI?3Xkj19z zOrEW&iRCnp;?CN_-$LhO6h|k!80Wz?EUdm6$i{`0QCkcLahn~;=SxT6 zYwl*C*vR3Z2g?VPv0)Poj*{&9s|fuj8-)MuR+0bU;}Yas{PQW?HoJJ96?XO8=5}ZmPjMBcI_r=sD?Voi(YK+gT0O4_CxzqtLGI93eNOReuW)YD+n+f7Z*+ULol=L{ z3uyf$gUK%#M7n3=k90ZR`H$U0N0cw)cdX$mLY-Dn#C<6L7dK`5_iFyjA!+0t^z1*q zQ^D2&@E4R{*ADRM68~?%G_kd_7ckbf{q#2p-M`xmtk5*dCIHo(r_#ppRdap1Y?xz|5=Q!PejDJ|}V|V_ALL*~qr7sV#SEu>N7^YDGINNLf z=jQqks=o{`qiY19k&KP^+)Ql{Kq2dTH0BEve0*-5s?2l0US;A0dhb2X+&}DKNgsqn?~S6!4C;` zxSu)8a{sZ{KSh2XVjqnDrH&o+|Iz{a-%5NC{7Z{Z!-|_2*s0_GrRir^z{=s%7XKlF z?mzOR|B@k~`*C_%8GX#!=wp(9O&Xd;+8SWVr~i2ns{ex*jf$!ouBC&y`9D5rKX^;% zS{jiW0H_3o$o_{(|Cu}VF$q2gdt)nGPGKQoVOm;7Mp|0t41pX0=;-L#K57D7I@L?(%IV-CVU;NAGp=m@d4XtqL z{z}pR=m_Gf<8rbI)6=rCGBC0-(Xo8!!N(`aB1kXDCdkIeM^DEf{NWn^f0R$B7qrqB zFb3$G**RF?((u#sG4KluvI#IT2{E$>3bXJrv9PkRGP8V`n2&{@;nQ{P>~(GJ|5{SI P4@aSeCL$7&7KZ*mzQ%fu literal 0 HcmV?d00001 diff --git a/tests/unit/data/document.txt b/tests/unit/data/document.txt new file mode 100644 index 000000000..96c906756 --- /dev/null +++ b/tests/unit/data/document.txt @@ -0,0 +1 @@ +foo bar \ No newline at end of file diff --git a/tests/unit/phpunit.xml b/tests/unit/phpunit.xml new file mode 100644 index 000000000..d68c43cf7 --- /dev/null +++ b/tests/unit/phpunit.xml @@ -0,0 +1,21 @@ + + + + . + + + + + + + ../.. + + + + diff --git a/tests/unit/testcase.php b/tests/unit/testcase.php new file mode 100644 index 000000000..cd8c7085e --- /dev/null +++ b/tests/unit/testcase.php @@ -0,0 +1,140 @@ +. + * + */ + +namespace OCA\Search_Lucene\Tests\Unit; + +use OC\Files\Storage\Storage; +use OC\Files\View; +use PHPUnit_Framework_TestCase; + +abstract class TestCase extends PHPUnit_Framework_TestCase { + + /** + * @var \OC\Files\Storage\Storage $storage + */ + private $storage; + + /** + * + * @var string $userName user name + */ + private $userName; + + /** + * @var \OC\Files\Cache\Scanner + */ + protected $scanner; + + //for search lucene + public function setUp() { + + // reset backend + \OC_User::clearBackends(); + \OC_User::useBackend('database'); + + // create test user + $this->userName = 'test'; + \OC_User::deleteUser($this->userName); + \OC_User::createUser($this->userName, $this->userName); + + \OC_Util::tearDownFS(); + \OC_User::setUserId(''); + \OC\Files\Filesystem::tearDown(); + \OC_Util::setupFS($this->userName); + \OC_User::setUserId($this->userName); + + $view = new \OC_FilesystemView('/' . $this->userName . '/files'); + + // setup files + $filesToCopy = array( + 'documents' => array( + 'document.pdf', + 'document.docx', + 'document.odt', + 'document.txt', + ), + /* + 'music' => array( + 'projekteva-letitrain.mp3', + ), + 'photos' => array( + 'photo.jpg', + ), + 'videos' => array( + 'BigBuckBunny_320x180.mp4', + ), + */ + ); + $count = 0; + foreach($filesToCopy as $folder => $files) { + foreach($files as $file) { + $imgData = file_get_contents(__DIR__ . '/data/' . $file); + $view->mkdir($folder); + $path = $folder . '/' . $file; + $view->file_put_contents($path, $imgData); + + // set mtime to get fixed sorting with respect to recentFiles + $count++; + $view->touch($path, 1000 + $count); + } + } + + list($storage, $internalPath) = $view->resolvePath(''); + /** @var $storage Storage */ + $this->storage = $storage; + $this->scanner = $storage->getScanner(); + + // hookup scanner + /* + $this->scanner->listen('\OC\Files\Cache\Scanner', 'postScanFile', function($path, $storage) { + $h = new Hooks(); + $h->postScanFile($path, $storage); + }); + */ + + $this->scanner->scan(''); + } + + public function tearDown() { + if (is_null($this->storage)) { + return; + } + $cache = $this->storage->getCache(); + $ids = $cache->getAll(); + $permissionsCache = $this->storage->getPermissionsCache(); + $permissionsCache->removeMultiple($ids, \OC_User::getUser()); + $cache->clear(); + } + + protected function getFileId($path) { + + $view = new View('/' . $this->userName . '/files'); + $fileInfo = $view->getFileInfo($path); + + if (! empty($fileInfo)) { + return $fileInfo['fileid']; + } + + return null; + } +} diff --git a/tests/unit/testsearchprovider.php b/tests/unit/testsearchprovider.php new file mode 100644 index 000000000..da3d338cd --- /dev/null +++ b/tests/unit/testsearchprovider.php @@ -0,0 +1,258 @@ +. + * + */ + +namespace OCA\Search_Lucene\Tests\Unit; + +class DummyIndex implements \Zend_Search_Lucene_Interface { + public function addDocument(\Zend_Search_Lucene_Document $document) { + + } + + public function addReference() { + + } + + public function closeTermsStream() { + + } + + public function commit() { + + } + + public function count() { + + } + + public function currentTerm() { + + } + + public function delete($id) { + + } + + public function docFreq(\Zend_Search_Lucene_Index_Term $term) { + + } + + public function find($query) { + + } + + public function getDirectory() { + + } + + public function getDocument($id) { + + } + + public function getFieldNames($indexed = false) { + + } + + public function getFormatVersion() { + + } + + public function getMaxBufferedDocs() { + + } + + public function getMaxMergeDocs() { + + } + + public function getMergeFactor() { + + } + + public function getSimilarity() { + + } + + public function hasDeletions() { + + } + + public function hasTerm(\Zend_Search_Lucene_Index_Term $term) { + + } + + public function isDeleted($id) { + + } + + public function maxDoc() { + + } + + public function nextTerm() { + + } + + public function norm($id, $fieldName) { + + } + + public function numDocs() { + + } + + public function optimize() { + + } + + public function removeReference() { + + } + + public function resetTermsStream() { + + } + + public function setFormatVersion($formatVersion) { + + } + + public function setMaxBufferedDocs($maxBufferedDocs) { + + } + + public function setMaxMergeDocs($maxMergeDocs) { + + } + + public function setMergeFactor($mergeFactor) { + + } + + public function skipTo(\Zend_Search_Lucene_Index_Term $prefix) { + + } + + public function termDocs(\Zend_Search_Lucene_Index_Term $term, $docsFilter = null) { + + } + + public function termDocsFilter(\Zend_Search_Lucene_Index_Term $term, $docsFilter = null) { + + } + + public function termFreqs(\Zend_Search_Lucene_Index_Term $term, $docsFilter = null) { + + } + + public function termPositions(\Zend_Search_Lucene_Index_Term $term, $docsFilter = null) { + + } + + public function terms() { + + } + + public function undeleteAll() { + + } + + public static function getActualGeneration(\Zend_Search_Lucene_Storage_Directory $directory) { + + } + + public static function getDefaultSearchField() { + + } + + public static function getResultSetLimit() { + + } + + public static function getSegmentFileName($generation) { + + } + + public static function setDefaultSearchField($fieldName) { + + } + + public static function setResultSetLimit($limit) { + + } + +} + +class TestSearchProvider extends TestCase { + + /** + * @dataProvider searchResultDataProvider + */ + function testAsOCSearchResult(\Zend_Search_Lucene_Search_QueryHit $hit, $name, $text, $link, $type, $container) { + + $searchResult = \OCA\Search_Lucene\SearchProvider::asOCSearchResult($hit); + + $this->assertInstanceOf('OC_Search_Result', $searchResult); + $this->assertEquals($searchResult->name, $name); + $this->assertEquals($searchResult->text, $text); + //$this->assertEquals($searchResult->link, $link); + $this->assertEquals($searchResult->type, $type); + $this->assertEquals($searchResult->container, $container); + } + + public function searchResultDataProvider() { + + $index = new DummyIndex(); + $hit1 = new \Zend_Search_Lucene_Search_QueryHit($index); + $hit1->mimetype = 'text/plain'; + $hit1->path = 'documents/document.txt'; + $hit1->size = 123; + $hit1->score = 0.4; + + $hit2 = new \Zend_Search_Lucene_Search_QueryHit($index); + $hit2->mimetype = 'application/pdf'; + $hit2->path = 'documents/document.pdf'; + $hit2->size = 1234; + $hit2->score = 0.31; + + $hit3 = new \Zend_Search_Lucene_Search_QueryHit($index); + $hit3->mimetype = 'audio/mp3'; + $hit3->path = 'documents/document.mp3'; + $hit3->size = 12341234; + $hit3->score = 0.299; + + $hit4 = new \Zend_Search_Lucene_Search_QueryHit($index); + $hit4->mimetype = 'image/jpg'; + $hit4->path = 'documents/document.jpg'; + $hit4->size = 1234123; + $hit4->score = 0.001; + + return array( + // name, text, link, type, container + //FIXME search result should not contain translated strings + array($hit1,'document.txt', 'documents, 123 B, Score: 0.40', 'FIXME', 'Text', 'documents'), + array($hit2,'document.pdf', 'documents, 1 kB, Score: 0.31', 'FIXME', 'Files', 'documents'), + array($hit3,'document.mp3', 'documents, 11.8 MB, Score: 0.30', 'FIXME', 'Music', 'documents'), + array($hit4,'document.jpg', 'documents, 1.2 MB, Score: 0.00', 'FIXME', 'Images', 'documents'), + ); + } +} diff --git a/tests/unit/teststatus.php b/tests/unit/teststatus.php new file mode 100644 index 000000000..24f315456 --- /dev/null +++ b/tests/unit/teststatus.php @@ -0,0 +1,160 @@ +. + * + */ + +namespace OCA\Search_Lucene\Tests\Unit; + +use OCA\Search_Lucene\Status; + + +class TestStatus extends TestCase { + + /** + * @dataProvider statusDataProvider + */ + function testFromFileIdNull($fileName) { + + // preparation + $fileId = $this->getFileId($fileName); + $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); + + // run test + $status = Status::fromFileId($fileId); + + $this->assertInstanceOf('OCA\Search_Lucene\Status', $status); + $this->assertEquals($fileId, $status->getFileId()); + $this->assertEquals(null, $status->getStatus()); + } + + /** + * @dataProvider statusDataProvider + */ + function testMarkNew($fileName) { + + // preparation + $fileId = $this->getFileId($fileName); + $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); + + // run test + $status = new Status($fileId); + $status->markNew(); + + $this->assertInstanceOf('OCA\Search_Lucene\Status', $status); + $this->assertEquals($fileId, $status->getFileId()); + $this->assertEquals(Status::STATUS_NEW, $status->getStatus()); + + //check after loading from db + $status2 = Status::fromFileId($fileId); + + $this->assertInstanceOf('OCA\Search_Lucene\Status', $status2); + $this->assertEquals($fileId, $status2->getFileId()); + $this->assertEquals(Status::STATUS_NEW, $status2->getStatus()); + + } + + /** + * @dataProvider statusDataProvider + */ + function testMarkSkipped($fileName) { + + // preparation + $fileId = $this->getFileId($fileName); + $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); + + // run test + $status = new Status($fileId); + $status->markSkipped(); + + $this->assertInstanceOf('OCA\Search_Lucene\Status', $status); + $this->assertEquals($fileId, $status->getFileId()); + $this->assertEquals(Status::STATUS_SKIPPED, $status->getStatus()); + + //check after loading from db + $status2 = Status::fromFileId($fileId); + + $this->assertInstanceOf('OCA\Search_Lucene\Status', $status2); + $this->assertEquals($fileId, $status2->getFileId()); + $this->assertEquals(Status::STATUS_SKIPPED, $status2->getStatus()); + + } + + /** + * @dataProvider statusDataProvider + */ + function testMarkIndexed($fileName) { + + // preparation + $fileId = $this->getFileId($fileName); + $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); + + // run test + $status = new Status($fileId); + $status->markIndexed(); + + $this->assertInstanceOf('OCA\Search_Lucene\Status', $status); + $this->assertEquals($fileId, $status->getFileId()); + $this->assertEquals(Status::STATUS_INDEXED, $status->getStatus()); + + //check after loading from db + $status2 = Status::fromFileId($fileId); + + $this->assertInstanceOf('OCA\Search_Lucene\Status', $status2); + $this->assertEquals($fileId, $status2->getFileId()); + $this->assertEquals(Status::STATUS_INDEXED, $status2->getStatus()); + + } + + /** + * @dataProvider statusDataProvider + */ + function testMarkError($fileName) { + + // preparation + $fileId = $this->getFileId($fileName); + $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); + + // run test + $status = new Status($fileId); + $status->markError(); + + $this->assertInstanceOf('OCA\Search_Lucene\Status', $status); + $this->assertEquals($fileId, $status->getFileId()); + $this->assertEquals(Status::STATUS_ERROR, $status->getStatus()); + + //check after loading from db + $status2 = Status::fromFileId($fileId); + + $this->assertInstanceOf('OCA\Search_Lucene\Status', $status2); + $this->assertEquals($fileId, $status2->getFileId()); + $this->assertEquals(Status::STATUS_ERROR, $status2->getStatus()); + + } + + public function statusDataProvider() { + return array( + array('/documents/document.pdf'), + array('/documents/document.docx'), + array('/documents/document.odt'), + array('/documents/document.txt'), + ); + } +} From 0e338661af0b29267a4f585b48cb9e51886346d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 29 Jan 2014 19:18:41 +0100 Subject: [PATCH 09/51] Update README.md --- tests/unit/README.md | 48 ++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/tests/unit/README.md b/tests/unit/README.md index 2bbd13ec3..4d0e83e7c 100644 --- a/tests/unit/README.md +++ b/tests/unit/README.md @@ -5,52 +5,56 @@ hooks.php --------- make sure -- indexFile() registers an IndexJob -- renameFile() deletes the file from lucene index and adds an IndexJob -- deleteFile() deletes the file from lucene index +- [ ] indexFile() registers an IndexJob +- [ ] renameFile() deletes the file from lucene index and adds an IndexJob +- [ ] deleteFile() deletes the file from lucene index indexer.php ----------- -indexFile() adds file to the index if possible (different filetypes) -indexFiles() indexes files and changes status accordingly +- [ ] indexFile() adds file to the index if possible (different filetypes) +- [ ] indexFiles() indexes files and changes status accordingly indexjob.php ------------ -- run() sets up correct FS and indexes all unindexed files +- [ ] run() sets up correct FS and indexes all unindexed files lucene.php ---------- -- openOrCreate() creates the index on the fly, opens existing index, check readonly index? -- optimizeIndex() optimizes the index? -- updateFile() deletes the old entry and adds a new one -- deleteFile() deletes the old entry -- find() finds an entry, try various queries +- [ ] openOrCreate() creates the index on the fly, opens existing index, check readonly index? +- [ ] optimizeIndex() optimizes the index? +- [ ] updateFile() deletes the old entry and adds a new one +- [ ] deleteFile() deletes the old entry +- [ ] find() finds an entry, try various queries optimizejob.php --------------- -- run() cleans up entries without a fileid (removing old pk entries), optimizes the index +- [ ] run() cleans up entries without a fileid (removing old pk entries), optimizes the index searchprovider.php ------------------ -- creates valid oc search result objects +- [x] creates valid oc search result objects status.php ---------- -- setUp() create folder and two files -- fromFileId() loads a status -- markNew() sets status to N -- markIndexed() sets status to I -- markSkipped() sets status to S -- markError() sets status to E -- delete() deletes a status -- getUnindexed() return list of unindexed file ids -- getDeleted() returns list of deleted file ids +- [x] fromFileId() loads a status +- [x] markNew() sets status to N +- [x] markIndexed() sets status to I +- [x] markSkipped() sets status to S +- [x] markError() sets status to E +- [ ] delete() deletes a status +- [ ] getUnindexed() return list of unindexed file ids +- [ ] getDeleted() returns list of deleted file ids + +testcase.php +------------ + +- [x] setUp() create folder and two files From f61cbfe87485f770de05a0ebf355d4762a7465f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 11 Feb 2014 16:34:15 +0100 Subject: [PATCH 10/51] Search in Cyrillic works correctly. --- lib/lucene.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lucene.php b/lib/lucene.php index ba6bb5b89..353b19471 100644 --- a/lib/lucene.php +++ b/lib/lucene.php @@ -46,7 +46,7 @@ private function openOrCreate() { //let lucene search for numbers as well as words \Zend_Search_Lucene_Analysis_Analyzer::setDefault( - new \Zend_Search_Lucene_Analysis_Analyzer_Common_TextNum_CaseInsensitive() + new \Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive() ); // Create index From 7efe61ddcbf7173229c552e07c8d32c6d6119224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 2 May 2014 10:49:07 +0200 Subject: [PATCH 11/51] fix unit tests --- appinfo/app.php | 2 ++ tests/unit/bootstrap.php | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/appinfo/app.php b/appinfo/app.php index fdf462664..7c0bbbd17 100644 --- a/appinfo/app.php +++ b/appinfo/app.php @@ -47,6 +47,8 @@ OC::$CLASSPATH['Zend_Search_Lucene_Document_Xlsx'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Document/Xlsx.php'; OC::$CLASSPATH['Zend_Search_Lucene_Document_Pptx'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Document/Pptx.php'; OC::$CLASSPATH['Zend_Search_Lucene_Analysis_Analyzer'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Analysis/Analyzer.php'; +OC::$CLASSPATH['Zend_Search_Lucene_Interface'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Interface.php'; +OC::$CLASSPATH['Zend_Search_Lucene_Search_QueryHit'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Search/QueryHit.php'; OC::$CLASSPATH['getID3'] = 'getid3/getid3.php'; diff --git a/tests/unit/bootstrap.php b/tests/unit/bootstrap.php index a25813a26..60e22d23e 100644 --- a/tests/unit/bootstrap.php +++ b/tests/unit/bootstrap.php @@ -3,7 +3,9 @@ global $RUNTIME_NOAPPS; $RUNTIME_NOAPPS = true; -define('PHPUNIT_RUN', 1); +if (!defined('PHPUNIT_RUN')) { + define('PHPUNIT_RUN', 1); +} require_once __DIR__.'/../../../../lib/base.php'; @@ -15,8 +17,5 @@ $dir = __DIR__.'/../../3rdparty'; set_include_path(get_include_path() . PATH_SEPARATOR . $dir); -OC::$CLASSPATH['Zend_Search_Lucene_Interface'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Interface.php'; -OC::$CLASSPATH['Zend_Search_Lucene_Search_QueryHit'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Search/QueryHit.php'; - OC_Hook::clear(); OC_Log::$enabled = false; From 04b94c9bc2bcfc0539f8a6b4a1ebe04f47b369a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 2 May 2014 10:49:34 +0200 Subject: [PATCH 12/51] use OC:: --- lib/searchprovider.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/searchprovider.php b/lib/searchprovider.php index 55fe34c1b..3cf347308 100644 --- a/lib/searchprovider.php +++ b/lib/searchprovider.php @@ -105,7 +105,7 @@ public static function asOCSearchResult(\Zend_Search_Lucene_Search_QueryHit $hit $url = Util::linkTo('files', 'index.php') . '?dir='.$hit->path; break; default: - $url = \OC::getRouter()->generate('download', array('file'=>$hit->path)); + $url = \OC::$server->getRouter()->generate('download', array('file'=>$hit->path)); } return new \OC_Search_Result( From add41e303381c304540fdf0f5bf5081e13013adf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 2 Jul 2014 11:43:14 +0200 Subject: [PATCH 13/51] we need to reindex all with v6 --- appinfo/preupdate.php | 35 +++++------------------------------ appinfo/update.php | 8 -------- 2 files changed, 5 insertions(+), 38 deletions(-) diff --git a/appinfo/preupdate.php b/appinfo/preupdate.php index 5808f4793..170a6f312 100644 --- a/appinfo/preupdate.php +++ b/appinfo/preupdate.php @@ -2,35 +2,10 @@ $currentVersion=OCP\Config::getAppValue('search_lucene', 'installed_version'); -if (version_compare($currentVersion, '0.5.2', '<')) { - - //delete duplicate id entries - - $dbtype = OCP\Config::getSystemValue('dbtype', 'sqlite3'); - - if ($dbtype === 'mysql') { - // fix MySQL ERROR 1093 (HY000), see http://stackoverflow.com/a/12969601 - $sql = 'DELETE FROM `*PREFIX*lucene_status` - WHERE `fileid` IN ( - SELECT `fileid` FROM ( - SELECT `fileid` - FROM `*PREFIX*lucene_status` - GROUP BY `fileid` - HAVING count(`status`) > 1 - ) AS `mysqlerr1093hack` - ) - '; - } else { - $sql = 'DELETE FROM `*PREFIX*lucene_status` - WHERE `fileid` IN ( - SELECT `fileid` - FROM `*PREFIX*lucene_status` - GROUP BY `fileid` - HAVING count(`status`) > 1 - ) - '; - } - - $stmt = OCP\DB::prepare($sql); +if (version_compare($currentVersion, '0.6.0', '<')) { + //force reindexing of files + $stmt = OCP\DB::prepare('DELETE FROM `*PREFIX*lucene_status`'); $stmt->execute(); + //FIXME wipe index on disk because primary key changed + } \ No newline at end of file diff --git a/appinfo/update.php b/appinfo/update.php index ccc5a218f..ac691cfef 100644 --- a/appinfo/update.php +++ b/appinfo/update.php @@ -7,11 +7,3 @@ $stmt = OCP\DB::prepare('DELETE FROM `*PREFIX*queuedtasks` WHERE `app`=?'); $stmt->execute(array('search_lucene')); } - -if (version_compare($currentVersion, '0.6.0', '<')) { - //force reindexing of files - $stmt = OCP\DB::prepare('DELETE FROM `*PREFIX*lucene_status`'); - $stmt->execute(); - //FIXME wipe index on disk because primary key changed - -} \ No newline at end of file From 8e6bdc3f3ba70301303487e3bd7ceeccf264e794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 2 Jul 2014 18:24:28 +0200 Subject: [PATCH 14/51] OC7 updates, minor fixes, force recreation of index and status table for v0.6.0 --- appinfo/app.php | 13 +- appinfo/preupdate.php | 2 - js/checker.js | 5 + lib/indexer.php | 16 +- lib/lucene.php | 26 ++- lib/searchprovider.php | 62 +----- result/content.php | 58 ++++++ tests/unit/testcase.php | 7 +- tests/unit/testsearchprovider.php | 305 ++++++++++-------------------- 9 files changed, 201 insertions(+), 293 deletions(-) create mode 100644 result/content.php diff --git a/appinfo/app.php b/appinfo/app.php index 7c0bbbd17..3a4755f5e 100644 --- a/appinfo/app.php +++ b/appinfo/app.php @@ -27,15 +27,6 @@ //add 3rdparty folder to include path $dir = dirname(dirname(__FILE__)).'/3rdparty'; set_include_path(get_include_path() . PATH_SEPARATOR . $dir); -/* -OC::$CLASSPATH['OCA\Search_Lucene\Lucene'] = 'search_lucene/lib/lucene.php'; -OC::$CLASSPATH['OCA\Search_Lucene\Indexer'] = 'search_lucene/lib/indexer.php'; -OC::$CLASSPATH['OCA\Search_Lucene\Hooks'] = 'search_lucene/lib/hooks.php'; -*/ -OC::$CLASSPATH['OCA\Search_Lucene\Document\Pdf'] = 'search_lucene/document/Pdf.php'; -OC::$CLASSPATH['OCA\Search_Lucene\Document\OpenDocument'] = 'search_lucene/document/OpenDocument.php'; -OC::$CLASSPATH['OCA\Search_Lucene\Document\Odt'] = 'search_lucene/document/Odt.php'; -OC::$CLASSPATH['OCA\Search_Lucene\Document\Ods'] = 'search_lucene/document/Ods.php'; OC::$CLASSPATH['Zend_Search_Lucene'] = 'search_lucene/3rdparty/Zend/Search/Lucene.php'; OC::$CLASSPATH['Zend_Search_Lucene_Index_Term'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Index/Term.php'; @@ -64,8 +55,8 @@ // --- replace default file search provider ----------------------------------------------- //remove other providers -OC_Search::removeProvider('OC_Search_Provider_File'); -OC_Search::registerProvider('OCA\Search_Lucene\SearchProvider'); +\OC::$server->getSearch()->removeProvider('OC_Search_Provider_File'); +\OC::$server->getSearch()->registerProvider('OCA\Search_Lucene\SearchProvider'); // add background job for index optimization: diff --git a/appinfo/preupdate.php b/appinfo/preupdate.php index 170a6f312..20e5f307b 100644 --- a/appinfo/preupdate.php +++ b/appinfo/preupdate.php @@ -6,6 +6,4 @@ //force reindexing of files $stmt = OCP\DB::prepare('DELETE FROM `*PREFIX*lucene_status`'); $stmt->execute(); - //FIXME wipe index on disk because primary key changed - } \ No newline at end of file diff --git a/js/checker.js b/js/checker.js index 84b218f3e..ea276d037 100644 --- a/js/checker.js +++ b/js/checker.js @@ -53,5 +53,10 @@ $(document).ready(function () { //hovering over it shows the current file //clicking it stops the indexer: ⌛ + OC.search.resultTypes.content = 'In' // translated + + OC.search.customResults.content = function (row, item){ + row.find('td.result .text').text(t('search_lucene', 'Score: {score}', {score: Math.round(item.score*100)/100})); + }; }); diff --git a/lib/indexer.php b/lib/indexer.php index a35d04ae0..a31f84cee 100644 --- a/lib/indexer.php +++ b/lib/indexer.php @@ -32,7 +32,7 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) $skippedDirs = explode(';', \OCP\Config::getUserValue(\OCP\User::getUser(), 'search_lucene', 'skipped_dirs', '.git;.svn;.CVS;.bzr')); foreach ($fileIds as $id) { - $skipped = false; + $skip = false; $fileStatus = \OCA\Search_Lucene\Status::fromFileId($id); @@ -43,7 +43,7 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) $fileStatus->markError(); $path = \OC\Files\Filesystem::getPath($id); - + if (empty($path)) { $skip = true; } else { @@ -55,14 +55,13 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) break; } } - $skip = false; } if ($skip) { $fileStatus->markSkipped(); \OCP\Util::writeLog('search_lucene', 'skipping file '.$id.':'.$path, - \OCP\Util::ERROR); + \OCP\Util::DEBUG); continue; } if ($eventSource) { @@ -77,7 +76,7 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) $eventSource->send('error', $path); } } - } catch (Exception $e) { //sqlite might report database locked errors when stock filescan is in progress + } catch (\Exception $e) { //sqlite might report database locked errors when stock filescan is in progress //this also catches db locked exception that might come up when using sqlite \OCP\Util::writeLog('search_lucene', $e->getMessage() . ' Trace:\n' . $e->getTraceAsString(), @@ -183,12 +182,11 @@ public function indexFile($path = '') { // Store filecache id as unique id to lookup by when deleting $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $data['fileid'])); - // Store filename - $doc->addField(\Zend_Search_Lucene_Field::Text('filename', $data['name'], 'UTF-8')); - - // Store document path to identify it in the search results + // Store document path for the search results $doc->addField(\Zend_Search_Lucene_Field::Text('path', $path, 'UTF-8')); + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', $data['mtime'])); + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('size', $data['size'])); $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', $mimeType)); diff --git a/lib/lucene.php b/lib/lucene.php index 353b19471..547ac22a0 100644 --- a/lib/lucene.php +++ b/lib/lucene.php @@ -16,7 +16,7 @@ class Lucene { * used as signalclass in OC_Hooks::emit() */ const CLASSNAME = 'Lucene'; - + public $user; public $index; @@ -24,13 +24,13 @@ public function __construct($user) { $this->user = $user; $this->index = self::openOrCreate(); } - + private function getIndexURL () { // TODO profile: encrypt the index on logout, decrypt on login //return OCP\Files::getStorage('search_lucene'); return \OC_User::getHome($this->user) . '/lucene_index'; } - + /** * opens or creates the users lucene index * @@ -52,13 +52,25 @@ private function openOrCreate() { // Create index $indexUrl = $this->getIndexURL(); - if (file_exists($indexUrl)) { + + // can we use the index? + if (file_exists($indexUrl.'/v0.6.0')) { + // correct index present $index = \Zend_Search_Lucene::open($indexUrl); + } else if (file_exists($indexUrl)) { + Util::writeLog( + 'search_lucene', + 'recreating outdated lucene index', + Util::INFO + ); + \OC_Helper::rmdirr($indexUrl); + $index = \Zend_Search_Lucene::create($indexUrl); + touch($indexUrl.'/v0.6.0'); } else { $index = \Zend_Search_Lucene::create($indexUrl); - //todo index all user files + touch($indexUrl.'/v0.6.0'); } - } catch ( Exception $e ) { + } catch ( \Exception $e ) { Util::writeLog( 'search_lucene', $e->getMessage().' Trace:\n'.$e->getTraceAsString(), @@ -100,7 +112,7 @@ public function optimizeIndex() { * * @author Jörn Dreyer * - * @param Zend_Search_Lucene_Document $doc the document to store for the path + * @param \Zend_Search_Lucene_Document $doc the document to store for the path * @param int $fileid fileid to update * * @return void diff --git a/lib/searchprovider.php b/lib/searchprovider.php index 3cf347308..ef7cc6d18 100644 --- a/lib/searchprovider.php +++ b/lib/searchprovider.php @@ -16,7 +16,7 @@ /** * @author Jörn Dreyer */ -class SearchProvider extends \OC_Search_Provider { +class SearchProvider extends \OCP\Search\Provider { /** * performs a search on the users index @@ -24,7 +24,7 @@ class SearchProvider extends \OC_Search_Provider { * @author Jörn Dreyer * * @param string $query lucene search query - * @return array of OC_Search_Result + * @return array of \OCP\Search\Result */ public function search($query){ $results=array(); @@ -50,10 +50,10 @@ public function search($query){ //limit results. we cant show more than ~30 anyway. TODO use paging later for ($i = 0; $i < 30 && $i < count($hits); $i++) { - $results[] = self::asOCSearchResult($hits[$i]); + $results[] = new \OCA\Search_Lucene\Result\Content($hits[$i]); } - } catch ( Exception $e ) { + } catch ( \Exception $e ) { Util::writeLog( 'search_lucene', $e->getMessage().' Trace:\n'.$e->getTraceAsString(), @@ -65,60 +65,6 @@ public function search($query){ return $results; } - /** - * converts a zend lucene search object to a OC_SearchResult - * - * Example: - * - * Text | Some Document.txt - * | /path/to/file, 148kb, Score: 0.55 - * - * @author Jörn Dreyer - * - * @param Zend_Search_Lucene_Search_QueryHit $hit The Lucene Search Result - * @return OC_Search_Result an OC_Search_Result - */ - public static function asOCSearchResult(\Zend_Search_Lucene_Search_QueryHit $hit) { - - $mimeBase = self::baseTypeOf($hit->mimetype); - - switch($mimeBase){ - case 'audio': - $type='Music'; - break; - case 'text': - $type='Text'; - break; - case 'image': - $type='Images'; - break; - default: - if ($hit->mimetype=='application/xml') { - $type='Text'; - } else { - $type='Files'; - } - } - - switch ($hit->mimetype) { - case 'httpd/unix-directory': - $url = Util::linkTo('files', 'index.php') . '?dir='.$hit->path; - break; - default: - $url = \OC::$server->getRouter()->generate('download', array('file'=>$hit->path)); - } - - return new \OC_Search_Result( - basename($hit->path), - dirname($hit->path) - . ', ' . \OCP\Util::humanFileSize($hit->size) - . ', Score: ' . number_format($hit->score, 2), - $url, - $type, - dirname($hit->path) - ); - } - /** * get the base type of a mimetype string * diff --git a/result/content.php b/result/content.php new file mode 100644 index 000000000..03acaf3aa --- /dev/null +++ b/result/content.php @@ -0,0 +1,58 @@ +. + * + */ + +namespace OCA\Search_Lucene\Result; + +/** + * A found file + */ +class Content extends \OC\Search\Result\File { + + /** + * Type name; translated in templates + * @var string + */ + public $type = 'content'; + + /** + * @var float + */ + public $score; + + /** + * Create a new content search result + * @param array $data file data given by provider + */ + public function __construct(\Zend_Search_Lucene_Search_QueryHit $hit) { + $this->id = (string)$hit->fileid; + $this->path = $hit->path; + $this->name = basename($hit->path); + $this->size = (int)$hit->size; + $this->score = $hit->score; + $this->link = \OCP\Util::linkTo( + 'files', + 'index.php', + array('dir' => dirname($hit->path), 'file' => basename($hit->path)) + ); + $this->permissions = self::get_permissions($hit->path); + $this->modified = (int)$hit->mtime; + $this->mime_type = $hit->mimetype; + } + +} diff --git a/tests/unit/testcase.php b/tests/unit/testcase.php index cd8c7085e..937d05fc7 100644 --- a/tests/unit/testcase.php +++ b/tests/unit/testcase.php @@ -63,7 +63,7 @@ public function setUp() { \OC_Util::setupFS($this->userName); \OC_User::setUserId($this->userName); - $view = new \OC_FilesystemView('/' . $this->userName . '/files'); + $view = new \OC\Files\View('/' . $this->userName . '/files'); // setup files $filesToCopy = array( @@ -121,9 +121,10 @@ public function tearDown() { } $cache = $this->storage->getCache(); $ids = $cache->getAll(); - $permissionsCache = $this->storage->getPermissionsCache(); - $permissionsCache->removeMultiple($ids, \OC_User::getUser()); $cache->clear(); + foreach ($ids as $id) { + \OCA\Search_Lucene\Status::delete($id); + } } protected function getFileId($path) { diff --git a/tests/unit/testsearchprovider.php b/tests/unit/testsearchprovider.php index da3d338cd..a8e2600f5 100644 --- a/tests/unit/testsearchprovider.php +++ b/tests/unit/testsearchprovider.php @@ -24,182 +24,55 @@ namespace OCA\Search_Lucene\Tests\Unit; class DummyIndex implements \Zend_Search_Lucene_Interface { + var $document = array(); public function addDocument(\Zend_Search_Lucene_Document $document) { - + $this->document[$document->id] = $document; } - - public function addReference() { - - } - - public function closeTermsStream() { - - } - - public function commit() { - - } - - public function count() { - - } - - public function currentTerm() { - - } - - public function delete($id) { - - } - - public function docFreq(\Zend_Search_Lucene_Index_Term $term) { - - } - - public function find($query) { - - } - - public function getDirectory() { - - } - public function getDocument($id) { - - } - - public function getFieldNames($indexed = false) { - - } - - public function getFormatVersion() { - - } - - public function getMaxBufferedDocs() { - - } - - public function getMaxMergeDocs() { - - } - - public function getMergeFactor() { - - } - - public function getSimilarity() { - - } - - public function hasDeletions() { - - } - - public function hasTerm(\Zend_Search_Lucene_Index_Term $term) { - - } - - public function isDeleted($id) { - - } - - public function maxDoc() { - - } - - public function nextTerm() { - - } - - public function norm($id, $fieldName) { - - } - - public function numDocs() { - - } - - public function optimize() { - - } - - public function removeReference() { - - } - - public function resetTermsStream() { - - } - - public function setFormatVersion($formatVersion) { - - } - - public function setMaxBufferedDocs($maxBufferedDocs) { - - } - - public function setMaxMergeDocs($maxMergeDocs) { - - } - - public function setMergeFactor($mergeFactor) { - - } - - public function skipTo(\Zend_Search_Lucene_Index_Term $prefix) { - - } - - public function termDocs(\Zend_Search_Lucene_Index_Term $term, $docsFilter = null) { - - } - - public function termDocsFilter(\Zend_Search_Lucene_Index_Term $term, $docsFilter = null) { - - } - - public function termFreqs(\Zend_Search_Lucene_Index_Term $term, $docsFilter = null) { - - } - - public function termPositions(\Zend_Search_Lucene_Index_Term $term, $docsFilter = null) { - - } - - public function terms() { - - } - - public function undeleteAll() { - - } - - public static function getActualGeneration(\Zend_Search_Lucene_Storage_Directory $directory) { - - } - - public static function getDefaultSearchField() { - - } - - public static function getResultSetLimit() { - - } - - public static function getSegmentFileName($generation) { - - } - - public static function setDefaultSearchField($fieldName) { - - } - - public static function setResultSetLimit($limit) { - - } - + return $this->document[$id]; + } + public function addReference() {} + public function closeTermsStream() {} + public function commit() {} + public function count() {} + public function currentTerm() {} + public function delete($id) {} + public function docFreq(\Zend_Search_Lucene_Index_Term $term) {} + public function find($query) {} + public function getDirectory() {} + public function getFieldNames($indexed = false) {} + public function getFormatVersion() {} + public function getMaxBufferedDocs() {} + public function getMaxMergeDocs() {} + public function getMergeFactor() {} + public function getSimilarity() {} + public function hasDeletions() {} + public function hasTerm(\Zend_Search_Lucene_Index_Term $term) {} + public function isDeleted($id) {} + public function maxDoc() {} + public function nextTerm() {} + public function norm($id, $fieldName) {} + public function numDocs() {} + public function optimize() {} + public function removeReference() {} + public function resetTermsStream() {} + public function setFormatVersion($formatVersion) {} + public function setMaxBufferedDocs($maxBufferedDocs) {} + public function setMaxMergeDocs($maxMergeDocs) {} + public function setMergeFactor($mergeFactor) {} + public function skipTo(\Zend_Search_Lucene_Index_Term $prefix) {} + public function termDocs(\Zend_Search_Lucene_Index_Term $term, $docsFilter = null) {} + public function termDocsFilter(\Zend_Search_Lucene_Index_Term $term, $docsFilter = null) {} + public function termFreqs(\Zend_Search_Lucene_Index_Term $term, $docsFilter = null) {} + public function termPositions(\Zend_Search_Lucene_Index_Term $term, $docsFilter = null) {} + public function terms() {} + public function undeleteAll() {} + public static function getActualGeneration(\Zend_Search_Lucene_Storage_Directory $directory) {} + public static function getDefaultSearchField() {} + public static function getResultSetLimit() {} + public static function getSegmentFileName($generation) {} + public static function setDefaultSearchField($fieldName) {} + public static function setResultSetLimit($limit) {} } class TestSearchProvider extends TestCase { @@ -207,52 +80,78 @@ class TestSearchProvider extends TestCase { /** * @dataProvider searchResultDataProvider */ - function testAsOCSearchResult(\Zend_Search_Lucene_Search_QueryHit $hit, $name, $text, $link, $type, $container) { + function testSearchLuceneResultContent(\Zend_Search_Lucene_Search_QueryHit $hit, $fileId, $name, $path, $size, $score, $mimeType, $modified, $container) { - $searchResult = \OCA\Search_Lucene\SearchProvider::asOCSearchResult($hit); + $searchResult = new \OCA\Search_Lucene\Result\Content($hit); - $this->assertInstanceOf('OC_Search_Result', $searchResult); + $this->assertInstanceOf('OCA\Search_Lucene\Result\Content', $searchResult); + $this->assertEquals($searchResult->id, $fileId); + $this->assertEquals($searchResult->type, 'content'); + $this->assertEquals($searchResult->path, $path); $this->assertEquals($searchResult->name, $name); - $this->assertEquals($searchResult->text, $text); - //$this->assertEquals($searchResult->link, $link); - $this->assertEquals($searchResult->type, $type); + $this->assertEquals($searchResult->mime_type, $mimeType); + $this->assertEquals($searchResult->size, $size); + $this->assertEquals($searchResult->score, $score); + $this->assertEquals($searchResult->modified, $modified); $this->assertEquals($searchResult->container, $container); } - + public function searchResultDataProvider() { - + $index = new DummyIndex(); + + $doc1 = new \Zend_Search_Lucene_Document(); + $doc1->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 1)); + $doc1->addField(\Zend_Search_Lucene_Field::Text('path', 'documents/document.txt', 'UTF-8')); + $doc1->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', 1234567)); + $doc1->addField(\Zend_Search_Lucene_Field::unIndexed('size', 123)); + $doc1->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', 'text/plain')); + $index->addDocument($doc1); + + $doc2 = new \Zend_Search_Lucene_Document(); + $doc2->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 2)); + $doc2->addField(\Zend_Search_Lucene_Field::Text('path', 'documents/document.pdf', 'UTF-8')); + $doc2->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', 1234567)); + $doc2->addField(\Zend_Search_Lucene_Field::unIndexed('size', 1234)); + $doc2->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', 'application/pdf')); + $index->addDocument($doc2); + + $doc3 = new \Zend_Search_Lucene_Document(); + $doc3->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 3)); + $doc3->addField(\Zend_Search_Lucene_Field::Text('path', 'documents/document.mp3', 'UTF-8')); + $doc3->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', 1234567)); + $doc3->addField(\Zend_Search_Lucene_Field::unIndexed('size', 12341234)); + $doc3->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', 'audio/mp3')); + $index->addDocument($doc3); + + $doc4 = new \Zend_Search_Lucene_Document(); + $doc4->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 4)); + $doc4->addField(\Zend_Search_Lucene_Field::Text('path', 'documents/document.jpg', 'UTF-8')); + $doc4->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', 1234567)); + $doc4->addField(\Zend_Search_Lucene_Field::unIndexed('size', 1234123)); + $doc4->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', 'image/jpg')); + $index->addDocument($doc4); + + + $hit1 = new \Zend_Search_Lucene_Search_QueryHit($index); - $hit1->mimetype = 'text/plain'; - $hit1->path = 'documents/document.txt'; - $hit1->size = 123; $hit1->score = 0.4; - + $hit2 = new \Zend_Search_Lucene_Search_QueryHit($index); - $hit2->mimetype = 'application/pdf'; - $hit2->path = 'documents/document.pdf'; - $hit2->size = 1234; $hit2->score = 0.31; - + $hit3 = new \Zend_Search_Lucene_Search_QueryHit($index); - $hit3->mimetype = 'audio/mp3'; - $hit3->path = 'documents/document.mp3'; - $hit3->size = 12341234; $hit3->score = 0.299; - + $hit4 = new \Zend_Search_Lucene_Search_QueryHit($index); - $hit4->mimetype = 'image/jpg'; - $hit4->path = 'documents/document.jpg'; - $hit4->size = 1234123; $hit4->score = 0.001; - + return array( - // name, text, link, type, container - //FIXME search result should not contain translated strings - array($hit1,'document.txt', 'documents, 123 B, Score: 0.40', 'FIXME', 'Text', 'documents'), - array($hit2,'document.pdf', 'documents, 1 kB, Score: 0.31', 'FIXME', 'Files', 'documents'), - array($hit3,'document.mp3', 'documents, 11.8 MB, Score: 0.30', 'FIXME', 'Music', 'documents'), - array($hit4,'document.jpg', 'documents, 1.2 MB, Score: 0.00', 'FIXME', 'Images', 'documents'), + // hit, name, size, score, mime_type, container + array($hit1, '1', 'document.txt', 'documents/document.txt', 123, 0.4, 'text/plain', 1234567, 'documents'), + array($hit2, '2', 'document.pdf', 'documents/document.pdf', 1234, 0.31, 'application/pdf', 1234567, 'documents'), + array($hit3, '3', 'document.mp3', 'documents/document.mp3', 12341234, 0.299, 'audio/mp3', 1234567, 'documents'), + array($hit4, '4', 'document.jpg', 'documents/document.jpg', 1234123, 0.001, 'image/jpg', 1234567, 'documents'), ); } } From bf5a91ca714b3c890efefeba182e237ebdb7f938 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 3 Jul 2014 19:15:54 +0200 Subject: [PATCH 15/51] update comment, fix whitespace --- lib/indexer.php | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/indexer.php b/lib/indexer.php index a31f84cee..2d921024d 100644 --- a/lib/indexer.php +++ b/lib/indexer.php @@ -150,9 +150,7 @@ public function indexFile($path = '') { $doc = Pdf::loadPdf($this->view->file_get_contents($path)); - // commented the mimetype checks, as the zend classes only understand docx and not doc files. - // FIXME distinguish doc and docx, xls and xlsx, ppt and pptx, in oc core mimetype helper ... - //} else if ('application/msword' === $mimeType) { + // the zend classes only understand docx and not doc files } else if (strtolower(substr($data['name'], -5)) === '.docx') { $doc = \Zend_Search_Lucene_Document_Docx::loadDocxFile($localFile); @@ -178,7 +176,6 @@ public function indexFile($path = '') { } } - // Store filecache id as unique id to lookup by when deleting $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $data['fileid'])); @@ -191,7 +188,6 @@ public function indexFile($path = '') { $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', $mimeType)); - $this->lucene->updateFile($doc, $data['fileid']); return true; From 9511aea65e77c4c74b76dce6bf624bd6260c497e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 4 Jul 2014 17:01:43 +0200 Subject: [PATCH 16/51] move to \OC\Files\FileInfo, fix phpdoc, use pathinfo to get extension, log number of deleted files with debug level --- lib/hooks.php | 20 ++++++++++++++------ lib/indexer.php | 31 ++++++++++++++++++------------- result/content.php | 2 +- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/lib/hooks.php b/lib/hooks.php index bc0c4fa02..1336cb664 100644 --- a/lib/hooks.php +++ b/lib/hooks.php @@ -78,9 +78,12 @@ public static function renameFile(array $param) { } if (!empty($param['newpath'])) { $view = new \OC\Files\View('/' . $user . '/files'); + /** @var \OC\Files\FileInfo $info */ $info = $view->getFileInfo($param['newpath']); - Status::fromFileId($info['fileid'])->markNew(); - self::indexFile(array('path'=>$param['newpath'])); + if ($info) { + Status::fromFileId($info->getId())->markNew(); + self::indexFile(array('path'=>$param['newpath'])); + } } } @@ -98,18 +101,23 @@ static public function deleteFile(array $param) { $lucene = new Lucene($user); $deletedIds = Status::getDeleted(); $count = 0; - foreach ($deletedIds as $fileid) { + foreach ($deletedIds as $fileId) { Util::writeLog( 'search_lucene', - 'deleting status for ('.$fileid.') ', + 'deleting status for ('.$fileId.') ', Util::DEBUG ); //delete status - \OCA\Search_Lucene\Status::delete($fileid); + Status::delete($fileId); //delete from lucene - $count += $lucene->deleteFile($fileid); + $count += $lucene->deleteFile($fileId); } + Util::writeLog( + 'search_lucene', + 'removed '.$count.' files from index', + Util::DEBUG + ); } diff --git a/lib/indexer.php b/lib/indexer.php index 2d921024d..5fcfb88bf 100644 --- a/lib/indexer.php +++ b/lib/indexer.php @@ -118,9 +118,14 @@ public function indexFile($path = '') { } // the cache already knows mime and other basic stuff + /** @var \OC\Files\FileInfo $data */ $data = $this->view->getFileInfo($path); - if (isset($data['mimetype'])) { - $mimeType = $data['mimetype']; + + if (isset($data)) { + + // we decide how to index on mime type or file extension + $mimeType = $data->getMimetype(); + $fileExtension = strtolower(pathinfo($data->getName(), PATHINFO_EXTENSION)); // initialize plain lucene document $doc = new \Zend_Search_Lucene_Document(); @@ -131,7 +136,7 @@ public function indexFile($path = '') { if ( $localFile ) { //try to use special lucene document types - if ('text/plain' === $mimeType) { + if ('text/plain' === $data->getMimetype()) { $body = $this->view->file_get_contents($path); @@ -151,25 +156,25 @@ public function indexFile($path = '') { $doc = Pdf::loadPdf($this->view->file_get_contents($path)); // the zend classes only understand docx and not doc files - } else if (strtolower(substr($data['name'], -5)) === '.docx') { + } else if ($fileExtension === 'docx') { $doc = \Zend_Search_Lucene_Document_Docx::loadDocxFile($localFile); //} else if ('application/msexcel' === $mimeType) { - } else if (strtolower(substr($data['name'], -5)) === '.xlsx') { + } else if ($fileExtension === 'xlsx') { $doc = \Zend_Search_Lucene_Document_Xlsx::loadXlsxFile($localFile); //} else if ('application/mspowerpoint' === $mimeType) { - } else if (strtolower(substr($data['name'], -5)) === '.pptx') { + } else if ($fileExtension === 'pptx') { $doc = \Zend_Search_Lucene_Document_Pptx::loadPptxFile($localFile); - } else if (strtolower(substr($data['name'], -4)) === '.odt') { + } else if ($fileExtension === 'odt') { $doc = Odt::loadOdtFile($localFile); - } else if (strtolower(substr($data['name'], -4)) === '.ods') { + } else if ($fileExtension === 'ods') { $doc = Ods::loadOdsFile($localFile); @@ -177,25 +182,25 @@ public function indexFile($path = '') { } // Store filecache id as unique id to lookup by when deleting - $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $data['fileid'])); + $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $data->getId())); // Store document path for the search results $doc->addField(\Zend_Search_Lucene_Field::Text('path', $path, 'UTF-8')); - $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', $data['mtime'])); + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', $data->getMTime())); - $doc->addField(\Zend_Search_Lucene_Field::unIndexed('size', $data['size'])); + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('size', $data->getSize())); $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', $mimeType)); - $this->lucene->updateFile($doc, $data['fileid']); + $this->lucene->updateFile($doc, $data->getId()); return true; } else { Util::writeLog( 'search_lucene', - 'need mimetype for content extraction', + 'need file info object for content extraction', Util::ERROR ); return false; diff --git a/result/content.php b/result/content.php index 03acaf3aa..e2beb1a1b 100644 --- a/result/content.php +++ b/result/content.php @@ -37,7 +37,7 @@ class Content extends \OC\Search\Result\File { /** * Create a new content search result - * @param array $data file data given by provider + * @param \Zend_Search_Lucene_Search_QueryHit $hit file data given by provider */ public function __construct(\Zend_Search_Lucene_Search_QueryHit $hit) { $this->id = (string)$hit->fileid; From 03a89da2206c6a890e7b9a612fe74c49bfb73022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Sun, 6 Jul 2014 20:24:19 +0200 Subject: [PATCH 17/51] move to \OCP\Files\Folder interface, fix reindexing of files, whitespace --- ajax/lucene.php | 15 ++-- lib/hooks.php | 31 ++++---- lib/indexer.php | 164 +++++++++++++++++++++-------------------- lib/indexjob.php | 39 ++++++---- lib/lucene.php | 91 +++++++++++++++-------- lib/optimizejob.php | 5 +- lib/searchprovider.php | 6 +- lib/status.php | 25 +++++-- lib/util.php | 48 ++++++++++++ 9 files changed, 263 insertions(+), 161 deletions(-) create mode 100644 lib/util.php diff --git a/ajax/lucene.php b/ajax/lucene.php index 1adbe3b4c..95417daa8 100644 --- a/ajax/lucene.php +++ b/ajax/lucene.php @@ -6,11 +6,7 @@ session_write_close(); function index() { - $user = OCP\User::getUser(); - - OC_Util::tearDownFS(); - OC_Util::setupFS($user); - + if ( isset($_GET['fileid']) ){ $fileIds = array($_GET['fileid']); } else { @@ -20,11 +16,10 @@ function index() { $eventSource = new OC_EventSource(); $eventSource->send('count', count($fileIds)); - - $view = new OC\Files\View('/' . $user . '/files'); - $lucene = new OCA\Search_Lucene\Lucene($user); - - $indexer = new OCA\Search_Lucene\Indexer($view, $lucene); + $folder = OCA\Search_Lucene\Util::setUpUserFolder(); + $lucene = new OCA\Search_Lucene\Lucene(); + + $indexer = new OCA\Search_Lucene\Indexer($folder, $lucene); $indexer->indexFiles($fileIds, $eventSource); $eventSource->send('done', ''); diff --git a/lib/hooks.php b/lib/hooks.php index 1336cb664..499e50ec3 100644 --- a/lib/hooks.php +++ b/lib/hooks.php @@ -48,9 +48,15 @@ class Hooks { * @param $param array from postWriteFile-Hook */ public static function indexFile(array $param) { + //FIXME use fileid to index correct file, otherwise there will never be an update $user = \OCP\User::getUser(); if (!empty($user)) { - $arguments = array('user' => $user); + $userFolder = \OC::$server->getUserFolder(); + $folder = $userFolder->get($param['path']); + $arguments = array( + 'user' => $user, + 'fileId' => $folder->getId() + ); //Add Background Job: BackgroundJob::registerJob( '\OCA\Search_Lucene\IndexJob', $arguments ); } else { @@ -70,20 +76,16 @@ public static function indexFile(array $param) { * @param $param array from postRenameFile-Hook */ public static function renameFile(array $param) { - $user = \OCP\User::getUser(); if (!empty($param['oldpath'])) { //delete from lucene index - $lucene = new Lucene($user); + $lucene = new Lucene(); $lucene->deleteFile($param['oldpath']); } if (!empty($param['newpath'])) { - $view = new \OC\Files\View('/' . $user . '/files'); - /** @var \OC\Files\FileInfo $info */ - $info = $view->getFileInfo($param['newpath']); - if ($info) { - Status::fromFileId($info->getId())->markNew(); - self::indexFile(array('path'=>$param['newpath'])); - } + $userFolder = \OC::$server->getUserFolder(); + $folder = $userFolder->get($param['newpath']); + Status::fromFileId($folder->getId())->markNew(); + self::indexFile(array('path'=>$param['newpath'])); } } @@ -97,8 +99,7 @@ public static function renameFile(array $param) { static public function deleteFile(array $param) { // we cannot use post_delete as $param would not contain the id // of the deleted file and we could not fetch it with getId - $user = \OCP\User::getUser(); - $lucene = new Lucene($user); + $lucene = new Lucene(); $deletedIds = Status::getDeleted(); $count = 0; foreach ($deletedIds as $fileId) { @@ -111,7 +112,7 @@ static public function deleteFile(array $param) { Status::delete($fileId); //delete from lucene $count += $lucene->deleteFile($fileId); - + } Util::writeLog( 'search_lucene', @@ -120,7 +121,7 @@ static public function deleteFile(array $param) { ); } - + /** * was used by backgroundjobs to index individual files * @@ -131,5 +132,5 @@ static public function deleteFile(array $param) { * @param $param array from deleteFile-Hook */ static public function doIndexFile(array $param) {/* ignore */} - + } diff --git a/lib/indexer.php b/lib/indexer.php index 5fcfb88bf..93fd6b9cb 100644 --- a/lib/indexer.php +++ b/lib/indexer.php @@ -2,11 +2,10 @@ namespace OCA\Search_Lucene; -use \OC\Files\Filesystem; use OCA\Search_Lucene\Document\Ods; use OCA\Search_Lucene\Document\Odt; use OCA\Search_Lucene\Document\Pdf; -use \OCP\Util; +use OCP\Util; /** * @author Jörn Dreyer @@ -18,19 +17,25 @@ class Indexer { * used as signalclass in OC_Hooks::emit() */ const CLASSNAME = 'Indexer'; - - private $view; + + /** + * @var \OCP\Files\Folder + */ + private $folder; private $lucene; - public function __construct(\OC\Files\View $view, Lucene $lucene) { - $this->view = $view; - $this->lucene = $lucene; + public function __construct(\OCP\Files\Folder $folder, Lucene $lucene) { + $this->folder = $folder; + $this->lucene = $lucene; } - + public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) { - - $skippedDirs = explode(';', \OCP\Config::getUserValue(\OCP\User::getUser(), 'search_lucene', 'skipped_dirs', '.git;.svn;.CVS;.bzr')); - + + $skippedDirs = explode( + ';', + \OCP\Config::getUserValue(\OCP\User::getUser(), 'search_lucene', 'skipped_dirs', '.git;.svn;.CVS;.bzr') + ); + foreach ($fileIds as $id) { $skip = false; @@ -42,7 +47,17 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) // the file again $fileStatus->markError(); - $path = \OC\Files\Filesystem::getPath($id); + // FIXME use Folder + /** @var \OCP\Files\Node $folder */ + $folder = \OC::$server->getUserFolder()->getById($id); + // TODO why does getById return an array?!?! + if (empty($folder)) { + $path = null; + } else { + $folder = $folder[0]; + //$path = \OC\Files\Filesystem::getPath($id); + $path = $folder->getPath(); + } if (empty($path)) { $skip = true; @@ -56,7 +71,7 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) } } } - + if ($skip) { $fileStatus->markSkipped(); \OCP\Util::writeLog('search_lucene', @@ -67,7 +82,7 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) if ($eventSource) { $eventSource->send('indexing', $path); } - + if ($this->indexFile($path)) { $fileStatus->markIndexed(); } else { @@ -98,110 +113,103 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) * * @param string $path the path of the file * - * @return bool + * @return bool true when something was stored in the index, false otherwise (eg, folders are not indexed) + * @throws \Exception indicating an error */ public function indexFile($path = '') { - if (!Filesystem::isValidPath($path)) { - return; - } - if (empty($path)) { - //ignore the empty path element - return false; - } - - if(!$this->view->file_exists($path)) { - Util::writeLog('search_lucene', - 'file '.$path.' vanished, ignoring', - Util::DEBUG); - return true; - } + try { - // the cache already knows mime and other basic stuff - /** @var \OC\Files\FileInfo $data */ - $data = $this->view->getFileInfo($path); + // the cache already knows mime and other basic stuff + /** @var \OCP\Files\Node $data */ + $node = $this->folder->get($path); - if (isset($data)) { + if ($node instanceof \OCP\Files\File) { - // we decide how to index on mime type or file extension - $mimeType = $data->getMimetype(); - $fileExtension = strtolower(pathinfo($data->getName(), PATHINFO_EXTENSION)); + // we decide how to index on mime type or file extension + $mimeType = $node->getMimetype(); + $fileExtension = strtolower(pathinfo($node->getName(), PATHINFO_EXTENSION)); - // initialize plain lucene document - $doc = new \Zend_Search_Lucene_Document(); + // initialize plain lucene document + $doc = new \Zend_Search_Lucene_Document(); - // index content for local files only - $localFile = $this->view->getLocalFile($path); + // index content for local files only + $storage = $node->getStorage(); - if ( $localFile ) { - //try to use special lucene document types + $internalPath = $node->getInternalPath(); + $path = $node->getPath(); - if ('text/plain' === $data->getMimetype()) { + if ($storage->isLocal()) { - $body = $this->view->file_get_contents($path); + //try to use special lucene document types - if ($body != '') { - $doc->addField(\Zend_Search_Lucene_Field::UnStored('body', $body)); - } + if ('text/plain' === $mimeType) { - // FIXME other text files? c, php, java ... + $body = $node->getContent(); - } else if ('text/html' === $mimeType) { + if ($body != '') { + $doc->addField(\Zend_Search_Lucene_Field::UnStored('body', $body)); + } - //TODO could be indexed, even if not local - $doc = \Zend_Search_Lucene_Document_Html::loadHTML($this->view->file_get_contents($path)); + // FIXME other text files? c, php, java ... - } else if ('application/pdf' === $mimeType) { + } else if ('text/html' === $mimeType) { - $doc = Pdf::loadPdf($this->view->file_get_contents($path)); + //TODO could be indexed, even if not local + $doc = \Zend_Search_Lucene_Document_Html::loadHTML($node->getContent()); - // the zend classes only understand docx and not doc files - } else if ($fileExtension === 'docx') { + } else if ('application/pdf' === $mimeType) { - $doc = \Zend_Search_Lucene_Document_Docx::loadDocxFile($localFile); + $doc = Pdf::loadPdf($node->getContent()); - //} else if ('application/msexcel' === $mimeType) { - } else if ($fileExtension === 'xlsx') { + // the zend classes only understand docx and not doc files + } else if ($fileExtension === 'docx') { - $doc = \Zend_Search_Lucene_Document_Xlsx::loadXlsxFile($localFile); + $doc = \Zend_Search_Lucene_Document_Docx::loadDocxFile($path); - //} else if ('application/mspowerpoint' === $mimeType) { - } else if ($fileExtension === 'pptx') { + //} else if ('application/msexcel' === $mimeType) { + } else if ($fileExtension === 'xlsx') { - $doc = \Zend_Search_Lucene_Document_Pptx::loadPptxFile($localFile); + $doc = \Zend_Search_Lucene_Document_Xlsx::loadXlsxFile($path); - } else if ($fileExtension === 'odt') { + //} else if ('application/mspowerpoint' === $mimeType) { + } else if ($fileExtension === 'pptx') { - $doc = Odt::loadOdtFile($localFile); + $doc = \Zend_Search_Lucene_Document_Pptx::loadPptxFile($path); - } else if ($fileExtension === 'ods') { + } else if ($fileExtension === 'odt') { - $doc = Ods::loadOdsFile($localFile); + $doc = Odt::loadOdtFile($path); + } else if ($fileExtension === 'ods') { + + $doc = Ods::loadOdsFile($path); + + } } - } - // Store filecache id as unique id to lookup by when deleting - $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $data->getId())); + // Store filecache id as unique id to lookup by when deleting + $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $node->getId())); - // Store document path for the search results - $doc->addField(\Zend_Search_Lucene_Field::Text('path', $path, 'UTF-8')); + // Store document path for the search results + $doc->addField(\Zend_Search_Lucene_Field::Text('path', $path, 'UTF-8')); - $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', $data->getMTime())); + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', $node->getMTime())); - $doc->addField(\Zend_Search_Lucene_Field::unIndexed('size', $data->getSize())); + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('size', $node->getSize())); - $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', $mimeType)); + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', $mimeType)); - $this->lucene->updateFile($doc, $data->getId()); + $this->lucene->updateFile($doc, $data->getId()); + } return true; - } else { + } catch (\Exception $ex) { Util::writeLog( 'search_lucene', - 'need file info object for content extraction', - Util::ERROR + $ex->getCode().':'.$ex->getMessage(), + Util::DEBUG ); return false; } diff --git a/lib/indexjob.php b/lib/indexjob.php index 3ebe95262..227ff2e13 100644 --- a/lib/indexjob.php +++ b/lib/indexjob.php @@ -3,23 +3,32 @@ namespace OCA\Search_Lucene; class IndexJob extends \OC\BackgroundJob\Job { + + public function run($arguments){ - if (!empty($arguments['user'])) { + if (isset($arguments['user'])) { $user = $arguments['user']; - \OC_Util::tearDownFS(); - \OC_Util::setupFS($user); - $fileIds = Status::getUnindexed(); - \OCP\Util::writeLog( - 'search_lucene', - 'background job indexing '.count($fileIds).' files for '.$user, - \OCP\Util::DEBUG - ); - $view = new \OC\Files\View('/' . $user . '/files'); - $lucene = new \OCA\Search_Lucene\Lucene($user); - - $indexer = new Indexer($view, $lucene); - $indexer->indexFiles($fileIds); + $folder = Util::setUpUserFolder($user); + if ($folder) { + $fileIds = Status::getUnindexed(); + + // we might have to update an already indexed file + if (isset($arguments['fileId']) && ! in_array($arguments['fileId'], $fileIds)) { + $fileIds[] = $arguments['fileId']; + } + + \OCP\Util::writeLog( + 'search_lucene', + 'background job indexing '.count($fileIds).' files for '.$user, + \OCP\Util::DEBUG + ); + + $lucene = new Lucene(); + $indexer = new Indexer($folder, $lucene); + + $indexer->indexFiles($fileIds); + } } else { \OCP\Util::writeLog( 'search_lucene', @@ -27,5 +36,5 @@ public function run($arguments){ \OCP\Util::DEBUG ); } - } + } } diff --git a/lib/lucene.php b/lib/lucene.php index 547ac22a0..cca806416 100644 --- a/lib/lucene.php +++ b/lib/lucene.php @@ -2,8 +2,6 @@ namespace OCA\Search_Lucene; -use \OC\Files\Filesystem; -use \OCP\User; use \OCP\Util; /** @@ -17,76 +15,109 @@ class Lucene { */ const CLASSNAME = 'Lucene'; - public $user; public $index; - public function __construct($user) { - $this->user = $user; - $this->index = self::openOrCreate(); + /** + * The default location of '/lucene_index' can be overridden by passing in a different folder + * + * @param \OCP\Files\Folder $indexFolder location of the lucene_index + */ + public function __construct(\OCP\Files\Folder $indexFolder = null) { + if (is_null($indexFolder)) { + $indexFolder = $this->getIndexFolder(); + } + $this->index = $this->openOrCreate($indexFolder); } - private function getIndexURL () { + /** + * @return null|\OCP\Files\Folder + */ + private function getIndexFolder() { + // TODO profile: encrypt the index on logout, decrypt on login //return OCP\Files::getStorage('search_lucene'); - return \OC_User::getHome($this->user) . '/lucene_index'; + // FIXME \OC::$server->getAppFolder() returns '/search' + //$indexFolder = \OC::$server->getAppFolder(); + + $root = \OC::$server->getRootFolder(); + $dir = '/'.\OCP\User::getUser(); + $userFolder = null; + if(!$root->nodeExists($dir)) { + $userFolder = $root->newFolder($dir); + } else { + $userFolder = $root->get($dir); + } + $dir = 'lucene_index'; + $indexFolder = null; + if(!$userFolder->nodeExists($dir)) { + $indexFolder = $userFolder->newFolder($dir); + } else { + $indexFolder = $userFolder->get($dir); + } + + return $indexFolder; } /** - * opens or creates the users lucene index + * opens or creates the given lucene index * - * stores the index in //lucene_index + * stores the index in $indexFolder * * @author Jörn Dreyer * - * @return Zend_Search_Lucene_Interface + * @param \OCP\Files\Folder $indexFolder + * @return \Zend_Search_Lucene_Interface + * @throws \Exception */ - private function openOrCreate() { + private function openOrCreate(\OCP\Files\Folder $indexFolder) { + + if (is_null($indexFolder)) { + throw new \Exception('No Index folder given'); + } try { - + + $storage = $indexFolder->getStorage(); + $localPath = $storage->getLocalFile($indexFolder->getInternalPath()); + //let lucene search for numbers as well as words \Zend_Search_Lucene_Analysis_Analyzer::setDefault( new \Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive() ); - - // Create index - - $indexUrl = $this->getIndexURL(); // can we use the index? - if (file_exists($indexUrl.'/v0.6.0')) { + if ($indexFolder->nodeExists('/v0.6.0')) { // correct index present - $index = \Zend_Search_Lucene::open($indexUrl); - } else if (file_exists($indexUrl)) { + $index = \Zend_Search_Lucene::open($localPath); + } else if (file_exists($indexFolder)) { Util::writeLog( 'search_lucene', 'recreating outdated lucene index', Util::INFO ); - \OC_Helper::rmdirr($indexUrl); - $index = \Zend_Search_Lucene::create($indexUrl); - touch($indexUrl.'/v0.6.0'); + $indexFolder->delete(); + $index = \Zend_Search_Lucene::create($localPath); + touch($indexFolder.'/v0.6.0'); } else { - $index = \Zend_Search_Lucene::create($indexUrl); - touch($indexUrl.'/v0.6.0'); + $index = \Zend_Search_Lucene::create($localPath); + touch($indexFolder.'/v0.6.0'); } + return $index; } catch ( \Exception $e ) { Util::writeLog( 'search_lucene', $e->getMessage().' Trace:\n'.$e->getTraceAsString(), Util::ERROR ); - return null; } - - return $index; + return null; + } /** * optimizes the lucene index - * - * + * * @author Jörn Dreyer * * @return void diff --git a/lib/optimizejob.php b/lib/optimizejob.php index 602880340..123b41b7a 100644 --- a/lib/optimizejob.php +++ b/lib/optimizejob.php @@ -3,12 +3,11 @@ namespace OCA\Search_Lucene; class OptimizeJob extends \OC\BackgroundJob\TimedJob { - - + public function __construct() { $this->setInterval(86400); //execute at most once a day } - + public function run($arguments){ if (!empty($arguments['user'])) { $user = $arguments['user']; diff --git a/lib/searchprovider.php b/lib/searchprovider.php index ef7cc6d18..e33447563 100644 --- a/lib/searchprovider.php +++ b/lib/searchprovider.php @@ -9,8 +9,6 @@ namespace OCA\Search_Lucene; -use \OC\Files\Filesystem; -use \OCP\User; use \OCP\Util; /** @@ -39,7 +37,7 @@ public function search($query){ // TODO add end user guide for search terms ... //} try { - $lucene = new Lucene(\OCP\User::getUser()); + $lucene = new Lucene(); //default is 3, 0 needed to keep current search behaviour //Zend_Search_Lucene_Search_Query_Wildcard::setMinPrefixLength(0); @@ -50,7 +48,7 @@ public function search($query){ //limit results. we cant show more than ~30 anyway. TODO use paging later for ($i = 0; $i < 30 && $i < count($hits); $i++) { - $results[] = new \OCA\Search_Lucene\Result\Content($hits[$i]); + $results[] = new Result\Content($hits[$i]); } } catch ( \Exception $e ) { diff --git a/lib/status.php b/lib/status.php index 3ac12f036..62e7795a5 100644 --- a/lib/status.php +++ b/lib/status.php @@ -14,7 +14,7 @@ class Status { const STATUS_INDEXED = 'I'; const STATUS_SKIPPED = 'S'; const STATUS_ERROR = 'E'; - + private $fileId; private $status; @@ -22,6 +22,7 @@ public function __construct($fileId, $status = null) { $this->fileId = $fileId; $this->status = $status; } + public static function fromFileId($fileId) { $status = self::get($fileId); if ($status) { @@ -30,29 +31,36 @@ public static function fromFileId($fileId) { return new Status($fileId, null); } } + public function getFileId() { return $this->fileId; } + public function getStatus() { return $this->status; } + // always write status to db immediately public function markNew() { $this->status = self::STATUS_NEW; return $this->store(); } + public function markIndexed() { $this->status = self::STATUS_INDEXED; return $this->store(); } + public function markSkipped() { $this->status = self::STATUS_SKIPPED; return $this->store(); } + public function markError() { $this->status = self::STATUS_ERROR; return $this->store(); } + private function store() { $savedStatus = self::get($this->fileId); if ($savedStatus) { @@ -61,6 +69,7 @@ private function store() { return self::insert($this->fileId, $this->status); } } + public static function delete($fileId) { $query = \OC_DB::prepare(' DELETE FROM `*PREFIX*lucene_status` WHERE `fileid` = ? @@ -82,12 +91,14 @@ private static function get($fileId) { return null; } } + private static function insert($fileId, $status) { $query = \OC_DB::prepare(' INSERT INTO `*PREFIX*lucene_status` VALUES (?,?) '); return $query->execute(array($fileId, $status)); } + private static function update($fileId, $status) { $query = \OC_DB::prepare(' UPDATE `*PREFIX*lucene_status` @@ -96,7 +107,7 @@ private static function update($fileId, $status) { '); return $query->execute(array($status, $fileId)); } - + /** * get the list of all unindexed files for the user * @@ -155,7 +166,7 @@ static public function getUnindexed() { static public function getDeleted() { $files = array(); - + $query = \OCP\DB::prepare(' SELECT `*PREFIX*lucene_status`.`fileid` FROM `*PREFIX*lucene_status` @@ -163,13 +174,15 @@ static public function getDeleted() { ON `*PREFIX*filecache`.`fileid` = `*PREFIX*lucene_status`.`fileid` WHERE `*PREFIX*filecache`.`fileid` IS NULL '); - + $result = $query->execute(); - + while ($row = $result->fetchRow()) { $files[] = $row['fileid']; } - + return $files; + } + } diff --git a/lib/util.php b/lib/util.php new file mode 100644 index 000000000..98f8f9843 --- /dev/null +++ b/lib/util.php @@ -0,0 +1,48 @@ +getRootFolder(); + $folder = null; + + if(!$root->nodeExists($dir)) { + $folder = $root->newFolder($dir); + } else { + $folder = $root->get($dir); + } + + $dir = '/files'; + if(!$folder->nodeExists($dir)) { + $folder = $folder->newFolder($dir); + } else { + $folder = $folder->get($dir); + } + + return $folder; + + } + +} From b1b70eebf6c8b9607bc21b4d9b6ab68d1ce2e32b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 7 Jul 2014 22:36:26 +0200 Subject: [PATCH 18/51] fix case of open document classes, handle index problems with exceptions, fix reindexing of edited files --- document/{Ods.php => ods.php} | 0 document/{Odt.php => odt.php} | 0 .../{OpenDocument.php => opendocument.php} | 0 document/{Pdf.php => pdf.php} | 0 lib/hooks.php | 21 +- lib/indexer.php | 191 ++++++++---------- lib/indexjob.php | 6 +- lib/lucene.php | 66 ++---- lib/notindexedexception.php | 5 + lib/optimizejob.php | 17 +- lib/skippedexception.php | 5 + lib/status.php | 6 + lib/util.php | 47 ++++- 13 files changed, 174 insertions(+), 190 deletions(-) rename document/{Ods.php => ods.php} (100%) rename document/{Odt.php => odt.php} (100%) rename document/{OpenDocument.php => opendocument.php} (100%) rename document/{Pdf.php => pdf.php} (100%) create mode 100644 lib/notindexedexception.php create mode 100644 lib/skippedexception.php diff --git a/document/Ods.php b/document/ods.php similarity index 100% rename from document/Ods.php rename to document/ods.php diff --git a/document/Odt.php b/document/odt.php similarity index 100% rename from document/Odt.php rename to document/odt.php diff --git a/document/OpenDocument.php b/document/opendocument.php similarity index 100% rename from document/OpenDocument.php rename to document/opendocument.php diff --git a/document/Pdf.php b/document/pdf.php similarity index 100% rename from document/Pdf.php rename to document/pdf.php diff --git a/lib/hooks.php b/lib/hooks.php index 499e50ec3..fa2280de4 100644 --- a/lib/hooks.php +++ b/lib/hooks.php @@ -48,17 +48,24 @@ class Hooks { * @param $param array from postWriteFile-Hook */ public static function indexFile(array $param) { - //FIXME use fileid to index correct file, otherwise there will never be an update $user = \OCP\User::getUser(); if (!empty($user)) { + + // mark written file as new $userFolder = \OC::$server->getUserFolder(); - $folder = $userFolder->get($param['path']); - $arguments = array( - 'user' => $user, - 'fileId' => $folder->getId() - ); + $node = $userFolder->get($param['path']); + $status = Status::fromFileId($node->getId()); + + // only index files + if ($node instanceof \OCP\Files\File) { + /** @var \OCP\Files\File $node */ + $status->markNew(); + } else { + $status->markSkipped(); + } + //Add Background Job: - BackgroundJob::registerJob( '\OCA\Search_Lucene\IndexJob', $arguments ); + BackgroundJob::registerJob( '\OCA\Search_Lucene\IndexJob', array('user' => $user) ); } else { \OCP\Util::writeLog( 'search_lucene', diff --git a/lib/indexer.php b/lib/indexer.php index 93fd6b9cb..4ee466006 100644 --- a/lib/indexer.php +++ b/lib/indexer.php @@ -37,71 +37,68 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) ); foreach ($fileIds as $id) { - $skip = false; - $fileStatus = \OCA\Search_Lucene\Status::fromFileId($id); + $fileStatus = Status::fromFileId($id); - try{ + try { // before we start mark the file as error so we know there // was a problem in case the php execution dies and we don't try // the file again $fileStatus->markError(); - // FIXME use Folder /** @var \OCP\Files\Node $folder */ - $folder = \OC::$server->getUserFolder()->getById($id); - // TODO why does getById return an array?!?! - if (empty($folder)) { - $path = null; + $nodes = \OC::$server->getUserFolder()->getById($id); + // getById can return more than one id because the containing storage might be mounted more than once + // Since we only want to index the file once, we only use the first entry + + if (isset($nodes[0])) { + /** @var \OCP\Files\File $node */ + $node = $nodes[0]; } else { - $folder = $folder[0]; - //$path = \OC\Files\Filesystem::getPath($id); - $path = $folder->getPath(); + throw new \Exception('no file found for fileid '.$id); } - if (empty($path)) { - $skip = true; - } else { - foreach ($skippedDirs as $skippedDir) { - if (strpos($path, '/' . $skippedDir . '/') !== false //contains dir - || strrpos($path, '/' . $skippedDir) === strlen($path) - (strlen($skippedDir) + 1) // ends with dir - ) { - $skip = true; - break; - } - } + if ( ! $node instanceof \OCP\Files\File ) { + throw new NotIndexedException(); } - if ($skip) { - $fileStatus->markSkipped(); - \OCP\Util::writeLog('search_lucene', - 'skipping file '.$id.':'.$path, - \OCP\Util::DEBUG); - continue; + $path = $node->getPath(); + + foreach ($skippedDirs as $skippedDir) { + if (strpos($path, '/' . $skippedDir . '/') !== false //contains dir + || strrpos($path, '/' . $skippedDir) === strlen($path) - (strlen($skippedDir) + 1) // ends with dir + ) { + throw new SkippedException('skipping file '.$id.':'.$path); + } } + if ($eventSource) { $eventSource->send('indexing', $path); } - if ($this->indexFile($path)) { + if ($this->indexFile($node)) { $fileStatus->markIndexed(); - } else { - \OCP\JSON::error(array('message' => 'Could not index file '.$id.':'.$path)); - if ($eventSource) { - $eventSource->send('error', $path); - } } - } catch (\Exception $e) { //sqlite might report database locked errors when stock filescan is in progress + + } catch (NotIndexedException $e) { + + $fileStatus->markUnIndexed(); + + } catch (SkippedException $e) { + + $fileStatus->markSkipped(); + \OCP\Util::writeLog('search_lucene', $e->getMessage(), \OCP\Util::DEBUG); + + } catch (\Exception $e) { + //sqlite might report database locked errors when stock filescan is in progress //this also catches db locked exception that might come up when using sqlite \OCP\Util::writeLog('search_lucene', $e->getMessage() . ' Trace:\n' . $e->getTraceAsString(), \OCP\Util::ERROR); - \OCP\JSON::error(array('message' => 'Could not index file.')); - if ($eventSource) { - $eventSource->send('error', $e->getMessage()); - } - //try to mark the file as new to let it reindex - $fileStatus->markNew(); // Add UI to trigger rescan of files with status 'E'rror? + $fileStatus->markError(); // Add UI to trigger rescan of files with status 'E'rror? + if ($eventSource) { + $eventSource->send('error', $e->getMessage()); + } } } } @@ -111,108 +108,92 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) * * @author Jörn Dreyer * - * @param string $path the path of the file + * @param \OCP\Files\File $file the file to be indexed * * @return bool true when something was stored in the index, false otherwise (eg, folders are not indexed) - * @throws \Exception indicating an error + * @throws \OCA\Search_Lucene\NotIndexedException when an unsupported file type is encvountered */ - public function indexFile($path = '') { - - try { + public function indexFile(\OCP\Files\File $file) { - // the cache already knows mime and other basic stuff - /** @var \OCP\Files\Node $data */ - $node = $this->folder->get($path); + // we decide how to index on mime type or file extension + $mimeType = $file->getMimeType(); + $fileExtension = strtolower(pathinfo($file->getName(), PATHINFO_EXTENSION)); - if ($node instanceof \OCP\Files\File) { + // initialize plain lucene document + $doc = new \Zend_Search_Lucene_Document(); - // we decide how to index on mime type or file extension - $mimeType = $node->getMimetype(); - $fileExtension = strtolower(pathinfo($node->getName(), PATHINFO_EXTENSION)); + // index content for local files only + $storage = $file->getStorage(); - // initialize plain lucene document - $doc = new \Zend_Search_Lucene_Document(); - - // index content for local files only - $storage = $node->getStorage(); - - $internalPath = $node->getInternalPath(); - $path = $node->getPath(); + if ($storage->isLocal()) { - if ($storage->isLocal()) { + $path = $storage->getLocalFile($file->getInternalPath()); - //try to use special lucene document types + //try to use special lucene document types - if ('text/plain' === $mimeType) { + if ('text/plain' === $mimeType) { - $body = $node->getContent(); + $body = $file->getContent(); - if ($body != '') { - $doc->addField(\Zend_Search_Lucene_Field::UnStored('body', $body)); - } + if ($body != '') { + $doc->addField(\Zend_Search_Lucene_Field::UnStored('body', $body)); + } - // FIXME other text files? c, php, java ... + // FIXME other text files? c, php, java ... - } else if ('text/html' === $mimeType) { + } else if ('text/html' === $mimeType) { - //TODO could be indexed, even if not local - $doc = \Zend_Search_Lucene_Document_Html::loadHTML($node->getContent()); + //TODO could be indexed, even if not local + $doc = \Zend_Search_Lucene_Document_Html::loadHTML($file->getContent()); - } else if ('application/pdf' === $mimeType) { + } else if ('application/pdf' === $mimeType) { - $doc = Pdf::loadPdf($node->getContent()); + $doc = Pdf::loadPdf($file->getContent()); - // the zend classes only understand docx and not doc files - } else if ($fileExtension === 'docx') { + // the zend classes only understand docx and not doc files + } else if ($fileExtension === 'docx') { - $doc = \Zend_Search_Lucene_Document_Docx::loadDocxFile($path); + $doc = \Zend_Search_Lucene_Document_Docx::loadDocxFile($path); - //} else if ('application/msexcel' === $mimeType) { - } else if ($fileExtension === 'xlsx') { + //} else if ('application/msexcel' === $mimeType) { + } else if ($fileExtension === 'xlsx') { - $doc = \Zend_Search_Lucene_Document_Xlsx::loadXlsxFile($path); + $doc = \Zend_Search_Lucene_Document_Xlsx::loadXlsxFile($path); - //} else if ('application/mspowerpoint' === $mimeType) { - } else if ($fileExtension === 'pptx') { + //} else if ('application/mspowerpoint' === $mimeType) { + } else if ($fileExtension === 'pptx') { - $doc = \Zend_Search_Lucene_Document_Pptx::loadPptxFile($path); + $doc = \Zend_Search_Lucene_Document_Pptx::loadPptxFile($path); - } else if ($fileExtension === 'odt') { + } else if ($fileExtension === 'odt') { - $doc = Odt::loadOdtFile($path); + $doc = Odt::loadOdtFile($path); - } else if ($fileExtension === 'ods') { + } else if ($fileExtension === 'ods') { - $doc = Ods::loadOdsFile($path); + $doc = Ods::loadOdsFile($path); - } + } else { + throw new NotIndexedException(); } + } - // Store filecache id as unique id to lookup by when deleting - $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $node->getId())); + // Store filecache id as unique id to lookup by when deleting + $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $file->getId())); - // Store document path for the search results - $doc->addField(\Zend_Search_Lucene_Field::Text('path', $path, 'UTF-8')); + // Store document path for the search results + $doc->addField(\Zend_Search_Lucene_Field::Text('path', $path, 'UTF-8')); - $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', $node->getMTime())); + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', $file->getMTime())); - $doc->addField(\Zend_Search_Lucene_Field::unIndexed('size', $node->getSize())); + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('size', $file->getSize())); - $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', $mimeType)); + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', $mimeType)); - $this->lucene->updateFile($doc, $data->getId()); - } + $this->lucene->updateFile($doc, $file->getId()); return true; - } catch (\Exception $ex) { - Util::writeLog( - 'search_lucene', - $ex->getCode().':'.$ex->getMessage(), - Util::DEBUG - ); - return false; - } } } diff --git a/lib/indexjob.php b/lib/indexjob.php index 227ff2e13..61e53f13a 100644 --- a/lib/indexjob.php +++ b/lib/indexjob.php @@ -11,12 +11,8 @@ public function run($arguments){ $folder = Util::setUpUserFolder($user); if ($folder) { - $fileIds = Status::getUnindexed(); - // we might have to update an already indexed file - if (isset($arguments['fileId']) && ! in_array($arguments['fileId'], $fileIds)) { - $fileIds[] = $arguments['fileId']; - } + $fileIds = Status::getUnindexed(); \OCP\Util::writeLog( 'search_lucene', diff --git a/lib/lucene.php b/lib/lucene.php index cca806416..de7365a6f 100644 --- a/lib/lucene.php +++ b/lib/lucene.php @@ -2,8 +2,6 @@ namespace OCA\Search_Lucene; -use \OCP\Util; - /** * @author Jörn Dreyer */ @@ -24,40 +22,11 @@ class Lucene { */ public function __construct(\OCP\Files\Folder $indexFolder = null) { if (is_null($indexFolder)) { - $indexFolder = $this->getIndexFolder(); + $indexFolder = Util::setUpIndexFolder(); } $this->index = $this->openOrCreate($indexFolder); } - /** - * @return null|\OCP\Files\Folder - */ - private function getIndexFolder() { - - // TODO profile: encrypt the index on logout, decrypt on login - //return OCP\Files::getStorage('search_lucene'); - // FIXME \OC::$server->getAppFolder() returns '/search' - //$indexFolder = \OC::$server->getAppFolder(); - - $root = \OC::$server->getRootFolder(); - $dir = '/'.\OCP\User::getUser(); - $userFolder = null; - if(!$root->nodeExists($dir)) { - $userFolder = $root->newFolder($dir); - } else { - $userFolder = $root->get($dir); - } - $dir = 'lucene_index'; - $indexFolder = null; - if(!$userFolder->nodeExists($dir)) { - $indexFolder = $userFolder->newFolder($dir); - } else { - $indexFolder = $userFolder->get($dir); - } - - return $indexFolder; - } - /** * opens or creates the given lucene index * @@ -86,28 +55,25 @@ private function openOrCreate(\OCP\Files\Folder $indexFolder) { ); // can we use the index? - if ($indexFolder->nodeExists('/v0.6.0')) { + if ($indexFolder->nodeExists('v0.6.0')) { // correct index present $index = \Zend_Search_Lucene::open($localPath); - } else if (file_exists($indexFolder)) { - Util::writeLog( + } else { + \OCP\Util::writeLog( 'search_lucene', 'recreating outdated lucene index', - Util::INFO + \OCP\Util::INFO ); $indexFolder->delete(); $index = \Zend_Search_Lucene::create($localPath); - touch($indexFolder.'/v0.6.0'); - } else { - $index = \Zend_Search_Lucene::create($localPath); - touch($indexFolder.'/v0.6.0'); + $indexFolder->newFile('v0.6.0'); } return $index; } catch ( \Exception $e ) { - Util::writeLog( + \OCP\Util::writeLog( 'search_lucene', $e->getMessage().' Trace:\n'.$e->getTraceAsString(), - Util::ERROR + \OCP\Util::ERROR ); } @@ -124,10 +90,10 @@ private function openOrCreate(\OCP\Files\Folder $indexFolder) { */ public function optimizeIndex() { - Util::writeLog( + \OCP\Util::writeLog( 'search_lucene', 'optimizing index', - Util::DEBUG + \OCP\Util::DEBUG ); $this->index->optimize(); @@ -156,10 +122,10 @@ public function updateFile( // TODO profile perfomance for searching before adding to index $this->deleteFile($fileid); - Util::writeLog( + \OCP\Util::writeLog( 'search_lucene', 'adding ' . $fileid .' '.json_encode($doc), - Util::DEBUG + \OCP\Util::DEBUG ); // Add document to the index @@ -182,17 +148,17 @@ public function deleteFile($fileid) { $hits = $this->index->find( 'fileid:' . $fileid ); - Util::writeLog( + \OCP\Util::writeLog( 'search_lucene', 'found ' . count($hits) . ' hits for fileid ' . $fileid, - Util::DEBUG + \OCP\Util::DEBUG ); foreach ($hits as $hit) { - Util::writeLog( + \OCP\Util::writeLog( 'search_lucene', 'removing ' . $hit->id . ':' . $hit->path . ' from index', - Util::DEBUG + \OCP\Util::DEBUG ); $this->index->delete($hit); } diff --git a/lib/notindexedexception.php b/lib/notindexedexception.php new file mode 100644 index 000000000..53c8c23ac --- /dev/null +++ b/lib/notindexedexception.php @@ -0,0 +1,5 @@ +find('pk:*'); - foreach ($hits as $hit) { - \OCP\Util::writeLog( - 'search_lucene', - 'deleting deprecated index document for ' . $hit->id . ':' . $hit->path , - \OCP\Util::DEBUG - ); - $lucene->index->delete($hit); - } + $folder = Util::setUpIndexFolder($user); + $lucene = new Lucene($folder); $lucene->optimizeIndex(); } else { diff --git a/lib/skippedexception.php b/lib/skippedexception.php new file mode 100644 index 000000000..185e5a6df --- /dev/null +++ b/lib/skippedexception.php @@ -0,0 +1,5 @@ +store(); } + public function markUnIndexed() { + $this->status = self::STATUS_UNINDEXED; + return $this->store(); + } + public function markError() { $this->status = self::STATUS_ERROR; return $this->store(); diff --git a/lib/util.php b/lib/util.php index 98f8f9843..0af89c70f 100644 --- a/lib/util.php +++ b/lib/util.php @@ -12,6 +12,45 @@ class Util { * @return \OCP\Files\Folder */ static function setUpUserFolder($user = null) { + $userHome = self::setUpUserHome($user); + + $dir = 'files'; + $folder = null; + if(!$userHome->nodeExists($dir)) { + $folder = $userHome->newFolder($dir); + } else { + $folder = $userHome->get($dir); + } + + return $folder; + + } + + /** + * @return null|\OCP\Files\Folder + */ + static function setUpIndexFolder($user = null) { + $userHome = self::setUpUserHome($user); + // TODO profile: encrypt the index on logout, decrypt on login + //return OCP\Files::getStorage('search_lucene'); + // FIXME \OC::$server->getAppFolder() returns '/search' + //$indexFolder = \OC::$server->getAppFolder(); + + $dir = 'lucene_index'; + $folder = null; + if(!$userHome->nodeExists($dir)) { + $folder = $userHome->newFolder($dir); + } else { + $folder = $userHome->get($dir); + } + + return $folder; + } + + /** + * @return null|\OCP\Files\Folder + */ + static function setUpUserHome($user = null) { if (is_null($user)) { $user = \OCP\User::getUser(); } @@ -34,15 +73,7 @@ static function setUpUserFolder($user = null) { $folder = $root->get($dir); } - $dir = '/files'; - if(!$folder->nodeExists($dir)) { - $folder = $folder->newFolder($dir); - } else { - $folder = $folder->get($dir); - } - return $folder; } - } From 68b182f39481cac33e0ab65ae7524e34c8c32294 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 8 Jul 2014 14:09:51 +0200 Subject: [PATCH 19/51] fix handling of storage wrappers --- lib/status.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/status.php b/lib/status.php index 3e530c827..8d422ae9a 100644 --- a/lib/status.php +++ b/lib/status.php @@ -149,7 +149,7 @@ static public function getUnindexed() { Util::DEBUG); } //only index local files for now - if ($storage instanceof \OC\Files\Storage\Local) { + if ($storage->instanceOfStorage('\OC\Files\Storage\Local')) { $cache = $storage->getCache(); $numericId = $cache->getNumericStorageId(); From ff22b87465f4da40ffb765fb47aab0ff61b5edc9 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 8 Jul 2014 14:11:13 +0200 Subject: [PATCH 20/51] dont commit the index on every write when indexing a large number of files --- lib/indexer.php | 8 +++++--- lib/lucene.php | 13 ++++++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/indexer.php b/lib/indexer.php index 4ee466006..d35d8f6a4 100644 --- a/lib/indexer.php +++ b/lib/indexer.php @@ -76,7 +76,7 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) $eventSource->send('indexing', $path); } - if ($this->indexFile($node)) { + if ($this->indexFile($node, false)) { $fileStatus->markIndexed(); } @@ -101,6 +101,7 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) } } } + $this->lucene->commit(); } /** @@ -109,11 +110,12 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) * @author Jörn Dreyer * * @param \OCP\Files\File $file the file to be indexed + * @param bool $commit * * @return bool true when something was stored in the index, false otherwise (eg, folders are not indexed) * @throws \OCA\Search_Lucene\NotIndexedException when an unsupported file type is encvountered */ - public function indexFile(\OCP\Files\File $file) { + public function indexFile(\OCP\Files\File $file, $commit = true) { // we decide how to index on mime type or file extension $mimeType = $file->getMimeType(); @@ -190,7 +192,7 @@ public function indexFile(\OCP\Files\File $file) { $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', $mimeType)); - $this->lucene->updateFile($doc, $file->getId()); + $this->lucene->updateFile($doc, $file->getId(), $commit); return true; diff --git a/lib/lucene.php b/lib/lucene.php index de7365a6f..18457d4cb 100644 --- a/lib/lucene.php +++ b/lib/lucene.php @@ -111,12 +111,14 @@ public function optimizeIndex() { * * @param \Zend_Search_Lucene_Document $doc the document to store for the path * @param int $fileid fileid to update + * @param bool $commit * * @return void */ public function updateFile( \Zend_Search_Lucene_Document $doc, - $fileid + $fileid, + $commit = true ) { // TODO profile perfomance for searching before adding to index @@ -131,8 +133,9 @@ public function updateFile( // Add document to the index $this->index->addDocument($doc); - $this->index->commit(); - + if ($commit) { + $this->index->commit(); + } } /** @@ -170,4 +173,8 @@ public function find ($query) { return $this->index->find($query); } + public function commit () { + $this->index->commit(); + } + } From d4b6e7fb5866857900b34f7be7f8562f4c2e9fcd Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 8 Jul 2014 14:11:43 +0200 Subject: [PATCH 21/51] dont skip files just because we cant index their body --- lib/indexer.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/indexer.php b/lib/indexer.php index d35d8f6a4..fcd060156 100644 --- a/lib/indexer.php +++ b/lib/indexer.php @@ -175,8 +175,6 @@ public function indexFile(\OCP\Files\File $file, $commit = true) { $doc = Ods::loadOdsFile($path); - } else { - throw new NotIndexedException(); } } From e1c38d667ded247a100a6ca8e812a733a23fac06 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 8 Jul 2014 15:11:08 +0200 Subject: [PATCH 22/51] also index the body of other text files --- lib/indexer.php | 90 +++++++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 47 deletions(-) diff --git a/lib/indexer.php b/lib/indexer.php index fcd060156..126dd5c6b 100644 --- a/lib/indexer.php +++ b/lib/indexer.php @@ -5,7 +5,6 @@ use OCA\Search_Lucene\Document\Ods; use OCA\Search_Lucene\Document\Odt; use OCA\Search_Lucene\Document\Pdf; -use OCP\Util; /** * @author Jörn Dreyer @@ -117,82 +116,79 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) */ public function indexFile(\OCP\Files\File $file, $commit = true) { - // we decide how to index on mime type or file extension - $mimeType = $file->getMimeType(); - $fileExtension = strtolower(pathinfo($file->getName(), PATHINFO_EXTENSION)); + // we decide how to index on mime type or file extension + $mimeType = $file->getMimeType(); + $fileExtension = strtolower(pathinfo($file->getName(), PATHINFO_EXTENSION)); - // initialize plain lucene document - $doc = new \Zend_Search_Lucene_Document(); + // initialize plain lucene document + $doc = new \Zend_Search_Lucene_Document(); - // index content for local files only - $storage = $file->getStorage(); + // index content for local files only + $storage = $file->getStorage(); - if ($storage->isLocal()) { + if ($storage->isLocal()) { - $path = $storage->getLocalFile($file->getInternalPath()); + $path = $storage->getLocalFile($file->getInternalPath()); - //try to use special lucene document types + //try to use special lucene document types - if ('text/plain' === $mimeType) { + if ('text/html' === $mimeType) { - $body = $file->getContent(); + //TODO could be indexed, even if not local + $doc = \Zend_Search_Lucene_Document_Html::loadHTML($file->getContent()); + } else if ('text/' === substr($mimeType, 0, 5)) { - if ($body != '') { - $doc->addField(\Zend_Search_Lucene_Field::UnStored('body', $body)); - } - - // FIXME other text files? c, php, java ... - - } else if ('text/html' === $mimeType) { + $body = $file->getContent(); - //TODO could be indexed, even if not local - $doc = \Zend_Search_Lucene_Document_Html::loadHTML($file->getContent()); + if ($body != '') { + $doc->addField(\Zend_Search_Lucene_Field::UnStored('body', $body)); + } - } else if ('application/pdf' === $mimeType) { + } else if ('application/pdf' === $mimeType) { - $doc = Pdf::loadPdf($file->getContent()); + $doc = Pdf::loadPdf($file->getContent()); - // the zend classes only understand docx and not doc files - } else if ($fileExtension === 'docx') { + // the zend classes only understand docx and not doc files + } else if ($fileExtension === 'docx') { - $doc = \Zend_Search_Lucene_Document_Docx::loadDocxFile($path); + $doc = \Zend_Search_Lucene_Document_Docx::loadDocxFile($path); - //} else if ('application/msexcel' === $mimeType) { - } else if ($fileExtension === 'xlsx') { + //} else if ('application/msexcel' === $mimeType) { + } else if ($fileExtension === 'xlsx') { - $doc = \Zend_Search_Lucene_Document_Xlsx::loadXlsxFile($path); + $doc = \Zend_Search_Lucene_Document_Xlsx::loadXlsxFile($path); - //} else if ('application/mspowerpoint' === $mimeType) { - } else if ($fileExtension === 'pptx') { + //} else if ('application/mspowerpoint' === $mimeType) { + } else if ($fileExtension === 'pptx') { - $doc = \Zend_Search_Lucene_Document_Pptx::loadPptxFile($path); + $doc = \Zend_Search_Lucene_Document_Pptx::loadPptxFile($path); - } else if ($fileExtension === 'odt') { + } else if ($fileExtension === 'odt') { - $doc = Odt::loadOdtFile($path); + $doc = Odt::loadOdtFile($path); - } else if ($fileExtension === 'ods') { + } else if ($fileExtension === 'ods') { - $doc = Ods::loadOdsFile($path); + $doc = Ods::loadOdsFile($path); - } } + } - // Store filecache id as unique id to lookup by when deleting - $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $file->getId())); + // Store filecache id as unique id to lookup by when deleting + $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $file->getId())); - // Store document path for the search results - $doc->addField(\Zend_Search_Lucene_Field::Text('path', $path, 'UTF-8')); + // Store document path for the search results + $doc->addField(\Zend_Search_Lucene_Field::Text('path', $path, 'UTF-8')); - $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', $file->getMTime())); + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', $file->getMTime())); - $doc->addField(\Zend_Search_Lucene_Field::unIndexed('size', $file->getSize())); + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('size', $file->getSize())); - $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', $mimeType)); + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', $mimeType)); - $this->lucene->updateFile($doc, $file->getId(), $commit); + $this->lucene->updateFile($doc, $file->getId(), $commit); - return true; + return true; } From e4d79d4805b88a7b2c89672490c48ad3c7a97e3f Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 8 Jul 2014 15:11:50 +0200 Subject: [PATCH 23/51] index the owncloud path, not the local path --- lib/indexer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/indexer.php b/lib/indexer.php index 126dd5c6b..dce97f820 100644 --- a/lib/indexer.php +++ b/lib/indexer.php @@ -178,7 +178,7 @@ public function indexFile(\OCP\Files\File $file, $commit = true) { $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $file->getId())); // Store document path for the search results - $doc->addField(\Zend_Search_Lucene_Field::Text('path', $path, 'UTF-8')); + $doc->addField(\Zend_Search_Lucene_Field::Text('path', $file->getPath(), 'UTF-8')); $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', $file->getMTime())); From 4049020cf91b025f2b44dc7e2e01d8ed48e3c34e Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 8 Jul 2014 15:14:54 +0200 Subject: [PATCH 24/51] index body of tex files --- lib/indexer.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/indexer.php b/lib/indexer.php index dce97f820..53bf542a6 100644 --- a/lib/indexer.php +++ b/lib/indexer.php @@ -136,7 +136,8 @@ public function indexFile(\OCP\Files\File $file, $commit = true) { //TODO could be indexed, even if not local $doc = \Zend_Search_Lucene_Document_Html::loadHTML($file->getContent()); - } else if ('text/' === substr($mimeType, 0, 5)) { + } else if ('text/' === substr($mimeType, 0, 5) + || 'application/x-tex' === $mimeType) { $body = $file->getContent(); From c7c0d70e44989452737a8cca0961c83f620a8d27 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 8 Jul 2014 15:22:09 +0200 Subject: [PATCH 25/51] return relative path for the search result --- result/content.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/result/content.php b/result/content.php index e2beb1a1b..28449e056 100644 --- a/result/content.php +++ b/result/content.php @@ -18,6 +18,7 @@ */ namespace OCA\Search_Lucene\Result; +use OC\Files\Filesystem; /** * A found file @@ -41,16 +42,16 @@ class Content extends \OC\Search\Result\File { */ public function __construct(\Zend_Search_Lucene_Search_QueryHit $hit) { $this->id = (string)$hit->fileid; - $this->path = $hit->path; - $this->name = basename($hit->path); + $this->path = Filesystem::getView()->getRelativePath($hit->path); + $this->name = basename($this->path); $this->size = (int)$hit->size; $this->score = $hit->score; $this->link = \OCP\Util::linkTo( 'files', 'index.php', - array('dir' => dirname($hit->path), 'file' => basename($hit->path)) + array('dir' => dirname($this->path), 'file' => basename($this->path)) ); - $this->permissions = self::get_permissions($hit->path); + $this->permissions = self::get_permissions($this->path); $this->modified = (int)$hit->mtime; $this->mime_type = $hit->mimetype; } From 6fcb5d30bd56c627f18ef6b97f677c72ba17ee71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 9 Jul 2014 10:35:36 +0200 Subject: [PATCH 26/51] use appframework, zend loader, di --- 3rdparty/Zend/Loader.php | 343 +++++++++ 3rdparty/Zend/Loader/Autoloader.php | 589 ++++++++++++++++ 3rdparty/Zend/Loader/Autoloader/Interface.php | 43 ++ 3rdparty/Zend/Loader/Autoloader/Resource.php | 472 +++++++++++++ 3rdparty/Zend/Loader/AutoloaderFactory.php | 211 ++++++ 3rdparty/Zend/Loader/ClassMapAutoloader.php | 248 +++++++ 3rdparty/Zend/Loader/Exception.php | 35 + .../Exception/InvalidArgumentException.php | 34 + 3rdparty/Zend/Loader/PluginLoader.php | 506 ++++++++++++++ .../Zend/Loader/PluginLoader/Exception.php | 39 ++ .../Zend/Loader/PluginLoader/Interface.php | 75 ++ 3rdparty/Zend/Loader/SplAutoloader.php | 75 ++ 3rdparty/Zend/Loader/StandardAutoloader.php | 368 ++++++++++ COPYING | 661 ++++++++++++++++++ ajax/lucene.php | 44 -- appinfo/app.php | 71 +- appinfo/application.php | 100 +++ appinfo/classpath.php | 14 + appinfo/preupdate.php | 9 + appinfo/routes.php | 27 + appinfo/update.php | 11 +- controller/apicontroller.php | 69 ++ core/db.php | 40 ++ lib/util.php => core/files.php | 33 +- core/logger.php | 49 ++ db/status.php | 98 +++ db/statusmapper.php | 264 +++++++ document/ods.php | 9 + document/odt.php | 9 + document/opendocument.php | 9 + document/pdf.php | 13 +- lib/hooks.php => hooks/files.php | 91 ++- jobs/indexjob.php | 42 ++ jobs/optimizejob.php | 37 + js/checker.js | 6 +- lib/indexer.php | 196 ------ lib/indexjob.php | 36 - lib/lucene.php | 180 ----- lib/notindexedexception.php | 5 - lib/optimizejob.php | 31 - lib/skippedexception.php | 5 - lib/status.php | 194 ----- lucene/index.php | 147 ++++ lucene/indexer.php | 206 ++++++ lucene/notindexedexception.php | 14 + lucene/skippedexception.php | 14 + .../luceneprovider.php | 38 +- result/content.php => search/luceneresult.php | 24 +- .../pdf2text.php => utility/pdfparser.php | 5 +- 49 files changed, 5005 insertions(+), 834 deletions(-) create mode 100644 3rdparty/Zend/Loader.php create mode 100644 3rdparty/Zend/Loader/Autoloader.php create mode 100644 3rdparty/Zend/Loader/Autoloader/Interface.php create mode 100644 3rdparty/Zend/Loader/Autoloader/Resource.php create mode 100644 3rdparty/Zend/Loader/AutoloaderFactory.php create mode 100644 3rdparty/Zend/Loader/ClassMapAutoloader.php create mode 100644 3rdparty/Zend/Loader/Exception.php create mode 100644 3rdparty/Zend/Loader/Exception/InvalidArgumentException.php create mode 100644 3rdparty/Zend/Loader/PluginLoader.php create mode 100644 3rdparty/Zend/Loader/PluginLoader/Exception.php create mode 100644 3rdparty/Zend/Loader/PluginLoader/Interface.php create mode 100644 3rdparty/Zend/Loader/SplAutoloader.php create mode 100644 3rdparty/Zend/Loader/StandardAutoloader.php create mode 100644 COPYING delete mode 100644 ajax/lucene.php create mode 100644 appinfo/application.php create mode 100644 appinfo/classpath.php create mode 100644 appinfo/routes.php create mode 100644 controller/apicontroller.php create mode 100644 core/db.php rename lib/util.php => core/files.php (64%) create mode 100644 core/logger.php create mode 100644 db/status.php create mode 100644 db/statusmapper.php rename lib/hooks.php => hooks/files.php (55%) create mode 100644 jobs/indexjob.php create mode 100644 jobs/optimizejob.php delete mode 100644 lib/indexer.php delete mode 100644 lib/indexjob.php delete mode 100644 lib/lucene.php delete mode 100644 lib/notindexedexception.php delete mode 100644 lib/optimizejob.php delete mode 100644 lib/skippedexception.php delete mode 100644 lib/status.php create mode 100644 lucene/index.php create mode 100644 lucene/indexer.php create mode 100644 lucene/notindexedexception.php create mode 100644 lucene/skippedexception.php rename lib/searchprovider.php => search/luceneprovider.php (67%) rename result/content.php => search/luceneresult.php (52%) rename 3rdparty/pdf2text.php => utility/pdfparser.php (98%) diff --git a/3rdparty/Zend/Loader.php b/3rdparty/Zend/Loader.php new file mode 100644 index 000000000..a2b3de1c0 --- /dev/null +++ b/3rdparty/Zend/Loader.php @@ -0,0 +1,343 @@ + $dir) { + if ($dir == '.') { + $dirs[$key] = $dirPath; + } else { + $dir = rtrim($dir, '\\/'); + $dirs[$key] = $dir . DIRECTORY_SEPARATOR . $dirPath; + } + } + $file = basename($file); + self::loadFile($file, $dirs, true); + } else { + self::loadFile($file, null, true); + } + + if (!class_exists($class, false) && !interface_exists($class, false)) { + require_once 'Zend/Exception.php'; + throw new Zend_Exception("File \"$file\" does not exist or class \"$class\" was not found in the file"); + } + } + + /** + * Loads a PHP file. This is a wrapper for PHP's include() function. + * + * $filename must be the complete filename, including any + * extension such as ".php". Note that a security check is performed that + * does not permit extended characters in the filename. This method is + * intended for loading Zend Framework files. + * + * If $dirs is a string or an array, it will search the directories + * in the order supplied, and attempt to load the first matching file. + * + * If the file was not found in the $dirs, or if no $dirs were specified, + * it will attempt to load it from PHP's include_path. + * + * If $once is TRUE, it will use include_once() instead of include(). + * + * @param string $filename + * @param string|array $dirs - OPTIONAL either a path or array of paths + * to search. + * @param boolean $once + * @return boolean + * @throws Zend_Exception + */ + public static function loadFile($filename, $dirs = null, $once = false) + { + self::_securityCheck($filename); + + /** + * Search in provided directories, as well as include_path + */ + $incPath = false; + if (!empty($dirs) && (is_array($dirs) || is_string($dirs))) { + if (is_array($dirs)) { + $dirs = implode(PATH_SEPARATOR, $dirs); + } + $incPath = get_include_path(); + set_include_path($dirs . PATH_SEPARATOR . $incPath); + } + + /** + * Try finding for the plain filename in the include_path. + */ + if ($once) { + include_once $filename; + } else { + include $filename; + } + + /** + * If searching in directories, reset include_path + */ + if ($incPath) { + set_include_path($incPath); + } + + return true; + } + + /** + * Returns TRUE if the $filename is readable, or FALSE otherwise. + * This function uses the PHP include_path, where PHP's is_readable() + * does not. + * + * Note from ZF-2900: + * If you use custom error handler, please check whether return value + * from error_reporting() is zero or not. + * At mark of fopen() can not suppress warning if the handler is used. + * + * @param string $filename + * @return boolean + */ + public static function isReadable($filename) + { + if (is_readable($filename)) { + // Return early if the filename is readable without needing the + // include_path + return true; + } + + if (strtoupper(substr(PHP_OS, 0, 3)) == 'WIN' + && preg_match('/^[a-z]:/i', $filename) + ) { + // If on windows, and path provided is clearly an absolute path, + // return false immediately + return false; + } + + foreach (self::explodeIncludePath() as $path) { + if ($path == '.') { + if (is_readable($filename)) { + return true; + } + continue; + } + $file = $path . '/' . $filename; + if (is_readable($file)) { + return true; + } + } + return false; + } + + /** + * Explode an include path into an array + * + * If no path provided, uses current include_path. Works around issues that + * occur when the path includes stream schemas. + * + * @param string|null $path + * @return array + */ + public static function explodeIncludePath($path = null) + { + if (null === $path) { + $path = get_include_path(); + } + + if (PATH_SEPARATOR == ':') { + // On *nix systems, include_paths which include paths with a stream + // schema cannot be safely explode'd, so we have to be a bit more + // intelligent in the approach. + $paths = preg_split('#:(?!//)#', $path); + } else { + $paths = explode(PATH_SEPARATOR, $path); + } + return $paths; + } + + /** + * spl_autoload() suitable implementation for supporting class autoloading. + * + * Attach to spl_autoload() using the following: + * + * spl_autoload_register(array('Zend_Loader', 'autoload')); + * + * + * @deprecated Since 1.8.0 + * @param string $class + * @return string|false Class name on success; false on failure + */ + public static function autoload($class) + { + trigger_error(__CLASS__ . '::' . __METHOD__ . ' is deprecated as of 1.8.0 and will be removed with 2.0.0; use Zend_Loader_Autoloader instead', E_USER_NOTICE); + try { + @self::loadClass($class); + return $class; + } catch (Exception $e) { + return false; + } + } + + /** + * Register {@link autoload()} with spl_autoload() + * + * @deprecated Since 1.8.0 + * @param string $class (optional) + * @param boolean $enabled (optional) + * @return void + * @throws Zend_Exception if spl_autoload() is not found + * or if the specified class does not have an autoload() method. + */ + public static function registerAutoload($class = 'Zend_Loader', $enabled = true) + { + trigger_error(__CLASS__ . '::' . __METHOD__ . ' is deprecated as of 1.8.0 and will be removed with 2.0.0; use Zend_Loader_Autoloader instead', E_USER_NOTICE); + require_once 'Zend/Loader/Autoloader.php'; + $autoloader = Zend_Loader_Autoloader::getInstance(); + $autoloader->setFallbackAutoloader(true); + + if ('Zend_Loader' != $class) { + self::loadClass($class); + $methods = get_class_methods($class); + if (!in_array('autoload', (array) $methods)) { + require_once 'Zend/Exception.php'; + throw new Zend_Exception("The class \"$class\" does not have an autoload() method"); + } + + $callback = array($class, 'autoload'); + + if ($enabled) { + $autoloader->pushAutoloader($callback); + } else { + $autoloader->removeAutoloader($callback); + } + } + } + + /** + * Ensure that filename does not contain exploits + * + * @param string $filename + * @return void + * @throws Zend_Exception + */ + protected static function _securityCheck($filename) + { + /** + * Security check + */ + if (preg_match('/[^a-z0-9\\/\\\\_.:-]/i', $filename)) { + require_once 'Zend/Exception.php'; + throw new Zend_Exception('Security check: Illegal character in filename'); + } + } + + /** + * Attempt to include() the file. + * + * include() is not prefixed with the @ operator because if + * the file is loaded and contains a parse error, execution + * will halt silently and this is difficult to debug. + * + * Always set display_errors = Off on production servers! + * + * @param string $filespec + * @param boolean $once + * @return boolean + * @deprecated Since 1.5.0; use loadFile() instead + */ + protected static function _includeFile($filespec, $once = false) + { + if ($once) { + return include_once $filespec; + } else { + return include $filespec ; + } + } + + /** + * Standardise the filename. + * + * Convert the supplied filename into the namespace-aware standard, + * based on the Framework Interop Group reference implementation: + * http://groups.google.com/group/php-standards/web/psr-0-final-proposal + * + * The filename must be formatted as "$file.php". + * + * @param string $file - The file name to be loaded. + * @return string + */ + public static function standardiseFile($file) + { + $fileName = ltrim($file, '\\'); + $file = ''; + $namespace = ''; + if ($lastNsPos = strripos($fileName, '\\')) { + $namespace = substr($fileName, 0, $lastNsPos); + $fileName = substr($fileName, $lastNsPos + 1); + $file = str_replace('\\', DIRECTORY_SEPARATOR, $namespace) . DIRECTORY_SEPARATOR; + } + $file .= str_replace('_', DIRECTORY_SEPARATOR, $fileName) . '.php'; + return $file; + } +} diff --git a/3rdparty/Zend/Loader/Autoloader.php b/3rdparty/Zend/Loader/Autoloader.php new file mode 100644 index 000000000..ed5a2c43d --- /dev/null +++ b/3rdparty/Zend/Loader/Autoloader.php @@ -0,0 +1,589 @@ + true, + 'ZendX_' => true, + ); + + /** + * @var array Namespace-specific autoloaders + */ + protected $_namespaceAutoloaders = array(); + + /** + * @var bool Whether or not to suppress file not found warnings + */ + protected $_suppressNotFoundWarnings = false; + + /** + * @var null|string + */ + protected $_zfPath; + + /** + * Retrieve singleton instance + * + * @return Zend_Loader_Autoloader + */ + public static function getInstance() + { + if (null === self::$_instance) { + self::$_instance = new self(); + } + return self::$_instance; + } + + /** + * Reset the singleton instance + * + * @return void + */ + public static function resetInstance() + { + self::$_instance = null; + } + + /** + * Autoload a class + * + * @param string $class + * @return bool + */ + public static function autoload($class) + { + $self = self::getInstance(); + + foreach ($self->getClassAutoloaders($class) as $autoloader) { + if ($autoloader instanceof Zend_Loader_Autoloader_Interface) { + if ($autoloader->autoload($class)) { + return true; + } + } elseif (is_array($autoloader)) { + if (call_user_func($autoloader, $class)) { + return true; + } + } elseif (is_string($autoloader) || is_callable($autoloader)) { + if ($autoloader($class)) { + return true; + } + } + } + + return false; + } + + /** + * Set the default autoloader implementation + * + * @param string|array $callback PHP callback + * @return void + */ + public function setDefaultAutoloader($callback) + { + if (!is_callable($callback)) { + throw new Zend_Loader_Exception('Invalid callback specified for default autoloader'); + } + + $this->_defaultAutoloader = $callback; + return $this; + } + + /** + * Retrieve the default autoloader callback + * + * @return string|array PHP Callback + */ + public function getDefaultAutoloader() + { + return $this->_defaultAutoloader; + } + + /** + * Set several autoloader callbacks at once + * + * @param array $autoloaders Array of PHP callbacks (or Zend_Loader_Autoloader_Interface implementations) to act as autoloaders + * @return Zend_Loader_Autoloader + */ + public function setAutoloaders(array $autoloaders) + { + $this->_autoloaders = $autoloaders; + return $this; + } + + /** + * Get attached autoloader implementations + * + * @return array + */ + public function getAutoloaders() + { + return $this->_autoloaders; + } + + /** + * Return all autoloaders for a given namespace + * + * @param string $namespace + * @return array + */ + public function getNamespaceAutoloaders($namespace) + { + $namespace = (string) $namespace; + if (!array_key_exists($namespace, $this->_namespaceAutoloaders)) { + return array(); + } + return $this->_namespaceAutoloaders[$namespace]; + } + + /** + * Register a namespace to autoload + * + * @param string|array $namespace + * @return Zend_Loader_Autoloader + */ + public function registerNamespace($namespace) + { + if (is_string($namespace)) { + $namespace = (array) $namespace; + } elseif (!is_array($namespace)) { + throw new Zend_Loader_Exception('Invalid namespace provided'); + } + + foreach ($namespace as $ns) { + if (!isset($this->_namespaces[$ns])) { + $this->_namespaces[$ns] = true; + } + } + return $this; + } + + /** + * Unload a registered autoload namespace + * + * @param string|array $namespace + * @return Zend_Loader_Autoloader + */ + public function unregisterNamespace($namespace) + { + if (is_string($namespace)) { + $namespace = (array) $namespace; + } elseif (!is_array($namespace)) { + throw new Zend_Loader_Exception('Invalid namespace provided'); + } + + foreach ($namespace as $ns) { + if (isset($this->_namespaces[$ns])) { + unset($this->_namespaces[$ns]); + } + } + return $this; + } + + /** + * Get a list of registered autoload namespaces + * + * @return array + */ + public function getRegisteredNamespaces() + { + return array_keys($this->_namespaces); + } + + public function setZfPath($spec, $version = 'latest') + { + $path = $spec; + if (is_array($spec)) { + if (!isset($spec['path'])) { + throw new Zend_Loader_Exception('No path specified for ZF'); + } + $path = $spec['path']; + if (isset($spec['version'])) { + $version = $spec['version']; + } + } + + $this->_zfPath = $this->_getVersionPath($path, $version); + set_include_path(implode(PATH_SEPARATOR, array( + $this->_zfPath, + get_include_path(), + ))); + return $this; + } + + public function getZfPath() + { + return $this->_zfPath; + } + + /** + * Get or set the value of the "suppress not found warnings" flag + * + * @param null|bool $flag + * @return bool|Zend_Loader_Autoloader Returns boolean if no argument is passed, object instance otherwise + */ + public function suppressNotFoundWarnings($flag = null) + { + if (null === $flag) { + return $this->_suppressNotFoundWarnings; + } + $this->_suppressNotFoundWarnings = (bool) $flag; + return $this; + } + + /** + * Indicate whether or not this autoloader should be a fallback autoloader + * + * @param bool $flag + * @return Zend_Loader_Autoloader + */ + public function setFallbackAutoloader($flag) + { + $this->_fallbackAutoloader = (bool) $flag; + return $this; + } + + /** + * Is this instance acting as a fallback autoloader? + * + * @return bool + */ + public function isFallbackAutoloader() + { + return $this->_fallbackAutoloader; + } + + /** + * Get autoloaders to use when matching class + * + * Determines if the class matches a registered namespace, and, if so, + * returns only the autoloaders for that namespace. Otherwise, it returns + * all non-namespaced autoloaders. + * + * @param string $class + * @return array Array of autoloaders to use + */ + public function getClassAutoloaders($class) + { + $namespace = false; + $autoloaders = array(); + + // Add concrete namespaced autoloaders + foreach (array_keys($this->_namespaceAutoloaders) as $ns) { + if ('' == $ns) { + continue; + } + if (0 === strpos($class, $ns)) { + if ((false === $namespace) || (strlen($ns) > strlen($namespace))) { + $namespace = $ns; + $autoloaders = $this->getNamespaceAutoloaders($ns); + } + } + } + + // Add internal namespaced autoloader + foreach ($this->getRegisteredNamespaces() as $ns) { + if (0 === strpos($class, $ns)) { + $namespace = $ns; + $autoloaders[] = $this->_internalAutoloader; + break; + } + } + + // Add non-namespaced autoloaders + $autoloadersNonNamespace = $this->getNamespaceAutoloaders(''); + if (count($autoloadersNonNamespace)) { + foreach ($autoloadersNonNamespace as $ns) { + $autoloaders[] = $ns; + } + unset($autoloadersNonNamespace); + } + + // Add fallback autoloader + if (!$namespace && $this->isFallbackAutoloader()) { + $autoloaders[] = $this->_internalAutoloader; + } + + return $autoloaders; + } + + /** + * Add an autoloader to the beginning of the stack + * + * @param object|array|string $callback PHP callback or Zend_Loader_Autoloader_Interface implementation + * @param string|array $namespace Specific namespace(s) under which to register callback + * @return Zend_Loader_Autoloader + */ + public function unshiftAutoloader($callback, $namespace = '') + { + $autoloaders = $this->getAutoloaders(); + array_unshift($autoloaders, $callback); + $this->setAutoloaders($autoloaders); + + $namespace = (array) $namespace; + foreach ($namespace as $ns) { + $autoloaders = $this->getNamespaceAutoloaders($ns); + array_unshift($autoloaders, $callback); + $this->_setNamespaceAutoloaders($autoloaders, $ns); + } + + return $this; + } + + /** + * Append an autoloader to the autoloader stack + * + * @param object|array|string $callback PHP callback or Zend_Loader_Autoloader_Interface implementation + * @param string|array $namespace Specific namespace(s) under which to register callback + * @return Zend_Loader_Autoloader + */ + public function pushAutoloader($callback, $namespace = '') + { + $autoloaders = $this->getAutoloaders(); + array_push($autoloaders, $callback); + $this->setAutoloaders($autoloaders); + + $namespace = (array) $namespace; + foreach ($namespace as $ns) { + $autoloaders = $this->getNamespaceAutoloaders($ns); + array_push($autoloaders, $callback); + $this->_setNamespaceAutoloaders($autoloaders, $ns); + } + + return $this; + } + + /** + * Remove an autoloader from the autoloader stack + * + * @param object|array|string $callback PHP callback or Zend_Loader_Autoloader_Interface implementation + * @param null|string|array $namespace Specific namespace(s) from which to remove autoloader + * @return Zend_Loader_Autoloader + */ + public function removeAutoloader($callback, $namespace = null) + { + if (null === $namespace) { + $autoloaders = $this->getAutoloaders(); + if (false !== ($index = array_search($callback, $autoloaders, true))) { + unset($autoloaders[$index]); + $this->setAutoloaders($autoloaders); + } + + foreach ($this->_namespaceAutoloaders as $ns => $autoloaders) { + if (false !== ($index = array_search($callback, $autoloaders, true))) { + unset($autoloaders[$index]); + $this->_setNamespaceAutoloaders($autoloaders, $ns); + } + } + } else { + $namespace = (array) $namespace; + foreach ($namespace as $ns) { + $autoloaders = $this->getNamespaceAutoloaders($ns); + if (false !== ($index = array_search($callback, $autoloaders, true))) { + unset($autoloaders[$index]); + $this->_setNamespaceAutoloaders($autoloaders, $ns); + } + } + } + + return $this; + } + + /** + * Constructor + * + * Registers instance with spl_autoload stack + * + * @return void + */ + protected function __construct() + { + spl_autoload_register(array(__CLASS__, 'autoload')); + $this->_internalAutoloader = array($this, '_autoload'); + } + + /** + * Internal autoloader implementation + * + * @param string $class + * @return bool + */ + protected function _autoload($class) + { + $callback = $this->getDefaultAutoloader(); + try { + if ($this->suppressNotFoundWarnings()) { + @call_user_func($callback, $class); + } else { + call_user_func($callback, $class); + } + return $class; + } catch (Zend_Exception $e) { + return false; + } + } + + /** + * Set autoloaders for a specific namespace + * + * @param array $autoloaders + * @param string $namespace + * @return Zend_Loader_Autoloader + */ + protected function _setNamespaceAutoloaders(array $autoloaders, $namespace = '') + { + $namespace = (string) $namespace; + $this->_namespaceAutoloaders[$namespace] = $autoloaders; + return $this; + } + + /** + * Retrieve the filesystem path for the requested ZF version + * + * @param string $path + * @param string $version + * @return void + */ + protected function _getVersionPath($path, $version) + { + $type = $this->_getVersionType($version); + + if ($type == 'latest') { + $version = 'latest'; + } + + $availableVersions = $this->_getAvailableVersions($path, $version); + if (empty($availableVersions)) { + throw new Zend_Loader_Exception('No valid ZF installations discovered'); + } + + $matchedVersion = array_pop($availableVersions); + return $matchedVersion; + } + + /** + * Retrieve the ZF version type + * + * @param string $version + * @return string "latest", "major", "minor", or "specific" + * @throws Zend_Loader_Exception if version string contains too many dots + */ + protected function _getVersionType($version) + { + if (strtolower($version) == 'latest') { + return 'latest'; + } + + $parts = explode('.', $version); + $count = count($parts); + if (1 == $count) { + return 'major'; + } + if (2 == $count) { + return 'minor'; + } + if (3 < $count) { + throw new Zend_Loader_Exception('Invalid version string provided'); + } + return 'specific'; + } + + /** + * Get available versions for the version type requested + * + * @param string $path + * @param string $version + * @return array + */ + protected function _getAvailableVersions($path, $version) + { + if (!is_dir($path)) { + throw new Zend_Loader_Exception('Invalid ZF path provided'); + } + + $path = rtrim($path, '/'); + $path = rtrim($path, '\\'); + $versionLen = strlen($version); + $versions = array(); + $dirs = glob("$path/*", GLOB_ONLYDIR); + foreach ((array) $dirs as $dir) { + $dirName = substr($dir, strlen($path) + 1); + if (!preg_match('/^(?:ZendFramework-)?(\d+\.\d+\.\d+((a|b|pl|pr|p|rc)\d+)?)(?:-minimal)?$/i', $dirName, $matches)) { + continue; + } + + $matchedVersion = $matches[1]; + + if (('latest' == $version) + || ((strlen($matchedVersion) >= $versionLen) + && (0 === strpos($matchedVersion, $version))) + ) { + $versions[$matchedVersion] = $dir . '/library'; + } + } + + uksort($versions, 'version_compare'); + return $versions; + } +} diff --git a/3rdparty/Zend/Loader/Autoloader/Interface.php b/3rdparty/Zend/Loader/Autoloader/Interface.php new file mode 100644 index 000000000..768b734fe --- /dev/null +++ b/3rdparty/Zend/Loader/Autoloader/Interface.php @@ -0,0 +1,43 @@ +toArray(); + } + if (!is_array($options)) { + require_once 'Zend/Loader/Exception.php'; + throw new Zend_Loader_Exception('Options must be passed to resource loader constructor'); + } + + $this->setOptions($options); + + $namespace = $this->getNamespace(); + if ((null === $namespace) + || (null === $this->getBasePath()) + ) { + require_once 'Zend/Loader/Exception.php'; + throw new Zend_Loader_Exception('Resource loader requires both a namespace and a base path for initialization'); + } + + if (!empty($namespace)) { + $namespace .= '_'; + } + require_once 'Zend/Loader/Autoloader.php'; + Zend_Loader_Autoloader::getInstance()->unshiftAutoloader($this, $namespace); + } + + /** + * Overloading: methods + * + * Allow retrieving concrete resource object instances using 'get()' + * syntax. Example: + * + * $loader = new Zend_Loader_Autoloader_Resource(array( + * 'namespace' => 'Stuff_', + * 'basePath' => '/path/to/some/stuff', + * )) + * $loader->addResourceType('Model', 'models', 'Model'); + * + * $foo = $loader->getModel('Foo'); // get instance of Stuff_Model_Foo class + * + * + * @param string $method + * @param array $args + * @return mixed + * @throws Zend_Loader_Exception if method not beginning with 'get' or not matching a valid resource type is called + */ + public function __call($method, $args) + { + if ('get' == substr($method, 0, 3)) { + $type = strtolower(substr($method, 3)); + if (!$this->hasResourceType($type)) { + require_once 'Zend/Loader/Exception.php'; + throw new Zend_Loader_Exception("Invalid resource type $type; cannot load resource"); + } + if (empty($args)) { + require_once 'Zend/Loader/Exception.php'; + throw new Zend_Loader_Exception("Cannot load resources; no resource specified"); + } + $resource = array_shift($args); + return $this->load($resource, $type); + } + + require_once 'Zend/Loader/Exception.php'; + throw new Zend_Loader_Exception("Method '$method' is not supported"); + } + + /** + * Helper method to calculate the correct class path + * + * @param string $class + * @return False if not matched other wise the correct path + */ + public function getClassPath($class) + { + $segments = explode('_', $class); + $namespaceTopLevel = $this->getNamespace(); + $namespace = ''; + + if (!empty($namespaceTopLevel)) { + $namespace = array(); + $topLevelSegments = count(explode('_', $namespaceTopLevel)); + for ($i = 0; $i < $topLevelSegments; $i++) { + $namespace[] = array_shift($segments); + } + $namespace = implode('_', $namespace); + if ($namespace != $namespaceTopLevel) { + // wrong prefix? we're done + return false; + } + } + + if (count($segments) < 2) { + // assumes all resources have a component and class name, minimum + return false; + } + + $final = array_pop($segments); + $component = $namespace; + $lastMatch = false; + do { + $segment = array_shift($segments); + $component .= empty($component) ? $segment : '_' . $segment; + if (isset($this->_components[$component])) { + $lastMatch = $component; + } + } while (count($segments)); + + if (!$lastMatch) { + return false; + } + + $final = substr($class, strlen($lastMatch) + 1); + $path = $this->_components[$lastMatch]; + $classPath = $path . '/' . str_replace('_', '/', $final) . '.php'; + + if (Zend_Loader::isReadable($classPath)) { + return $classPath; + } + + return false; + } + + /** + * Attempt to autoload a class + * + * @param string $class + * @return mixed False if not matched, otherwise result if include operation + */ + public function autoload($class) + { + $classPath = $this->getClassPath($class); + if (false !== $classPath) { + return include $classPath; + } + return false; + } + + /** + * Set class state from options + * + * @param array $options + * @return Zend_Loader_Autoloader_Resource + */ + public function setOptions(array $options) + { + // Set namespace first, see ZF-10836 + if (isset($options['namespace'])) { + $this->setNamespace($options['namespace']); + unset($options['namespace']); + } + + $methods = get_class_methods($this); + foreach ($options as $key => $value) { + $method = 'set' . ucfirst($key); + if (in_array($method, $methods)) { + $this->$method($value); + } + } + return $this; + } + + /** + * Set namespace that this autoloader handles + * + * @param string $namespace + * @return Zend_Loader_Autoloader_Resource + */ + public function setNamespace($namespace) + { + $this->_namespace = rtrim((string) $namespace, '_'); + return $this; + } + + /** + * Get namespace this autoloader handles + * + * @return string + */ + public function getNamespace() + { + return $this->_namespace; + } + + /** + * Set base path for this set of resources + * + * @param string $path + * @return Zend_Loader_Autoloader_Resource + */ + public function setBasePath($path) + { + $this->_basePath = (string) $path; + return $this; + } + + /** + * Get base path to this set of resources + * + * @return string + */ + public function getBasePath() + { + return $this->_basePath; + } + + /** + * Add resource type + * + * @param string $type identifier for the resource type being loaded + * @param string $path path relative to resource base path containing the resource types + * @param null|string $namespace sub-component namespace to append to base namespace that qualifies this resource type + * @return Zend_Loader_Autoloader_Resource + */ + public function addResourceType($type, $path, $namespace = null) + { + $type = strtolower($type); + if (!isset($this->_resourceTypes[$type])) { + if (null === $namespace) { + require_once 'Zend/Loader/Exception.php'; + throw new Zend_Loader_Exception('Initial definition of a resource type must include a namespace'); + } + $namespaceTopLevel = $this->getNamespace(); + $namespace = ucfirst(trim($namespace, '_')); + $this->_resourceTypes[$type] = array( + 'namespace' => empty($namespaceTopLevel) ? $namespace : $namespaceTopLevel . '_' . $namespace, + ); + } + if (!is_string($path)) { + require_once 'Zend/Loader/Exception.php'; + throw new Zend_Loader_Exception('Invalid path specification provided; must be string'); + } + $this->_resourceTypes[$type]['path'] = $this->getBasePath() . '/' . rtrim($path, '\/'); + + $component = $this->_resourceTypes[$type]['namespace']; + $this->_components[$component] = $this->_resourceTypes[$type]['path']; + return $this; + } + + /** + * Add multiple resources at once + * + * $types should be an associative array of resource type => specification + * pairs. Each specification should be an associative array containing + * minimally the 'path' key (specifying the path relative to the resource + * base path) and optionally the 'namespace' key (indicating the subcomponent + * namespace to append to the resource namespace). + * + * As an example: + * + * $loader->addResourceTypes(array( + * 'model' => array( + * 'path' => 'models', + * 'namespace' => 'Model', + * ), + * 'form' => array( + * 'path' => 'forms', + * 'namespace' => 'Form', + * ), + * )); + * + * + * @param array $types + * @return Zend_Loader_Autoloader_Resource + */ + public function addResourceTypes(array $types) + { + foreach ($types as $type => $spec) { + if (!is_array($spec)) { + require_once 'Zend/Loader/Exception.php'; + throw new Zend_Loader_Exception('addResourceTypes() expects an array of arrays'); + } + if (!isset($spec['path'])) { + require_once 'Zend/Loader/Exception.php'; + throw new Zend_Loader_Exception('addResourceTypes() expects each array to include a paths element'); + } + $paths = $spec['path']; + $namespace = null; + if (isset($spec['namespace'])) { + $namespace = $spec['namespace']; + } + $this->addResourceType($type, $paths, $namespace); + } + return $this; + } + + /** + * Overwrite existing and set multiple resource types at once + * + * @see Zend_Loader_Autoloader_Resource::addResourceTypes() + * @param array $types + * @return Zend_Loader_Autoloader_Resource + */ + public function setResourceTypes(array $types) + { + $this->clearResourceTypes(); + return $this->addResourceTypes($types); + } + + /** + * Retrieve resource type mappings + * + * @return array + */ + public function getResourceTypes() + { + return $this->_resourceTypes; + } + + /** + * Is the requested resource type defined? + * + * @param string $type + * @return bool + */ + public function hasResourceType($type) + { + return isset($this->_resourceTypes[$type]); + } + + /** + * Remove the requested resource type + * + * @param string $type + * @return Zend_Loader_Autoloader_Resource + */ + public function removeResourceType($type) + { + if ($this->hasResourceType($type)) { + $namespace = $this->_resourceTypes[$type]['namespace']; + unset($this->_components[$namespace]); + unset($this->_resourceTypes[$type]); + } + return $this; + } + + /** + * Clear all resource types + * + * @return Zend_Loader_Autoloader_Resource + */ + public function clearResourceTypes() + { + $this->_resourceTypes = array(); + $this->_components = array(); + return $this; + } + + /** + * Set default resource type to use when calling load() + * + * @param string $type + * @return Zend_Loader_Autoloader_Resource + */ + public function setDefaultResourceType($type) + { + if ($this->hasResourceType($type)) { + $this->_defaultResourceType = $type; + } + return $this; + } + + /** + * Get default resource type to use when calling load() + * + * @return string|null + */ + public function getDefaultResourceType() + { + return $this->_defaultResourceType; + } + + /** + * Object registry and factory + * + * Loads the requested resource of type $type (or uses the default resource + * type if none provided). If the resource has been loaded previously, + * returns the previous instance; otherwise, instantiates it. + * + * @param string $resource + * @param string $type + * @return object + * @throws Zend_Loader_Exception if resource type not specified or invalid + */ + public function load($resource, $type = null) + { + if (null === $type) { + $type = $this->getDefaultResourceType(); + if (empty($type)) { + require_once 'Zend/Loader/Exception.php'; + throw new Zend_Loader_Exception('No resource type specified'); + } + } + if (!$this->hasResourceType($type)) { + require_once 'Zend/Loader/Exception.php'; + throw new Zend_Loader_Exception('Invalid resource type specified'); + } + $namespace = $this->_resourceTypes[$type]['namespace']; + $class = $namespace . '_' . ucfirst($resource); + if (!isset($this->_resources[$class])) { + $this->_resources[$class] = new $class; + } + return $this->_resources[$class]; + } +} diff --git a/3rdparty/Zend/Loader/AutoloaderFactory.php b/3rdparty/Zend/Loader/AutoloaderFactory.php new file mode 100644 index 000000000..3d872faed --- /dev/null +++ b/3rdparty/Zend/Loader/AutoloaderFactory.php @@ -0,0 +1,211 @@ + + * array( + * '' => $autoloaderOptions, + * ) + * + * + * The factory will then loop through and instantiate each autoloader with + * the specified options, and register each with the spl_autoloader. + * + * You may retrieve the concrete autoloader instances later using + * {@link getRegisteredAutoloaders()}. + * + * Note that the class names must be resolvable on the include_path or via + * the Zend library, using PSR-0 rules (unless the class has already been + * loaded). + * + * @param array|Traversable $options (optional) options to use. Defaults to Zend_Loader_StandardAutoloader + * @return void + * @throws Zend_Loader_Exception_InvalidArgumentException for invalid options + * @throws Zend_Loader_Exception_InvalidArgumentException for unloadable autoloader classes + */ + public static function factory($options = null) + { + if (null === $options) { + if (!isset(self::$loaders[self::STANDARD_AUTOLOADER])) { + $autoloader = self::getStandardAutoloader(); + $autoloader->register(); + self::$loaders[self::STANDARD_AUTOLOADER] = $autoloader; + } + + // Return so we don't hit the next check's exception (we're done here anyway) + return; + } + + if (!is_array($options) && !($options instanceof Traversable)) { + require_once 'Exception/InvalidArgumentException.php'; + throw new Zend_Loader_Exception_InvalidArgumentException( + 'Options provided must be an array or Traversable' + ); + } + + foreach ($options as $class => $options) { + if (!isset(self::$loaders[$class])) { + $autoloader = self::getStandardAutoloader(); + if (!class_exists($class) && !$autoloader->autoload($class)) { + require_once 'Exception/InvalidArgumentException.php'; + throw new Zend_Loader_Exception_InvalidArgumentException(sprintf( + 'Autoloader class "%s" not loaded', + $class + )); + } + + // unfortunately is_subclass_of is broken on some 5.3 versions + // additionally instanceof is also broken for this use case + if (version_compare(PHP_VERSION, '5.3.7', '>=')) { + if (!is_subclass_of($class, 'Zend_Loader_SplAutoloader')) { + require_once 'Exception/InvalidArgumentException.php'; + throw new Zend_Loader_Exception_InvalidArgumentException(sprintf( + 'Autoloader class %s must implement Zend\\Loader\\SplAutoloader', + $class + )); + } + } + + if ($class === self::STANDARD_AUTOLOADER) { + $autoloader->setOptions($options); + } else { + $autoloader = new $class($options); + } + $autoloader->register(); + self::$loaders[$class] = $autoloader; + } else { + self::$loaders[$class]->setOptions($options); + } + } + } + + /** + * Get an list of all autoloaders registered with the factory + * + * Returns an array of autoloader instances. + * + * @return array + */ + public static function getRegisteredAutoloaders() + { + return self::$loaders; + } + + /** + * Retrieves an autoloader by class name + * + * @param string $class + * @return Zend_Loader_SplAutoloader + * @throws Zend_Loader_Exception_InvalidArgumentException for non-registered class + */ + public static function getRegisteredAutoloader($class) + { + if (!isset(self::$loaders[$class])) { + require_once 'Exception/InvalidArgumentException.php'; + throw new Zend_Loader_Exception_InvalidArgumentException(sprintf('Autoloader class "%s" not loaded', $class)); + } + return self::$loaders[$class]; + } + + /** + * Unregisters all autoloaders that have been registered via the factory. + * This will NOT unregister autoloaders registered outside of the fctory. + * + * @return void + */ + public static function unregisterAutoloaders() + { + foreach (self::getRegisteredAutoloaders() as $class => $autoloader) { + spl_autoload_unregister(array($autoloader, 'autoload')); + unset(self::$loaders[$class]); + } + } + + /** + * Unregister a single autoloader by class name + * + * @param string $autoloaderClass + * @return bool + */ + public static function unregisterAutoloader($autoloaderClass) + { + if (!isset(self::$loaders[$autoloaderClass])) { + return false; + } + + $autoloader = self::$loaders[$autoloaderClass]; + spl_autoload_unregister(array($autoloader, 'autoload')); + unset(self::$loaders[$autoloaderClass]); + return true; + } + + /** + * Get an instance of the standard autoloader + * + * Used to attempt to resolve autoloader classes, using the + * StandardAutoloader. The instance is marked as a fallback autoloader, to + * allow resolving autoloaders not under the "Zend" or "Zend" namespaces. + * + * @return Zend_Loader_SplAutoloader + */ + protected static function getStandardAutoloader() + { + if (null !== self::$standardAutoloader) { + return self::$standardAutoloader; + } + + // Extract the filename from the classname + $stdAutoloader = substr(strrchr(self::STANDARD_AUTOLOADER, '_'), 1); + + if (!class_exists(self::STANDARD_AUTOLOADER)) { + require_once dirname(__FILE__) . "/$stdAutoloader.php"; + } + $loader = new Zend_Loader_StandardAutoloader(); + self::$standardAutoloader = $loader; + return self::$standardAutoloader; + } +} diff --git a/3rdparty/Zend/Loader/ClassMapAutoloader.php b/3rdparty/Zend/Loader/ClassMapAutoloader.php new file mode 100644 index 000000000..c1923ac5f --- /dev/null +++ b/3rdparty/Zend/Loader/ClassMapAutoloader.php @@ -0,0 +1,248 @@ +setOptions($options); + } + } + + /** + * Configure the autoloader + * + * Proxies to {@link registerAutoloadMaps()}. + * + * @param array|Traversable $options + * @return Zend_Loader_ClassMapAutoloader + */ + public function setOptions($options) + { + $this->registerAutoloadMaps($options); + return $this; + } + + /** + * Register an autoload map + * + * An autoload map may be either an associative array, or a file returning + * an associative array. + * + * An autoload map should be an associative array containing + * classname/file pairs. + * + * @param string|array $location + * @return Zend_Loader_ClassMapAutoloader + */ + public function registerAutoloadMap($map) + { + if (is_string($map)) { + $location = $map; + if ($this === ($map = $this->loadMapFromFile($location))) { + return $this; + } + } + + if (!is_array($map)) { + require_once dirname(__FILE__) . '/Exception/InvalidArgumentException.php'; + throw new Zend_Loader_Exception_InvalidArgumentException('Map file provided does not return a map'); + } + + $this->map = array_merge($this->map, $map); + + if (isset($location)) { + $this->mapsLoaded[] = $location; + } + + return $this; + } + + /** + * Register many autoload maps at once + * + * @param array $locations + * @return Zend_Loader_ClassMapAutoloader + */ + public function registerAutoloadMaps($locations) + { + if (!is_array($locations) && !($locations instanceof Traversable)) { + require_once dirname(__FILE__) . '/Exception/InvalidArgumentException.php'; + throw new Zend_Loader_Exception_InvalidArgumentException('Map list must be an array or implement Traversable'); + } + foreach ($locations as $location) { + $this->registerAutoloadMap($location); + } + return $this; + } + + /** + * Retrieve current autoload map + * + * @return array + */ + public function getAutoloadMap() + { + return $this->map; + } + + /** + * Defined by Autoloadable + * + * @param string $class + * @return void + */ + public function autoload($class) + { + if (isset($this->map[$class])) { + require_once $this->map[$class]; + } + } + + /** + * Register the autoloader with spl_autoload registry + * + * @return void + */ + public function register() + { + if (version_compare(PHP_VERSION, '5.3.0', '>=')) { + spl_autoload_register(array($this, 'autoload'), true, true); + } else { + spl_autoload_register(array($this, 'autoload'), true); + } + } + + /** + * Load a map from a file + * + * If the map has been previously loaded, returns the current instance; + * otherwise, returns whatever was returned by calling include() on the + * location. + * + * @param string $location + * @return Zend_Loader_ClassMapAutoloader|mixed + * @throws Zend_Loader_Exception_InvalidArgumentException for nonexistent locations + */ + protected function loadMapFromFile($location) + { + if (!file_exists($location)) { + require_once dirname(__FILE__) . '/Exception/InvalidArgumentException.php'; + throw new Zend_Loader_Exception_InvalidArgumentException('Map file provided does not exist'); + } + + if (!$path = self::realPharPath($location)) { + $path = realpath($location); + } + + if (in_array($path, $this->mapsLoaded)) { + // Already loaded this map + return $this; + } + + $map = include $path; + + return $map; + } + + /** + * Resolve the real_path() to a file within a phar. + * + * @see https://bugs.php.net/bug.php?id=52769 + * @param string $path + * @return string + */ + public static function realPharPath($path) + { + if (strpos($path, 'phar:///') !== 0) { + return; + } + + $parts = explode('/', str_replace(array('/','\\'), '/', substr($path, 8))); + $parts = array_values(array_filter($parts, array(__CLASS__, 'concatPharParts'))); + + array_walk($parts, array(__CLASS__, 'resolvePharParentPath'), $parts); + + if (file_exists($realPath = 'phar:///' . implode('/', $parts))) { + return $realPath; + } + } + + /** + * Helper callback for filtering phar paths + * + * @param string $part + * @return bool + */ + public static function concatPharParts($part) + { + return ($part !== '' && $part !== '.'); + } + + /** + * Helper callback to resolve a parent path in a Phar archive + * + * @param string $value + * @param int $key + * @param array $parts + * @return void + */ + public static function resolvePharParentPath($value, $key, &$parts) + { + if ($value !== '...') { + return; + } + unset($parts[$key], $parts[$key-1]); + $parts = array_values($parts); + } +} diff --git a/3rdparty/Zend/Loader/Exception.php b/3rdparty/Zend/Loader/Exception.php new file mode 100644 index 000000000..394788bcb --- /dev/null +++ b/3rdparty/Zend/Loader/Exception.php @@ -0,0 +1,35 @@ +_useStaticRegistry = $staticRegistryName; + if(!isset(self::$_staticPrefixToPaths[$staticRegistryName])) { + self::$_staticPrefixToPaths[$staticRegistryName] = array(); + } + if(!isset(self::$_staticLoadedPlugins[$staticRegistryName])) { + self::$_staticLoadedPlugins[$staticRegistryName] = array(); + } + } + + foreach ($prefixToPaths as $prefix => $path) { + $this->addPrefixPath($prefix, $path); + } + } + + /** + * Format prefix for internal use + * + * @param string $prefix + * @return string + */ + protected function _formatPrefix($prefix) + { + if($prefix == "") { + return $prefix; + } + + $nsSeparator = (false !== strpos($prefix, '\\'))?'\\':'_'; + $prefix = rtrim($prefix, $nsSeparator) . $nsSeparator; + //if $nsSeprator == "\" and the prefix ends in "_\" remove trailing \ + //https://github.com/zendframework/zf1/issues/152 + if(($nsSeparator == "\\") && (substr($prefix,-2) == "_\\")) { + $prefix = substr($prefix, 0, -1); + } + return $prefix; + } + + /** + * Add prefixed paths to the registry of paths + * + * @param string $prefix + * @param string $path + * @return Zend_Loader_PluginLoader + */ + public function addPrefixPath($prefix, $path) + { + if (!is_string($prefix) || !is_string($path)) { + require_once 'Zend/Loader/PluginLoader/Exception.php'; + throw new Zend_Loader_PluginLoader_Exception('Zend_Loader_PluginLoader::addPrefixPath() method only takes strings for prefix and path.'); + } + + $prefix = $this->_formatPrefix($prefix); + $path = rtrim($path, '/\\') . '/'; + + if ($this->_useStaticRegistry) { + self::$_staticPrefixToPaths[$this->_useStaticRegistry][$prefix][] = $path; + } else { + if (!isset($this->_prefixToPaths[$prefix])) { + $this->_prefixToPaths[$prefix] = array(); + } + if (!in_array($path, $this->_prefixToPaths[$prefix])) { + $this->_prefixToPaths[$prefix][] = $path; + } + } + return $this; + } + + /** + * Get path stack + * + * @param string $prefix + * @return false|array False if prefix does not exist, array otherwise + */ + public function getPaths($prefix = null) + { + if ((null !== $prefix) && is_string($prefix)) { + $prefix = $this->_formatPrefix($prefix); + if ($this->_useStaticRegistry) { + if (isset(self::$_staticPrefixToPaths[$this->_useStaticRegistry][$prefix])) { + return self::$_staticPrefixToPaths[$this->_useStaticRegistry][$prefix]; + } + + return false; + } + + if (isset($this->_prefixToPaths[$prefix])) { + return $this->_prefixToPaths[$prefix]; + } + + return false; + } + + if ($this->_useStaticRegistry) { + return self::$_staticPrefixToPaths[$this->_useStaticRegistry]; + } + + return $this->_prefixToPaths; + } + + /** + * Clear path stack + * + * @param string $prefix + * @return bool False only if $prefix does not exist + */ + public function clearPaths($prefix = null) + { + if ((null !== $prefix) && is_string($prefix)) { + $prefix = $this->_formatPrefix($prefix); + if ($this->_useStaticRegistry) { + if (isset(self::$_staticPrefixToPaths[$this->_useStaticRegistry][$prefix])) { + unset(self::$_staticPrefixToPaths[$this->_useStaticRegistry][$prefix]); + return true; + } + + return false; + } + + if (isset($this->_prefixToPaths[$prefix])) { + unset($this->_prefixToPaths[$prefix]); + return true; + } + + return false; + } + + if ($this->_useStaticRegistry) { + self::$_staticPrefixToPaths[$this->_useStaticRegistry] = array(); + } else { + $this->_prefixToPaths = array(); + } + + return true; + } + + /** + * Remove a prefix (or prefixed-path) from the registry + * + * @param string $prefix + * @param string $path OPTIONAL + * @return Zend_Loader_PluginLoader + */ + public function removePrefixPath($prefix, $path = null) + { + $prefix = $this->_formatPrefix($prefix); + if ($this->_useStaticRegistry) { + $registry =& self::$_staticPrefixToPaths[$this->_useStaticRegistry]; + } else { + $registry =& $this->_prefixToPaths; + } + + if (!isset($registry[$prefix])) { + require_once 'Zend/Loader/PluginLoader/Exception.php'; + throw new Zend_Loader_PluginLoader_Exception('Prefix ' . $prefix . ' was not found in the PluginLoader.'); + } + + if ($path != null) { + $pos = array_search($path, $registry[$prefix]); + if (false === $pos) { + require_once 'Zend/Loader/PluginLoader/Exception.php'; + throw new Zend_Loader_PluginLoader_Exception('Prefix ' . $prefix . ' / Path ' . $path . ' was not found in the PluginLoader.'); + } + unset($registry[$prefix][$pos]); + } else { + unset($registry[$prefix]); + } + + return $this; + } + + /** + * Normalize plugin name + * + * @param string $name + * @return string + */ + protected function _formatName($name) + { + return ucfirst((string) $name); + } + + /** + * Whether or not a Plugin by a specific name is loaded + * + * @param string $name + * @return Zend_Loader_PluginLoader + */ + public function isLoaded($name) + { + $name = $this->_formatName($name); + if ($this->_useStaticRegistry) { + return isset(self::$_staticLoadedPlugins[$this->_useStaticRegistry][$name]); + } + + return isset($this->_loadedPlugins[$name]); + } + + /** + * Return full class name for a named plugin + * + * @param string $name + * @return string|false False if class not found, class name otherwise + */ + public function getClassName($name) + { + $name = $this->_formatName($name); + if ($this->_useStaticRegistry + && isset(self::$_staticLoadedPlugins[$this->_useStaticRegistry][$name]) + ) { + return self::$_staticLoadedPlugins[$this->_useStaticRegistry][$name]; + } elseif (isset($this->_loadedPlugins[$name])) { + return $this->_loadedPlugins[$name]; + } + + return false; + } + + /** + * Get path to plugin class + * + * @param mixed $name + * @return string|false False if not found + */ + public function getClassPath($name) + { + $name = $this->_formatName($name); + if ($this->_useStaticRegistry + && !empty(self::$_staticLoadedPluginPaths[$this->_useStaticRegistry][$name]) + ) { + return self::$_staticLoadedPluginPaths[$this->_useStaticRegistry][$name]; + } elseif (!empty($this->_loadedPluginPaths[$name])) { + return $this->_loadedPluginPaths[$name]; + } + + if ($this->isLoaded($name)) { + $class = $this->getClassName($name); + $r = new ReflectionClass($class); + $path = $r->getFileName(); + if ($this->_useStaticRegistry) { + self::$_staticLoadedPluginPaths[$this->_useStaticRegistry][$name] = $path; + } else { + $this->_loadedPluginPaths[$name] = $path; + } + return $path; + } + + return false; + } + + /** + * Load a plugin via the name provided + * + * @param string $name + * @param bool $throwExceptions Whether or not to throw exceptions if the + * class is not resolved + * @return string|false Class name of loaded class; false if $throwExceptions + * if false and no class found + * @throws Zend_Loader_Exception if class not found + */ + public function load($name, $throwExceptions = true) + { + $name = $this->_formatName($name); + if ($this->isLoaded($name)) { + return $this->getClassName($name); + } + + if ($this->_useStaticRegistry) { + $registry = self::$_staticPrefixToPaths[$this->_useStaticRegistry]; + } else { + $registry = $this->_prefixToPaths; + } + + $registry = array_reverse($registry, true); + $found = false; + if (false !== strpos($name, '\\')) { + $classFile = str_replace('\\', DIRECTORY_SEPARATOR, $name) . '.php'; + } else { + $classFile = str_replace('_', DIRECTORY_SEPARATOR, $name) . '.php'; + } + $incFile = self::getIncludeFileCache(); + foreach ($registry as $prefix => $paths) { + $className = $prefix . $name; + + if (class_exists($className, false)) { + $found = true; + break; + } + + $paths = array_reverse($paths, true); + + foreach ($paths as $path) { + $loadFile = $path . $classFile; + if (Zend_Loader::isReadable($loadFile)) { + include_once $loadFile; + if (class_exists($className, false)) { + if (null !== $incFile) { + self::_appendIncFile($loadFile); + } + $found = true; + break 2; + } + } + } + } + + if (!$found) { + if (!$throwExceptions) { + return false; + } + + $message = "Plugin by name '$name' was not found in the registry; used paths:"; + foreach ($registry as $prefix => $paths) { + $message .= "\n$prefix: " . implode(PATH_SEPARATOR, $paths); + } + require_once 'Zend/Loader/PluginLoader/Exception.php'; + throw new Zend_Loader_PluginLoader_Exception($message); + } + + if ($this->_useStaticRegistry) { + self::$_staticLoadedPlugins[$this->_useStaticRegistry][$name] = $className; + } else { + $this->_loadedPlugins[$name] = $className; + } + return $className; + } + + /** + * Set path to class file cache + * + * Specify a path to a file that will add include_once statements for each + * plugin class loaded. This is an opt-in feature for performance purposes. + * + * @param string $file + * @return void + * @throws Zend_Loader_PluginLoader_Exception if file is not writeable or path does not exist + */ + public static function setIncludeFileCache($file) + { + if (!empty(self::$_includeFileCacheHandler)) { + flock(self::$_includeFileCacheHandler, LOCK_UN); + fclose(self::$_includeFileCacheHandler); + } + + self::$_includeFileCacheHandler = null; + + if (null === $file) { + self::$_includeFileCache = null; + return; + } + + if (!file_exists($file) && !file_exists(dirname($file))) { + require_once 'Zend/Loader/PluginLoader/Exception.php'; + throw new Zend_Loader_PluginLoader_Exception('Specified file does not exist and/or directory does not exist (' . $file . ')'); + } + if (file_exists($file) && !is_writable($file)) { + require_once 'Zend/Loader/PluginLoader/Exception.php'; + throw new Zend_Loader_PluginLoader_Exception('Specified file is not writeable (' . $file . ')'); + } + if (!file_exists($file) && file_exists(dirname($file)) && !is_writable(dirname($file))) { + require_once 'Zend/Loader/PluginLoader/Exception.php'; + throw new Zend_Loader_PluginLoader_Exception('Specified file is not writeable (' . $file . ')'); + } + + self::$_includeFileCache = $file; + } + + /** + * Retrieve class file cache path + * + * @return string|null + */ + public static function getIncludeFileCache() + { + return self::$_includeFileCache; + } + + /** + * Append an include_once statement to the class file cache + * + * @param string $incFile + * @return void + */ + protected static function _appendIncFile($incFile) + { + if (!isset(self::$_includeFileCacheHandler)) { + self::$_includeFileCacheHandler = fopen(self::$_includeFileCache, 'ab'); + + if (!flock(self::$_includeFileCacheHandler, LOCK_EX | LOCK_NB, $wouldBlock) || $wouldBlock) { + self::$_includeFileCacheHandler = false; + } + } + + if (false !== self::$_includeFileCacheHandler) { + $line = "\n"; + fwrite(self::$_includeFileCacheHandler, $line, strlen($line)); + } + } +} diff --git a/3rdparty/Zend/Loader/PluginLoader/Exception.php b/3rdparty/Zend/Loader/PluginLoader/Exception.php new file mode 100644 index 000000000..50e9d8c2f --- /dev/null +++ b/3rdparty/Zend/Loader/PluginLoader/Exception.php @@ -0,0 +1,39 @@ + + * spl_autoload_register(array($this, 'autoload')); + * + * + * @return void + */ + public function register(); +} diff --git a/3rdparty/Zend/Loader/StandardAutoloader.php b/3rdparty/Zend/Loader/StandardAutoloader.php new file mode 100644 index 000000000..31a5c1e45 --- /dev/null +++ b/3rdparty/Zend/Loader/StandardAutoloader.php @@ -0,0 +1,368 @@ +setOptions($options); + } + } + + /** + * Configure autoloader + * + * Allows specifying both "namespace" and "prefix" pairs, using the + * following structure: + * + * array( + * 'namespaces' => array( + * 'Zend' => '/path/to/Zend/library', + * 'Doctrine' => '/path/to/Doctrine/library', + * ), + * 'prefixes' => array( + * 'Phly_' => '/path/to/Phly/library', + * ), + * 'fallback_autoloader' => true, + * ) + * + * + * @param array|Traversable $options + * @return Zend_Loader_StandardAutoloader + */ + public function setOptions($options) + { + if (!is_array($options) && !($options instanceof Traversable)) { + require_once dirname(__FILE__) . '/Exception/InvalidArgumentException.php'; + throw new Zend_Loader_Exception_InvalidArgumentException('Options must be either an array or Traversable'); + } + + foreach ($options as $type => $pairs) { + switch ($type) { + case self::AUTOREGISTER_ZF: + if ($pairs) { + $this->registerPrefix('Zend', dirname(dirname(__FILE__))); + } + break; + case self::LOAD_NS: + if (is_array($pairs) || $pairs instanceof Traversable) { + $this->registerNamespaces($pairs); + } + break; + case self::LOAD_PREFIX: + if (is_array($pairs) || $pairs instanceof Traversable) { + $this->registerPrefixes($pairs); + } + break; + case self::ACT_AS_FALLBACK: + $this->setFallbackAutoloader($pairs); + break; + default: + // ignore + } + } + return $this; + } + + /** + * Set flag indicating fallback autoloader status + * + * @param bool $flag + * @return Zend_Loader_StandardAutoloader + */ + public function setFallbackAutoloader($flag) + { + $this->fallbackAutoloaderFlag = (bool) $flag; + return $this; + } + + /** + * Is this autoloader acting as a fallback autoloader? + * + * @return bool + */ + public function isFallbackAutoloader() + { + return $this->fallbackAutoloaderFlag; + } + + /** + * Register a namespace/directory pair + * + * @param string $namespace + * @param string $directory + * @return Zend_Loader_StandardAutoloader + */ + public function registerNamespace($namespace, $directory) + { + $namespace = rtrim($namespace, self::NS_SEPARATOR). self::NS_SEPARATOR; + $this->namespaces[$namespace] = $this->normalizeDirectory($directory); + return $this; + } + + /** + * Register many namespace/directory pairs at once + * + * @param array $namespaces + * @return Zend_Loader_StandardAutoloader + */ + public function registerNamespaces($namespaces) + { + if (!is_array($namespaces) && !$namespaces instanceof Traversable) { + require_once dirname(__FILE__) . '/Exception/InvalidArgumentException.php'; + throw new Zend_Loader_Exception_InvalidArgumentException('Namespace pairs must be either an array or Traversable'); + } + + foreach ($namespaces as $namespace => $directory) { + $this->registerNamespace($namespace, $directory); + } + return $this; + } + + /** + * Register a prefix/directory pair + * + * @param string $prefix + * @param string $directory + * @return Zend_Loader_StandardAutoloader + */ + public function registerPrefix($prefix, $directory) + { + $prefix = rtrim($prefix, self::PREFIX_SEPARATOR). self::PREFIX_SEPARATOR; + $this->prefixes[$prefix] = $this->normalizeDirectory($directory); + return $this; + } + + /** + * Register many namespace/directory pairs at once + * + * @param array $prefixes + * @return Zend_Loader_StandardAutoloader + */ + public function registerPrefixes($prefixes) + { + if (!is_array($prefixes) && !$prefixes instanceof Traversable) { + require_once dirname(__FILE__) . '/Exception/InvalidArgumentException.php'; + throw new Zend_Loader_Exception_InvalidArgumentException('Prefix pairs must be either an array or Traversable'); + } + + foreach ($prefixes as $prefix => $directory) { + $this->registerPrefix($prefix, $directory); + } + return $this; + } + + /** + * Defined by Autoloadable; autoload a class + * + * @param string $class + * @return false|string + */ + public function autoload($class) + { + $isFallback = $this->isFallbackAutoloader(); + if (false !== strpos($class, self::NS_SEPARATOR)) { + if ($this->loadClass($class, self::LOAD_NS)) { + return $class; + } elseif ($isFallback) { + return $this->loadClass($class, self::ACT_AS_FALLBACK); + } + return false; + } + if (false !== strpos($class, self::PREFIX_SEPARATOR)) { + if ($this->loadClass($class, self::LOAD_PREFIX)) { + return $class; + } elseif ($isFallback) { + return $this->loadClass($class, self::ACT_AS_FALLBACK); + } + return false; + } + if ($isFallback) { + return $this->loadClass($class, self::ACT_AS_FALLBACK); + } + return false; + } + + /** + * Register the autoloader with spl_autoload + * + * @return void + */ + public function register() + { + spl_autoload_register(array($this, 'autoload')); + } + + /** + * Error handler + * + * Used by {@link loadClass} during fallback autoloading in PHP versions + * prior to 5.3.0. + * + * @param mixed $errno + * @param mixed $errstr + * @return void + */ + public function handleError($errno, $errstr) + { + $this->error = true; + } + + /** + * Transform the class name to a filename + * + * @param string $class + * @param string $directory + * @return string + */ + protected function transformClassNameToFilename($class, $directory) + { + // $class may contain a namespace portion, in which case we need + // to preserve any underscores in that portion. + $matches = array(); + preg_match('/(?P.+\\\)?(?P[^\\\]+$)/', $class, $matches); + + $class = (isset($matches['class'])) ? $matches['class'] : ''; + $namespace = (isset($matches['namespace'])) ? $matches['namespace'] : ''; + + return $directory + . str_replace(self::NS_SEPARATOR, '/', $namespace) + . str_replace(self::PREFIX_SEPARATOR, '/', $class) + . '.php'; + } + + /** + * Load a class, based on its type (namespaced or prefixed) + * + * @param string $class + * @param string $type + * @return void + */ + protected function loadClass($class, $type) + { + if (!in_array($type, array(self::LOAD_NS, self::LOAD_PREFIX, self::ACT_AS_FALLBACK))) { + require_once dirname(__FILE__) . '/Exception/InvalidArgumentException.php'; + throw new Zend_Loader_Exception_InvalidArgumentException(); + } + + // Fallback autoloading + if ($type === self::ACT_AS_FALLBACK) { + // create filename + $filename = $this->transformClassNameToFilename($class, ''); + if (version_compare(PHP_VERSION, '5.3.2', '>=')) { + $resolvedName = stream_resolve_include_path($filename); + if ($resolvedName !== false) { + return include $resolvedName; + } + return false; + } + $this->error = false; + set_error_handler(array($this, 'handleError'), E_WARNING); + include $filename; + restore_error_handler(); + if ($this->error) { + return false; + } + return class_exists($class, false); + } + + // Namespace and/or prefix autoloading + foreach ($this->$type as $leader => $path) { + if (0 === strpos($class, $leader)) { + // Trim off leader (namespace or prefix) + $trimmedClass = substr($class, strlen($leader)); + + // create filename + $filename = $this->transformClassNameToFilename($trimmedClass, $path); + if (file_exists($filename)) { + return include $filename; + } + return false; + } + } + return false; + } + + /** + * Normalize the directory to include a trailing directory separator + * + * @param string $directory + * @return string + */ + protected function normalizeDirectory($directory) + { + $last = $directory[strlen($directory) - 1]; + if (in_array($last, array('/', '\\'))) { + $directory[strlen($directory) - 1] = DIRECTORY_SEPARATOR; + return $directory; + } + $directory .= DIRECTORY_SEPARATOR; + return $directory; + } + +} diff --git a/COPYING b/COPYING new file mode 100644 index 000000000..2def0e883 --- /dev/null +++ b/COPYING @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. \ No newline at end of file diff --git a/ajax/lucene.php b/ajax/lucene.php deleted file mode 100644 index 95417daa8..000000000 --- a/ajax/lucene.php +++ /dev/null @@ -1,44 +0,0 @@ -send('count', count($fileIds)); - - $folder = OCA\Search_Lucene\Util::setUpUserFolder(); - $lucene = new OCA\Search_Lucene\Lucene(); - - $indexer = new OCA\Search_Lucene\Indexer($folder, $lucene); - $indexer->indexFiles($fileIds, $eventSource); - - $eventSource->send('done', ''); - $eventSource->close(); -} - -function handleOptimize() { - OCA\Search_Lucene\Lucene::optimizeIndex(); -} - -if ($_GET['operation']) { - switch ($_GET['operation']) { - case 'index': - index(); - break; - case 'optimize': - handleOptimize(); - break; - default: - OCP\JSON::error(array('cause' => 'Unknown operation')); - } -} diff --git a/appinfo/app.php b/appinfo/app.php index 3a4755f5e..c99c33e96 100644 --- a/appinfo/app.php +++ b/appinfo/app.php @@ -1,51 +1,13 @@ -* @copyright 2012 Jörn Dreyer -* @license http://www.gnu.org/licenses/agpl-3.0 GNU Affero General Public License (AGPL) 3.0 -* -* This library is free software; you can redistribute it and/or -* modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE -* License as published by the Free Software Foundation; either -* version 3 of the License, or any later version. -* -* This library is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU AFFERO GENERAL PUBLIC LICENSE for more details. -* -* You should have received a copy of the GNU Lesser General Public -* License along with this library. If not, see . -* -*/ - -// --- Register classes ----------------------------------------------- - -//add 3rdparty folder to include path -$dir = dirname(dirname(__FILE__)).'/3rdparty'; -set_include_path(get_include_path() . PATH_SEPARATOR . $dir); - -OC::$CLASSPATH['Zend_Search_Lucene'] = 'search_lucene/3rdparty/Zend/Search/Lucene.php'; -OC::$CLASSPATH['Zend_Search_Lucene_Index_Term'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Index/Term.php'; -OC::$CLASSPATH['Zend_Search_Lucene_Search_Query_Term'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Search/Query/Term.php'; -OC::$CLASSPATH['Zend_Search_Lucene_Field'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Field.php'; -OC::$CLASSPATH['Zend_Search_Lucene_Document'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Document.php'; -OC::$CLASSPATH['Zend_Search_Lucene_Document_Html'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Document/Html.php'; -OC::$CLASSPATH['Zend_Search_Lucene_Document_Docx'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Document/Docx.php'; -OC::$CLASSPATH['Zend_Search_Lucene_Document_Xlsx'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Document/Xlsx.php'; -OC::$CLASSPATH['Zend_Search_Lucene_Document_Pptx'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Document/Pptx.php'; -OC::$CLASSPATH['Zend_Search_Lucene_Analysis_Analyzer'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Analysis/Analyzer.php'; -OC::$CLASSPATH['Zend_Search_Lucene_Interface'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Interface.php'; -OC::$CLASSPATH['Zend_Search_Lucene_Search_QueryHit'] = 'search_lucene/3rdparty/Zend/Search/Lucene/Search/QueryHit.php'; - -OC::$CLASSPATH['getID3'] = 'getid3/getid3.php'; - -OC::$CLASSPATH['App_Search_Helper_PdfParser'] = 'search_lucene/3rdparty/pdf2text.php'; - -OC::$CLASSPATH['Zend_Pdf'] = 'search_lucene/3rdparty/Zend/Pdf.php'; + * ownCloud - search_lucene + * + * This file is licensed under the Affero General Public License version 3 or + * later. See the COPYING file. + * + * @author Jörn Friedrich Dreyer + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ // --- always add js & css ----------------------------------------------- @@ -55,14 +17,13 @@ // --- replace default file search provider ----------------------------------------------- //remove other providers -\OC::$server->getSearch()->removeProvider('OC_Search_Provider_File'); -\OC::$server->getSearch()->registerProvider('OCA\Search_Lucene\SearchProvider'); +\OC::$server->getSearch()->registerProvider('OCA\Search_Lucene\Search\LuceneProvider'); // add background job for index optimization: $arguments = array('user' => \OCP\User::getUser()); //Add Background Job: -\OCP\BackgroundJob::registerJob( '\OCA\Search_Lucene\OptimizeJob', $arguments ); +\OCP\BackgroundJob::registerJob( 'OCA\Search_Lucene\Jobs\OptimizeJob', $arguments ); // --- add hooks ----------------------------------------------- @@ -72,19 +33,19 @@ OCP\Util::connectHook( OC\Files\Filesystem::CLASSNAME, OC\Files\Filesystem::signal_post_write, - 'OCA\Search_Lucene\Hooks', - OCA\Search_Lucene\Hooks::handle_post_write); + 'OCA\Search_Lucene\Hooks\Files', + OCA\Search_Lucene\Hooks\Files::handle_post_write); //connect to the filesystem for renaming OCP\Util::connectHook( OC\Files\Filesystem::CLASSNAME, OC\Files\Filesystem::signal_post_rename, - 'OCA\Search_Lucene\Hooks', - OCA\Search_Lucene\Hooks::handle_post_rename); + 'OCA\Search_Lucene\Hooks\Files', + OCA\Search_Lucene\Hooks\Files::handle_post_rename); //listen for file deletions to clean the database OCP\Util::connectHook( OC\Files\Filesystem::CLASSNAME, 'post_delete', //FIXME add referenceable constant in core - 'OCA\Search_Lucene\Hooks', - OCA\Search_Lucene\Hooks::handle_delete); + 'OCA\Search_Lucene\Hooks\Files', + OCA\Search_Lucene\Hooks\Files::handle_delete); diff --git a/appinfo/application.php b/appinfo/application.php new file mode 100644 index 000000000..ec7018db4 --- /dev/null +++ b/appinfo/application.php @@ -0,0 +1,100 @@ + + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ + +namespace OCA\Search_Lucene\AppInfo; + +use OCA\Search_Lucene\Controller\ApiController; +use OCA\Search_Lucene\Core\Db; +use OCA\Search_Lucene\Core\Logger; +use OCA\Search_Lucene\Db\StatusMapper; +use OCA\Search_Lucene\Lucene\Index; +use OCA\Search_Lucene\Lucene\Indexer; +use OCA\Search_Lucene\Core\Files; +use OCP\AppFramework\App; + +//add 3rdparty folder to include path +set_include_path(get_include_path() . PATH_SEPARATOR . __DIR__.'/../3rdparty'); + +class Application extends App { + + public function __construct (array $urlParams=array()) { + parent::__construct('search_lucene', $urlParams); + + $container = $this->getContainer(); + + require_once __DIR__ . '/../3rdparty/Zend/Loader/Autoloader.php'; + \Zend_Loader_Autoloader::getInstance(); + + /** + * Controller + */ + $container->registerService('ApiController', function($c) { + return new ApiController( + $c->query('AppName'), + $c->query('Request'), + $c->query('StatusMapper'), + $c->query('Index'), + $c->query('Indexer') + ); + }); + + $container->registerService('Index', function($c) { + $index = new Index( + $c->query('FileUtility'), + $c->query('Logger') + ); + $index->openOrCreate(); + + return $index; + }); + + $container->registerService('Indexer', function($c) { + return new Indexer( + $c->query('FileUtility'), + $c->query('Index'), + $c->query('StatusMapper'), + $c->query('Logger') + ); + }); + + /** + * Mappers + */ + $container->registerService('StatusMapper', function($c) { + return new StatusMapper( + $c->query('Db'), + $c->query('Logger') + ); + }); + + /** + * Core + */ + $container->registerService('UserId', function() { + return \OCP\User::getUser(); + }); + + $container->registerService('Logger', function($c) { + return new Logger($c->query('AppName')); + }); + + $container->registerService('Db', function() { + return new Db(); + }); + + $container->registerService('FileUtility', function($c) { + return new Files($c->query('UserId')); + }); + + } + + +} \ No newline at end of file diff --git a/appinfo/classpath.php b/appinfo/classpath.php new file mode 100644 index 000000000..be46a4b08 --- /dev/null +++ b/appinfo/classpath.php @@ -0,0 +1,14 @@ + + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ + +OC::$CLASSPATH['getID3'] = 'getid3/getid3.php'; + +OC::$CLASSPATH['App_Search_Helper_PdfParser'] = 'search_lucene/3rdparty/pdf2text.php'; diff --git a/appinfo/preupdate.php b/appinfo/preupdate.php index 20e5f307b..177f22033 100644 --- a/appinfo/preupdate.php +++ b/appinfo/preupdate.php @@ -1,4 +1,13 @@ + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ $currentVersion=OCP\Config::getAppValue('search_lucene', 'installed_version'); diff --git a/appinfo/routes.php b/appinfo/routes.php new file mode 100644 index 000000000..bd39e7450 --- /dev/null +++ b/appinfo/routes.php @@ -0,0 +1,27 @@ + + * @copyright Jörn Friedrich Dreyer 2014 + */ + +namespace OCA\Search_Lucene\AppInfo; + +/** + * Create your routes in here. The name is the lowercase name of the controller + * without the controller part, the stuff after the hash is the method. + * e.g. page#index -> PageController->index() + * + * The controller class has to be registered in the application.php file since + * it's instantiated in there + */ +$application = new Application(); + +$application->registerRoutes($this, array('routes' => array( + array('name' => 'api#index', 'url' => '/indexer/index', 'verb' => 'GET'), + array('name' => 'api#optimize', 'url' => '/indexer/optimize', 'verb' => 'POST'), +))); diff --git a/appinfo/update.php b/appinfo/update.php index ac691cfef..9488b0835 100644 --- a/appinfo/update.php +++ b/appinfo/update.php @@ -1,6 +1,15 @@ + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ -$currentVersion=OCP\Config::getAppValue('search_lucene', 'installed_version'); +$currentVersion = OCP\Config::getAppValue('search_lucene', 'installed_version'); if (version_compare($currentVersion, '0.5.0', '<')) { //clear old background jobs diff --git a/controller/apicontroller.php b/controller/apicontroller.php new file mode 100644 index 000000000..8788800f7 --- /dev/null +++ b/controller/apicontroller.php @@ -0,0 +1,69 @@ + + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ + +namespace OCA\Search_Lucene\Controller; + + +use OCA\Search_Lucene\Db\StatusMapper; +use OCA\Search_Lucene\Http\EventSourceResponse; +use OCA\Search_Lucene\Lucene\Index; +use OCA\Search_Lucene\Lucene\Indexer; +use \OCP\IRequest; +use \OCP\AppFramework\Controller; + +class ApiController extends Controller { + + private $mapper; + private $index; + private $indexer; + + public function __construct($appName, IRequest $request, StatusMapper $mapper, Index $index, Indexer $indexer) { + parent::__construct($appName, $request); + $this->mapper = $mapper; + $this->index = $index; + $this->indexer = $indexer; + } + + + /** + * index the given fileIds or, if not given, all unindexed files + * @NoAdminRequired + */ + public function index($fileId) { + if ( isset($fileId) ){ + $fileIds = array($fileId); + } else { + $fileIds = $this->mapper->getUnindexed(); + } + + $eventSource = new \OC_EventSource(); + $eventSource->send('count', count($fileIds)); + + $this->indexer->indexFiles($fileIds, $eventSource); + + $eventSource->send('done', ''); + $eventSource->close(); + + // end script execution to prevent app framework from sending headers after + // the eventsource is closed + exit(); + } + + + /** + * Optimize the index + * @NoAdminRequired + */ + public function optimize() { + $this->index->optimizeIndex(); + } + +} \ No newline at end of file diff --git a/core/db.php b/core/db.php new file mode 100644 index 000000000..3ce3560c5 --- /dev/null +++ b/core/db.php @@ -0,0 +1,40 @@ + + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ + +namespace OCA\Search_Lucene\Core; + +use OCP\IDb; + +class Db implements IDb { + + /** + * Used to abstract the owncloud database access away + * @param string $sql the sql query with ? placeholder for params + * @param int $limit the maximum number of rows + * @param int $offset from which row we want to start + * @return \OCP\DB a query object + */ + public function prepareQuery($sql, $limit=null, $offset=null){ + return \OCP\DB::prepare($sql, $limit, $offset); + } + + + /** + * Used to get the id of the just inserted element + * @param string $tableName the name of the table where we inserted the item + * @return int the id of the inserted element + */ + public function getInsertId($tableName){ + return \OCP\DB::insertid($tableName); + } + + +} diff --git a/lib/util.php b/core/files.php similarity index 64% rename from lib/util.php rename to core/files.php index 0af89c70f..978962706 100644 --- a/lib/util.php +++ b/core/files.php @@ -1,9 +1,23 @@ + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ -namespace OCA\Search_Lucene; +namespace OCA\Search_Lucene\Core; -class Util { +class Files { + private $userId; + + public function __construct($userId){ + $this->userId = $userId; + } /** * Returns a folder for the users 'files' folder * Warning, this will tear down the current filesystem @@ -11,8 +25,8 @@ class Util { * @param string $user the user id * @return \OCP\Files\Folder */ - static function setUpUserFolder($user = null) { - $userHome = self::setUpUserHome($user); + public function setUpUserFolder($user = null) { + $userHome = $this->setUpUserHome($user); $dir = 'files'; $folder = null; @@ -29,8 +43,8 @@ static function setUpUserFolder($user = null) { /** * @return null|\OCP\Files\Folder */ - static function setUpIndexFolder($user = null) { - $userHome = self::setUpUserHome($user); + public function setUpIndexFolder($user = null) { + $userHome = $this->setUpUserHome($user); // TODO profile: encrypt the index on logout, decrypt on login //return OCP\Files::getStorage('search_lucene'); // FIXME \OC::$server->getAppFolder() returns '/search' @@ -50,16 +64,17 @@ static function setUpIndexFolder($user = null) { /** * @return null|\OCP\Files\Folder */ - static function setUpUserHome($user = null) { + public function setUpUserHome($user = null) { if (is_null($user)) { - $user = \OCP\User::getUser(); + $user = $this->userId; } if (!\OCP\User::userExists($user)) { return null; } - if ($user !== \OCP\User::getUser()) { + if ($user !== $this->userId) { \OC_Util::tearDownFS(); \OC_User::setUserId($user); + $this->userId = $user; } \OC_Util::setupFS($user); diff --git a/core/logger.php b/core/logger.php new file mode 100644 index 000000000..4026be53c --- /dev/null +++ b/core/logger.php @@ -0,0 +1,49 @@ + + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ + +namespace OCA\Search_Lucene\Core; + +class Logger { + + protected $appName; + + public function __construct($appName) { + $this->appName = $appName; + } + + /** + * Writes a function into the error log + * @param string $msg the error message to be logged + * @param int $level the error level + */ + public function log($msg, $level=null){ + switch($level){ + case 'debug': + $level = \OCP\Util::DEBUG; + break; + case 'info': + $level = \OCP\Util::INFO; + break; + case 'warn': + $level = \OCP\Util::WARN; + break; + case 'fatal': + $level = \OCP\Util::FATAL; + break; + default: + $level = \OCP\Util::ERROR; + break; + } + \OCP\Util::writeLog($this->appName, $msg, $level); + } + +} + diff --git a/db/status.php b/db/status.php new file mode 100644 index 000000000..a62ac9295 --- /dev/null +++ b/db/status.php @@ -0,0 +1,98 @@ + + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ + +namespace OCA\Search_Lucene\Db; + +use OCP\AppFramework\Db\Entity; + +/** + * @method integer getFileId() + * @method void setFileId(integer $fileId) + * @method string getStatus() + * @method setStatus(string $status) + */ +class Status extends Entity { + + const STATUS_NEW = 'N'; + const STATUS_INDEXED = 'I'; + const STATUS_SKIPPED = 'S'; + const STATUS_UNINDEXED = 'U'; + const STATUS_ERROR = 'E'; + + public $fileId; + public $status; + + // we use fileid as the primary key + private $_fieldTypes = array('fileId' => 'integer'); + + /** + * @return an array with attribute and type + */ + public function getFieldTypes() { + return $this->_fieldTypes; + } + + /** + * Adds type information for a field so that its automatically casted to + * that value once its being returned from the database + * @param string $fieldName the name of the attribute + * @param string $type the type which will be used to call settype() + */ + protected function addType($fieldName, $type){ + $this->_fieldTypes[$fieldName] = $type; + } + + // we need to overwrite the setter because it would otherwise use _fieldTypes of the Entity class + protected function setter($name, $args) { + // setters should only work for existing attributes + if(property_exists($this, $name)){ + if($this->$name === $args[0]) { + return; + } + $this->markFieldUpdated($name); + + // if type definition exists, cast to correct type + if($args[0] !== null && array_key_exists($name, $this->_fieldTypes)) { + settype($args[0], $this->_fieldTypes[$name]); + } + $this->$name = $args[0]; + + } else { + throw new \BadFunctionCallException($name . + ' is not a valid attribute'); + } + } + + /** + * Transform a database columnname to a property + * @param string $columnName the name of the column + * @return string the property name + */ + public function columnToProperty($columnName) { + if ($columnName === 'fileid') { + $property = 'fileId'; + } else { + $property = parent::columnToProperty($columnName); + } + return $property; + } + + /** + * Transform a property to a database column name + * for search_lucene we don't magically insert a _ for CamelCase + * @param string $property the name of the property + * @return string the column name + */ + public function propertyToColumn($property){ + $column = strtolower($property); + return $column; + } +} diff --git a/db/statusmapper.php b/db/statusmapper.php new file mode 100644 index 000000000..2056cf638 --- /dev/null +++ b/db/statusmapper.php @@ -0,0 +1,264 @@ + + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ + +namespace OCA\Search_Lucene\Db; + +use OC\Files\Filesystem; +use OCA\Search_Lucene\Core\Db; +use OCA\Search_Lucene\Core\Logger; +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\AppFramework\Db\Entity; +use OCP\AppFramework\Db\Mapper; + +/** + * @author Jörn Dreyer + */ +class StatusMapper extends Mapper { + + const STATUS_NEW = 'N'; + const STATUS_INDEXED = 'I'; + const STATUS_SKIPPED = 'S'; + const STATUS_UNINDEXED = 'U'; + const STATUS_ERROR = 'E'; + + private $logger; + + public function __construct(Db $db, Logger $logger){ + parent::__construct($db, 'lucene_status', '\OCA\Search_Lucene\Db\Status'); + $this->logger = $logger; + } + + + /** + * Deletes a status from the table + * @param Entity $entity the status that should be deleted + */ + public function delete(Entity $entity){ + $sql = 'DELETE FROM `' . $this->tableName . '` WHERE `fileid` = ?'; + $this->execute($sql, array($entity->getFileId())); + } + + /** + * Creates a new entry in the db from an entity + * @param Entity $entity the entity that should be created + * @return Entity the saved entity with the set id + */ + public function insert(Entity $entity){ + // get updated fields to save, fields have to be set using a setter to + // be saved + $properties = $entity->getUpdatedFields(); + $values = ''; + $columns = ''; + $params = array(); + + // build the fields + $i = 0; + foreach($properties as $property => $updated) { + $column = $entity->propertyToColumn($property); + $getter = 'get' . ucfirst($property); + + $columns .= '`' . $column . '`'; + $values .= '?'; + + // only append colon if there are more entries + if($i < count($properties)-1){ + $columns .= ','; + $values .= ','; + } + + array_push($params, $entity->$getter()); + $i++; + + } + + $sql = 'INSERT INTO `' . $this->tableName . '`(' . + $columns . ') VALUES(' . $values . ')'; + + $this->execute($sql, $params); + + $entity->setFileId((int) $this->db->getInsertId($this->tableName)); + return $entity; + } + + /** + * Updates an entry in the db from a status + * @throws \InvalidArgumentException if entity has no id + * @param Entity $entity the status that should be created + */ + public function update(Entity $entity){ + // if entity wasn't changed it makes no sense to run a db query + $properties = $entity->getUpdatedFields(); + if(count($properties) === 0) { + return $entity; + } + + // entity needs an id + $fileId = $entity->getFileId(); + if($fileId === null){ + throw new \InvalidArgumentException( + 'Entity which should be updated has no fileId'); + } + + // get updated fields to save, fields have to be set using a setter to + // be saved + // dont update the fileId field + unset($properties['fileId']); + + $columns = ''; + $params = array(); + + // build the fields + $i = 0; + foreach($properties as $property => $updated) { + + $column = $entity->propertyToColumn($property); + $getter = 'get' . ucfirst($property); + + $columns .= '`' . $column . '` = ?'; + + // only append colon if there are more entries + if($i < count($properties)-1){ + $columns .= ','; + } + + array_push($params, $entity->$getter()); + $i++; + } + + $sql = 'UPDATE `' . $this->tableName . '` SET ' . + $columns . ' WHERE `fileid` = ?'; + array_push($params, $fileId); + + $this->execute($sql, $params); + } + + + /** + * get the list of all unindexed files for the user + * + * @return array + */ + public function getUnindexed() { + $files = array(); + $absoluteRoot = Filesystem::getView()->getAbsolutePath('/'); + $mounts = Filesystem::getMountPoints($absoluteRoot); + $mount = Filesystem::getMountPoint($absoluteRoot); + if (!in_array($mount, $mounts)) { + $mounts[] = $mount; + } + + $query = $this->db->prepareQuery(' + SELECT `*PREFIX*filecache`.`fileid` + FROM `*PREFIX*filecache` + LEFT JOIN `*PREFIX*lucene_status` + ON `*PREFIX*filecache`.`fileid` = `*PREFIX*lucene_status`.`fileid` + WHERE `storage` = ? + AND `status` IS NULL OR `status` = ? + '); + + foreach ($mounts as $mount) { + if (is_string($mount)) { + $storage = Filesystem::getStorage($mount); + } else if ($mount instanceof \OC\Files\Mount\Mount) { + $storage = $mount->getStorage(); + } else { + $storage = null; + $this->logger->log( + 'expected string or instance of \OC\Files\Mount\Mount got ' . json_encode($mount), 'debug' + ); + } + //only index local files for now + if ($storage->isLocal()) { + $cache = $storage->getCache(); + $numericId = $cache->getNumericStorageId(); + + $result = $query->execute(array($numericId, self::STATUS_NEW)); + + while ($row = $result->fetchRow()) { + $files[] = $row['fileid']; + } + } + } + return $files; + } + + + /** + * @param $fileId + * @return Status + */ + public function getOrCreateFromFileId($fileId) { + $sql = ' + SELECT `fileid`, `status` + FROM ' . $this->tableName . ' + WHERE `fileid` = ? + '; + try { + return $this->findEntity($sql, array($fileId)); + } catch (DoesNotExistException $e) { + $status = new Status(); + $status->setFileId($fileId); + $status->setStatus(Status::STATUS_NEW); + return $this->insert($status); + } + } + + // always write status to db immediately + public function markNew(Status $status) { + $status->setStatus(Status::STATUS_NEW); + return $this->update($status); + } + + public function markIndexed(Status $status) { + $status->setStatus(Status::STATUS_INDEXED); + return $this->update($status); + } + + public function markSkipped(Status $status) { + $status->setStatus(Status::STATUS_SKIPPED); + return $this->update($status); + } + + public function markUnIndexed(Status $status) { + $status->setStatus(Status::STATUS_UNINDEXED); + return $this->update($status); + } + + public function markError(Status $status) { + $status->setStatus(Status::STATUS_ERROR); + return $this->update($status); + } + + /** + * @return int[] + */ + public function getDeleted() { + $files = array(); + + $query = $this->db->prepareQuery(' + SELECT `*PREFIX*lucene_status`.`fileid` + FROM `*PREFIX*lucene_status` + LEFT JOIN `*PREFIX*filecache` + ON `*PREFIX*filecache`.`fileid` = `*PREFIX*lucene_status`.`fileid` + WHERE `*PREFIX*filecache`.`fileid` IS NULL + '); + + $result = $query->execute(); + + while ($row = $result->fetchRow()) { + $files[] = $row['fileid']; + } + + return $files; + + } + +} diff --git a/document/ods.php b/document/ods.php index 0fe15938e..956f030de 100644 --- a/document/ods.php +++ b/document/ods.php @@ -1,4 +1,13 @@ + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ namespace OCA\Search_Lucene\Document; /** diff --git a/document/odt.php b/document/odt.php index 8869c2cbd..2a78181ea 100644 --- a/document/odt.php +++ b/document/odt.php @@ -1,4 +1,13 @@ + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ namespace OCA\Search_Lucene\Document; /** diff --git a/document/opendocument.php b/document/opendocument.php index 4abbdf6b7..ac139a984 100644 --- a/document/opendocument.php +++ b/document/opendocument.php @@ -1,4 +1,13 @@ + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ namespace OCA\Search_Lucene\Document; /** diff --git a/document/pdf.php b/document/pdf.php index 865e32874..34ae5db55 100644 --- a/document/pdf.php +++ b/document/pdf.php @@ -1,7 +1,17 @@ + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ namespace OCA\Search_Lucene\Document; +use OCA\Search_Lucene\Utility\PdfParser; use \OCP\Util; /** * PDF document @@ -36,10 +46,9 @@ private function __construct($data, $storeContent) { //TODO handle PDF 1.6 metadata Zend_Pdf::getMetadata() //do the content extraction - $pdfParse = new \App_Search_Helper_PdfParser(); + $pdfParse = new PdfParser(); $body = $pdfParse->pdf2txt($zendpdf->render()); - if ($body != '') { // Store contents if ($storeContent) { diff --git a/lib/hooks.php b/hooks/files.php similarity index 55% rename from lib/hooks.php rename to hooks/files.php index fa2280de4..b6ee040fc 100644 --- a/lib/hooks.php +++ b/hooks/files.php @@ -1,21 +1,25 @@ + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ -namespace OCA\Search_Lucene; +namespace OCA\Search_Lucene\Hooks; +use OCA\Search_Lucene\AppInfo\Application; +use OCA\Search_Lucene\Db\StatusMapper; use \OCP\BackgroundJob; -use \OCP\Util; /** * * @author Jörn Dreyer */ -class Hooks { - - /** - * classname which used for hooks handling - * used as signalclass in OC_Hooks::emit() - */ - const CLASSNAME = 'Hooks'; +class Files { /** * handle for indexing file @@ -48,29 +52,33 @@ class Hooks { * @param $param array from postWriteFile-Hook */ public static function indexFile(array $param) { - $user = \OCP\User::getUser(); - if (!empty($user)) { + + $app = new Application(); + $container = $app->getContainer(); + $userId = $container->query('UserId'); + + if (!empty($userId)) { // mark written file as new $userFolder = \OC::$server->getUserFolder(); $node = $userFolder->get($param['path']); - $status = Status::fromFileId($node->getId()); + /** @var StatusMapper $mapper */ + $mapper = $container->query('StatusMapper'); + $status = $mapper->getOrCreateFromFileId($node->getId()); // only index files if ($node instanceof \OCP\Files\File) { - /** @var \OCP\Files\File $node */ - $status->markNew(); + $mapper->markNew($status); } else { - $status->markSkipped(); + $mapper->markSkipped($status); } //Add Background Job: - BackgroundJob::registerJob( '\OCA\Search_Lucene\IndexJob', array('user' => $user) ); + BackgroundJob::registerJob( 'OCA\Search_Lucene\Jobs\IndexJob', array('user' => $userId) ); } else { - \OCP\Util::writeLog( - 'search_lucene', + $container->query('Logger')->log( 'Hook indexFile could not determine user when called with param '.json_encode($param), - \OCP\Util::DEBUG + 'debug' ); } } @@ -83,16 +91,25 @@ public static function indexFile(array $param) { * @param $param array from postRenameFile-Hook */ public static function renameFile(array $param) { + $app = new Application(); + $container = $app->getContainer(); + if (!empty($param['oldpath'])) { //delete from lucene index - $lucene = new Lucene(); - $lucene->deleteFile($param['oldpath']); + $container->query('Index')->deleteFile($param['oldpath']); } + if (!empty($param['newpath'])) { $userFolder = \OC::$server->getUserFolder(); - $folder = $userFolder->get($param['newpath']); - Status::fromFileId($folder->getId())->markNew(); - self::indexFile(array('path'=>$param['newpath'])); + $node = $userFolder->get($param['newpath']); + + // only index files + if ($node instanceof \OCP\Files\File) { + $mapper = $container->query('StatusMapper'); + $mapper->getOrCreateFromFileId($node->getId()); + self::indexFile(array('path'=>$param['newpath'])); + } + } } @@ -106,26 +123,24 @@ public static function renameFile(array $param) { static public function deleteFile(array $param) { // we cannot use post_delete as $param would not contain the id // of the deleted file and we could not fetch it with getId - $lucene = new Lucene(); - $deletedIds = Status::getDeleted(); + $app = new Application(); + $container = $app->getContainer(); + + $index = $container->query('Index'); + $mapper = $container->query('StatusMapper'); + $logger = $container->query('Logger'); + + $deletedIds = $mapper->getDeleted(); $count = 0; foreach ($deletedIds as $fileId) { - Util::writeLog( - 'search_lucene', - 'deleting status for ('.$fileId.') ', - Util::DEBUG - ); + $logger->log( 'deleting status for ('.$fileId.') ', 'debug' ); //delete status - Status::delete($fileId); + $mapper->delete($fileId); //delete from lucene - $count += $lucene->deleteFile($fileId); + $count += $index->deleteFile($fileId); } - Util::writeLog( - 'search_lucene', - 'removed '.$count.' files from index', - Util::DEBUG - ); + $logger->log( 'removed '.$count.' files from index', 'debug' ); } diff --git a/jobs/indexjob.php b/jobs/indexjob.php new file mode 100644 index 000000000..fa2d3c427 --- /dev/null +++ b/jobs/indexjob.php @@ -0,0 +1,42 @@ + + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ + +namespace OCA\Search_Lucene\Jobs; + +use OCA\Search_Lucene\AppInfo\Application; + +class IndexJob extends \OC\BackgroundJob\Job { + + public function run($arguments){ + $app = new Application(); + $container = $app->getContainer(); + if (isset($arguments['user'])) { + $userId = $arguments['user']; + + + $folder = $container->query('FileUtility')->setUpUserFolder($userId); + + if ($folder) { + + $fileIds = $container->query('StatusMapper')->getUnindexed(); + + $container->query('Logger')-> + log('background job indexing '.count($fileIds).' files for '.$userId, 'debug' ); + + $container->query('Indexer')->indexFiles($fileIds); + + } + } else { + $container->query('Logger')-> + log('indexer job did not receive user in arguments: '.json_encode($arguments), 'debug' ); + } + } +} diff --git a/jobs/optimizejob.php b/jobs/optimizejob.php new file mode 100644 index 000000000..53ed34540 --- /dev/null +++ b/jobs/optimizejob.php @@ -0,0 +1,37 @@ + + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ + +namespace OCA\Search_Lucene\Jobs; + +use OCA\Search_Lucene\AppInfo\Application; + +class OptimizeJob extends \OC\BackgroundJob\TimedJob { + + public function __construct() { + $this->setInterval(86400); //execute at most once a day + } + + public function run($arguments){ + $app = new Application(); + $container = $app->getContainer(); + + if (!empty($arguments['user'])) { + $userId = $arguments['user']; + $container->query('Logger')->log('background job optimizing index for '.$userId, 'debug' ); + $folder = $container->query('FileUtility')->setUpIndexFolder($userId); + //TODO use folder? + $container->query('Index')->optimizeIndex(); + } else { + $container->query('Logger')-> + log('indexer job did not receive user in arguments: '.json_encode($arguments), 'debug' ); + } + } +} diff --git a/js/checker.js b/js/checker.js index ea276d037..c0e84e123 100644 --- a/js/checker.js +++ b/js/checker.js @@ -5,7 +5,7 @@ function luceneIndexFiles() { } t('search_lucene', 'Indexing... {count} files left', {count: 0}); //preload translations luceneIndexFiles.active = true; - updateEventSource = new OC.EventSource(OC.filePath('search_lucene', 'ajax', 'lucene.php'), {operation: 'index'}); + updateEventSource = new OC.EventSource(OC.generateUrl('/apps/search_lucene/indexer/index', {})); updateEventSource.listen('count', function (unIndexedCount) { count = unIndexedCount; if (count > 0) { @@ -53,9 +53,9 @@ $(document).ready(function () { //hovering over it shows the current file //clicking it stops the indexer: ⌛ - OC.search.resultTypes.content = 'In' // translated + OC.search.resultTypes.lucene = t('search_lucene', 'In'); - OC.search.customResults.content = function (row, item){ + OC.search.customResults.lucene = function (row, item){ row.find('td.result .text').text(t('search_lucene', 'Score: {score}', {score: Math.round(item.score*100)/100})); }; diff --git a/lib/indexer.php b/lib/indexer.php deleted file mode 100644 index 53bf542a6..000000000 --- a/lib/indexer.php +++ /dev/null @@ -1,196 +0,0 @@ - - */ -class Indexer { - - /** - * classname which used for hooks handling - * used as signalclass in OC_Hooks::emit() - */ - const CLASSNAME = 'Indexer'; - - /** - * @var \OCP\Files\Folder - */ - private $folder; - private $lucene; - - public function __construct(\OCP\Files\Folder $folder, Lucene $lucene) { - $this->folder = $folder; - $this->lucene = $lucene; - } - - public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) { - - $skippedDirs = explode( - ';', - \OCP\Config::getUserValue(\OCP\User::getUser(), 'search_lucene', 'skipped_dirs', '.git;.svn;.CVS;.bzr') - ); - - foreach ($fileIds as $id) { - - $fileStatus = Status::fromFileId($id); - - try { - // before we start mark the file as error so we know there - // was a problem in case the php execution dies and we don't try - // the file again - $fileStatus->markError(); - - /** @var \OCP\Files\Node $folder */ - $nodes = \OC::$server->getUserFolder()->getById($id); - // getById can return more than one id because the containing storage might be mounted more than once - // Since we only want to index the file once, we only use the first entry - - if (isset($nodes[0])) { - /** @var \OCP\Files\File $node */ - $node = $nodes[0]; - } else { - throw new \Exception('no file found for fileid '.$id); - } - - if ( ! $node instanceof \OCP\Files\File ) { - throw new NotIndexedException(); - } - - $path = $node->getPath(); - - foreach ($skippedDirs as $skippedDir) { - if (strpos($path, '/' . $skippedDir . '/') !== false //contains dir - || strrpos($path, '/' . $skippedDir) === strlen($path) - (strlen($skippedDir) + 1) // ends with dir - ) { - throw new SkippedException('skipping file '.$id.':'.$path); - } - } - - if ($eventSource) { - $eventSource->send('indexing', $path); - } - - if ($this->indexFile($node, false)) { - $fileStatus->markIndexed(); - } - - } catch (NotIndexedException $e) { - - $fileStatus->markUnIndexed(); - - } catch (SkippedException $e) { - - $fileStatus->markSkipped(); - \OCP\Util::writeLog('search_lucene', $e->getMessage(), \OCP\Util::DEBUG); - - } catch (\Exception $e) { - //sqlite might report database locked errors when stock filescan is in progress - //this also catches db locked exception that might come up when using sqlite - \OCP\Util::writeLog('search_lucene', - $e->getMessage() . ' Trace:\n' . $e->getTraceAsString(), - \OCP\Util::ERROR); - $fileStatus->markError(); // Add UI to trigger rescan of files with status 'E'rror? - if ($eventSource) { - $eventSource->send('error', $e->getMessage()); - } - } - } - $this->lucene->commit(); - } - - /** - * index a file - * - * @author Jörn Dreyer - * - * @param \OCP\Files\File $file the file to be indexed - * @param bool $commit - * - * @return bool true when something was stored in the index, false otherwise (eg, folders are not indexed) - * @throws \OCA\Search_Lucene\NotIndexedException when an unsupported file type is encvountered - */ - public function indexFile(\OCP\Files\File $file, $commit = true) { - - // we decide how to index on mime type or file extension - $mimeType = $file->getMimeType(); - $fileExtension = strtolower(pathinfo($file->getName(), PATHINFO_EXTENSION)); - - // initialize plain lucene document - $doc = new \Zend_Search_Lucene_Document(); - - // index content for local files only - $storage = $file->getStorage(); - - if ($storage->isLocal()) { - - $path = $storage->getLocalFile($file->getInternalPath()); - - //try to use special lucene document types - - if ('text/html' === $mimeType) { - - //TODO could be indexed, even if not local - $doc = \Zend_Search_Lucene_Document_Html::loadHTML($file->getContent()); - } else if ('text/' === substr($mimeType, 0, 5) - || 'application/x-tex' === $mimeType) { - - $body = $file->getContent(); - - if ($body != '') { - $doc->addField(\Zend_Search_Lucene_Field::UnStored('body', $body)); - } - - } else if ('application/pdf' === $mimeType) { - - $doc = Pdf::loadPdf($file->getContent()); - - // the zend classes only understand docx and not doc files - } else if ($fileExtension === 'docx') { - - $doc = \Zend_Search_Lucene_Document_Docx::loadDocxFile($path); - - //} else if ('application/msexcel' === $mimeType) { - } else if ($fileExtension === 'xlsx') { - - $doc = \Zend_Search_Lucene_Document_Xlsx::loadXlsxFile($path); - - //} else if ('application/mspowerpoint' === $mimeType) { - } else if ($fileExtension === 'pptx') { - - $doc = \Zend_Search_Lucene_Document_Pptx::loadPptxFile($path); - - } else if ($fileExtension === 'odt') { - - $doc = Odt::loadOdtFile($path); - - } else if ($fileExtension === 'ods') { - - $doc = Ods::loadOdsFile($path); - - } - } - - // Store filecache id as unique id to lookup by when deleting - $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $file->getId())); - - // Store document path for the search results - $doc->addField(\Zend_Search_Lucene_Field::Text('path', $file->getPath(), 'UTF-8')); - - $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', $file->getMTime())); - - $doc->addField(\Zend_Search_Lucene_Field::unIndexed('size', $file->getSize())); - - $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', $mimeType)); - - $this->lucene->updateFile($doc, $file->getId(), $commit); - - return true; - - } - -} diff --git a/lib/indexjob.php b/lib/indexjob.php deleted file mode 100644 index 61e53f13a..000000000 --- a/lib/indexjob.php +++ /dev/null @@ -1,36 +0,0 @@ -indexFiles($fileIds); - } - } else { - \OCP\Util::writeLog( - 'search_lucene', - 'indexer job did not receive user in arguments: '.json_encode($arguments), - \OCP\Util::DEBUG - ); - } - } -} diff --git a/lib/lucene.php b/lib/lucene.php deleted file mode 100644 index 18457d4cb..000000000 --- a/lib/lucene.php +++ /dev/null @@ -1,180 +0,0 @@ - - */ -class Lucene { - - /** - * classname which used for hooks handling - * used as signalclass in OC_Hooks::emit() - */ - const CLASSNAME = 'Lucene'; - - public $index; - - /** - * The default location of '/lucene_index' can be overridden by passing in a different folder - * - * @param \OCP\Files\Folder $indexFolder location of the lucene_index - */ - public function __construct(\OCP\Files\Folder $indexFolder = null) { - if (is_null($indexFolder)) { - $indexFolder = Util::setUpIndexFolder(); - } - $this->index = $this->openOrCreate($indexFolder); - } - - /** - * opens or creates the given lucene index - * - * stores the index in $indexFolder - * - * @author Jörn Dreyer - * - * @param \OCP\Files\Folder $indexFolder - * @return \Zend_Search_Lucene_Interface - * @throws \Exception - */ - private function openOrCreate(\OCP\Files\Folder $indexFolder) { - - if (is_null($indexFolder)) { - throw new \Exception('No Index folder given'); - } - - try { - - $storage = $indexFolder->getStorage(); - $localPath = $storage->getLocalFile($indexFolder->getInternalPath()); - - //let lucene search for numbers as well as words - \Zend_Search_Lucene_Analysis_Analyzer::setDefault( - new \Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive() - ); - - // can we use the index? - if ($indexFolder->nodeExists('v0.6.0')) { - // correct index present - $index = \Zend_Search_Lucene::open($localPath); - } else { - \OCP\Util::writeLog( - 'search_lucene', - 'recreating outdated lucene index', - \OCP\Util::INFO - ); - $indexFolder->delete(); - $index = \Zend_Search_Lucene::create($localPath); - $indexFolder->newFile('v0.6.0'); - } - return $index; - } catch ( \Exception $e ) { - \OCP\Util::writeLog( - 'search_lucene', - $e->getMessage().' Trace:\n'.$e->getTraceAsString(), - \OCP\Util::ERROR - ); - } - - return null; - - } - - /** - * optimizes the lucene index - * - * @author Jörn Dreyer - * - * @return void - */ - public function optimizeIndex() { - - \OCP\Util::writeLog( - 'search_lucene', - 'optimizing index', - \OCP\Util::DEBUG - ); - - $this->index->optimize(); - - } - - /** - * upates a file in the lucene index - * - * 1. the file is deleted from the index - * 2. the file is readded to the index - * 3. the file is marked as index in the status table - * - * @author Jörn Dreyer - * - * @param \Zend_Search_Lucene_Document $doc the document to store for the path - * @param int $fileid fileid to update - * @param bool $commit - * - * @return void - */ - public function updateFile( - \Zend_Search_Lucene_Document $doc, - $fileid, - $commit = true - ) { - - // TODO profile perfomance for searching before adding to index - $this->deleteFile($fileid); - - \OCP\Util::writeLog( - 'search_lucene', - 'adding ' . $fileid .' '.json_encode($doc), - \OCP\Util::DEBUG - ); - - // Add document to the index - $this->index->addDocument($doc); - - if ($commit) { - $this->index->commit(); - } - } - - /** - * removes a file frome the lucene index - * - * @author Jörn Dreyer - * - * @param int $fileid fileid to remove from the index - * - * @return int count of deleted documents in the index - */ - public function deleteFile($fileid) { - - $hits = $this->index->find( 'fileid:' . $fileid ); - - \OCP\Util::writeLog( - 'search_lucene', - 'found ' . count($hits) . ' hits for fileid ' . $fileid, - \OCP\Util::DEBUG - ); - - foreach ($hits as $hit) { - \OCP\Util::writeLog( - 'search_lucene', - 'removing ' . $hit->id . ':' . $hit->path . ' from index', - \OCP\Util::DEBUG - ); - $this->index->delete($hit); - } - - return count($hits); - } - - public function find ($query) { - return $this->index->find($query); - } - - public function commit () { - $this->index->commit(); - } - -} diff --git a/lib/notindexedexception.php b/lib/notindexedexception.php deleted file mode 100644 index 53c8c23ac..000000000 --- a/lib/notindexedexception.php +++ /dev/null @@ -1,5 +0,0 @@ -setInterval(86400); //execute at most once a day - } - - public function run($arguments){ - if (!empty($arguments['user'])) { - $user = $arguments['user']; - \OCP\Util::writeLog( - 'search_lucene', - 'background job optimizing index for '.$user, - \OCP\Util::DEBUG - ); - $folder = Util::setUpIndexFolder($user); - $lucene = new Lucene($folder); - - $lucene->optimizeIndex(); - } else { - \OCP\Util::writeLog( - 'search_lucene', - 'optimize job did not receive user in arguments: '.json_encode($arguments), - \OCP\Util::DEBUG - ); - } - } -} diff --git a/lib/skippedexception.php b/lib/skippedexception.php deleted file mode 100644 index 185e5a6df..000000000 --- a/lib/skippedexception.php +++ /dev/null @@ -1,5 +0,0 @@ - - */ -class Status { - - const STATUS_NEW = 'N'; - const STATUS_INDEXED = 'I'; - const STATUS_SKIPPED = 'S'; - const STATUS_UNINDEXED = 'U'; - const STATUS_ERROR = 'E'; - - private $fileId; - private $status; - - public function __construct($fileId, $status = null) { - $this->fileId = $fileId; - $this->status = $status; - } - - public static function fromFileId($fileId) { - $status = self::get($fileId); - if ($status) { - return new Status($fileId, $status); - } else { - return new Status($fileId, null); - } - } - - public function getFileId() { - return $this->fileId; - } - - public function getStatus() { - return $this->status; - } - - // always write status to db immediately - public function markNew() { - $this->status = self::STATUS_NEW; - return $this->store(); - } - - public function markIndexed() { - $this->status = self::STATUS_INDEXED; - return $this->store(); - } - - public function markSkipped() { - $this->status = self::STATUS_SKIPPED; - return $this->store(); - } - - public function markUnIndexed() { - $this->status = self::STATUS_UNINDEXED; - return $this->store(); - } - - public function markError() { - $this->status = self::STATUS_ERROR; - return $this->store(); - } - - private function store() { - $savedStatus = self::get($this->fileId); - if ($savedStatus) { - return self::update($this->fileId, $this->status); - } else { - return self::insert($this->fileId, $this->status); - } - } - - public static function delete($fileId) { - $query = \OC_DB::prepare(' - DELETE FROM `*PREFIX*lucene_status` WHERE `fileid` = ? - '); - return $query->execute(array($fileId)); - } - - private static function get($fileId) { - $query = \OC_DB::prepare(' - SELECT `status` - FROM `*PREFIX*lucene_status` - WHERE `fileid` = ? - '); - $result = $query->execute(array($fileId)); - $row = $result->fetchRow(); - if ($row) { - return $row['status']; - } else { - return null; - } - } - - private static function insert($fileId, $status) { - $query = \OC_DB::prepare(' - INSERT INTO `*PREFIX*lucene_status` VALUES (?,?) - '); - return $query->execute(array($fileId, $status)); - } - - private static function update($fileId, $status) { - $query = \OC_DB::prepare(' - UPDATE `*PREFIX*lucene_status` - SET `status` = ? - WHERE `fileid` = ? - '); - return $query->execute(array($status, $fileId)); - } - - /** - * get the list of all unindexed files for the user - * - * @return array - */ - static public function getUnindexed() { - $files = array(); - $absoluteRoot = Filesystem::getView()->getAbsolutePath('/'); - $mounts = Filesystem::getMountPoints($absoluteRoot); - $mount = Filesystem::getMountPoint($absoluteRoot); - if (!in_array($mount, $mounts)) { - $mounts[] = $mount; - } - - $query = \OCP\DB::prepare(' - SELECT `*PREFIX*filecache`.`fileid` - FROM `*PREFIX*filecache` - LEFT JOIN `*PREFIX*lucene_status` - ON `*PREFIX*filecache`.`fileid` = `*PREFIX*lucene_status`.`fileid` - WHERE `storage` = ? - AND `status` IS NULL OR `status` = ? - '); - - foreach ($mounts as $mount) { - if (is_string($mount)) { - $storage = Filesystem::getStorage($mount); - } else if ($mount instanceof \OC\Files\Mount\Mount) { - $storage = $mount->getStorage(); - } else { - $storage = null; - Util::writeLog('search_lucene', - 'expected string or instance of \OC\Files\Mount\Mount got ' . json_encode($mount), - Util::DEBUG); - } - //only index local files for now - if ($storage->instanceOfStorage('\OC\Files\Storage\Local')) { - $cache = $storage->getCache(); - $numericId = $cache->getNumericStorageId(); - - $result = $query->execute(array($numericId, self::STATUS_NEW)); - if (\OCP\DB::isError($result)) { - Util::writeLog( - 'search_lucene', - 'failed to find unindexed files: '.\OCP\DB::getErrorMessage($result), - Util::WARN - ); - return false; - } - while ($row = $result->fetchRow()) { - $files[] = $row['fileid']; - } - } - } - return $files; - } - - static public function getDeleted() { - $files = array(); - - $query = \OCP\DB::prepare(' - SELECT `*PREFIX*lucene_status`.`fileid` - FROM `*PREFIX*lucene_status` - LEFT JOIN `*PREFIX*filecache` - ON `*PREFIX*filecache`.`fileid` = `*PREFIX*lucene_status`.`fileid` - WHERE `*PREFIX*filecache`.`fileid` IS NULL - '); - - $result = $query->execute(); - - while ($row = $result->fetchRow()) { - $files[] = $row['fileid']; - } - - return $files; - - } - -} diff --git a/lucene/index.php b/lucene/index.php new file mode 100644 index 000000000..3285a72e7 --- /dev/null +++ b/lucene/index.php @@ -0,0 +1,147 @@ + + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ + +namespace OCA\Search_Lucene\Lucene; +use OCA\Search_Lucene\Core\Files; +use OCA\Search_Lucene\Core\Logger; + +/** + * @author Jörn Dreyer + */ +class Index { + + public $files; + /** + * @var \Zend_Search_Lucene + */ + public $index; + public $logger; + + public function __construct(Files $files, Logger $logger) { + $this->files = $files; + $this->logger = $logger; + } + + /** + * opens or creates the given lucene index + * + * @author Jörn Dreyer + * + * @throws \Exception + */ + public function openOrCreate() { + + $indexFolder = $this->files->setUpIndexFolder(); + + if (is_null($indexFolder)) { + throw new \Exception('Could not set up index folder'); + } + + $storage = $indexFolder->getStorage(); + $localPath = $storage->getLocalFile($indexFolder->getInternalPath()); + + //let lucene search for numbers as well as words + \Zend_Search_Lucene_Analysis_Analyzer::setDefault( + new \Zend_Search_Lucene_Analysis_Analyzer_Common_Utf8Num_CaseInsensitive() + ); + + // can we use the index? + if ($indexFolder->nodeExists('v0.6.0')) { + // correct index present + $this->index = \Zend_Search_Lucene::open($localPath); + } else { + $this->logger->log( 'recreating outdated lucene index', 'info' ); + $indexFolder->delete(); + $this->index = \Zend_Search_Lucene::create($localPath); + $indexFolder->newFile('v0.6.0'); + } + + } + + /** + * optimizes the lucene index + * + * @author Jörn Dreyer + * + * @return void + */ + public function optimizeIndex() { + $this->logger->log( 'optimizing index', 'debug' ); + $this->index->optimize(); + } + + /** + * upates a file in the lucene index + * + * 1. the file is deleted from the index + * 2. the file is readded to the index + * 3. the file is marked as index in the status table + * + * @author Jörn Dreyer + * + * @param \Zend_Search_Lucene_Document $doc the document to store for the path + * @param int $fileId fileid to update + * @param bool $commit + * + * @return void + */ + public function updateFile( + \Zend_Search_Lucene_Document $doc, + $fileId, + $commit = true + ) { + + // TODO profile perfomance for searching before adding to index + $this->deleteFile($fileId); + + $this->logger->log( 'adding ' . $fileId .' '.json_encode($doc), 'debug' ); + + // Add document to the index + $this->index->addDocument($doc); + + if ($commit) { + $this->index->commit(); + } + + } + + /** + * removes a file from the lucene index + * + * @author Jörn Dreyer + * + * @param int $fileId fileid to remove from the index + * + * @return int count of deleted documents in the index + */ + public function deleteFile($fileId) { + + $hits = $this->index->find( 'fileId:' . $fileId ); + + $this->logger->log( 'found ' . count($hits) . ' hits for fileId ' . $fileId, 'debug' ); + + foreach ($hits as $hit) { + $this->logger->log( 'removing ' . $hit->id . ':' . $hit->path . ' from index', 'debug' ); + $this->index->delete($hit); + } + + return count($hits); + } + + public function find ($query) { + return $this->index->find($query); + } + + public function commit () { + $this->index->commit(); + } + +} diff --git a/lucene/indexer.php b/lucene/indexer.php new file mode 100644 index 000000000..91b28b07d --- /dev/null +++ b/lucene/indexer.php @@ -0,0 +1,206 @@ + + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ + +namespace OCA\Search_Lucene\Lucene; + +use OCA\Search_Lucene\Core\Files; +use OCA\Search_Lucene\Core\Logger; +use OCA\Search_Lucene\Db\Status; +use OCA\Search_Lucene\Db\StatusMapper; +use OCA\Search_Lucene\Document\Ods; +use OCA\Search_Lucene\Document\Odt; +use OCA\Search_Lucene\Document\Pdf; + +/** + * @author Jörn Dreyer + */ +class Indexer { + + private $files; + private $index; + private $mapper; + + public function __construct(Files $files, Index $index, StatusMapper $mapper, Logger $logger) { + $this->files = $files; + $this->index = $index; + $this->mapper = $mapper; + } + + public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) { + + $skippedDirs = explode( + ';', + \OCP\Config::getUserValue(\OCP\User::getUser(), 'search_lucene', 'skipped_dirs', '.git;.svn;.CVS;.bzr') + ); + + foreach ($fileIds as $id) { + + $fileStatus = $this->mapper->getOrCreateFromFileId($id); + + try { + // before we start mark the file as error so we know there + // was a problem in case the php execution dies and we don't try + // the file again + $this->mapper->markError($fileStatus); + + /** @var \OCP\Files\Node $folder */ + $nodes = \OC::$server->getUserFolder()->getById($id); + // getById can return more than one id because the containing storage might be mounted more than once + // Since we only want to index the file once, we only use the first entry + + if (isset($nodes[0])) { + /** @var \OCP\Files\File $node */ + $node = $nodes[0]; + } else { + throw new \Exception('no file found for fileid '.$id); + } + + if ( ! $node instanceof \OCP\Files\File ) { + throw new NotIndexedException(); + } + + $path = $node->getPath(); + + foreach ($skippedDirs as $skippedDir) { + if (strpos($path, '/' . $skippedDir . '/') !== false //contains dir + || strrpos($path, '/' . $skippedDir) === strlen($path) - (strlen($skippedDir) + 1) // ends with dir + ) { + throw new SkippedException('skipping file '.$id.':'.$path); + } + } + + if ($eventSource) { + $eventSource->send('indexing', $path); + } + + if ($this->indexFile($node, false)) { + $this->mapper->markIndexed($fileStatus); + } + + } catch (NotIndexedException $e) { + + $this->mapper->markUnIndexed($fileStatus); + + } catch (SkippedException $e) { + + $this->mapper->markSkipped($fileStatus); + \OCP\Util::writeLog('search_lucene', $e->getMessage(), \OCP\Util::DEBUG); + + } catch (\Exception $e) { + //sqlite might report database locked errors when stock filescan is in progress + //this also catches db locked exception that might come up when using sqlite + \OCP\Util::writeLog('search_lucene', + $e->getMessage() . ' Trace:\n' . $e->getTraceAsString(), + \OCP\Util::ERROR); + $this->mapper->markError($fileStatus); + // TODO Add UI to trigger rescan of files with status 'E'rror? + if ($eventSource) { + $eventSource->send('error', $e->getMessage()); + } + } + } + + $this->index->commit(); + } + + /** + * index a file + * + * @author Jörn Dreyer + * + * @param \OCP\Files\File $file the file to be indexed + * @param bool $commit + * + * @return bool true when something was stored in the index, false otherwise (eg, folders are not indexed) + * @throws \OCA\Search_Lucene\Lucene\NotIndexedException when an unsupported file type is encountered + */ + public function indexFile(\OCP\Files\File $file, $commit = true) { + + // we decide how to index on mime type or file extension + $mimeType = $file->getMimeType(); + $fileExtension = strtolower(pathinfo($file->getName(), PATHINFO_EXTENSION)); + + // initialize plain lucene document + $doc = new \Zend_Search_Lucene_Document(); + + // index content for local files only + $storage = $file->getStorage(); + + if ($storage->isLocal()) { + + $path = $storage->getLocalFile($file->getInternalPath()); + + //try to use special lucene document types + + if ('text/html' === $mimeType) { + + //TODO could be indexed, even if not local + $doc = \Zend_Search_Lucene_Document_Html::loadHTML($file->getContent()); + } else if ('text/' === substr($mimeType, 0, 5) + || 'application/x-tex' === $mimeType) { + + $body = $file->getContent(); + + if ($body != '') { + $doc->addField(\Zend_Search_Lucene_Field::UnStored('body', $body)); + } + + } else if ('application/pdf' === $mimeType) { + + $doc = Pdf::loadPdf($file->getContent()); + + // the zend classes only understand docx and not doc files + } else if ($fileExtension === 'docx') { + + $doc = \Zend_Search_Lucene_Document_Docx::loadDocxFile($path); + + //} else if ('application/msexcel' === $mimeType) { + } else if ($fileExtension === 'xlsx') { + + $doc = \Zend_Search_Lucene_Document_Xlsx::loadXlsxFile($path); + + //} else if ('application/mspowerpoint' === $mimeType) { + } else if ($fileExtension === 'pptx') { + + $doc = \Zend_Search_Lucene_Document_Pptx::loadPptxFile($path); + + } else if ($fileExtension === 'odt') { + + $doc = Odt::loadOdtFile($path); + + } else if ($fileExtension === 'ods') { + + $doc = Ods::loadOdsFile($path); + + } else { + throw new NotIndexedException(); + } + } + + // Store filecache id as unique id to lookup by when deleting + $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $file->getId())); + + // Store document path for the search results + $doc->addField(\Zend_Search_Lucene_Field::Text('path', $file->getPath(), 'UTF-8')); + + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', $file->getMTime())); + + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('size', $file->getSize())); + + $doc->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', $mimeType)); + + $this->index->updateFile($doc, $file->getId(), $commit); + + return true; + + } + +} diff --git a/lucene/notindexedexception.php b/lucene/notindexedexception.php new file mode 100644 index 000000000..ee269b78d --- /dev/null +++ b/lucene/notindexedexception.php @@ -0,0 +1,14 @@ + + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ + +namespace OCA\Search_Lucene\Lucene; + +class NotIndexedException extends \Exception {} \ No newline at end of file diff --git a/lucene/skippedexception.php b/lucene/skippedexception.php new file mode 100644 index 000000000..a20a240ab --- /dev/null +++ b/lucene/skippedexception.php @@ -0,0 +1,14 @@ + + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ + +namespace OCA\Search_Lucene\Lucene; + +class SkippedException extends \Exception {} \ No newline at end of file diff --git a/lib/searchprovider.php b/search/luceneprovider.php similarity index 67% rename from lib/searchprovider.php rename to search/luceneprovider.php index e33447563..21de1a239 100644 --- a/lib/searchprovider.php +++ b/search/luceneprovider.php @@ -1,20 +1,22 @@ + * @copyright Jörn Friedrich Dreyer 2012-2014 */ +namespace OCA\Search_Lucene\Search; -namespace OCA\Search_Lucene; - -use \OCP\Util; +use OCA\Search_Lucene\AppInfo\Application; /** * @author Jörn Dreyer */ -class SearchProvider extends \OCP\Search\Provider { +class LuceneProvider extends \OCP\Search\Provider { /** * performs a search on the users index @@ -22,9 +24,13 @@ class SearchProvider extends \OCP\Search\Provider { * @author Jörn Dreyer * * @param string $query lucene search query - * @return array of \OCP\Search\Result + * @return \OCA\Search_Lucene\Search\LuceneResult[] */ public function search($query){ + + $app = new Application(); + $container = $app->getContainer(); + $results=array(); if ( $query !== null ) { // * query * kills performance for bigger indexes @@ -37,26 +43,22 @@ public function search($query){ // TODO add end user guide for search terms ... //} try { - $lucene = new Lucene(); + $index = $container->query('Index'); //default is 3, 0 needed to keep current search behaviour //Zend_Search_Lucene_Search_Query_Wildcard::setMinPrefixLength(0); //$term = new Zend_Search_Lucene_Index_Term($query); //$query = new Zend_Search_Lucene_Search_Query_Term($term); - $hits = $lucene->find($query); + $hits = $index->find($query); //limit results. we cant show more than ~30 anyway. TODO use paging later for ($i = 0; $i < 30 && $i < count($hits); $i++) { - $results[] = new Result\Content($hits[$i]); + $results[] = new LuceneResult($hits[$i]); } } catch ( \Exception $e ) { - Util::writeLog( - 'search_lucene', - $e->getMessage().' Trace:\n'.$e->getTraceAsString(), - Util::ERROR - ); + $container->query('Logger')->log( $e->getMessage().' Trace:\n'.$e->getTraceAsString(), 'error' ); } } diff --git a/result/content.php b/search/luceneresult.php similarity index 52% rename from result/content.php rename to search/luceneresult.php index 28449e056..6ff5e2070 100644 --- a/result/content.php +++ b/search/luceneresult.php @@ -1,35 +1,27 @@ . + * This file is licensed under the Affero General Public License version 3 or + * later. See the COPYING file. * + * @author Jörn Friedrich Dreyer + * @copyright Jörn Friedrich Dreyer 2012-2014 */ -namespace OCA\Search_Lucene\Result; +namespace OCA\Search_Lucene\Search; use OC\Files\Filesystem; /** * A found file */ -class Content extends \OC\Search\Result\File { +class LuceneResult extends \OC\Search\Result\File { /** * Type name; translated in templates * @var string */ - public $type = 'content'; + public $type = 'lucene'; /** * @var float diff --git a/3rdparty/pdf2text.php b/utility/pdfparser.php similarity index 98% rename from 3rdparty/pdf2text.php rename to utility/pdfparser.php index 757c67e26..290bf0c7b 100644 --- a/3rdparty/pdf2text.php +++ b/utility/pdfparser.php @@ -16,7 +16,10 @@ * @link https://github.com/philipnorton42/PDFSearch * @link http://www.hashbangcode.com/blog/zend-lucene-and-pdf-documents-part-2-pdf-data-extraction-437.html */ -class App_Search_Helper_PdfParser { + +namespace OCA\Search_Lucene\Utility; + +class PdfParser { /** * Convert a PDF into text. From eda9bf8f81fdfbaa35012a16555f053086abf7da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 9 Jul 2014 12:29:04 +0200 Subject: [PATCH 27/51] exclude files otside the users 'files' folder --- db/status.php | 1 + db/statusmapper.php | 8 +++++++- lucene/indexer.php | 6 +++++- lucene/vanishedexception.php | 14 ++++++++++++++ search/luceneresult.php | 2 +- 5 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 lucene/vanishedexception.php diff --git a/db/status.php b/db/status.php index a62ac9295..24ead3cb7 100644 --- a/db/status.php +++ b/db/status.php @@ -25,6 +25,7 @@ class Status extends Entity { const STATUS_INDEXED = 'I'; const STATUS_SKIPPED = 'S'; const STATUS_UNINDEXED = 'U'; + const STATUS_VANISHED = 'V'; const STATUS_ERROR = 'E'; public $fileId; diff --git a/db/statusmapper.php b/db/statusmapper.php index 2056cf638..402bcf114 100644 --- a/db/statusmapper.php +++ b/db/statusmapper.php @@ -161,7 +161,8 @@ public function getUnindexed() { LEFT JOIN `*PREFIX*lucene_status` ON `*PREFIX*filecache`.`fileid` = `*PREFIX*lucene_status`.`fileid` WHERE `storage` = ? - AND `status` IS NULL OR `status` = ? + AND ( `status` IS NULL OR `status` = ? ) + AND `path` LIKE \'files/%\' '); foreach ($mounts as $mount) { @@ -232,6 +233,11 @@ public function markUnIndexed(Status $status) { return $this->update($status); } + public function markVanished(Status $status) { + $status->setStatus(Status::STATUS_VANISHED); + return $this->update($status); + } + public function markError(Status $status) { $status->setStatus(Status::STATUS_ERROR); return $this->update($status); diff --git a/lucene/indexer.php b/lucene/indexer.php index 91b28b07d..c204cb3a4 100644 --- a/lucene/indexer.php +++ b/lucene/indexer.php @@ -60,7 +60,7 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) /** @var \OCP\Files\File $node */ $node = $nodes[0]; } else { - throw new \Exception('no file found for fileid '.$id); + throw new VanishedException($id); } if ( ! $node instanceof \OCP\Files\File ) { @@ -85,6 +85,10 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) $this->mapper->markIndexed($fileStatus); } + } catch (VanishedException $e) { + + $this->mapper->markVanished($fileStatus); + } catch (NotIndexedException $e) { $this->mapper->markUnIndexed($fileStatus); diff --git a/lucene/vanishedexception.php b/lucene/vanishedexception.php new file mode 100644 index 000000000..845eaa575 --- /dev/null +++ b/lucene/vanishedexception.php @@ -0,0 +1,14 @@ + + * @copyright Jörn Friedrich Dreyer 2012-2014 + */ + +namespace OCA\Search_Lucene\Lucene; + +class VanishedException extends \Exception {} \ No newline at end of file diff --git a/search/luceneresult.php b/search/luceneresult.php index 6ff5e2070..8eeffe452 100644 --- a/search/luceneresult.php +++ b/search/luceneresult.php @@ -41,7 +41,7 @@ public function __construct(\Zend_Search_Lucene_Search_QueryHit $hit) { $this->link = \OCP\Util::linkTo( 'files', 'index.php', - array('dir' => dirname($this->path), 'file' => basename($this->path)) + array('dir' => dirname($this->path), 'file' => $this->name) ); $this->permissions = self::get_permissions($this->path); $this->modified = (int)$hit->mtime; From 5124296e6084d11f51fdd5d823daaac3d9558077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 9 Jul 2014 12:50:42 +0200 Subject: [PATCH 28/51] cleanup classpath and use declarations --- appinfo/classpath.php | 14 -------------- controller/apicontroller.php | 1 - lucene/indexer.php | 1 - 3 files changed, 16 deletions(-) delete mode 100644 appinfo/classpath.php diff --git a/appinfo/classpath.php b/appinfo/classpath.php deleted file mode 100644 index be46a4b08..000000000 --- a/appinfo/classpath.php +++ /dev/null @@ -1,14 +0,0 @@ - - * @copyright Jörn Friedrich Dreyer 2012-2014 - */ - -OC::$CLASSPATH['getID3'] = 'getid3/getid3.php'; - -OC::$CLASSPATH['App_Search_Helper_PdfParser'] = 'search_lucene/3rdparty/pdf2text.php'; diff --git a/controller/apicontroller.php b/controller/apicontroller.php index 8788800f7..7a77a1ac7 100644 --- a/controller/apicontroller.php +++ b/controller/apicontroller.php @@ -13,7 +13,6 @@ use OCA\Search_Lucene\Db\StatusMapper; -use OCA\Search_Lucene\Http\EventSourceResponse; use OCA\Search_Lucene\Lucene\Index; use OCA\Search_Lucene\Lucene\Indexer; use \OCP\IRequest; diff --git a/lucene/indexer.php b/lucene/indexer.php index c204cb3a4..816d3d1d3 100644 --- a/lucene/indexer.php +++ b/lucene/indexer.php @@ -13,7 +13,6 @@ use OCA\Search_Lucene\Core\Files; use OCA\Search_Lucene\Core\Logger; -use OCA\Search_Lucene\Db\Status; use OCA\Search_Lucene\Db\StatusMapper; use OCA\Search_Lucene\Document\Ods; use OCA\Search_Lucene\Document\Odt; From f972a263819a46ecd0ae653afd8b1ae91604c963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 9 Jul 2014 12:51:14 +0200 Subject: [PATCH 29/51] whitespace --- appinfo/routes.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appinfo/routes.php b/appinfo/routes.php index bd39e7450..3a59412f1 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -23,5 +23,5 @@ $application->registerRoutes($this, array('routes' => array( array('name' => 'api#index', 'url' => '/indexer/index', 'verb' => 'GET'), - array('name' => 'api#optimize', 'url' => '/indexer/optimize', 'verb' => 'POST'), + array('name' => 'api#optimize', 'url' => '/indexer/optimize', 'verb' => 'POST'), ))); From 4a30b277ea042244b7fff74213a2f0bae8aeb2fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 9 Jul 2014 12:51:36 +0200 Subject: [PATCH 30/51] remove unused constants --- db/statusmapper.php | 6 ------ 1 file changed, 6 deletions(-) diff --git a/db/statusmapper.php b/db/statusmapper.php index 402bcf114..b8bc9ab7a 100644 --- a/db/statusmapper.php +++ b/db/statusmapper.php @@ -23,12 +23,6 @@ */ class StatusMapper extends Mapper { - const STATUS_NEW = 'N'; - const STATUS_INDEXED = 'I'; - const STATUS_SKIPPED = 'S'; - const STATUS_UNINDEXED = 'U'; - const STATUS_ERROR = 'E'; - private $logger; public function __construct(Db $db, Logger $logger){ From 8c2a75ac3706793baf046bd20e0432e49df62a41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 9 Jul 2014 12:51:54 +0200 Subject: [PATCH 31/51] update tests --- tests/unit/testcase.php | 11 ++- tests/unit/testsearchprovider.php | 4 +- tests/unit/teststatus.php | 133 +++++++++++++++++++++++++----- 3 files changed, 123 insertions(+), 25 deletions(-) diff --git a/tests/unit/testcase.php b/tests/unit/testcase.php index 937d05fc7..e8fc6e4ed 100644 --- a/tests/unit/testcase.php +++ b/tests/unit/testcase.php @@ -25,6 +25,9 @@ use OC\Files\Storage\Storage; use OC\Files\View; +use OCA\Search_Lucene\AppInfo\Application; +use OCA\Search_Lucene\Db\Status; +use OCA\Search_Lucene\Db\StatusMapper; use PHPUnit_Framework_TestCase; abstract class TestCase extends PHPUnit_Framework_TestCase { @@ -122,8 +125,14 @@ public function tearDown() { $cache = $this->storage->getCache(); $ids = $cache->getAll(); $cache->clear(); + $app = new Application(); + $container = $app->getContainer(); + /** @var StatusMapper $mapper */ + $mapper = $container->query('StatusMapper'); foreach ($ids as $id) { - \OCA\Search_Lucene\Status::delete($id); + $status = new Status(); + $status->setFileId($id); + $mapper->delete($status); } } diff --git a/tests/unit/testsearchprovider.php b/tests/unit/testsearchprovider.php index a8e2600f5..031994641 100644 --- a/tests/unit/testsearchprovider.php +++ b/tests/unit/testsearchprovider.php @@ -82,9 +82,9 @@ class TestSearchProvider extends TestCase { */ function testSearchLuceneResultContent(\Zend_Search_Lucene_Search_QueryHit $hit, $fileId, $name, $path, $size, $score, $mimeType, $modified, $container) { - $searchResult = new \OCA\Search_Lucene\Result\Content($hit); + $searchResult = new \OCA\Search_Lucene\Search\LuceneResult($hit); - $this->assertInstanceOf('OCA\Search_Lucene\Result\Content', $searchResult); + $this->assertInstanceOf('OCA\Search_Lucene\Search\LuceneResult', $searchResult); $this->assertEquals($searchResult->id, $fileId); $this->assertEquals($searchResult->type, 'content'); $this->assertEquals($searchResult->path, $path); diff --git a/tests/unit/teststatus.php b/tests/unit/teststatus.php index 24f315456..187722c68 100644 --- a/tests/unit/teststatus.php +++ b/tests/unit/teststatus.php @@ -23,8 +23,9 @@ namespace OCA\Search_Lucene\Tests\Unit; -use OCA\Search_Lucene\Status; - +use OCA\Search_Lucene\AppInfo\Application; +use OCA\Search_Lucene\Db\StatusMapper; +use OCA\Search_Lucene\Db\Status; class TestStatus extends TestCase { @@ -37,10 +38,15 @@ function testFromFileIdNull($fileName) { $fileId = $this->getFileId($fileName); $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); + $app = new Application(); + $container = $app->getContainer(); + /** @var StatusMapper $mapper */ + $mapper = $container->query('StatusMapper'); + // run test - $status = Status::fromFileId($fileId); + $status = $mapper->getOrCreateFromFileId($fileId); - $this->assertInstanceOf('OCA\Search_Lucene\Status', $status); + $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status); $this->assertEquals($fileId, $status->getFileId()); $this->assertEquals(null, $status->getStatus()); } @@ -54,18 +60,24 @@ function testMarkNew($fileName) { $fileId = $this->getFileId($fileName); $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); + $app = new Application(); + $container = $app->getContainer(); + /** @var StatusMapper $mapper */ + $mapper = $container->query('StatusMapper'); + // run test - $status = new Status($fileId); - $status->markNew(); + $status = new Status(); + $status->setFileId($fileId); + $mapper->markNew($status); - $this->assertInstanceOf('OCA\Search_Lucene\Status', $status); + $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status); $this->assertEquals($fileId, $status->getFileId()); $this->assertEquals(Status::STATUS_NEW, $status->getStatus()); //check after loading from db - $status2 = Status::fromFileId($fileId); + $status2 = $mapper->getOrCreateFromFileId($fileId); - $this->assertInstanceOf('OCA\Search_Lucene\Status', $status2); + $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status2); $this->assertEquals($fileId, $status2->getFileId()); $this->assertEquals(Status::STATUS_NEW, $status2->getStatus()); @@ -80,18 +92,23 @@ function testMarkSkipped($fileName) { $fileId = $this->getFileId($fileName); $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); + $app = new Application(); + $container = $app->getContainer(); + /** @var StatusMapper $mapper */ + $mapper = $container->query('StatusMapper'); + // run test $status = new Status($fileId); - $status->markSkipped(); + $mapper->markSkipped($status); - $this->assertInstanceOf('OCA\Search_Lucene\Status', $status); + $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status); $this->assertEquals($fileId, $status->getFileId()); $this->assertEquals(Status::STATUS_SKIPPED, $status->getStatus()); //check after loading from db - $status2 = Status::fromFileId($fileId); + $status2 = $mapper->getOrCreateFromFileId($fileId); - $this->assertInstanceOf('OCA\Search_Lucene\Status', $status2); + $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status2); $this->assertEquals($fileId, $status2->getFileId()); $this->assertEquals(Status::STATUS_SKIPPED, $status2->getStatus()); @@ -106,18 +123,23 @@ function testMarkIndexed($fileName) { $fileId = $this->getFileId($fileName); $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); + $app = new Application(); + $container = $app->getContainer(); + /** @var StatusMapper $mapper */ + $mapper = $container->query('StatusMapper'); + // run test $status = new Status($fileId); - $status->markIndexed(); + $mapper->markIndexed($status); - $this->assertInstanceOf('OCA\Search_Lucene\Status', $status); + $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status); $this->assertEquals($fileId, $status->getFileId()); $this->assertEquals(Status::STATUS_INDEXED, $status->getStatus()); //check after loading from db - $status2 = Status::fromFileId($fileId); + $status2 = $mapper->getOrCreateFromFileId($fileId); - $this->assertInstanceOf('OCA\Search_Lucene\Status', $status2); + $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status2); $this->assertEquals($fileId, $status2->getFileId()); $this->assertEquals(Status::STATUS_INDEXED, $status2->getStatus()); @@ -132,21 +154,88 @@ function testMarkError($fileName) { $fileId = $this->getFileId($fileName); $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); + $app = new Application(); + $container = $app->getContainer(); + /** @var StatusMapper $mapper */ + $mapper = $container->query('StatusMapper'); + // run test $status = new Status($fileId); - $status->markError(); + $mapper->markError($status); - $this->assertInstanceOf('OCA\Search_Lucene\Status', $status); + $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status); $this->assertEquals($fileId, $status->getFileId()); $this->assertEquals(Status::STATUS_ERROR, $status->getStatus()); //check after loading from db - $status2 = Status::fromFileId($fileId); + $status2 = $mapper->getOrCreateFromFileId($fileId); - $this->assertInstanceOf('OCA\Search_Lucene\Status', $status2); + $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status2); $this->assertEquals($fileId, $status2->getFileId()); $this->assertEquals(Status::STATUS_ERROR, $status2->getStatus()); - + + } + + /** + * @dataProvider statusDataProvider + */ + function testMarkVanished($fileName) { + + // preparation + $fileId = $this->getFileId($fileName); + $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); + + $app = new Application(); + $container = $app->getContainer(); + /** @var StatusMapper $mapper */ + $mapper = $container->query('StatusMapper'); + + // run test + $status = new Status($fileId); + $mapper->markVanished($status); + + $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status); + $this->assertEquals($fileId, $status->getFileId()); + $this->assertEquals(Status::STATUS_VANISHED, $status->getStatus()); + + //check after loading from db + $status2 = $mapper->getOrCreateFromFileId($fileId); + + $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status2); + $this->assertEquals($fileId, $status2->getFileId()); + $this->assertEquals(Status::STATUS_VANISHED, $status2->getStatus()); + + } + + /** + * @dataProvider statusDataProvider + */ + function testMarkUnIndexed($fileName) { + + // preparation + $fileId = $this->getFileId($fileName); + $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); + + $app = new Application(); + $container = $app->getContainer(); + /** @var StatusMapper $mapper */ + $mapper = $container->query('StatusMapper'); + + // run test + $status = new Status($fileId); + $mapper->markUnIndexed($status); + + $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status); + $this->assertEquals($fileId, $status->getFileId()); + $this->assertEquals(Status::STATUS_UNINDEXED, $status->getStatus()); + + //check after loading from db + $status2 = $mapper->getOrCreateFromFileId($fileId); + + $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status2); + $this->assertEquals($fileId, $status2->getFileId()); + $this->assertEquals(Status::STATUS_UNINDEXED, $status2->getStatus()); + } public function statusDataProvider() { From 17e0fcaaf7f969692cb34fb5e51d384cb577811c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 9 Jul 2014 14:30:49 +0200 Subject: [PATCH 32/51] add readme --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000..61983881d --- /dev/null +++ b/README.md @@ -0,0 +1,22 @@ +# Search Lucene + +[![Build Status](https://secure.travis-ci.org/owncloud/search_lucene.png)](http://travis-ci.org/owncloud/search_lucene) +[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/owncloud/search_lucene/badges/quality-score.png)](https://scrutinizer-ci.com/g/owncloud/search_lucene/) + +The Search Lucene app adds a full text search for files stored in ownCloud. It is based on +[Zend Search Lucene](http://framework.zend.com/manual/1.12/en/zend.search.lucene.html) and +can index content from plain text, .docx, .xlsx, .pptx, .odt, .ods and .pdf files. The source +code is [available on GitHub](https://github.com/owncloud/search_lucene) + +# Maintainers + +Maintainers wanted for additional features! + +* [Jörn Friedrich Dreyer](https://github.com/butonic) + +# Known limitations + +* Does not work with Encryption: the background indexing process does not have access to the + key needed to decrypt files when the user is not logged in. +* Does not index files in external storage. For performance reasons. +* Not all PDF versions can be indexed. The text extraction used for it is incompatible with newer PDF versions. \ No newline at end of file From 655cbb968cd81776b88de4517315eeafd4e56fe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 9 Jul 2014 14:31:19 +0200 Subject: [PATCH 33/51] remove index job after execution --- jobs/indexjob.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/indexjob.php b/jobs/indexjob.php index fa2d3c427..f46cb031d 100644 --- a/jobs/indexjob.php +++ b/jobs/indexjob.php @@ -13,7 +13,7 @@ use OCA\Search_Lucene\AppInfo\Application; -class IndexJob extends \OC\BackgroundJob\Job { +class IndexJob extends \OC\BackgroundJob\QueuedJob { public function run($arguments){ $app = new Application(); From 4136750d261853c9b9e2b791ebccd101524b5c1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 9 Jul 2014 14:49:58 +0200 Subject: [PATCH 34/51] fix wrong reference of static member --- db/statusmapper.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/statusmapper.php b/db/statusmapper.php index b8bc9ab7a..e5bfba8dc 100644 --- a/db/statusmapper.php +++ b/db/statusmapper.php @@ -175,7 +175,7 @@ public function getUnindexed() { $cache = $storage->getCache(); $numericId = $cache->getNumericStorageId(); - $result = $query->execute(array($numericId, self::STATUS_NEW)); + $result = $query->execute(array($numericId, Status::STATUS_NEW)); while ($row = $result->fetchRow()) { $files[] = $row['fileid']; From bb44c42199c9a4fa6b8e3b35b07f0279251cf165 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 15 Jul 2014 13:50:11 +0200 Subject: [PATCH 35/51] fix unit tests, use db from servercontainer, use a wrapper for the logger to automatically set the app name --- appinfo/application.php | 9 +- core/db.php | 40 ------ core/logger.php | 49 ++++--- db/statusmapper.php | 10 +- hooks/files.php | 11 -- jobs/indexjob.php | 6 +- jobs/optimizejob.php | 5 +- lucene/index.php | 17 ++- lucene/indexer.php | 24 +++- search/luceneprovider.php | 2 +- search/luceneresult.php | 36 ++++- tests/unit/bootstrap.php | 4 - tests/unit/testcase.php | 3 + tests/unit/testsearchprovider.php | 61 +++++---- tests/unit/teststatus.php | 217 +++++------------------------- 15 files changed, 173 insertions(+), 321 deletions(-) delete mode 100644 core/db.php diff --git a/appinfo/application.php b/appinfo/application.php index ec7018db4..b8671f5ff 100644 --- a/appinfo/application.php +++ b/appinfo/application.php @@ -83,11 +83,14 @@ public function __construct (array $urlParams=array()) { }); $container->registerService('Logger', function($c) { - return new Logger($c->query('AppName')); + return new Logger( + $c->query('AppName'), + $c->query('ServerContainer')->getLogger() + ); }); - $container->registerService('Db', function() { - return new Db(); + $container->registerService('Db', function($c) { + return $c->query('ServerContainer')->getDb(); }); $container->registerService('FileUtility', function($c) { diff --git a/core/db.php b/core/db.php deleted file mode 100644 index 3ce3560c5..000000000 --- a/core/db.php +++ /dev/null @@ -1,40 +0,0 @@ - - * @copyright Jörn Friedrich Dreyer 2012-2014 - */ - -namespace OCA\Search_Lucene\Core; - -use OCP\IDb; - -class Db implements IDb { - - /** - * Used to abstract the owncloud database access away - * @param string $sql the sql query with ? placeholder for params - * @param int $limit the maximum number of rows - * @param int $offset from which row we want to start - * @return \OCP\DB a query object - */ - public function prepareQuery($sql, $limit=null, $offset=null){ - return \OCP\DB::prepare($sql, $limit, $offset); - } - - - /** - * Used to get the id of the just inserted element - * @param string $tableName the name of the table where we inserted the item - * @return int the id of the inserted element - */ - public function getInsertId($tableName){ - return \OCP\DB::insertid($tableName); - } - - -} diff --git a/core/logger.php b/core/logger.php index 4026be53c..18623223e 100644 --- a/core/logger.php +++ b/core/logger.php @@ -10,40 +10,39 @@ */ namespace OCA\Search_Lucene\Core; +use OCP\ILogger; -class Logger { +/** + * Class Logger + * + * inserts the app name when not set in context + * + * @package OCA\Search_Lucene\Core + */ +class Logger extends \OC\Log { - protected $appName; + private $appName; + private $logger; - public function __construct($appName) { + public function __construct($appName, ILogger $logger) { $this->appName = $appName; + $this->logger = $logger; } /** - * Writes a function into the error log - * @param string $msg the error message to be logged - * @param int $level the error level + * Logs with an arbitrary level. + * + * @param mixed $level + * @param string $message + * @param array $context + * + * @return void */ - public function log($msg, $level=null){ - switch($level){ - case 'debug': - $level = \OCP\Util::DEBUG; - break; - case 'info': - $level = \OCP\Util::INFO; - break; - case 'warn': - $level = \OCP\Util::WARN; - break; - case 'fatal': - $level = \OCP\Util::FATAL; - break; - default: - $level = \OCP\Util::ERROR; - break; + public function log($level, $message, array $context = array()) { + if (empty($context['app'])) { + $context['app'] = $this->appName; } - \OCP\Util::writeLog($this->appName, $msg, $level); + $this->logger->log($level, $message, $context); } - } diff --git a/db/statusmapper.php b/db/statusmapper.php index e5bfba8dc..593498217 100644 --- a/db/statusmapper.php +++ b/db/statusmapper.php @@ -13,10 +13,11 @@ use OC\Files\Filesystem; use OCA\Search_Lucene\Core\Db; -use OCA\Search_Lucene\Core\Logger; use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Db\Entity; use OCP\AppFramework\Db\Mapper; +use OCP\IDb; +use OCP\ILogger; /** * @author Jörn Dreyer @@ -25,7 +26,7 @@ class StatusMapper extends Mapper { private $logger; - public function __construct(Db $db, Logger $logger){ + public function __construct(IDb $db, ILogger $logger){ parent::__construct($db, 'lucene_status', '\OCA\Search_Lucene\Db\Status'); $this->logger = $logger; } @@ -166,9 +167,8 @@ public function getUnindexed() { $storage = $mount->getStorage(); } else { $storage = null; - $this->logger->log( - 'expected string or instance of \OC\Files\Mount\Mount got ' . json_encode($mount), 'debug' - ); + $this->logger-> + debug( 'expected string or instance of \OC\Files\Mount\Mount got ' . json_encode($mount) ); } //only index local files for now if ($storage->isLocal()) { diff --git a/hooks/files.php b/hooks/files.php index b6ee040fc..f4d9c08d2 100644 --- a/hooks/files.php +++ b/hooks/files.php @@ -144,15 +144,4 @@ static public function deleteFile(array $param) { } - /** - * was used by backgroundjobs to index individual files - * - * @deprecated since version 0.6.0 - * - * @author Jörn Dreyer - * - * @param $param array from deleteFile-Hook - */ - static public function doIndexFile(array $param) {/* ignore */} - } diff --git a/jobs/indexjob.php b/jobs/indexjob.php index f46cb031d..13344a5e6 100644 --- a/jobs/indexjob.php +++ b/jobs/indexjob.php @@ -28,15 +28,13 @@ public function run($arguments){ $fileIds = $container->query('StatusMapper')->getUnindexed(); - $container->query('Logger')-> - log('background job indexing '.count($fileIds).' files for '.$userId, 'debug' ); + $container->query('Logger')->debug('background job indexing '.count($fileIds).' files for '.$userId ); $container->query('Indexer')->indexFiles($fileIds); } } else { - $container->query('Logger')-> - log('indexer job did not receive user in arguments: '.json_encode($arguments), 'debug' ); + $container->query('Logger')->debug('indexer job did not receive user in arguments: '.json_encode($arguments)); } } } diff --git a/jobs/optimizejob.php b/jobs/optimizejob.php index 53ed34540..d6cf23b72 100644 --- a/jobs/optimizejob.php +++ b/jobs/optimizejob.php @@ -25,13 +25,14 @@ public function run($arguments){ if (!empty($arguments['user'])) { $userId = $arguments['user']; - $container->query('Logger')->log('background job optimizing index for '.$userId, 'debug' ); + $container->query('Logger')-> + debug('background job optimizing index for '.$userId ); $folder = $container->query('FileUtility')->setUpIndexFolder($userId); //TODO use folder? $container->query('Index')->optimizeIndex(); } else { $container->query('Logger')-> - log('indexer job did not receive user in arguments: '.json_encode($arguments), 'debug' ); + debug('indexer job did not receive user in arguments: '.json_encode($arguments) ); } } } diff --git a/lucene/index.php b/lucene/index.php index 3285a72e7..b3d35adc2 100644 --- a/lucene/index.php +++ b/lucene/index.php @@ -11,7 +11,7 @@ namespace OCA\Search_Lucene\Lucene; use OCA\Search_Lucene\Core\Files; -use OCA\Search_Lucene\Core\Logger; +use OCP\ILogger; /** * @author Jörn Dreyer @@ -23,9 +23,12 @@ class Index { * @var \Zend_Search_Lucene */ public $index; + /** + * @var \OCP\ILogger + */ public $logger; - public function __construct(Files $files, Logger $logger) { + public function __construct(Files $files, ILogger $logger) { $this->files = $files; $this->logger = $logger; } @@ -58,7 +61,7 @@ public function openOrCreate() { // correct index present $this->index = \Zend_Search_Lucene::open($localPath); } else { - $this->logger->log( 'recreating outdated lucene index', 'info' ); + $this->logger->info( 'recreating outdated lucene index' ); $indexFolder->delete(); $this->index = \Zend_Search_Lucene::create($localPath); $indexFolder->newFile('v0.6.0'); @@ -74,7 +77,7 @@ public function openOrCreate() { * @return void */ public function optimizeIndex() { - $this->logger->log( 'optimizing index', 'debug' ); + $this->logger->debug( 'optimizing index' ); $this->index->optimize(); } @@ -102,7 +105,7 @@ public function updateFile( // TODO profile perfomance for searching before adding to index $this->deleteFile($fileId); - $this->logger->log( 'adding ' . $fileId .' '.json_encode($doc), 'debug' ); + $this->logger->debug( 'adding ' . $fileId .' '.json_encode($doc) ); // Add document to the index $this->index->addDocument($doc); @@ -126,10 +129,10 @@ public function deleteFile($fileId) { $hits = $this->index->find( 'fileId:' . $fileId ); - $this->logger->log( 'found ' . count($hits) . ' hits for fileId ' . $fileId, 'debug' ); + $this->logger->debug( 'found ' . count($hits) . ' hits for fileId ' . $fileId ); foreach ($hits as $hit) { - $this->logger->log( 'removing ' . $hit->id . ':' . $hit->path . ' from index', 'debug' ); + $this->logger->debug( 'removing ' . $hit->id . ':' . $hit->path . ' from index' ); $this->index->delete($hit); } diff --git a/lucene/indexer.php b/lucene/indexer.php index 816d3d1d3..e3f8b5c40 100644 --- a/lucene/indexer.php +++ b/lucene/indexer.php @@ -12,25 +12,39 @@ namespace OCA\Search_Lucene\Lucene; use OCA\Search_Lucene\Core\Files; -use OCA\Search_Lucene\Core\Logger; use OCA\Search_Lucene\Db\StatusMapper; use OCA\Search_Lucene\Document\Ods; use OCA\Search_Lucene\Document\Odt; use OCA\Search_Lucene\Document\Pdf; +use OCP\ILogger; /** * @author Jörn Dreyer */ class Indexer { + /** + * @var \OCA\Search_Lucene\Core\Files + */ private $files; + /** + * @var Index + */ private $index; + /** + * @var \OCA\Search_Lucene\Db\StatusMapper + */ private $mapper; + /** + * @var \OCP\ILogger + */ + private $logger; - public function __construct(Files $files, Index $index, StatusMapper $mapper, Logger $logger) { + public function __construct(Files $files, Index $index, StatusMapper $mapper, ILogger $logger) { $this->files = $files; $this->index = $index; $this->mapper = $mapper; + $this->logger = $logger; } public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) { @@ -95,14 +109,12 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) } catch (SkippedException $e) { $this->mapper->markSkipped($fileStatus); - \OCP\Util::writeLog('search_lucene', $e->getMessage(), \OCP\Util::DEBUG); + $this->logger->debug( $e->getMessage() ); } catch (\Exception $e) { //sqlite might report database locked errors when stock filescan is in progress //this also catches db locked exception that might come up when using sqlite - \OCP\Util::writeLog('search_lucene', - $e->getMessage() . ' Trace:\n' . $e->getTraceAsString(), - \OCP\Util::ERROR); + $this->logger->error($e->getMessage() . ' Trace:\n' . $e->getTraceAsString() ); $this->mapper->markError($fileStatus); // TODO Add UI to trigger rescan of files with status 'E'rror? if ($eventSource) { diff --git a/search/luceneprovider.php b/search/luceneprovider.php index 21de1a239..5dcc2e6d1 100644 --- a/search/luceneprovider.php +++ b/search/luceneprovider.php @@ -58,7 +58,7 @@ public function search($query){ } } catch ( \Exception $e ) { - $container->query('Logger')->log( $e->getMessage().' Trace:\n'.$e->getTraceAsString(), 'error' ); + $container->query('Logger')->error( $e->getMessage().' Trace:\n'.$e->getTraceAsString() ); } } diff --git a/search/luceneresult.php b/search/luceneresult.php index 8eeffe452..e3d7b1c18 100644 --- a/search/luceneresult.php +++ b/search/luceneresult.php @@ -34,7 +34,7 @@ class LuceneResult extends \OC\Search\Result\File { */ public function __construct(\Zend_Search_Lucene_Search_QueryHit $hit) { $this->id = (string)$hit->fileid; - $this->path = Filesystem::getView()->getRelativePath($hit->path); + $this->path = $this->getRelativePath($hit->path); $this->name = basename($this->path); $this->size = (int)$hit->size; $this->score = $hit->score; @@ -43,9 +43,41 @@ public function __construct(\Zend_Search_Lucene_Search_QueryHit $hit) { 'index.php', array('dir' => dirname($this->path), 'file' => $this->name) ); - $this->permissions = self::get_permissions($this->path); + $this->permissions = $this->getPermissions($this->path); $this->modified = (int)$hit->mtime; $this->mime_type = $hit->mimetype; } + protected function getRelativePath ($path) { + $root = \OC::$server->getUserFolder(); + return $root->getRelativePath($path); + } + + /** + * Determine permissions for a given file path + * @param string $path + * @return int + */ + function getPermissions($path) { + // add read permissions + $permissions = \OCP\PERMISSION_READ; + // get directory + $fileinfo = pathinfo($path); + $dir = $fileinfo['dirname'] . '/'; + // add update permissions + if (Filesystem::isUpdatable($dir)) { + $permissions |= \OCP\PERMISSION_UPDATE; + } + // add delete permissions + if (Filesystem::isDeletable($dir)) { + $permissions |= \OCP\PERMISSION_DELETE; + } + // add share permissions + if (Filesystem::isSharable($dir)) { + $permissions |= \OCP\PERMISSION_SHARE; + } + // return + return $permissions; + } + } diff --git a/tests/unit/bootstrap.php b/tests/unit/bootstrap.php index 60e22d23e..9bc036d21 100644 --- a/tests/unit/bootstrap.php +++ b/tests/unit/bootstrap.php @@ -13,9 +13,5 @@ require_once('PHPUnit/Autoload.php'); } -//add 3rdparty folder to include path -$dir = __DIR__.'/../../3rdparty'; -set_include_path(get_include_path() . PATH_SEPARATOR . $dir); - OC_Hook::clear(); OC_Log::$enabled = false; diff --git a/tests/unit/testcase.php b/tests/unit/testcase.php index e8fc6e4ed..e220f2a53 100644 --- a/tests/unit/testcase.php +++ b/tests/unit/testcase.php @@ -116,6 +116,9 @@ public function setUp() { */ $this->scanner->scan(''); + + // init 3rdparty classloader + //new Application(); } public function tearDown() { diff --git a/tests/unit/testsearchprovider.php b/tests/unit/testsearchprovider.php index 031994641..5d56342d0 100644 --- a/tests/unit/testsearchprovider.php +++ b/tests/unit/testsearchprovider.php @@ -23,13 +23,21 @@ namespace OCA\Search_Lucene\Tests\Unit; + +//add 3rdparty folder to include path +set_include_path(get_include_path() . PATH_SEPARATOR . __DIR__.'/../../3rdparty'); + +//initialize zend autoloader +require_once __DIR__ . '/../../3rdparty/Zend/Loader/Autoloader.php'; +\Zend_Loader_Autoloader::getInstance(); + class DummyIndex implements \Zend_Search_Lucene_Interface { - var $document = array(); + var $documents = array(); public function addDocument(\Zend_Search_Lucene_Document $document) { - $this->document[$document->id] = $document; + $this->documents[] = $document; } public function getDocument($id) { - return $this->document[$id]; + return $this->documents[$id]; } public function addReference() {} public function closeTermsStream() {} @@ -85,15 +93,14 @@ function testSearchLuceneResultContent(\Zend_Search_Lucene_Search_QueryHit $hit, $searchResult = new \OCA\Search_Lucene\Search\LuceneResult($hit); $this->assertInstanceOf('OCA\Search_Lucene\Search\LuceneResult', $searchResult); - $this->assertEquals($searchResult->id, $fileId); - $this->assertEquals($searchResult->type, 'content'); - $this->assertEquals($searchResult->path, $path); - $this->assertEquals($searchResult->name, $name); - $this->assertEquals($searchResult->mime_type, $mimeType); - $this->assertEquals($searchResult->size, $size); - $this->assertEquals($searchResult->score, $score); - $this->assertEquals($searchResult->modified, $modified); - $this->assertEquals($searchResult->container, $container); + $this->assertEquals($fileId, $searchResult->id); + $this->assertEquals('lucene', $searchResult->type); + $this->assertEquals($path, $searchResult->path); + $this->assertEquals($name, $searchResult->name); + $this->assertEquals($mimeType, $searchResult->mime_type); + $this->assertEquals($size, $searchResult->size); + $this->assertEquals($score, $searchResult->score); + $this->assertEquals($modified, $searchResult->modified); } public function searchResultDataProvider() { @@ -101,57 +108,59 @@ public function searchResultDataProvider() { $index = new DummyIndex(); $doc1 = new \Zend_Search_Lucene_Document(); - $doc1->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 1)); - $doc1->addField(\Zend_Search_Lucene_Field::Text('path', 'documents/document.txt', 'UTF-8')); + $doc1->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 10)); + $doc1->addField(\Zend_Search_Lucene_Field::Text('path', '/test/files/documents/document.txt', 'UTF-8')); $doc1->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', 1234567)); $doc1->addField(\Zend_Search_Lucene_Field::unIndexed('size', 123)); $doc1->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', 'text/plain')); $index->addDocument($doc1); $doc2 = new \Zend_Search_Lucene_Document(); - $doc2->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 2)); - $doc2->addField(\Zend_Search_Lucene_Field::Text('path', 'documents/document.pdf', 'UTF-8')); + $doc2->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 20)); + $doc2->addField(\Zend_Search_Lucene_Field::Text('path', '/test/files/documents/document.pdf', 'UTF-8')); $doc2->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', 1234567)); $doc2->addField(\Zend_Search_Lucene_Field::unIndexed('size', 1234)); $doc2->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', 'application/pdf')); $index->addDocument($doc2); $doc3 = new \Zend_Search_Lucene_Document(); - $doc3->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 3)); - $doc3->addField(\Zend_Search_Lucene_Field::Text('path', 'documents/document.mp3', 'UTF-8')); + $doc3->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 30)); + $doc3->addField(\Zend_Search_Lucene_Field::Text('path', '/test/files/documents/document.mp3', 'UTF-8')); $doc3->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', 1234567)); $doc3->addField(\Zend_Search_Lucene_Field::unIndexed('size', 12341234)); $doc3->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', 'audio/mp3')); $index->addDocument($doc3); $doc4 = new \Zend_Search_Lucene_Document(); - $doc4->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 4)); - $doc4->addField(\Zend_Search_Lucene_Field::Text('path', 'documents/document.jpg', 'UTF-8')); + $doc4->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 40)); + $doc4->addField(\Zend_Search_Lucene_Field::Text('path', '/test/files/documents/document.jpg', 'UTF-8')); $doc4->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', 1234567)); $doc4->addField(\Zend_Search_Lucene_Field::unIndexed('size', 1234123)); $doc4->addField(\Zend_Search_Lucene_Field::unIndexed('mimetype', 'image/jpg')); $index->addDocument($doc4); - - $hit1 = new \Zend_Search_Lucene_Search_QueryHit($index); $hit1->score = 0.4; + $hit1->id = 0; $hit2 = new \Zend_Search_Lucene_Search_QueryHit($index); $hit2->score = 0.31; + $hit2->id = 1; $hit3 = new \Zend_Search_Lucene_Search_QueryHit($index); $hit3->score = 0.299; + $hit3->id = 2; $hit4 = new \Zend_Search_Lucene_Search_QueryHit($index); $hit4->score = 0.001; + $hit4->id = 3; return array( // hit, name, size, score, mime_type, container - array($hit1, '1', 'document.txt', 'documents/document.txt', 123, 0.4, 'text/plain', 1234567, 'documents'), - array($hit2, '2', 'document.pdf', 'documents/document.pdf', 1234, 0.31, 'application/pdf', 1234567, 'documents'), - array($hit3, '3', 'document.mp3', 'documents/document.mp3', 12341234, 0.299, 'audio/mp3', 1234567, 'documents'), - array($hit4, '4', 'document.jpg', 'documents/document.jpg', 1234123, 0.001, 'image/jpg', 1234567, 'documents'), + array($hit1, '10', 'document.txt', '/documents/document.txt', 123, 0.4, 'text/plain', 1234567, 'documents'), + array($hit2, '20', 'document.pdf', '/documents/document.pdf', 1234, 0.31, 'application/pdf', 1234567, 'documents'), + array($hit3, '30', 'document.mp3', '/documents/document.mp3', 12341234, 0.299, 'audio/mp3', 1234567, 'documents'), + array($hit4, '40', 'document.jpg', '/documents/document.jpg', 1234123, 0.001, 'image/jpg', 1234567, 'documents'), ); } } diff --git a/tests/unit/teststatus.php b/tests/unit/teststatus.php index 187722c68..b26f441e9 100644 --- a/tests/unit/teststatus.php +++ b/tests/unit/teststatus.php @@ -32,29 +32,7 @@ class TestStatus extends TestCase { /** * @dataProvider statusDataProvider */ - function testFromFileIdNull($fileName) { - - // preparation - $fileId = $this->getFileId($fileName); - $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); - - $app = new Application(); - $container = $app->getContainer(); - /** @var StatusMapper $mapper */ - $mapper = $container->query('StatusMapper'); - - // run test - $status = $mapper->getOrCreateFromFileId($fileId); - - $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status); - $this->assertEquals($fileId, $status->getFileId()); - $this->assertEquals(null, $status->getStatus()); - } - - /** - * @dataProvider statusDataProvider - */ - function testMarkNew($fileName) { + function testMarkingMethods($fileName, $method, $expectedStatus) { // preparation $fileId = $this->getFileId($fileName); @@ -68,182 +46,51 @@ function testMarkNew($fileName) { // run test $status = new Status(); $status->setFileId($fileId); - $mapper->markNew($status); - - $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status); - $this->assertEquals($fileId, $status->getFileId()); - $this->assertEquals(Status::STATUS_NEW, $status->getStatus()); - - //check after loading from db - $status2 = $mapper->getOrCreateFromFileId($fileId); - - $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status2); - $this->assertEquals($fileId, $status2->getFileId()); - $this->assertEquals(Status::STATUS_NEW, $status2->getStatus()); - - } - - /** - * @dataProvider statusDataProvider - */ - function testMarkSkipped($fileName) { - - // preparation - $fileId = $this->getFileId($fileName); - $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); - - $app = new Application(); - $container = $app->getContainer(); - /** @var StatusMapper $mapper */ - $mapper = $container->query('StatusMapper'); - - // run test - $status = new Status($fileId); - $mapper->markSkipped($status); - - $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status); - $this->assertEquals($fileId, $status->getFileId()); - $this->assertEquals(Status::STATUS_SKIPPED, $status->getStatus()); - - //check after loading from db - $status2 = $mapper->getOrCreateFromFileId($fileId); - - $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status2); - $this->assertEquals($fileId, $status2->getFileId()); - $this->assertEquals(Status::STATUS_SKIPPED, $status2->getStatus()); - - } - - /** - * @dataProvider statusDataProvider - */ - function testMarkIndexed($fileName) { - - // preparation - $fileId = $this->getFileId($fileName); - $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); - - $app = new Application(); - $container = $app->getContainer(); - /** @var StatusMapper $mapper */ - $mapper = $container->query('StatusMapper'); - - // run test - $status = new Status($fileId); - $mapper->markIndexed($status); - - $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status); - $this->assertEquals($fileId, $status->getFileId()); - $this->assertEquals(Status::STATUS_INDEXED, $status->getStatus()); - - //check after loading from db - $status2 = $mapper->getOrCreateFromFileId($fileId); - - $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status2); - $this->assertEquals($fileId, $status2->getFileId()); - $this->assertEquals(Status::STATUS_INDEXED, $status2->getStatus()); - - } - - /** - * @dataProvider statusDataProvider - */ - function testMarkError($fileName) { - - // preparation - $fileId = $this->getFileId($fileName); - $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); - - $app = new Application(); - $container = $app->getContainer(); - /** @var StatusMapper $mapper */ - $mapper = $container->query('StatusMapper'); - - // run test - $status = new Status($fileId); - $mapper->markError($status); - - $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status); - $this->assertEquals($fileId, $status->getFileId()); - $this->assertEquals(Status::STATUS_ERROR, $status->getStatus()); - - //check after loading from db - $status2 = $mapper->getOrCreateFromFileId($fileId); - - $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status2); - $this->assertEquals($fileId, $status2->getFileId()); - $this->assertEquals(Status::STATUS_ERROR, $status2->getStatus()); - - } - - /** - * @dataProvider statusDataProvider - */ - function testMarkVanished($fileName) { - - // preparation - $fileId = $this->getFileId($fileName); - $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); - - $app = new Application(); - $container = $app->getContainer(); - /** @var StatusMapper $mapper */ - $mapper = $container->query('StatusMapper'); - - // run test - $status = new Status($fileId); - $mapper->markVanished($status); - - $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status); - $this->assertEquals($fileId, $status->getFileId()); - $this->assertEquals(Status::STATUS_VANISHED, $status->getStatus()); - - //check after loading from db - $status2 = $mapper->getOrCreateFromFileId($fileId); - - $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status2); - $this->assertEquals($fileId, $status2->getFileId()); - $this->assertEquals(Status::STATUS_VANISHED, $status2->getStatus()); - - } - - /** - * @dataProvider statusDataProvider - */ - function testMarkUnIndexed($fileName) { - - // preparation - $fileId = $this->getFileId($fileName); - $this->assertNotNull($fileId, 'Precondition failed: file id not found!'); - - $app = new Application(); - $container = $app->getContainer(); - /** @var StatusMapper $mapper */ - $mapper = $container->query('StatusMapper'); - - // run test - $status = new Status($fileId); - $mapper->markUnIndexed($status); + $mapper->$method($status); $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status); $this->assertEquals($fileId, $status->getFileId()); - $this->assertEquals(Status::STATUS_UNINDEXED, $status->getStatus()); + $this->assertEquals($expectedStatus, $status->getStatus()); //check after loading from db $status2 = $mapper->getOrCreateFromFileId($fileId); $this->assertInstanceOf('OCA\Search_Lucene\Db\Status', $status2); $this->assertEquals($fileId, $status2->getFileId()); - $this->assertEquals(Status::STATUS_UNINDEXED, $status2->getStatus()); + $this->assertEquals($status->getFileId(), $status2->getFileId()); + $this->assertEquals($expectedStatus, $status2->getStatus()); } public function statusDataProvider() { return array( - array('/documents/document.pdf'), - array('/documents/document.docx'), - array('/documents/document.odt'), - array('/documents/document.txt'), + array('/documents/document.pdf', 'markNew', Status::STATUS_NEW), + array('/documents/document.pdf', 'markSkipped', Status::STATUS_SKIPPED), + array('/documents/document.pdf', 'markIndexed', Status::STATUS_INDEXED), + array('/documents/document.pdf', 'markUnIndexed', Status::STATUS_UNINDEXED), + array('/documents/document.pdf', 'markError', Status::STATUS_ERROR), + array('/documents/document.pdf', 'markVanished', Status::STATUS_VANISHED), + + array('/documents/document.docx', 'markNew', Status::STATUS_NEW), + array('/documents/document.docx', 'markSkipped', Status::STATUS_SKIPPED), + array('/documents/document.docx', 'markIndexed', Status::STATUS_INDEXED), + array('/documents/document.docx', 'markUnIndexed', Status::STATUS_UNINDEXED), + array('/documents/document.docx', 'markError', Status::STATUS_ERROR), + array('/documents/document.docx', 'markVanished', Status::STATUS_VANISHED), + + array('/documents/document.odt', 'markNew', Status::STATUS_NEW), + array('/documents/document.odt', 'markSkipped', Status::STATUS_SKIPPED), + array('/documents/document.odt', 'markIndexed', Status::STATUS_INDEXED), + array('/documents/document.odt', 'markUnIndexed', Status::STATUS_UNINDEXED), + array('/documents/document.odt', 'markError', Status::STATUS_ERROR), + array('/documents/document.odt', 'markVanished', Status::STATUS_VANISHED), + + array('/documents/document.txt', 'markNew', Status::STATUS_NEW), + array('/documents/document.txt', 'markSkipped', Status::STATUS_SKIPPED), + array('/documents/document.txt', 'markIndexed', Status::STATUS_INDEXED), + array('/documents/document.txt', 'markUnIndexed', Status::STATUS_UNINDEXED), + array('/documents/document.txt', 'markError', Status::STATUS_ERROR), + array('/documents/document.txt', 'markVanished', Status::STATUS_VANISHED), ); } } From 4576ad8bb586168f1158f17f789e87975b13cbf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 15 Jul 2014 14:06:10 +0200 Subject: [PATCH 36/51] inject root folder --- appinfo/application.php | 9 ++++++++- core/files.php | 17 ++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/appinfo/application.php b/appinfo/application.php index b8671f5ff..89d5fda9b 100644 --- a/appinfo/application.php +++ b/appinfo/application.php @@ -94,7 +94,14 @@ public function __construct (array $urlParams=array()) { }); $container->registerService('FileUtility', function($c) { - return new Files($c->query('UserId')); + return new Files( + $c->query('UserId'), + $c->query('RootFolder') + ); + }); + + $container->registerService('RootFolder', function($c) { + return $c->query('ServerContainer')->getRootFolder(); }); } diff --git a/core/files.php b/core/files.php index 978962706..bd7d2fe78 100644 --- a/core/files.php +++ b/core/files.php @@ -11,12 +11,20 @@ namespace OCA\Search_Lucene\Core; +use OC\Files\Node\Folder; + class Files { private $userId; - public function __construct($userId){ + /** + * @var \OC\Files\Node\Folder + */ + private $rootFolder; + + public function __construct($userId, Folder $rootFolder){ $this->userId = $userId; + $this->rootFolder = $rootFolder; } /** * Returns a folder for the users 'files' folder @@ -79,13 +87,12 @@ public function setUpUserHome($user = null) { \OC_Util::setupFS($user); $dir = '/' . $user; - $root = \OC::$server->getRootFolder(); $folder = null; - if(!$root->nodeExists($dir)) { - $folder = $root->newFolder($dir); + if(!$this->rootFolder->nodeExists($dir)) { + $folder = $this->rootFolder->newFolder($dir); } else { - $folder = $root->get($dir); + $folder = $this->rootFolder->get($dir); } return $folder; From e5d82cf53fe6577a1725ffcb4a3e08b8f0691e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 15 Jul 2014 14:09:01 +0200 Subject: [PATCH 37/51] clarify user -> userid --- core/files.php | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/core/files.php b/core/files.php index bd7d2fe78..e4af52c72 100644 --- a/core/files.php +++ b/core/files.php @@ -15,6 +15,9 @@ class Files { + /** + * @var string + */ private $userId; /** @@ -30,11 +33,11 @@ public function __construct($userId, Folder $rootFolder){ * Returns a folder for the users 'files' folder * Warning, this will tear down the current filesystem * - * @param string $user the user id + * @param string $userId * @return \OCP\Files\Folder */ - public function setUpUserFolder($user = null) { - $userHome = $this->setUpUserHome($user); + public function setUpUserFolder($userId = null) { + $userHome = $this->setUpUserHome($userId); $dir = 'files'; $folder = null; @@ -49,10 +52,11 @@ public function setUpUserFolder($user = null) { } /** + * @param string $userId * @return null|\OCP\Files\Folder */ - public function setUpIndexFolder($user = null) { - $userHome = $this->setUpUserHome($user); + public function setUpIndexFolder($userId = null) { + $userHome = $this->setUpUserHome($userId); // TODO profile: encrypt the index on logout, decrypt on login //return OCP\Files::getStorage('search_lucene'); // FIXME \OC::$server->getAppFolder() returns '/search' @@ -70,23 +74,24 @@ public function setUpIndexFolder($user = null) { } /** + * @param string $userId * @return null|\OCP\Files\Folder */ - public function setUpUserHome($user = null) { - if (is_null($user)) { - $user = $this->userId; + public function setUpUserHome($userId = null) { + if (is_null($userId)) { + $userId = $this->userId; } - if (!\OCP\User::userExists($user)) { + if (!\OCP\User::userExists($userId)) { return null; } - if ($user !== $this->userId) { + if ($userId !== $this->userId) { \OC_Util::tearDownFS(); - \OC_User::setUserId($user); - $this->userId = $user; + \OC_User::setUserId($userId); + $this->userId = $userId; } - \OC_Util::setupFS($user); + \OC_Util::setupFS($userId); - $dir = '/' . $user; + $dir = '/' . $userId; $folder = null; if(!$this->rootFolder->nodeExists($dir)) { From 2c0290666ee563c16bd3fd7223b460fb7f30808d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 21 Jul 2014 11:56:23 +0200 Subject: [PATCH 38/51] add setupexception --- core/setupexception.php | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 core/setupexception.php diff --git a/core/setupexception.php b/core/setupexception.php new file mode 100644 index 000000000..765cad73e --- /dev/null +++ b/core/setupexception.php @@ -0,0 +1,4 @@ + Date: Mon, 21 Jul 2014 11:56:57 +0200 Subject: [PATCH 39/51] only add optimize job when we know for which user --- appinfo/app.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/appinfo/app.php b/appinfo/app.php index c99c33e96..d63dbcccb 100644 --- a/appinfo/app.php +++ b/appinfo/app.php @@ -22,8 +22,11 @@ // add background job for index optimization: $arguments = array('user' => \OCP\User::getUser()); -//Add Background Job: -\OCP\BackgroundJob::registerJob( 'OCA\Search_Lucene\Jobs\OptimizeJob', $arguments ); + +//only when we know for which user: +if ($arguments['user']) { + \OCP\BackgroundJob::registerJob( 'OCA\Search_Lucene\Jobs\OptimizeJob', $arguments ); +} // --- add hooks ----------------------------------------------- From b43c3ea05e5c77750e8d8b6a41932e370aeba284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 21 Jul 2014 11:57:43 +0200 Subject: [PATCH 40/51] store logger in a local variable --- jobs/indexjob.php | 9 ++++++--- jobs/optimizejob.php | 11 +++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/jobs/indexjob.php b/jobs/indexjob.php index 13344a5e6..7ed44b3e1 100644 --- a/jobs/indexjob.php +++ b/jobs/indexjob.php @@ -18,23 +18,26 @@ class IndexJob extends \OC\BackgroundJob\QueuedJob { public function run($arguments){ $app = new Application(); $container = $app->getContainer(); + + /** @var Logger $logger */ + $logger = $container->query('Logger'); + if (isset($arguments['user'])) { $userId = $arguments['user']; - $folder = $container->query('FileUtility')->setUpUserFolder($userId); if ($folder) { $fileIds = $container->query('StatusMapper')->getUnindexed(); - $container->query('Logger')->debug('background job indexing '.count($fileIds).' files for '.$userId ); + $logger->debug('background job indexing '.count($fileIds).' files for '.$userId ); $container->query('Indexer')->indexFiles($fileIds); } } else { - $container->query('Logger')->debug('indexer job did not receive user in arguments: '.json_encode($arguments)); + $logger->debug('indexer job did not receive user in arguments: '.json_encode($arguments)); } } } diff --git a/jobs/optimizejob.php b/jobs/optimizejob.php index d6cf23b72..0c2dc9dac 100644 --- a/jobs/optimizejob.php +++ b/jobs/optimizejob.php @@ -22,17 +22,16 @@ public function __construct() { public function run($arguments){ $app = new Application(); $container = $app->getContainer(); + /** @var Logger $logger */ + $logger = $container->query('Logger'); if (!empty($arguments['user'])) { $userId = $arguments['user']; - $container->query('Logger')-> - debug('background job optimizing index for '.$userId ); - $folder = $container->query('FileUtility')->setUpIndexFolder($userId); - //TODO use folder? + $logger->debug('background job optimizing index for '.$userId ); + $container->query('FileUtility')->setUpIndexFolder($userId); $container->query('Index')->optimizeIndex(); } else { - $container->query('Logger')-> - debug('indexer job did not receive user in arguments: '.json_encode($arguments) ); + $logger->debug('indexer job did not receive user in arguments: '.json_encode($arguments) ); } } } From 144a1c64ebc3cab0f758a0c39b64c5a827b1afc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 21 Jul 2014 12:00:18 +0200 Subject: [PATCH 41/51] clean up dependency injection for logger, usermanager, usersession, skipped files and file setup --- appinfo/application.php | 19 +++++++-- controller/apicontroller.php | 1 + core/files.php | 82 +++++++++++++++++++----------------- hooks/files.php | 22 ++++++---- lucene/indexer.php | 24 +++++++---- 5 files changed, 89 insertions(+), 59 deletions(-) diff --git a/appinfo/application.php b/appinfo/application.php index 89d5fda9b..d5e78953e 100644 --- a/appinfo/application.php +++ b/appinfo/application.php @@ -46,6 +46,9 @@ public function __construct (array $urlParams=array()) { ); }); + /** + * Lucene + */ $container->registerService('Index', function($c) { $index = new Index( $c->query('FileUtility'), @@ -59,12 +62,21 @@ public function __construct (array $urlParams=array()) { $container->registerService('Indexer', function($c) { return new Indexer( $c->query('FileUtility'), + $c->query('ServerContainer'), $c->query('Index'), + $c->query('SkippedDirs'), $c->query('StatusMapper'), $c->query('Logger') ); }); + $container->registerService('SkippedDirs', function($c) { + return explode( + ';', + \OCP\Config::getUserValue($c->query('UserId'), 'search_lucene', 'skipped_dirs', '.git;.svn;.CVS;.bzr') + ); + }); + /** * Mappers */ @@ -78,8 +90,8 @@ public function __construct (array $urlParams=array()) { /** * Core */ - $container->registerService('UserId', function() { - return \OCP\User::getUser(); + $container->registerService('UserId', function($c) { + return $c->query('ServerContainer')->getUserSession()->getUser()->getUID(); }); $container->registerService('Logger', function($c) { @@ -95,7 +107,8 @@ public function __construct (array $urlParams=array()) { $container->registerService('FileUtility', function($c) { return new Files( - $c->query('UserId'), + $c->query('ServerContainer')->getUserManager(), + $c->query('ServerContainer')->getUserSession(), $c->query('RootFolder') ); }); diff --git a/controller/apicontroller.php b/controller/apicontroller.php index 7a77a1ac7..7bd66985a 100644 --- a/controller/apicontroller.php +++ b/controller/apicontroller.php @@ -43,6 +43,7 @@ public function index($fileId) { $fileIds = $this->mapper->getUnindexed(); } + //TODO use public api when available in \OCP\AppFramework\IApi $eventSource = new \OC_EventSource(); $eventSource->send('count', count($fileIds)); diff --git a/core/files.php b/core/files.php index e4af52c72..42aaa809b 100644 --- a/core/files.php +++ b/core/files.php @@ -10,23 +10,29 @@ */ namespace OCA\Search_Lucene\Core; - -use OC\Files\Node\Folder; +use OCP\Files\Folder; +use OCP\IUserManager; +use OCP\IUserSession; class Files { /** - * @var string + * @var \OCP\IUserManager + */ + private $userManager; + /** + * @var \OCP\IUserSession */ - private $userId; + private $userSession; /** * @var \OC\Files\Node\Folder */ private $rootFolder; - public function __construct($userId, Folder $rootFolder){ - $this->userId = $userId; + public function __construct(IUserManager $userManager, IUserSession $userSession, Folder $rootFolder){ + $this->userManager = $userManager; + $this->userSession = $userSession; $this->rootFolder = $rootFolder; } /** @@ -35,72 +41,70 @@ public function __construct($userId, Folder $rootFolder){ * * @param string $userId * @return \OCP\Files\Folder + * @throws SetUpException */ public function setUpUserFolder($userId = null) { - $userHome = $this->setUpUserHome($userId); - $dir = 'files'; - $folder = null; - if(!$userHome->nodeExists($dir)) { - $folder = $userHome->newFolder($dir); - } else { - $folder = $userHome->get($dir); - } + $userHome = $this->setUpUserHome($userId); - return $folder; + return $this->getOrCreateSubFolder($userHome, 'files'); } /** * @param string $userId * @return null|\OCP\Files\Folder + * @throws SetUpException */ public function setUpIndexFolder($userId = null) { - $userHome = $this->setUpUserHome($userId); // TODO profile: encrypt the index on logout, decrypt on login //return OCP\Files::getStorage('search_lucene'); // FIXME \OC::$server->getAppFolder() returns '/search' //$indexFolder = \OC::$server->getAppFolder(); - $dir = 'lucene_index'; - $folder = null; - if(!$userHome->nodeExists($dir)) { - $folder = $userHome->newFolder($dir); - } else { - $folder = $userHome->get($dir); - } + $userHome = $this->setUpUserHome($userId); - return $folder; + return $this->getOrCreateSubFolder($userHome, 'lucene_index'); } /** * @param string $userId * @return null|\OCP\Files\Folder + * @throws SetUpException */ public function setUpUserHome($userId = null) { + if (is_null($userId)) { - $userId = $this->userId; + $user = $this->userSession->getUser(); + } else { + $user = $this->userManager->get($userId); } - if (!\OCP\User::userExists($userId)) { - return null; + if (is_null($user) || !$this->userManager->userExists($user->getUID())) { + throw new SetUpException('could not set up user home for '.json_encode($user)); } - if ($userId !== $this->userId) { + //if ($user !== $this->userSession->getUser()) { + if ($user !== $this->userSession->getUser()) { \OC_Util::tearDownFS(); - \OC_User::setUserId($userId); - $this->userId = $userId; + $this->userSession->setUser($user); } - \OC_Util::setupFS($userId); + \OC_Util::setupFS($user->getUID()); - $dir = '/' . $userId; - $folder = null; + return $this->getOrCreateSubFolder($this->rootFolder, '/' . $user->getUID()); - if(!$this->rootFolder->nodeExists($dir)) { - $folder = $this->rootFolder->newFolder($dir); + } + + /** + * @param \OCP\Files\Folder $parent + * @param string $folderName + * @return null|\OCP\Files\Folder + * @throws SetUpException + */ + private function getOrCreateSubFolder(Folder $parent, $folderName) { + if($parent->nodeExists($folderName)) { + return $parent->get($folderName); } else { - $folder = $this->rootFolder->get($dir); + return $parent->newFolder($folderName); } - - return $folder; - } + } diff --git a/hooks/files.php b/hooks/files.php index f4d9c08d2..25419ddbb 100644 --- a/hooks/files.php +++ b/hooks/files.php @@ -12,8 +12,10 @@ namespace OCA\Search_Lucene\Hooks; use OCA\Search_Lucene\AppInfo\Application; +use OCA\Search_Lucene\Core\Logger; use OCA\Search_Lucene\Db\StatusMapper; -use \OCP\BackgroundJob; +use OCA\Search_Lucene\Lucene\Index; +use OCP\BackgroundJob; /** * @@ -60,7 +62,7 @@ public static function indexFile(array $param) { if (!empty($userId)) { // mark written file as new - $userFolder = \OC::$server->getUserFolder(); + $userFolder = $container->query('ServerContainer')->getUserFolder(); $node = $userFolder->get($param['path']); /** @var StatusMapper $mapper */ $mapper = $container->query('StatusMapper'); @@ -76,9 +78,8 @@ public static function indexFile(array $param) { //Add Background Job: BackgroundJob::registerJob( 'OCA\Search_Lucene\Jobs\IndexJob', array('user' => $userId) ); } else { - $container->query('Logger')->log( - 'Hook indexFile could not determine user when called with param '.json_encode($param), - 'debug' + $container->query('Logger')->debug( + 'Hook indexFile could not determine user when called with param '.json_encode($param) ); } } @@ -100,7 +101,7 @@ public static function renameFile(array $param) { } if (!empty($param['newpath'])) { - $userFolder = \OC::$server->getUserFolder(); + $userFolder = $container->query('ServerContainer')->getUserFolder(); $node = $userFolder->get($param['newpath']); // only index files @@ -126,21 +127,26 @@ static public function deleteFile(array $param) { $app = new Application(); $container = $app->getContainer(); + /** @var Index $index */ $index = $container->query('Index'); + + /** @var StatusMapper $mapper */ $mapper = $container->query('StatusMapper'); + + /** @var Logger $logger */ $logger = $container->query('Logger'); $deletedIds = $mapper->getDeleted(); $count = 0; foreach ($deletedIds as $fileId) { - $logger->log( 'deleting status for ('.$fileId.') ', 'debug' ); + $logger->debug( 'deleting status for ('.$fileId.') ' ); //delete status $mapper->delete($fileId); //delete from lucene $count += $index->deleteFile($fileId); } - $logger->log( 'removed '.$count.' files from index', 'debug' ); + $logger->debug( 'removed '.$count.' files from index' ); } diff --git a/lucene/indexer.php b/lucene/indexer.php index e3f8b5c40..6b5a08189 100644 --- a/lucene/indexer.php +++ b/lucene/indexer.php @@ -17,6 +17,7 @@ use OCA\Search_Lucene\Document\Odt; use OCA\Search_Lucene\Document\Pdf; use OCP\ILogger; +use OCP\IServerContainer; /** * @author Jörn Dreyer @@ -27,10 +28,18 @@ class Indexer { * @var \OCA\Search_Lucene\Core\Files */ private $files; + /** + * @var \OCP\IServerContainer + */ + private $server; /** * @var Index */ private $index; + /** + * @var array + */ + private $skippedDirs; /** * @var \OCA\Search_Lucene\Db\StatusMapper */ @@ -40,20 +49,17 @@ class Indexer { */ private $logger; - public function __construct(Files $files, Index $index, StatusMapper $mapper, ILogger $logger) { + public function __construct(Files $files, IServerContainer $server, Index $index, array $skippedDirs, StatusMapper $mapper, ILogger $logger) { $this->files = $files; + $this->server = $server; $this->index = $index; + $this->skippedDirs = $skippedDirs; $this->mapper = $mapper; $this->logger = $logger; } public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) { - $skippedDirs = explode( - ';', - \OCP\Config::getUserValue(\OCP\User::getUser(), 'search_lucene', 'skipped_dirs', '.git;.svn;.CVS;.bzr') - ); - foreach ($fileIds as $id) { $fileStatus = $this->mapper->getOrCreateFromFileId($id); @@ -64,8 +70,8 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) // the file again $this->mapper->markError($fileStatus); - /** @var \OCP\Files\Node $folder */ - $nodes = \OC::$server->getUserFolder()->getById($id); + /** @var \OCP\Files\Node[] $nodes */ + $nodes = $this->server->getUserFolder()->getById($id); // getById can return more than one id because the containing storage might be mounted more than once // Since we only want to index the file once, we only use the first entry @@ -82,7 +88,7 @@ public function indexFiles (array $fileIds, \OC_EventSource $eventSource = null) $path = $node->getPath(); - foreach ($skippedDirs as $skippedDir) { + foreach ($this->skippedDirs as $skippedDir) { if (strpos($path, '/' . $skippedDir . '/') !== false //contains dir || strrpos($path, '/' . $skippedDir) === strlen($path) - (strlen($skippedDir) + 1) // ends with dir ) { From ff6eb9f5e95b1d2830a846a2b6ef477bdae94f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 21 Jul 2014 12:01:25 +0200 Subject: [PATCH 42/51] use application container in test setup --- tests/unit/testcase.php | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/unit/testcase.php b/tests/unit/testcase.php index e220f2a53..b5cba34a5 100644 --- a/tests/unit/testcase.php +++ b/tests/unit/testcase.php @@ -51,20 +51,26 @@ abstract class TestCase extends PHPUnit_Framework_TestCase { //for search lucene public function setUp() { + $app = new Application(); + $container = $app->getContainer(); + // reset backend - \OC_User::clearBackends(); - \OC_User::useBackend('database'); + $um = $container->getServer()->getUserManager(); + $us = $container->getServer()->getUserSession(); + $um->clearBackends(); + $um->registerBackend(new \OC_User_Database()); // create test user $this->userName = 'test'; \OC_User::deleteUser($this->userName); - \OC_User::createUser($this->userName, $this->userName); + $um->createUser($this->userName, $this->userName); \OC_Util::tearDownFS(); - \OC_User::setUserId(''); + $us->setUser(null); \OC\Files\Filesystem::tearDown(); \OC_Util::setupFS($this->userName); - \OC_User::setUserId($this->userName); + + $us->setUser($um->get($this->userName)); $view = new \OC\Files\View('/' . $this->userName . '/files'); From 2e381ea57912e460168d4a108619e8b2587569f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 21 Jul 2014 12:29:02 +0200 Subject: [PATCH 43/51] update info.xml and remove html markup bits --- appinfo/info.xml | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/appinfo/info.xml b/appinfo/info.xml index d5e09f789..c71da992b 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -7,29 +7,36 @@ - 4.93 + 7 true Activate this app when you want to be able to find files by searching their content. The implementation does not search the files - directly but will build a so called - reverse index.

+ directly but will build a reverse index based on lucene. + Currently only your personal files will be indexed, - and only you index is searched, so you will not be able + and only your index is searched, so you will not be able to find files that have been shared with you or that - reside on remote mount points.

- Every time a file has been written owncloud will - index it in the background. Furthermore, when + reside on remote mount points. + + Every time a file has been written ownCloud will + index it with a background job. Furthermore, when clicking the search box it will check if any - files need indexing. This might add a lot of - background jobs after first enabling this app, - depending on the amount of files you have.

+ files need indexing. It might take a while to build + the index after first enabling this app, + depending on the amount of files you have. + Even thought file indexing is in progress you will still be able to retrieve file search results for any - indexed files.

- We currenty support plain text, HTML and PDF files. - MS Office 2007 and Open/Libre Office are on the roadmap. + indexed files. + + We currently support text, latex, HTML, PDF, MS Office 2007 + or newer and Open/Libre Office files. + + Depending on the file size and version of PDF / office files + indexing may fail. This is a known issue and help is welcome + to get these problems reproduced and sorted out. 166057 From 779480826dd32427205478a2ec9ab7445afbce5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 23 Jul 2014 12:41:29 +0200 Subject: [PATCH 44/51] consistently use fileId as key --- lucene/index.php | 6 +-- lucene/indexer.php | 2 +- search/luceneresult.php | 2 +- tests/unit/testcase.php | 2 +- tests/unit/testdocumentpdf.php | 86 +++++++++++++++++++++++++++++++ tests/unit/testsearchprovider.php | 8 +-- 6 files changed, 96 insertions(+), 10 deletions(-) create mode 100644 tests/unit/testdocumentpdf.php diff --git a/lucene/index.php b/lucene/index.php index b3d35adc2..6bd04425f 100644 --- a/lucene/index.php +++ b/lucene/index.php @@ -91,7 +91,7 @@ public function optimizeIndex() { * @author Jörn Dreyer * * @param \Zend_Search_Lucene_Document $doc the document to store for the path - * @param int $fileId fileid to update + * @param int $fileId file id to update * @param bool $commit * * @return void @@ -121,7 +121,7 @@ public function updateFile( * * @author Jörn Dreyer * - * @param int $fileId fileid to remove from the index + * @param int $fileId file id to remove from the index * * @return int count of deleted documents in the index */ @@ -129,7 +129,7 @@ public function deleteFile($fileId) { $hits = $this->index->find( 'fileId:' . $fileId ); - $this->logger->debug( 'found ' . count($hits) . ' hits for fileId ' . $fileId ); + $this->logger->debug( 'found ' . count($hits) . ' hits for file id ' . $fileId ); foreach ($hits as $hit) { $this->logger->debug( 'removing ' . $hit->id . ':' . $hit->path . ' from index' ); diff --git a/lucene/indexer.php b/lucene/indexer.php index 6b5a08189..bbea29ad8 100644 --- a/lucene/indexer.php +++ b/lucene/indexer.php @@ -207,7 +207,7 @@ public function indexFile(\OCP\Files\File $file, $commit = true) { } // Store filecache id as unique id to lookup by when deleting - $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileid', $file->getId())); + $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileId', $file->getId())); // Store document path for the search results $doc->addField(\Zend_Search_Lucene_Field::Text('path', $file->getPath(), 'UTF-8')); diff --git a/search/luceneresult.php b/search/luceneresult.php index e3d7b1c18..036715efc 100644 --- a/search/luceneresult.php +++ b/search/luceneresult.php @@ -33,7 +33,7 @@ class LuceneResult extends \OC\Search\Result\File { * @param \Zend_Search_Lucene_Search_QueryHit $hit file data given by provider */ public function __construct(\Zend_Search_Lucene_Search_QueryHit $hit) { - $this->id = (string)$hit->fileid; + $this->id = (string)$hit->fileId; $this->path = $this->getRelativePath($hit->path); $this->name = basename($this->path); $this->size = (int)$hit->size; diff --git a/tests/unit/testcase.php b/tests/unit/testcase.php index b5cba34a5..eb0f033fc 100644 --- a/tests/unit/testcase.php +++ b/tests/unit/testcase.php @@ -151,7 +151,7 @@ protected function getFileId($path) { $fileInfo = $view->getFileInfo($path); if (! empty($fileInfo)) { - return $fileInfo['fileid']; + return $fileInfo->getId(); } return null; diff --git a/tests/unit/testdocumentpdf.php b/tests/unit/testdocumentpdf.php new file mode 100644 index 000000000..366efbd1c --- /dev/null +++ b/tests/unit/testdocumentpdf.php @@ -0,0 +1,86 @@ +. + * + */ + +namespace OCA\Search_Lucene\Tests\Unit; + +use OCA\Search_Lucene\AppInfo\Application; +use OCA\Search_Lucene\Lucene\Index; + +class TestIndex extends TestCase { + + function testUpdate() { + + // preparation + $app = new Application(); + $container = $app->getContainer(); + + // get an index + /** @var Index $index */ + $index = $container->query('Index'); + + // add a document + $doc = new \Zend_Search_Lucene_Document(); + + $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileId', '1')); + $doc->addField(\Zend_Search_Lucene_Field::Text('path', '/somewhere/deep/down/the/rabbit/hole' , 'UTF-8')); + $doc->addField(\Zend_Search_Lucene_Field::Text('users', 'alice' , 'UTF-8')); + + $index->index->addDocument($doc); + $index->commit(); + + // search for it + $idTerm = new \Zend_Search_Lucene_Index_Term('1', 'fileId'); + $idQuery = new \Zend_Search_Lucene_Search_Query_Term($idTerm); + + $query = new \Zend_Search_Lucene_Search_Query_Boolean(); + $query->addSubquery($idQuery); + /** @var \Zend_Search_Lucene_Search_QueryHit $hit */ + $hits = $index->find($query); + // get the document from the query hit + $foundDoc = $hits[0]->getDocument(); + $this->assertEquals('alice', $foundDoc->getFieldValue('users')); + + // delete the document from the index + //$index->index->delete($hit); + + // change the 'users' key of the document + $foundDoc->addField(\Zend_Search_Lucene_Field::Text('users', 'bob' , 'UTF-8')); + $this->assertEquals('bob', $foundDoc->getFieldValue('users')); + + // add the document back to the index + $index->updateFile($foundDoc, '1'); + + + $idTerm2 = new \Zend_Search_Lucene_Index_Term('1', 'fileId'); + $idQuery2 = new \Zend_Search_Lucene_Search_Query_Term($idTerm2); + + $query2 = new \Zend_Search_Lucene_Search_Query_Boolean(); + $query2->addSubquery($idQuery2); + /** @var \Zend_Search_Lucene_Search_QueryHit $hit */ + $hits2 = $index->find($query2); + // get the document from the query hit + $foundDoc2 = $hits2[0]->getDocument(); + $this->assertEquals('alice', $foundDoc2->getFieldValue('users')); + + } +} diff --git a/tests/unit/testsearchprovider.php b/tests/unit/testsearchprovider.php index 5d56342d0..b02c94e08 100644 --- a/tests/unit/testsearchprovider.php +++ b/tests/unit/testsearchprovider.php @@ -108,7 +108,7 @@ public function searchResultDataProvider() { $index = new DummyIndex(); $doc1 = new \Zend_Search_Lucene_Document(); - $doc1->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 10)); + $doc1->addField(\Zend_Search_Lucene_Field::Keyword('fileId', 10)); $doc1->addField(\Zend_Search_Lucene_Field::Text('path', '/test/files/documents/document.txt', 'UTF-8')); $doc1->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', 1234567)); $doc1->addField(\Zend_Search_Lucene_Field::unIndexed('size', 123)); @@ -116,7 +116,7 @@ public function searchResultDataProvider() { $index->addDocument($doc1); $doc2 = new \Zend_Search_Lucene_Document(); - $doc2->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 20)); + $doc2->addField(\Zend_Search_Lucene_Field::Keyword('fileId', 20)); $doc2->addField(\Zend_Search_Lucene_Field::Text('path', '/test/files/documents/document.pdf', 'UTF-8')); $doc2->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', 1234567)); $doc2->addField(\Zend_Search_Lucene_Field::unIndexed('size', 1234)); @@ -124,7 +124,7 @@ public function searchResultDataProvider() { $index->addDocument($doc2); $doc3 = new \Zend_Search_Lucene_Document(); - $doc3->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 30)); + $doc3->addField(\Zend_Search_Lucene_Field::Keyword('fileId', 30)); $doc3->addField(\Zend_Search_Lucene_Field::Text('path', '/test/files/documents/document.mp3', 'UTF-8')); $doc3->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', 1234567)); $doc3->addField(\Zend_Search_Lucene_Field::unIndexed('size', 12341234)); @@ -132,7 +132,7 @@ public function searchResultDataProvider() { $index->addDocument($doc3); $doc4 = new \Zend_Search_Lucene_Document(); - $doc4->addField(\Zend_Search_Lucene_Field::Keyword('fileid', 40)); + $doc4->addField(\Zend_Search_Lucene_Field::Keyword('fileId', 40)); $doc4->addField(\Zend_Search_Lucene_Field::Text('path', '/test/files/documents/document.jpg', 'UTF-8')); $doc4->addField(\Zend_Search_Lucene_Field::unIndexed('mtime', 1234567)); $doc4->addField(\Zend_Search_Lucene_Field::unIndexed('size', 1234123)); From 086202636cdd24494be4084958f8a545c6418c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 23 Jul 2014 12:44:33 +0200 Subject: [PATCH 45/51] add correct test for index updates with document fetched from index --- tests/unit/testdocumentpdf.php | 58 ++--------------------- tests/unit/testindex.php | 86 ++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 53 deletions(-) create mode 100644 tests/unit/testindex.php diff --git a/tests/unit/testdocumentpdf.php b/tests/unit/testdocumentpdf.php index 366efbd1c..8f84a5d6c 100644 --- a/tests/unit/testdocumentpdf.php +++ b/tests/unit/testdocumentpdf.php @@ -23,64 +23,16 @@ namespace OCA\Search_Lucene\Tests\Unit; -use OCA\Search_Lucene\AppInfo\Application; -use OCA\Search_Lucene\Lucene\Index; +use OCA\Search_Lucene\Document\Pdf; -class TestIndex extends TestCase { +class TestDocumentPdf extends \PHPUnit_Framework_TestCase { function testUpdate() { - // preparation - $app = new Application(); - $container = $app->getContainer(); + $data = file_get_contents(__DIR__ . '/data/document.pdf'); - // get an index - /** @var Index $index */ - $index = $container->query('Index'); - - // add a document - $doc = new \Zend_Search_Lucene_Document(); - - $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileId', '1')); - $doc->addField(\Zend_Search_Lucene_Field::Text('path', '/somewhere/deep/down/the/rabbit/hole' , 'UTF-8')); - $doc->addField(\Zend_Search_Lucene_Field::Text('users', 'alice' , 'UTF-8')); - - $index->index->addDocument($doc); - $index->commit(); - - // search for it - $idTerm = new \Zend_Search_Lucene_Index_Term('1', 'fileId'); - $idQuery = new \Zend_Search_Lucene_Search_Query_Term($idTerm); - - $query = new \Zend_Search_Lucene_Search_Query_Boolean(); - $query->addSubquery($idQuery); - /** @var \Zend_Search_Lucene_Search_QueryHit $hit */ - $hits = $index->find($query); - // get the document from the query hit - $foundDoc = $hits[0]->getDocument(); - $this->assertEquals('alice', $foundDoc->getFieldValue('users')); - - // delete the document from the index - //$index->index->delete($hit); - - // change the 'users' key of the document - $foundDoc->addField(\Zend_Search_Lucene_Field::Text('users', 'bob' , 'UTF-8')); - $this->assertEquals('bob', $foundDoc->getFieldValue('users')); - - // add the document back to the index - $index->updateFile($foundDoc, '1'); - - - $idTerm2 = new \Zend_Search_Lucene_Index_Term('1', 'fileId'); - $idQuery2 = new \Zend_Search_Lucene_Search_Query_Term($idTerm2); - - $query2 = new \Zend_Search_Lucene_Search_Query_Boolean(); - $query2->addSubquery($idQuery2); - /** @var \Zend_Search_Lucene_Search_QueryHit $hit */ - $hits2 = $index->find($query2); - // get the document from the query hit - $foundDoc2 = $hits2[0]->getDocument(); - $this->assertEquals('alice', $foundDoc2->getFieldValue('users')); + $doc = Pdf::loadPdf($data, true); + echo $doc->getFieldValue('body'); } } diff --git a/tests/unit/testindex.php b/tests/unit/testindex.php new file mode 100644 index 000000000..f5c2c3b5d --- /dev/null +++ b/tests/unit/testindex.php @@ -0,0 +1,86 @@ +. + * + */ + +namespace OCA\Search_Lucene\Tests\Unit; + +use OCA\Search_Lucene\AppInfo\Application; +use OCA\Search_Lucene\Lucene\Index; + +class TestIndex extends TestCase { + + function testUpdate() { + + // preparation + $app = new Application(); + $container = $app->getContainer(); + + // get an index + /** @var Index $index */ + $index = $container->query('Index'); + + // add a document + $doc = new \Zend_Search_Lucene_Document(); + + $doc->addField(\Zend_Search_Lucene_Field::Keyword('fileId', '1')); + $doc->addField(\Zend_Search_Lucene_Field::Text('path', '/somewhere/deep/down/the/rabbit/hole' , 'UTF-8')); + $doc->addField(\Zend_Search_Lucene_Field::Text('users', 'alice' , 'UTF-8')); + + $index->index->addDocument($doc); + $index->commit(); + + // search for it + $idTerm = new \Zend_Search_Lucene_Index_Term('1', 'fileId'); + $idQuery = new \Zend_Search_Lucene_Search_Query_Term($idTerm); + + $query = new \Zend_Search_Lucene_Search_Query_Boolean(); + $query->addSubquery($idQuery); + /** @var \Zend_Search_Lucene_Search_QueryHit $hit */ + $hits = $index->find($query); + // get the document from the query hit + $foundDoc = $hits[0]->getDocument(); + $this->assertEquals('alice', $foundDoc->getFieldValue('users')); + + // delete the document from the index + //$index->index->delete($hit); + + // change the 'users' key of the document + $foundDoc->addField(\Zend_Search_Lucene_Field::Text('users', 'bob' , 'UTF-8')); + $this->assertEquals('bob', $foundDoc->getFieldValue('users')); + + // add the document back to the index + $index->updateFile($foundDoc, '1'); + + + $idTerm2 = new \Zend_Search_Lucene_Index_Term('1', 'fileId'); + $idQuery2 = new \Zend_Search_Lucene_Search_Query_Term($idTerm2); + + $query2 = new \Zend_Search_Lucene_Search_Query_Boolean(); + $query2->addSubquery($idQuery2); + /** @var \Zend_Search_Lucene_Search_QueryHit $hit */ + $hits2 = $index->find($query2); + // get the document from the query hit + $foundDoc2 = $hits2[0]->getDocument(); + $this->assertEquals('bob', $foundDoc2->getFieldValue('users')); + + } +} From f8856e009b8066f946b8b2e7d539dffe1e35dfe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Tue, 29 Jul 2014 13:58:47 +0200 Subject: [PATCH 46/51] adding scrutinizer.yml --- .scrutinizer.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .scrutinizer.yml diff --git a/.scrutinizer.yml b/.scrutinizer.yml new file mode 100644 index 000000000..dc1e74008 --- /dev/null +++ b/.scrutinizer.yml @@ -0,0 +1,11 @@ +filter: + excluded_paths: + - '3rdparty/*' + +imports: + - javascript + - php + +tools: + external_code_coverage: true + From a97fff3bae8e248c22e4845f6eeb7401e6efd4f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Tue, 29 Jul 2014 14:33:19 +0200 Subject: [PATCH 47/51] execute phpunit tests on travis-ci --- .travis.yml | 58 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index af69e92e6..67b0d2810 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,34 +1,44 @@ -# see http://about.travis-ci.org/docs/user/languages/php/ for more hints language: php - -# list any PHP version you want to test against php: - # using major version aliases - - # aliased to a recent 5.3.x version - 5.3 - # aliased to a recent 5.4.x version - 5.4 - # aliased to a recent 5.5.x version - 5.5 -# optionally specify a list of environments, for example to test different RDBMS -#env: -# - DB=mysql -# - DB=pgsql +env: + global: + - CORE_BRANCH=master + matrix: + - DB=sqlite + +branches: + only: + - master + - stable7 + +before_install: + - composer install + - wget https://raw.githubusercontent.com/owncloud/administration/master/travis-ci/before_install.sh + - bash ./before_install.sh seach_lucene $CORE_BRANCH $DB -# execute any number of scripts before the test run, custom env's are available as variables -#before_script: -# - if [[ "$DB" == "pgsql" ]]; then psql -c "DROP DATABASE IF EXISTS hello_world_test;" -U postgres; fi -# - if [[ "$DB" == "pgsql" ]]; then psql -c "create database hello_world_test;" -U postgres; fi -# - if [[ "$DB" == "mysql" ]]; then mysql -e "create database IF NOT EXISTS hello_world_test;" -uroot; fi +script: + # Test lint + - cd ../core/apps/seach_lucene + - sh -c "if [ '$DB' = 'sqlite' ]; then ant test; fi" -# omitting "script:" will default to phpunit -# use the $DB env variable to determine the phpunit.xml to use -#script: phpunit --configuration phpunit_$DB.xml --coverage-text -script: ant test + # Run phpunit tests + - cd tests/unit + - phpunit --configuration phpunit.xml -# configure notifications (email, IRC, campfire etc) -#notifications: -# irc: "irc.freenode.org#travis" + # Create coverage report + - wget https://scrutinizer-ci.com/ocular.phar + - php ocular.phar code-coverage:upload --format=php-clover clover.xml +matrix: + include: + - php: 5.4 + env: DB=mysql + - php: 5.4 + env: DB=pgsql + allow_failures: + - php: hhvm + fast_finish: true From 7a2c1d04b6e8824a7f852d83bd36623115220ab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Tue, 29 Jul 2014 15:20:12 +0200 Subject: [PATCH 48/51] disable composer for now --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 67b0d2810..6d8c7fbb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,13 +16,13 @@ branches: - stable7 before_install: - - composer install +# - composer install - wget https://raw.githubusercontent.com/owncloud/administration/master/travis-ci/before_install.sh - - bash ./before_install.sh seach_lucene $CORE_BRANCH $DB + - bash ./before_install.sh search_lucene $CORE_BRANCH $DB script: # Test lint - - cd ../core/apps/seach_lucene + - cd ../core/apps/search_lucene - sh -c "if [ '$DB' = 'sqlite' ]; then ant test; fi" # Run phpunit tests From c3244b683f4c75130e30d49d919e29991b2185cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Wed, 30 Jul 2014 12:13:42 +0200 Subject: [PATCH 49/51] fixing code coverage reporting --- tests/unit/phpunit.xml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/unit/phpunit.xml b/tests/unit/phpunit.xml index d68c43cf7..1ab31cd9c 100644 --- a/tests/unit/phpunit.xml +++ b/tests/unit/phpunit.xml @@ -11,11 +11,19 @@ - - - - ../.. - - + + + ../../../search_lucene + + ../../../search_lucene/3rdparty + ../../../search_lucene/l10n + ../../../search_lucene/tests + + + + + + + From d23a0ec504f826b04932afb987dd174998c78fd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 30 Jul 2014 13:22:54 +0200 Subject: [PATCH 50/51] add code coverage badge --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 61983881d..ade6dc420 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Build Status](https://secure.travis-ci.org/owncloud/search_lucene.png)](http://travis-ci.org/owncloud/search_lucene) [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/owncloud/search_lucene/badges/quality-score.png)](https://scrutinizer-ci.com/g/owncloud/search_lucene/) +[![Scrutinizer Code Coverage](https://scrutinizer-ci.com/g/owncloud/search_lucene/badges/coverage.png)](https://scrutinizer-ci.com/g/owncloud/search_lucene/) The Search Lucene app adds a full text search for files stored in ownCloud. It is based on [Zend Search Lucene](http://framework.zend.com/manual/1.12/en/zend.search.lucene.html) and @@ -19,4 +20,4 @@ Maintainers wanted for additional features! * Does not work with Encryption: the background indexing process does not have access to the key needed to decrypt files when the user is not logged in. * Does not index files in external storage. For performance reasons. -* Not all PDF versions can be indexed. The text extraction used for it is incompatible with newer PDF versions. \ No newline at end of file +* Not all PDF versions can be indexed. The text extraction used for it is incompatible with newer PDF versions. From e9a4b5a045198b10bf8ba414f21d22539234a969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 30 Jul 2014 22:29:18 +0200 Subject: [PATCH 51/51] fix delete hook --- db/statusmapper.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/db/statusmapper.php b/db/statusmapper.php index 593498217..ad83ae6b4 100644 --- a/db/statusmapper.php +++ b/db/statusmapper.php @@ -34,11 +34,14 @@ public function __construct(IDb $db, ILogger $logger){ /** * Deletes a status from the table - * @param Entity $entity the status that should be deleted + * @param Entity|integer $statusOrId the status that should be deleted */ - public function delete(Entity $entity){ + public function delete($statusOrId){ + if ($statusOrId instanceof Status) { + $statusOrId = $statusOrId->getFileId(); + } $sql = 'DELETE FROM `' . $this->tableName . '` WHERE `fileid` = ?'; - $this->execute($sql, array($entity->getFileId())); + $this->execute($sql, array($statusOrId)); } /**