FilesMetadata

Signed-off-by: Maxence Lange <maxence@artificial-owl.com>
nextcloud · Oct 4, 2023 · 59ed4a4 · 59ed4a4
1 parent 8f30f97
commit 59ed4a4
Show file tree

Hide file tree

Showing 12 changed files with 645 additions and 0 deletions.
diff --git a/core/Command/FilesMetadata/Get.php b/core/Command/FilesMetadata/Get.php
@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OC\Core\Command\FilesMetadata;
+
+use OCP\FilesMetadata\IFilesMetadataManager;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class Get extends Command {
+	public function __construct(
+		private IFilesMetadataManager $filesMetadataManager,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure() {
+		$this->setName('metadata:get')
+			 ->setDescription('update and returns up-to-date metadata')
+			 ->addArgument(
+				 'fileId',
+				 InputArgument::REQUIRED,
+				 'id of the file document'
+			 )
+			 ->addOption(
+				 'background',
+				 '',
+				 InputOption::VALUE_NONE,
+				 'emulate background jobs env'
+			 );
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output): int {
+		$fileId = (int) $input->getArgument('fileId');
+		$metadata = $this->filesMetadataManager->refreshMetadata($fileId);
+		$output->writeln(json_encode($metadata, JSON_PRETTY_PRINT));
+
+		return 0;
+	}
+}
diff --git a/core/Migrations/Version28000Date20231004103301.php b/core/Migrations/Version28000Date20231004103301.php
@@ -0,0 +1,73 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OC\Core\Migrations;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+class Version28000Date20231004103301 extends SimpleMigrationStep {
+
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		if (!$schema->hasTable('files_metadata')) {
+			$table = $schema->createTable('files_metadata');
+			$table->addColumn('id', Types::BIGINT, [
+				'autoincrement' => true,
+				'notnull' => true,
+				'length' => 15,
+				'unsigned' => true,
+			]);
+			$table->addColumn('file_id', Types::BIGINT, [
+				'notnull' => false,
+				'length' => 15,
+			]);
+			$table->addColumn('json', Types::TEXT);
+			$table->addColumn('sync_token', Types::STRING, [
+				'length' => 15,
+			]);
+			$table->addColumn('last_update', Types::DATETIME);
+
+			$table->setPrimaryKey(['id']);
+			$table->addUniqueIndex(['file_id'], 'files_meta_fileid');
+		}
+
+		if (!$schema->hasTable('files_metadata_index')) {
+			$table = $schema->createTable('files_metadata_index');
+			$table->addColumn('id', Types::BIGINT, [
+				'autoincrement' => true,
+				'notnull' => true,
+				'length' => 15,
+				'unsigned' => true,
+			]);
+			$table->addColumn('file_id', Types::BIGINT, [
+				'notnull' => false,
+				'length' => 15,
+			]);
+			$table->addColumn('k', Types::STRING, [
+				'notnull' => false,
+				'length' => 31,
+			]);
+			$table->addColumn('v', Types::STRING, [
+				'notnull' => false,
+				'length' => 63,
+			]);
+			$table->addColumn('v_int', Types::BIGINT, [
+				'notnull' => false,
+				'length' => 11,
+			]);
+			$table->addColumn('last_update', Types::DATETIME);
+
+			$table->setPrimaryKey(['id']);
+			$table->addIndex(['k', 'v', 'v_int'], 'files_meta_indexes');
+		}
+
+		return $schema;
+	}
+}
diff --git a/core/register_command.php b/core/register_command.php
@@ -214,6 +214,8 @@
 	$application->add(new OC\Core\Command\Security\RemoveCertificate(\OC::$server->getCertificateManager()));
 	$application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceAttempts::class));
 	$application->add(\OC::$server->get(\OC\Core\Command\Security\BruteforceResetAttempts::class));
+
+	$application->add(\OCP\Server::get(\OC\Core\Command\FilesMetadata\Get::class));
 } else {
 	$application->add(\OC::$server->get(\OC\Core\Command\Maintenance\Install::class));
 }
diff --git a/lib/private/FilesMetadata/Event/FilesMetadataEvent.php b/lib/private/FilesMetadata/Event/FilesMetadataEvent.php
@@ -0,0 +1,57 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OC\FilesMetadata\Event;
+
+use OCP\EventDispatcher\Event;
+use OCP\FilesMetadata\IFilesMetadata;
+
+class FilesMetadataEvent extends Event {
+	private bool $runAsBackgroundJob = false;
+
+	public function __construct(
+		private int $fileId,
+		private IFilesMetadata $metadata,
+	) {
+		parent::__construct();
+	}
+
+	/**
+	 * If an app prefer to update metadata on a background job, instead of
+	 * live process, just call this method.
+	 * A new event will be generated on next cron tick
+	 *
+	 * @return void
+	 */
+	public function requestBackgroundJob() {
+		$this->runAsBackgroundJob = true;
+	}
+
+	/**
+	 * return fileId
+	 *
+	 * @return int
+	 */
+	public function getFileId(): int {
+		return $this->fileId;
+	}
+
+	/**
+	 * return Metadata
+	 *
+	 * @return IFilesMetadata
+	 */
+	public function getMetadata(): IFilesMetadata {
+		return $this->metadata;
+	}
+
+	/**
+	 * return true if any app that catch this event requested a re-run as background job
+	 *
+	 * @return bool
+	 */
+	public function isRunAsBackgroundJobRequested(): bool {
+		return $this->runAsBackgroundJob;
+	}
+}
diff --git a/lib/private/FilesMetadata/FilesMetadataManager.php b/lib/private/FilesMetadata/FilesMetadataManager.php
@@ -0,0 +1,170 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OC\FilesMetadata;
+
+use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
+use OC\FilesMetadata\Event\FilesMetadataEvent;
+use OC\FilesMetadata\Model\FilesMetadata;
+use OCP\DB\Exception;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\EventDispatcher\IEventDispatcher;
+use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException;
+use OCP\FilesMetadata\IFilesMetadata;
+use OCP\FilesMetadata\IFilesMetadataManager;
+use OCP\FilesMetadata\IFilesMetadataQueryHelper;
+use OCP\IDBConnection;
+use Psr\Log\LoggerInterface;
+
+class FilesMetadataManager implements IFilesMetadataManager {
+	public const TABLE_METADATA = 'files_metadata';
+	public const TABLE_METADATA_INDEX = 'files_metadata_index';
+
+	public function __construct(
+		private IDBConnection $dbConnection,
+		private IEventDispatcher $eventDispatcher,
+		private FilesMetadataQueryHelper $filesMetadataQueryHelper,
+		private LoggerInterface $logger
+	) {
+	}
+
+	public function refreshMetadata(
+		int $fileId,
+		bool $asBackgroundJob = false,
+		bool $fromScratch = false
+	): IFilesMetadata {
+		$metadata = null;
+		if (!$fromScratch) {
+			try {
+				$metadata = $this->selectMetadata($fileId);
+			} catch (FilesMetadataNotFoundException $e) {
+			}
+		}
+
+		if (is_null($metadata)) {
+			$metadata = new FilesMetadata($fileId);
+		}
+
+		$event = new FilesMetadataEvent($fileId, $metadata);
+		$this->eventDispatcher->dispatchTyped($event);
+		$this->saveMetadata($event->getMetadata());
+
+		return $metadata;
+	}
+
+	/**
+	 * @param int $fileId
+	 * @param bool $createIfNeeded
+	 *
+	 * @return IFilesMetadata
+	 * @throws FilesMetadataNotFoundException
+	 */
+	public function getMetadata(int $fileId, bool $createIfNeeded = false): IFilesMetadata {
+		try {
+			return $this->selectMetadata($fileId);
+		} catch (FilesMetadataNotFoundException $e) {
+			if ($createIfNeeded) {
+				return $this->refreshMetadata($fileId);
+			}
+
+			throw $e;
+		}
+	}
+
+
+	public function saveMetadata(IFilesMetadata $filesMetadata): void {
+		if ($filesMetadata->getFileId() === 0 || !$filesMetadata->updated()) {
+			return;
+		}
+
+		try {
+			try {
+				$this->insertMetadata($filesMetadata);
+			} catch (UniqueConstraintViolationException $e) {
+				$this->updateMetadata($filesMetadata);
+			}
+		} catch (Exception $e) {
+			$this->logger->warning('exception while saveMetadata()', ['exception' => $e]);
+
+			return;
+		}
+
+//		$this->removeDeprecatedMetadata($filesMetadata);
+		// remove indexes from metadata_index that are not in the list of indexes anymore.
+		foreach ($filesMetadata->listIndexes() as $index) {
+			// foreach index, update entry in table metadata_index
+			// if no update, insert as new row
+			// !! we might want to get the type of the value to be indexed at one point !!
+		}
+	}
+
+	private function removeDeprecatedMetadata(IFilesMetadata $filesMetadata): void {
+		$qb = $this->dbConnection->getQueryBuilder();
+		$qb->delete(self::TABLE_METADATA_INDEX)
+		   ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)))
+		   ->andWhere($qb->expr()->notIn('file_id', $filesMetadata->listIndexes(), IQueryBuilder::PARAM_STR_ARRAY));
+		$qb->executeStatement();
+	}
+
+
+	public function getQueryHelper(): IFilesMetadataQueryHelper {
+		return $this->filesMetadataQueryHelper;
+	}
+
+	private function insertMetadata(IFilesMetadata $filesMetadata): void {
+		$qb = $this->dbConnection->getQueryBuilder();
+		$qb->insert(self::TABLE_METADATA)
+		   ->setValue('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT))
+		   ->setValue('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize())))
+		   ->setValue('sync_token', $qb->createNamedParameter('abc'))
+		   ->setValue('last_update', $qb->createFunction('NOW()'));
+		$qb->execute();
+	}
+
+	/**
+	 * @param IFilesMetadata $filesMetadata
+	 *
+	 * @return bool
+	 * @throws Exception
+	 */
+	private function updateMetadata(IFilesMetadata $filesMetadata): void {
+		// TODO check sync_token on update
+		$qb = $this->dbConnection->getQueryBuilder();
+		$qb->update(self::TABLE_METADATA)
+		   ->set('json', $qb->createNamedParameter(json_encode($filesMetadata->jsonSerialize())))
+		   ->set('sync_token', $qb->createNamedParameter('abc'))
+		   ->set('last_update', $qb->createFunction('NOW()'))
+		   ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($filesMetadata->getFileId(), IQueryBuilder::PARAM_INT)));
+		$qb->executeStatement();
+	}
+
+	/**
+	 * @param int $fileId
+	 *
+	 * @return IFilesMetadata
+	 * @throws FilesMetadataNotFoundException
+	 */
+	private function selectMetadata(int $fileId): IFilesMetadata {
+		try {
+			$qb = $this->dbConnection->getQueryBuilder();
+			$qb->select('json')->from(self::TABLE_METADATA);
+			$qb->where($qb->expr()->eq('file_id', $qb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)));
+			$result = $qb->executeQuery();
+			$data = $result->fetch();
+			$result->closeCursor();
+		} catch (Exception $e) {
+			$this->logger->warning('exception while getMetadataFromDatabase()', ['exception' => $e, 'fileId' => $fileId]);
+			throw new FilesMetadataNotFoundException();
+		}
+
+		if ($data === false) {
+			throw new FilesMetadataNotFoundException();
+		}
+
+		$metadata = new FilesMetadata($fileId);
+		$metadata->import($data['json'] ?? '');
+
+		return $metadata;
+	}
+}