From 241fbdb45b6dc238272bf54a0cda444781381e3a Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 11 Mar 2021 16:37:23 +0100 Subject: [PATCH 1/2] add a prefix index to filecache.path The reason that `filecache.path` hasn't had an index added is the mysql limitation of ~1kb for indexeded fields, which is to small for the `path`, however mysql supports indexing only the first N bytes of a column instead of the entire column, allowing us to add an index even if the column is to long. Because the index doesn't cover the entire column it can't be used in all situations where a normal index would be used, but it does cover the `path like 'folder/path/%'` queries that are used in various places. Sqlite and Postgresql don't support prefix indexes, but they also don't have the 1kb limit and DBAL handles the differences in index creation. Signed-off-by: Robin Appelman --- core/Application.php | 4 ++++ core/Command/Db/AddMissingIndices.php | 7 +++++++ core/Migrations/Version13000Date20170718121200.php | 1 + 3 files changed, 12 insertions(+) diff --git a/core/Application.php b/core/Application.php index 0a87e69859789..e7f3dd255e147 100644 --- a/core/Application.php +++ b/core/Application.php @@ -107,6 +107,10 @@ function (GenericEvent $event) use ($container) { if (!$table->hasIndex('fs_mtime')) { $subject->addHintForMissingSubject($table->getName(), 'fs_mtime'); } + + if (!$table->hasIndex('fs_storage_path_prefix')) { + $subject->addHintForMissingSubject($table->getName(), 'fs_storage_path_prefix'); + } } if ($schema->hasTable('twofactor_providers')) { diff --git a/core/Command/Db/AddMissingIndices.php b/core/Command/Db/AddMissingIndices.php index 41ce626405e62..691f2df559c12 100644 --- a/core/Command/Db/AddMissingIndices.php +++ b/core/Command/Db/AddMissingIndices.php @@ -135,6 +135,13 @@ private function addCoreIndexes(OutputInterface $output) { $updated = true; $output->writeln('Filecache table updated successfully.'); } + if (!$table->hasIndex('fs_storage_path_prefix')) { + $output->writeln('Adding additional path index to the filecache table, this can take some time...'); + $table->addIndex(['storage', 'path'], 'fs_storage_path_prefix', [], ['lengths' => [null, 64]]); + $this->connection->migrateToSchema($schema->getWrappedSchema()); + $updated = true; + $output->writeln('Filecache table updated successfully.'); + } } $output->writeln('Check indices of the twofactor_providers table.'); diff --git a/core/Migrations/Version13000Date20170718121200.php b/core/Migrations/Version13000Date20170718121200.php index a2616c95752f0..4956aefada071 100644 --- a/core/Migrations/Version13000Date20170718121200.php +++ b/core/Migrations/Version13000Date20170718121200.php @@ -260,6 +260,7 @@ public function changeSchema(IOutput $output, \Closure $schemaClosure, array $op $table->addIndex(['storage', 'mimepart'], 'fs_storage_mimepart'); $table->addIndex(['storage', 'size', 'fileid'], 'fs_storage_size'); $table->addIndex(['mtime'], 'fs_mtime'); + $table->addIndex(['storage', 'path'], 'fs_storage_path_prefix', [], ['lengths' => [null, 64]]); } if (!$schema->hasTable('group_user')) { From be702a59ec643a372741c8633d94b42681065a3e Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 21 Oct 2021 18:05:49 +0200 Subject: [PATCH 2/2] disable path prefix index on postgresql for now having the index work properly for the queries we need it for requires some additional options which dbal does not support at the momement. to prevent making it harder to add the correct index later on we don't create the index for now on postgresql Signed-off-by: Robin Appelman --- core/Application.php | 3 ++- core/Command/Db/AddMissingIndices.php | 3 ++- core/Migrations/Version13000Date20170718121200.php | 5 ++++- lib/private/DB/SchemaWrapper.php | 12 ++++++++++++ lib/public/DB/ISchemaWrapper.php | 14 +++++++++++++- 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/core/Application.php b/core/Application.php index e7f3dd255e147..2d5ff6c3118e9 100644 --- a/core/Application.php +++ b/core/Application.php @@ -31,6 +31,7 @@ namespace OC\Core; +use Doctrine\DBAL\Platforms\PostgreSQL94Platform; use OC\Authentication\Events\RemoteWipeFinished; use OC\Authentication\Events\RemoteWipeStarted; use OC\Authentication\Listeners\RemoteWipeActivityListener; @@ -108,7 +109,7 @@ function (GenericEvent $event) use ($container) { $subject->addHintForMissingSubject($table->getName(), 'fs_mtime'); } - if (!$table->hasIndex('fs_storage_path_prefix')) { + if (!$table->hasIndex('fs_storage_path_prefix') && !$schema->getDatabasePlatform() instanceof PostgreSQL94Platform) { $subject->addHintForMissingSubject($table->getName(), 'fs_storage_path_prefix'); } } diff --git a/core/Command/Db/AddMissingIndices.php b/core/Command/Db/AddMissingIndices.php index 691f2df559c12..a26b8d16f541b 100644 --- a/core/Command/Db/AddMissingIndices.php +++ b/core/Command/Db/AddMissingIndices.php @@ -32,6 +32,7 @@ namespace OC\Core\Command\Db; +use Doctrine\DBAL\Platforms\PostgreSQL94Platform; use OC\DB\SchemaWrapper; use OCP\IDBConnection; use Symfony\Component\Console\Command\Command; @@ -135,7 +136,7 @@ private function addCoreIndexes(OutputInterface $output) { $updated = true; $output->writeln('Filecache table updated successfully.'); } - if (!$table->hasIndex('fs_storage_path_prefix')) { + if (!$table->hasIndex('fs_storage_path_prefix') && !$schema->getDatabasePlatform() instanceof PostgreSQL94Platform) { $output->writeln('Adding additional path index to the filecache table, this can take some time...'); $table->addIndex(['storage', 'path'], 'fs_storage_path_prefix', [], ['lengths' => [null, 64]]); $this->connection->migrateToSchema($schema->getWrappedSchema()); diff --git a/core/Migrations/Version13000Date20170718121200.php b/core/Migrations/Version13000Date20170718121200.php index 4956aefada071..971971cfa3d3d 100644 --- a/core/Migrations/Version13000Date20170718121200.php +++ b/core/Migrations/Version13000Date20170718121200.php @@ -29,6 +29,7 @@ namespace OC\Core\Migrations; +use Doctrine\DBAL\Platforms\PostgreSQL94Platform; use Doctrine\DBAL\Types\Types; use OCP\DB\ISchemaWrapper; use OCP\IDBConnection; @@ -260,7 +261,9 @@ public function changeSchema(IOutput $output, \Closure $schemaClosure, array $op $table->addIndex(['storage', 'mimepart'], 'fs_storage_mimepart'); $table->addIndex(['storage', 'size', 'fileid'], 'fs_storage_size'); $table->addIndex(['mtime'], 'fs_mtime'); - $table->addIndex(['storage', 'path'], 'fs_storage_path_prefix', [], ['lengths' => [null, 64]]); + if (!$schema->getDatabasePlatform() instanceof PostgreSQL94Platform) { + $table->addIndex(['storage', 'path'], 'fs_storage_path_prefix', [], ['lengths' => [null, 64]]); + } } if (!$schema->hasTable('group_user')) { diff --git a/lib/private/DB/SchemaWrapper.php b/lib/private/DB/SchemaWrapper.php index e42535d64ab25..b75e502b2d9b7 100644 --- a/lib/private/DB/SchemaWrapper.php +++ b/lib/private/DB/SchemaWrapper.php @@ -24,6 +24,7 @@ namespace OC\DB; +use Doctrine\DBAL\Platforms\AbstractPlatform; use Doctrine\DBAL\Schema\Schema; use OCP\DB\ISchemaWrapper; use OCP\IDBConnection; @@ -133,4 +134,15 @@ public function dropTable($tableName) { public function getTables() { return $this->schema->getTables(); } + + /** + * Gets the DatabasePlatform for the database. + * + * @return AbstractPlatform + * + * @throws Exception + */ + public function getDatabasePlatform() { + return $this->connection->getDatabasePlatform(); + } } diff --git a/lib/public/DB/ISchemaWrapper.php b/lib/public/DB/ISchemaWrapper.php index 3d58a10d2d28e..6ee7d625402cd 100644 --- a/lib/public/DB/ISchemaWrapper.php +++ b/lib/public/DB/ISchemaWrapper.php @@ -23,6 +23,8 @@ namespace OCP\DB; +use Doctrine\DBAL\Platforms\AbstractPlatform; + /** * Interface ISchemaWrapper * @@ -82,7 +84,7 @@ public function getTables(); * @since 13.0.0 */ public function getTableNames(); - + /** * Gets all table names * @@ -90,4 +92,14 @@ public function getTableNames(); * @since 13.0.0 */ public function getTableNamesWithoutPrefix(); + + /** + * Gets the DatabasePlatform for the database. + * + * @return AbstractPlatform + * + * @throws Exception + * @since 23.0.0 + */ + public function getDatabasePlatform(); }