Merge pull request #28253 from owncloud/repair-filecache-mismatch-path

Add repair step to repair mismatch filecache paths

apps/files/appinfo/app.php

@@ -29,6 +29,9 @@
 
 \OC::$server->getSearch()->registerProvider('OC\Search\Provider\File', ['apps' => ['files']]);
 
+// instantiate to make sure services get registered
+$app = new \OCA\Files\AppInfo\Application();
+
 $templateManager = \OC_Helper::getFileTemplateManager();
 $templateManager->registerTemplate('text/html', 'core/templates/filetemplates/template.html');
 $templateManager->registerTemplate('application/vnd.oasis.opendocument.presentation', 'core/templates/filetemplates/template.odp');

apps/files/lib/AppInfo/Application.php

@@ -87,6 +87,13 @@ public function __construct(array $urlParams= []) {
 			);
 		});
 
+		$container->registerService('OCP\Lock\ILockingProvider', function(IContainer $c)  {
+			return $c->query('ServerContainer')->getLockingProvider();
+		});
+		$container->registerService('OCP\Files\IMimeTypeLoader', function(IContainer $c)  {
+			return $c->query('ServerContainer')->getMimeTypeLoader();
+		});
+
 		/*
 		 * Register capabilities
 		 */

apps/files/lib/Command/Scan.php

@@ -39,20 +39,40 @@
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Helper\Table;
+use OC\Repair\RepairMismatchFileCachePath;
+use OC\Migration\ConsoleOutput;
+use OCP\Lock\ILockingProvider;
+use OCP\Lock\LockedException;
+use OCP\Files\IMimeTypeLoader;
+use OCP\IConfig;
 
 class Scan extends Base {
 
 	/** @var IUserManager $userManager */
 	private $userManager;
+	/** @var ILockingProvider */
+	private $lockingProvider;
+	/** @var IMimeTypeLoader */
+	private $mimeTypeLoader;
+	/** @var IConfig */
+	private $config;
 	/** @var float */
 	protected $execTime = 0;
 	/** @var int */
 	protected $foldersCounter = 0;
 	/** @var int */
 	protected $filesCounter = 0;
 
-	public function __construct(IUserManager $userManager) {
+	public function __construct(
+		IUserManager $userManager,
+		ILockingProvider $lockingProvider,
+		IMimeTypeLoader $mimeTypeLoader,
+		IConfig $config
+	) {
 		$this->userManager = $userManager;
+		$this->lockingProvider = $lockingProvider;
+		$this->mimeTypeLoader = $mimeTypeLoader;
+		$this->config = $config;
 		parent::__construct();
 	}
 
@@ -90,6 +110,12 @@ protected function configure() {
 				null,
 				InputOption::VALUE_NONE,
 				'will rescan all files of all known users'
+			)
+			->addOption(
+				'repair',
+				null,
+				InputOption::VALUE_NONE,
+				'will repair detached filecache entries (slow)'
 			)->addOption(
 				'unscanned',
 				null,
@@ -107,9 +133,48 @@ public function checkScanWarning($fullPath, OutputInterface $output) {
 		}
 	}
 
-	protected function scanFiles($user, $path, $verbose, OutputInterface $output, $backgroundScan = false) {
+	/**
+	 * Repair all storages at once
+	 *
+	 * @param OutputInterface $output
+	 */
+	protected function repairAll(OutputInterface $output) {
+		$connection = $this->reconnectToDatabase($output);
+		$repairStep = new RepairMismatchFileCachePath(
+			$connection,
+			$this->mimeTypeLoader
+		);
+		$repairStep->setStorageNumericId(null);
+		$repairStep->setCountOnly(false);
+		$repairStep->run(new ConsoleOutput($output));
+	}
+
+	protected function scanFiles($user, $path, $verbose, OutputInterface $output, $backgroundScan = false, $shouldRepair = false) {
 		$connection = $this->reconnectToDatabase($output);
 		$scanner = new \OC\Files\Utils\Scanner($user, $connection, \OC::$server->getLogger());
+		if ($shouldRepair) {
+			$scanner->listen('\OC\Files\Utils\Scanner', 'beforeScanStorage', function ($storage) use ($output, $connection) {
+				try {
+					// FIXME: this will lock the storage even if there is nothing to repair
+					$storage->acquireLock('', ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
+				} catch (OCP\Lock\LockedException $e) {
+					$output->writeln("\t<error>Storage \"" . $storage->getCache()->getNumericStorageId() . '" cannot be repaired as it is currently in use, please try again later</error>');
+					return;
+				}
+				try {
+					$repairStep = new RepairMismatchFileCachePath(
+						$connection,
+						$this->mimeTypeLoader
+					);
+					$repairStep->setStorageNumericId($storage->getCache()->getNumericStorageId());
+					$repairStep->setCountOnly(false);
+					$repairStep->run(new ConsoleOutput($output));
+				} finally {
+					$storage->releaseLock('', ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
+				}
+			});
+		}
+
 		# check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception
 		# printout and count
 		if ($verbose) {
@@ -156,7 +221,7 @@ protected function scanFiles($user, $path, $verbose, OutputInterface $output, $b
 			if ($backgroundScan) {
 				$scanner->backgroundScan($path);
 			}else {
-				$scanner->scan($path);
+				$scanner->scan($path, $shouldRepair);
 			}
 		} catch (ForbiddenException $e) {
 			$output->writeln("<error>Home storage for user $user not writable</error>");
@@ -166,18 +231,32 @@ protected function scanFiles($user, $path, $verbose, OutputInterface $output, $b
 			$output->writeln('Interrupted by user');
 			return;
 		} catch (\Exception $e) {
-			$output->writeln('<error>Exception during scan: ' . $e->getMessage() . "\n" . $e->getTraceAsString() . '</error>');
+			$output->writeln('<error>Exception during scan: ' . get_class($e) . ': ' . $e->getMessage() . "\n" . $e->getTraceAsString() . '</error>');
 		}
 	}
 
 
 	protected function execute(InputInterface $input, OutputInterface $output) {
 		$inputPath = $input->getOption('path');
+		$shouldRepairStoragesIndividually = (bool) $input->getOption('repair');
+
 		if ($inputPath) {
 			$inputPath = '/' . trim($inputPath, '/');
 			list (, $user,) = explode('/', $inputPath, 3);
 			$users = [$user];
 		} else if ($input->getOption('all')) {
+			// we can only repair all storages in bulk (more efficient) if singleuser or maintenance mode
+			// is enabled to prevent concurrent user access
+			if ($input->getOption('repair') &&
+				($this->config->getSystemValue('singleuser', false) || $this->config->getSystemValue('maintenance', false))) {
+				// repair all storages at once
+				$this->repairAll($output);
+				// don't fix individually
+				$shouldRepairStoragesIndividually = false;
+			} else {
+				$output->writeln("<comment>Repairing every storage individually is slower than repairing in bulk</comment>");
+				$output->writeln("<comment>To repair in bulk, please switch to single user mode first: occ maintenance:singleuser --on</comment>");
+			}
 			$users = $this->userManager->search('');
 		} else {
 			$users = $input->getArgument('user_id');
@@ -223,9 +302,10 @@ protected function execute(InputInterface $input, OutputInterface $output) {
 			if ($this->userManager->userExists($user)) {
 				# add an extra line when verbose is set to optical separate users
 				if ($verbose) {$output->writeln(""); }
-				$output->writeln("Starting scan for user $user_count out of $users_total ($user)");
+				$r = $shouldRepairStoragesIndividually ? ' (and repair)' : '';
+				$output->writeln("Starting scan$r for user $user_count out of $users_total ($user)");
 				# full: printout data if $verbose was set
-				$this->scanFiles($user, $path, $verbose, $output, $input->getOption('unscanned'));
+				$this->scanFiles($user, $path, $verbose, $output, $input->getOption('unscanned'), $shouldRepairStoragesIndividually);
 			} else {
 				$output->writeln("<error>Unknown user $user_count $user</error>");
 			}

lib/private/Files/Utils/Scanner.php

@@ -211,6 +211,9 @@ public function scan($dir = '') {
 			if ($storage->instanceOfStorage('OCA\Files_Sharing\ISharedStorage')) {
 				continue;
 			}
+
+			$this->emit('\OC\Files\Utils\Scanner', 'beforeScanStorage', [$storage]);
+
 			$relativePath = $mount->getInternalPath($dir);
 			$scanner = $storage->getScanner();
 			$scanner->setUseTransactions(false);
@@ -247,6 +250,7 @@ public function scan($dir = '') {
 			if ($this->useTransaction) {
 				$this->db->commit();
 			}
+			$this->emit('\OC\Files\Utils\Scanner', 'afterScanStorage', [$storage]);
 		}
 	}
 

lib/private/Repair.php

@@ -53,6 +53,7 @@
 use Symfony\Component\EventDispatcher\EventDispatcher;
 use Symfony\Component\EventDispatcher\GenericEvent;
 use OC\Repair\MoveAvatarOutsideHome;
+use OC\Repair\RepairMismatchFileCachePath;
 
 class Repair implements IOutput{
 	/* @var IRepairStep[] */
@@ -126,6 +127,7 @@ public function addStep($repairStep) {
 	public static function getRepairSteps() {
 		return [
 			new RepairMimeTypes(\OC::$server->getConfig()),
+			new RepairMismatchFileCachePath(\OC::$server->getDatabaseConnection(), \OC::$server->getMimeTypeLoader()),
 			new FillETags(\OC::$server->getDatabaseConnection()),
 			new CleanTags(\OC::$server->getDatabaseConnection(), \OC::$server->getUserManager()),
 			new DropOldTables(\OC::$server->getDatabaseConnection()),