[stable9.1] Backport of filecache repair step OLD2

apps/files/appinfo/app.php

@@ -42,6 +42,9 @@
 
 \OC::$server->getSearch()->registerProvider('OC\Search\Provider\File', array('apps' => array('files')));
 
+// instantiate to make sure services get registered
+$app = new \OCA\Files\AppInfo\Application();
+
 $templateManager = \OC_Helper::getFileTemplateManager();
 $templateManager->registerTemplate('text/html', 'core/templates/filetemplates/template.html');
 $templateManager->registerTemplate('application/vnd.oasis.opendocument.presentation', 'core/templates/filetemplates/template.odp');

apps/files/appinfo/register_command.php

@@ -26,8 +26,11 @@
 $userManager = OC::$server->getUserManager();
 $shareManager = \OC::$server->getShareManager();
 $mountManager = \OC::$server->getMountManager();
+$lockingProvider = \OC::$server->getLockingProvider();
+$mimeTypeLoader = \OC::$server->getMimeTypeLoader();
+$config = \OC::$server->getConfig();
 
 /** @var Symfony\Component\Console\Application $application */
-$application->add(new OCA\Files\Command\Scan($userManager));
+$application->add(new OCA\Files\Command\Scan($userManager, $lockingProvider, $mimeTypeLoader, $config));
 $application->add(new OCA\Files\Command\DeleteOrphanedFiles($dbConnection));
 $application->add(new OCA\Files\Command\TransferOwnership($userManager, $shareManager, $mountManager));

apps/files/lib/AppInfo/Application.php

@@ -87,6 +87,13 @@ public function __construct(array $urlParams=array()) {
 			);
 		});
 
+		$container->registerService('OCP\Lock\ILockingProvider', function(IContainer $c)  {
+			return $c->query('ServerContainer')->getLockingProvider();
+		});
+		$container->registerService('OCP\Files\IMimeTypeLoader', function(IContainer $c)  {
+			return $c->query('ServerContainer')->getMimeTypeLoader();
+		});
+
 		/*
 		 * Register capabilities
 		 */

apps/files/lib/Command/Scan.php

@@ -39,20 +39,40 @@
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Helper\Table;
+use OC\Repair\RepairMismatchFileCachePath;
+use OC\Migration\ConsoleOutput;
+use OCP\Lock\ILockingProvider;
+use OCP\Lock\LockedException;
+use OCP\Files\IMimeTypeLoader;
+use OCP\IConfig;
 
 class Scan extends Base {
 
 	/** @var IUserManager $userManager */
 	private $userManager;
+	/** @var ILockingProvider */
+	private $lockingProvider;
+	/** @var IMimeTypeLoader */
+	private $mimeTypeLoader;
+	/** @var IConfig */
+	private $config;
 	/** @var float */
 	protected $execTime = 0;
 	/** @var int */
 	protected $foldersCounter = 0;
 	/** @var int */
 	protected $filesCounter = 0;
 
-	public function __construct(IUserManager $userManager) {
+	public function __construct(
+		IUserManager $userManager,
+		ILockingProvider $lockingProvider,
+		IMimeTypeLoader $mimeTypeLoader,
+		IConfig $config
+	) {
 		$this->userManager = $userManager;
+		$this->lockingProvider = $lockingProvider;
+		$this->mimeTypeLoader = $mimeTypeLoader;
+		$this->config = $config;
 		parent::__construct();
 	}
 
@@ -90,6 +110,12 @@ protected function configure() {
 				null,
 				InputOption::VALUE_NONE,
 				'will rescan all files of all known users'
+			)
+			->addOption(
+				'repair',
+				null,
+				InputOption::VALUE_NONE,
+				'will repair detached filecache entries (slow)'
 			)->addOption(
 				'unscanned',
 				null,
@@ -107,9 +133,48 @@ public function checkScanWarning($fullPath, OutputInterface $output) {
 		}
 	}
 
-	protected function scanFiles($user, $path, $verbose, OutputInterface $output, $backgroundScan = false) {
+	/**
+	 * Repair all storages at once
+	 *
+	 * @param OutputInterface $output
+	 */
+	protected function repairAll(OutputInterface $output) {
+		$connection = $this->reconnectToDatabase($output);
+		$repairStep = new RepairMismatchFileCachePath(
+			$connection,
+			$this->mimeTypeLoader
+		);
+		$repairStep->setStorageNumericId(null);
+		$repairStep->setCountOnly(false);
+		$repairStep->run(new ConsoleOutput($output));
+	}
+
+	protected function scanFiles($user, $path, $verbose, OutputInterface $output, $backgroundScan = false, $shouldRepair = false) {
 		$connection = $this->reconnectToDatabase($output);
 		$scanner = new \OC\Files\Utils\Scanner($user, $connection, \OC::$server->getLogger());
+		if ($shouldRepair) {
+			$scanner->listen('\OC\Files\Utils\Scanner', 'beforeScanStorage', function ($storage) use ($output, $connection) {
+				try {
+					// FIXME: this will lock the storage even if there is nothing to repair
+					$storage->acquireLock('', ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
+				} catch (LockedException $e) {
+					$output->writeln("\t<error>Storage \"" . $storage->getCache()->getNumericStorageId() . '" cannot be repaired as it is currently in use, please try again later</error>');
+					return;
+				}
+				try {
+					$repairStep = new RepairMismatchFileCachePath(
+						$connection,
+						$this->mimeTypeLoader
+					);
+					$repairStep->setStorageNumericId($storage->getCache()->getNumericStorageId());
+					$repairStep->setCountOnly(false);
+					$repairStep->run(new ConsoleOutput($output));
+				} finally {
+					$storage->releaseLock('', ILockingProvider::LOCK_EXCLUSIVE, $this->lockingProvider);
+				}
+			});
+		}
+
 		# check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception
 		# printout and count
 		if ($verbose) {
@@ -156,7 +221,7 @@ protected function scanFiles($user, $path, $verbose, OutputInterface $output, $b
 			if ($backgroundScan) {
 				$scanner->backgroundScan($path);
 			}else {
-				$scanner->scan($path);
+				$scanner->scan($path, $shouldRepair);
 			}
 		} catch (ForbiddenException $e) {
 			$output->writeln("<error>Home storage for user $user not writable</error>");
@@ -166,18 +231,32 @@ protected function scanFiles($user, $path, $verbose, OutputInterface $output, $b
 			$output->writeln('Interrupted by user');
 			return;
 		} catch (\Exception $e) {
-			$output->writeln('<error>Exception during scan: ' . $e->getMessage() . "\n" . $e->getTraceAsString() . '</error>');
+			$output->writeln('<error>Exception during scan: ' . get_class($e) . ': ' . $e->getMessage() . "\n" . $e->getTraceAsString() . '</error>');
 		}
 	}
 
 
 	protected function execute(InputInterface $input, OutputInterface $output) {
 		$inputPath = $input->getOption('path');
+		$shouldRepairStoragesIndividually = (bool) $input->getOption('repair');
+
 		if ($inputPath) {
 			$inputPath = '/' . trim($inputPath, '/');
 			list (, $user,) = explode('/', $inputPath, 3);
 			$users = array($user);
 		} else if ($input->getOption('all')) {
+			// we can only repair all storages in bulk (more efficient) if singleuser or maintenance mode
+			// is enabled to prevent concurrent user access
+			if ($input->getOption('repair') &&
+				($this->config->getSystemValue('singleuser', false) || $this->config->getSystemValue('maintenance', false))) {
+				// repair all storages at once
+				$this->repairAll($output);
+				// don't fix individually
+				$shouldRepairStoragesIndividually = false;
+			} else {
+				$output->writeln("<comment>Repairing every storage individually is slower than repairing in bulk</comment>");
+				$output->writeln("<comment>To repair in bulk, please switch to single user mode first: occ maintenance:singleuser --on</comment>");
+			}
 			$users = $this->userManager->search('');
 		} else {
 			$users = $input->getArgument('user_id');
@@ -223,9 +302,10 @@ protected function execute(InputInterface $input, OutputInterface $output) {
 			if ($this->userManager->userExists($user)) {
 				# add an extra line when verbose is set to optical separate users
 				if ($verbose) {$output->writeln(""); }
-				$output->writeln("Starting scan for user $user_count out of $users_total ($user)");
+				$r = $shouldRepairStoragesIndividually ? ' (and repair)' : '';
+				$output->writeln("Starting scan$r for user $user_count out of $users_total ($user)");
 				# full: printout data if $verbose was set
-				$this->scanFiles($user, $path, $verbose, $output, $input->getOption('unscanned'));
+				$this->scanFiles($user, $path, $verbose, $output, $input->getOption('unscanned'), $shouldRepairStoragesIndividually);
 			} else {
 				$output->writeln("<error>Unknown user $user_count $user</error>");
 			}

lib/private/Files/Utils/Scanner.php

@@ -210,6 +210,9 @@ public function scan($dir = '') {
 			if ($storage->instanceOfStorage('OCA\Files_Sharing\ISharedStorage')) {
 				continue;
 			}
+
+			$this->emit('\OC\Files\Utils\Scanner', 'beforeScanStorage', [$storage]);
+
 			$relativePath = $mount->getInternalPath($dir);
 			$scanner = $storage->getScanner();
 			$scanner->setUseTransactions(false);
@@ -246,6 +249,7 @@ public function scan($dir = '') {
 			if ($this->useTransaction) {
 				$this->db->commit();
 			}
+			$this->emit('\OC\Files\Utils\Scanner', 'afterScanStorage', [$storage]);
 		}
 	}
 

lib/private/Migration/ConsoleOutput.php

@@ -0,0 +1,91 @@
+<?php
+/**
+ * @author Thomas Müller <thomas.mueller@tmit.eu>
+ *
+ * @copyright Copyright (c) 2017, ownCloud GmbH
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OC\Migration;
+
+use OCP\Migration\IOutput;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * Class SimpleOutput
+ *
+ * Just a simple IOutput implementation with writes messages to the log file.
+ * Alternative implementations will write to the console or to the web ui (web update case)
+ *
+ * @package OC\Migration
+ */
+class ConsoleOutput implements IOutput {
+
+	/** @var OutputInterface */
+	private $output;
+
+	/** @var ProgressBar */
+	private $progressBar;
+
+	public function __construct(OutputInterface $output) {
+		$this->output = $output;
+	}
+
+	/**
+	 * @param string $message
+	 */
+	public function info($message) {
+		$this->output->writeln("<info>$message</info>");
+	}
+
+	/**
+	 * @param string $message
+	 */
+	public function warning($message) {
+		$this->output->writeln("<comment>$message</comment>");
+	}
+
+	/**
+	 * @param int $max
+	 */
+	public function startProgress($max = 0) {
+		if (!is_null($this->progressBar)) {
+			$this->progressBar->finish();
+		}
+		$this->progressBar = new ProgressBar($this->output);
+		$this->progressBar->start($max);
+	}
+
+	/**
+	 * @param int $step
+	 * @param string $description
+	 */
+	public function advance($step = 1, $description = '') {
+		if (!is_null($this->progressBar)) {
+			$this->progressBar = new ProgressBar($this->output);
+			$this->progressBar->start();
+		}
+		$this->progressBar->advance($step);
+	}
+
+	public function finishProgress() {
+		if (is_null($this->progressBar)) {
+			return;
+		}
+		$this->progressBar->finish();
+	}
+}

lib/private/Repair.php

@@ -52,6 +52,8 @@
 use OCP\Migration\IRepairStep;
 use Symfony\Component\EventDispatcher\EventDispatcher;
 use Symfony\Component\EventDispatcher\GenericEvent;
+use OC\Repair\MoveAvatarOutsideHome;
+use OC\Repair\RepairMismatchFileCachePath;
 
 class Repair implements IOutput{
 	/* @var IRepairStep[] */
@@ -126,6 +128,7 @@ public static function getRepairSteps() {
 		return [
 			new RepairMimeTypes(\OC::$server->getConfig()),
 			new AssetCache(),
+			new RepairMismatchFileCachePath(\OC::$server->getDatabaseConnection(), \OC::$server->getMimeTypeLoader()),
 			new FillETags(\OC::$server->getDatabaseConnection()),
 			new CleanTags(\OC::$server->getDatabaseConnection(), \OC::$server->getUserManager()),
 			new DropOldTables(\OC::$server->getDatabaseConnection()),