From 138d6d32ccb268f8b5f4773e0c9382bd4f487fa4 Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Wed, 13 Nov 2024 18:41:37 +0600 Subject: [PATCH 01/18] feat: copy files:scan command we want to run `occ files:scan` command without opening the files app. In this comment, we just copy the scan command file from nc-28.0.8 version, so we can run it in through ecloud account app. --- appinfo/info.xml | 1 + lib/Command/scan.php | 368 +++++++++++++++++++++++++++++++++++++++++++ test.php | 6 + 3 files changed, 375 insertions(+) create mode 100644 lib/Command/scan.php create mode 100644 test.php diff --git a/appinfo/info.xml b/appinfo/info.xml index 35a9b216..fa33f00c 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -28,5 +28,6 @@ OCA\EcloudAccounts\Command\Migrate2FASecrets OCA\EcloudAccounts\Command\MigrateWebmailAddressbooks OCA\EcloudAccounts\Command\MapActiveAttributetoLDAP + OCA\EcloudAccounts\Command\Scan diff --git a/lib/Command/scan.php b/lib/Command/scan.php new file mode 100644 index 00000000..119a3e12 --- /dev/null +++ b/lib/Command/scan.php @@ -0,0 +1,368 @@ + + * @author Blaok + * @author Christoph Wurst + * @author Daniel Kesselberg + * @author J0WI + * @author Joas Schilling + * @author Joel S + * @author Jörn Friedrich Dreyer + * @author martin.mattel@diemattels.at + * @author Maxence Lange + * @author Robin Appelman + * @author Roeland Jago Douma + * @author Thomas Müller + * @author Vincent Petry + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OCA\EcloudAccounts\Command; + +use OC\Core\Command\Base; +use OC\Core\Command\InterruptedException; +use OC\DB\Connection; +use OC\DB\ConnectionAdapter; +use OC\FilesMetadata\FilesMetadataManager; +use OC\ForbiddenException; +use OCA\EcloudAccounts\AppInfo\Application; +use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Events\FileCacheUpdated; +use OCP\Files\Events\NodeAddedToCache; +use OCP\Files\Events\NodeRemovedFromCache; +use OCP\Files\IRootFolder; +use OCP\Files\Mount\IMountPoint; +use OCP\Files\NotFoundException; +use OCP\Files\StorageNotAvailableException; +use OCP\FilesMetadata\IFilesMetadataManager; +use OCP\IUserManager; +use Psr\Log\LoggerInterface; +use Symfony\Component\Console\Helper\Table; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +class Scan extends Base +{ + protected float $execTime = 0; + protected int $foldersCounter = 0; + protected int $filesCounter = 0; + protected int $errorsCounter = 0; + protected int $newCounter = 0; + protected int $updatedCounter = 0; + protected int $removedCounter = 0; + + public function __construct( + private IUserManager $userManager, + private IRootFolder $rootFolder, + private FilesMetadataManager $filesMetadataManager, + private IEventDispatcher $eventDispatcher, + private LoggerInterface $logger, + ) { + parent::__construct(); + } + + protected function configure(): void + { + parent::configure(); + + $this + ->setName(Application::APP_ID . ':scan') + ->setDescription('rescan filesystem') + ->addArgument( + 'user_id', + InputArgument::OPTIONAL | InputArgument::IS_ARRAY, + 'will rescan all files of the given user(s)' + ) + ->addOption( + 'path', + 'p', + InputArgument::OPTIONAL, + 'limit rescan to this path, eg. --path="/alice/files/Music", the user_id is determined by the path and the user_id parameter and --all are ignored' + ) + ->addOption( + 'generate-metadata', + null, + InputOption::VALUE_OPTIONAL, + 'Generate metadata for all scanned files; if specified only generate for named value', + '' + ) + ->addOption( + 'all', + null, + InputOption::VALUE_NONE, + 'will rescan all files of all known users' + )->addOption( + 'unscanned', + null, + InputOption::VALUE_NONE, + 'only scan files which are marked as not fully scanned' + )->addOption( + 'shallow', + null, + InputOption::VALUE_NONE, + 'do not scan folders recursively' + )->addOption( + 'home-only', + null, + InputOption::VALUE_NONE, + 'only scan the home storage, ignoring any mounted external storage or share' + ); + } + + protected function scanFiles(string $user, string $path, ?string $scanMetadata, OutputInterface $output, bool $backgroundScan = false, bool $recursive = true, bool $homeOnly = false): void + { + $connection = $this->reconnectToDatabase($output); + $scanner = new \OC\Files\Utils\Scanner( + $user, + new ConnectionAdapter($connection), + \OC::$server->get(IEventDispatcher::class), + \OC::$server->get(LoggerInterface::class) + ); + + # check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception + $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function (string $path) use ($output, $scanMetadata) { + $output->writeln("\tFile\t$path", OutputInterface::VERBOSITY_VERBOSE); + ++$this->filesCounter; + $this->abortIfInterrupted(); + if ($scanMetadata !== null) { + $node = $this->rootFolder->get($path); + $this->filesMetadataManager->refreshMetadata( + $node, + ($scanMetadata !== '') ? IFilesMetadataManager::PROCESS_NAMED : IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND, + $scanMetadata + ); + } + }); + + $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output) { + $output->writeln("\tFolder\t$path", OutputInterface::VERBOSITY_VERBOSE); + ++$this->foldersCounter; + $this->abortIfInterrupted(); + }); + + $scanner->listen('\OC\Files\Utils\Scanner', 'StorageNotAvailable', function (StorageNotAvailableException $e) use ($output) { + $output->writeln('Error while scanning, storage not available (' . $e->getMessage() . ')', OutputInterface::VERBOSITY_VERBOSE); + ++$this->errorsCounter; + }); + + $scanner->listen('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', function ($fullPath) use ($output) { + $output->writeln("\tEntry \"" . $fullPath . '" will not be accessible due to incompatible encoding'); + ++$this->errorsCounter; + }); + + $this->eventDispatcher->addListener(NodeAddedToCache::class, function () { + ++$this->newCounter; + }); + $this->eventDispatcher->addListener(FileCacheUpdated::class, function () { + ++$this->updatedCounter; + }); + $this->eventDispatcher->addListener(NodeRemovedFromCache::class, function () { + ++$this->removedCounter; + }); + + try { + if ($backgroundScan) { + $scanner->backgroundScan($path); + } else { + $scanner->scan($path, $recursive, $homeOnly ? [$this, 'filterHomeMount'] : null); + } + } catch (ForbiddenException $e) { + $output->writeln("Home storage for user $user not writable or 'files' subdirectory missing"); + $output->writeln(' ' . $e->getMessage()); + $output->writeln('Make sure you\'re running the scan command only as the user the web server runs as'); + ++$this->errorsCounter; + } catch (InterruptedException $e) { + # exit the function if ctrl-c has been pressed + $output->writeln('Interrupted by user'); + } catch (NotFoundException $e) { + $output->writeln('Path not found: ' . $e->getMessage() . ''); + ++$this->errorsCounter; + } catch (\Exception $e) { + $output->writeln('Exception during scan: ' . $e->getMessage() . ''); + $output->writeln('' . $e->getTraceAsString() . ''); + ++$this->errorsCounter; + } + } + + public function filterHomeMount(IMountPoint $mountPoint): bool + { + // any mountpoint inside '/$user/files/' + return substr_count($mountPoint->getMountPoint(), '/') <= 3; + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $inputPath = $input->getOption('path'); + if ($inputPath) { + $inputPath = '/' . trim($inputPath, '/'); + [, $user,] = explode('/', $inputPath, 3); + $users = [$user]; + } elseif ($input->getOption('all')) { + $users = $this->userManager->search(''); + } else { + $users = $input->getArgument('user_id'); + } + + # check quantity of users to be process and show it on the command line + $users_total = count($users); + if ($users_total === 0) { + $output->writeln('Please specify the user id to scan, --all to scan for all users or --path=...'); + return self::FAILURE; + } + + $this->initTools($output); + + // getOption() logic on VALUE_OPTIONAL + $metadata = null; // null if --generate-metadata is not set, empty if option have no value, value if set + if ($input->getOption('generate-metadata') !== '') { + $metadata = $input->getOption('generate-metadata') ?? ''; + } + + $user_count = 0; + foreach ($users as $user) { + if (is_object($user)) { + $user = $user->getUID(); + } + $path = $inputPath ?: '/' . $user; + ++$user_count; + if ($this->userManager->userExists($user)) { + $output->writeln("Starting scan for user $user_count out of $users_total ($user)"); + $this->scanFiles($user, $path, $metadata, $output, $input->getOption('unscanned'), !$input->getOption('shallow'), $input->getOption('home-only')); + $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); + } else { + $output->writeln("Unknown user $user_count $user"); + $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); + } + + try { + $this->abortIfInterrupted(); + } catch (InterruptedException $e) { + break; + } + } + + $this->presentStats($output); + return self::SUCCESS; + } + + /** + * Initialises some useful tools for the Command + */ + protected function initTools(OutputInterface $output): void + { + // Start the timer + $this->execTime = -microtime(true); + // Convert PHP errors to exceptions + set_error_handler( + fn(int $severity, string $message, string $file, int $line): bool => + $this->exceptionErrorHandler($output, $severity, $message, $file, $line), + E_ALL + ); + } + + /** + * Processes PHP errors in order to be able to show them in the output + * + * @see https://www.php.net/manual/en/function.set-error-handler.php + * + * @param int $severity the level of the error raised + * @param string $message + * @param string $file the filename that the error was raised in + * @param int $line the line number the error was raised + */ + public function exceptionErrorHandler(OutputInterface $output, int $severity, string $message, string $file, int $line): bool + { + if (($severity === E_DEPRECATED) || ($severity === E_USER_DEPRECATED)) { + // Do not show deprecation warnings + return false; + } + $e = new \ErrorException($message, 0, $severity, $file, $line); + $output->writeln('Error during scan: ' . $e->getMessage() . ''); + $output->writeln('' . $e->getTraceAsString() . '', OutputInterface::VERBOSITY_VERY_VERBOSE); + ++$this->errorsCounter; + return true; + } + + protected function presentStats(OutputInterface $output): void + { + // Stop the timer + $this->execTime += microtime(true); + + $this->logger->info("Completed scan of {$this->filesCounter} files in {$this->foldersCounter} folder. Found {$this->newCounter} new, {$this->updatedCounter} updated and {$this->removedCounter} removed items"); + + $headers = [ + 'Folders', + 'Files', + 'New', + 'Updated', + 'Removed', + 'Errors', + 'Elapsed time', + ]; + $niceDate = $this->formatExecTime(); + $rows = [ + $this->foldersCounter, + $this->filesCounter, + $this->newCounter, + $this->updatedCounter, + $this->removedCounter, + $this->errorsCounter, + $niceDate, + ]; + $table = new Table($output); + $table + ->setHeaders($headers) + ->setRows([$rows]); + $table->render(); + } + + + /** + * Formats microtime into a human-readable format + */ + protected function formatExecTime(): string + { + $secs = (int)round($this->execTime); + # convert seconds into HH:MM:SS form + return sprintf('%02d:%02d:%02d', (int)($secs / 3600), ((int)($secs / 60) % 60), $secs % 60); + } + + protected function reconnectToDatabase(OutputInterface $output): Connection + { + /** @var Connection $connection */ + $connection = \OC::$server->get(Connection::class); + try { + $connection->close(); + } catch (\Exception $ex) { + $output->writeln("Error while disconnecting from database: {$ex->getMessage()}"); + } + while (!$connection->isConnected()) { + try { + $connection->connect(); + } catch (\Exception $ex) { + $output->writeln("Error while re-connecting to database: {$ex->getMessage()}"); + sleep(60); + } + } + return $connection; + } +} diff --git a/test.php b/test.php new file mode 100644 index 00000000..6c631020 --- /dev/null +++ b/test.php @@ -0,0 +1,6 @@ + Date: Wed, 13 Nov 2024 18:47:25 +0600 Subject: [PATCH 02/18] rnd: block storageWrapper --- lib/AppInfo/Application.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 1beceac8..3394d418 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -56,7 +56,7 @@ class Application extends App implements IBootstrap { } public function register(IRegistrationContext $context): void { - Util::connectHook('OC_Filesystem', 'preSetup', $this, 'addStorageWrapper'); + // Util::connectHook('OC_Filesystem', 'preSetup', $this, 'addStorageWrapper'); $context->registerEventListener(BeforeTemplateRenderedEvent::class, BeforeTemplateRenderedListener::class); $context->registerEventListener(BeforeUserDeletedEvent::class, BeforeUserDeletedListener::class); $context->registerEventListener(UserChangedEvent::class, UserChangedListener::class); -- GitLab From e7e6d29878757b38f6a219688c4f5d64a5cdbe54 Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Wed, 13 Nov 2024 19:00:26 +0600 Subject: [PATCH 03/18] fix: lint issue --- lib/Command/scan.php | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/lib/Command/scan.php b/lib/Command/scan.php index 119a3e12..eaa44151 100644 --- a/lib/Command/scan.php +++ b/lib/Command/scan.php @@ -1,5 +1,4 @@ * */ +declare(strict_types=1); namespace OCA\EcloudAccounts\Command; @@ -60,8 +60,7 @@ use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; -class Scan extends Base -{ +class Scan extends Base { protected float $execTime = 0; protected int $foldersCounter = 0; protected int $filesCounter = 0; @@ -80,12 +79,11 @@ class Scan extends Base parent::__construct(); } - protected function configure(): void - { + protected function configure(): void { parent::configure(); $this - ->setName(Application::APP_ID . ':scan') + ->setName(Application::APP_ID.':scan') ->setDescription('rescan filesystem') ->addArgument( 'user_id', @@ -128,8 +126,7 @@ class Scan extends Base ); } - protected function scanFiles(string $user, string $path, ?string $scanMetadata, OutputInterface $output, bool $backgroundScan = false, bool $recursive = true, bool $homeOnly = false): void - { + protected function scanFiles(string $user, string $path, ?string $scanMetadata, OutputInterface $output, bool $backgroundScan = false, bool $recursive = true, bool $homeOnly = false): void { $connection = $this->reconnectToDatabase($output); $scanner = new \OC\Files\Utils\Scanner( $user, @@ -203,14 +200,12 @@ class Scan extends Base } } - public function filterHomeMount(IMountPoint $mountPoint): bool - { + public function filterHomeMount(IMountPoint $mountPoint): bool { // any mountpoint inside '/$user/files/' return substr_count($mountPoint->getMountPoint(), '/') <= 3; } - protected function execute(InputInterface $input, OutputInterface $output): int - { + protected function execute(InputInterface $input, OutputInterface $output): int { $inputPath = $input->getOption('path'); if ($inputPath) { $inputPath = '/' . trim($inputPath, '/'); @@ -267,14 +262,13 @@ class Scan extends Base /** * Initialises some useful tools for the Command */ - protected function initTools(OutputInterface $output): void - { + protected function initTools(OutputInterface $output): void { // Start the timer $this->execTime = -microtime(true); // Convert PHP errors to exceptions set_error_handler( - fn(int $severity, string $message, string $file, int $line): bool => - $this->exceptionErrorHandler($output, $severity, $message, $file, $line), + fn (int $severity, string $message, string $file, int $line): bool => + $this->exceptionErrorHandler($output, $severity, $message, $file, $line), E_ALL ); } @@ -289,8 +283,7 @@ class Scan extends Base * @param string $file the filename that the error was raised in * @param int $line the line number the error was raised */ - public function exceptionErrorHandler(OutputInterface $output, int $severity, string $message, string $file, int $line): bool - { + public function exceptionErrorHandler(OutputInterface $output, int $severity, string $message, string $file, int $line): bool { if (($severity === E_DEPRECATED) || ($severity === E_USER_DEPRECATED)) { // Do not show deprecation warnings return false; @@ -302,8 +295,7 @@ class Scan extends Base return true; } - protected function presentStats(OutputInterface $output): void - { + protected function presentStats(OutputInterface $output): void { // Stop the timer $this->execTime += microtime(true); @@ -339,15 +331,13 @@ class Scan extends Base /** * Formats microtime into a human-readable format */ - protected function formatExecTime(): string - { + protected function formatExecTime(): string { $secs = (int)round($this->execTime); # convert seconds into HH:MM:SS form return sprintf('%02d:%02d:%02d', (int)($secs / 3600), ((int)($secs / 60) % 60), $secs % 60); } - protected function reconnectToDatabase(OutputInterface $output): Connection - { + protected function reconnectToDatabase(OutputInterface $output): Connection { /** @var Connection $connection */ $connection = \OC::$server->get(Connection::class); try { -- GitLab From 057cc71b66fe7b8a895bf9daaa6e384dd5a505fb Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Wed, 13 Nov 2024 19:12:33 +0600 Subject: [PATCH 04/18] rnd: remove dummy file --- test.php | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 test.php diff --git a/test.php b/test.php deleted file mode 100644 index 6c631020..00000000 --- a/test.php +++ /dev/null @@ -1,6 +0,0 @@ - Date: Wed, 13 Nov 2024 19:29:34 +0600 Subject: [PATCH 05/18] rnd: fix lint --- lib/AppInfo/Application.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 3394d418..fb24ac10 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -56,7 +56,7 @@ class Application extends App implements IBootstrap { } public function register(IRegistrationContext $context): void { - // Util::connectHook('OC_Filesystem', 'preSetup', $this, 'addStorageWrapper'); + // Util::connectHook('OC_Filesystem', 'preSetup', $this, 'addStorageWrapper'); $context->registerEventListener(BeforeTemplateRenderedEvent::class, BeforeTemplateRenderedListener::class); $context->registerEventListener(BeforeUserDeletedEvent::class, BeforeUserDeletedListener::class); $context->registerEventListener(UserChangedEvent::class, UserChangedListener::class); -- GitLab From f4540414c7faf7e3bbd82a65e434937d49f35102 Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Wed, 13 Nov 2024 19:45:46 +0600 Subject: [PATCH 06/18] fix: renmae scan command fileName --- lib/Command/{scan.php => Scan.php} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/Command/{scan.php => Scan.php} (100%) diff --git a/lib/Command/scan.php b/lib/Command/Scan.php similarity index 100% rename from lib/Command/scan.php rename to lib/Command/Scan.php -- GitLab From 281e3fba76afba364cb66526fb901245d253ac04 Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Fri, 15 Nov 2024 17:56:45 +0600 Subject: [PATCH 07/18] rnd: enable storageWrapper & don't block for specific request --- lib/AppInfo/Application.php | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index fb24ac10..b501ec79 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -42,7 +42,9 @@ use OCP\AppFramework\Bootstrap\IBootstrap; use OCP\AppFramework\Bootstrap\IRegistrationContext; use OCP\AppFramework\Http\Events\BeforeTemplateRenderedEvent; use OCP\Files\Storage\IStorage; +use OCP\IGroupManager; use OCP\IUserManager; +use OCP\IUserSession; use OCP\User\Events\BeforeUserDeletedEvent; use OCP\User\Events\PasswordUpdatedEvent; use OCP\User\Events\UserChangedEvent; @@ -56,13 +58,13 @@ class Application extends App implements IBootstrap { } public function register(IRegistrationContext $context): void { - // Util::connectHook('OC_Filesystem', 'preSetup', $this, 'addStorageWrapper'); + Util::connectHook('OC_Filesystem', 'preSetup', $this, 'addStorageWrapper'); $context->registerEventListener(BeforeTemplateRenderedEvent::class, BeforeTemplateRenderedListener::class); $context->registerEventListener(BeforeUserDeletedEvent::class, BeforeUserDeletedListener::class); $context->registerEventListener(UserChangedEvent::class, UserChangedListener::class); $context->registerEventListener(StateChanged::class, TwoFactorStateChangedListener::class); $context->registerEventListener(PasswordUpdatedEvent::class, PasswordUpdatedListener::class); - + $context->registerMiddleware(AccountMiddleware::class); } @@ -89,6 +91,25 @@ class Application extends App implements IBootstrap { * @return StorageWrapper|IStorage */ public function addStorageWrapperCallback($mountPoint, IStorage $storage) { + if (\OC::$CLI && (\OC::$REQUESTEDAPP === "ecloud-accounts" || \OC::$REQUESTEDAPP === "encryption")) { + return $storage; + } + + $userSession = \OC::$server->get(IUserSession::class); + $currentUser = $userSession->getUser(); + if ($currentUser !== null) { + $groupManager = \OC::$server->get(IGroupManager::class); + $groups = $groupManager->getUserGroups($currentUser); + + if (!empty($groups)) { + foreach ($groups as $group) { + if ($group->getGID() === "recovery_done") { + return $storage; + } + } + } + } + $instanceId = \OC::$server->getConfig()->getSystemValue('instanceid', ''); $appdataFolder = 'appdata_' . $instanceId; if ($mountPoint !== '/' && strpos($mountPoint, '/' . $appdataFolder) !== 0) { -- GitLab From 811ecabf437186ee30b1b8ab158e7b801862d8a7 Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Fri, 15 Nov 2024 18:57:53 +0600 Subject: [PATCH 08/18] rnd: add requestedApp param for test --- lib/AppInfo/Application.php | 2 +- lib/Command/Scan.php | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index b501ec79..a19e8eca 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -91,7 +91,7 @@ class Application extends App implements IBootstrap { * @return StorageWrapper|IStorage */ public function addStorageWrapperCallback($mountPoint, IStorage $storage) { - if (\OC::$CLI && (\OC::$REQUESTEDAPP === "ecloud-accounts" || \OC::$REQUESTEDAPP === "encryption")) { + if (\OC::$CLI && \OC::$REQUESTEDAPP === self::APP_ID) { return $storage; } diff --git a/lib/Command/Scan.php b/lib/Command/Scan.php index eaa44151..3a4ea6be 100644 --- a/lib/Command/Scan.php +++ b/lib/Command/Scan.php @@ -206,6 +206,7 @@ class Scan extends Base { } protected function execute(InputInterface $input, OutputInterface $output): int { + \OC::$REQUESTEDAPP = Application::APP_ID; $inputPath = $input->getOption('path'); if ($inputPath) { $inputPath = '/' . trim($inputPath, '/'); -- GitLab From 984a286d3d3687f44378e485501f7df77bfd031e Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Fri, 15 Nov 2024 19:33:26 +0600 Subject: [PATCH 09/18] rnd: add fix-encrypted-version command --- appinfo/info.xml | 1 + lib/Command/FixEncryptedVersion.php | 334 ++++++++++++++++++++++++++++ 2 files changed, 335 insertions(+) create mode 100644 lib/Command/FixEncryptedVersion.php diff --git a/appinfo/info.xml b/appinfo/info.xml index fa33f00c..c5d99471 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -29,5 +29,6 @@ OCA\EcloudAccounts\Command\MigrateWebmailAddressbooks OCA\EcloudAccounts\Command\MapActiveAttributetoLDAP OCA\EcloudAccounts\Command\Scan + OCA\EcloudAccounts\Command\FixEncryptedVersion diff --git a/lib/Command/FixEncryptedVersion.php b/lib/Command/FixEncryptedVersion.php new file mode 100644 index 00000000..bd6e75c7 --- /dev/null +++ b/lib/Command/FixEncryptedVersion.php @@ -0,0 +1,334 @@ + + * @author Ilja Neumann + * + * @copyright Copyright (c) 2019, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +declare(strict_types=1); + +namespace OCA\EcloudAccounts\Command; + +use OC\Files\Storage\Wrapper\Encryption; +use OC\Files\View; +use OC\ServerNotAvailableException; +use OCA\EcloudAccounts\AppInfo\Application; +use OCA\Encryption\Util; +use OCP\Files\IRootFolder; +use OCP\HintException; +use OCP\IConfig; +use OCP\IUser; +use OCP\IUserManager; +use Psr\Log\LoggerInterface; +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +class FixEncryptedVersion extends Command { + private bool $supportLegacy; + + public function __construct( + private IConfig $config, + private LoggerInterface $logger, + private IRootFolder $rootFolder, + private IUserManager $userManager, + private Util $util, + private View $view, + ) { + $this->supportLegacy = false; + + parent::__construct(); + } + + protected function configure(): void { + parent::configure(); + + $this + ->setName(Application::APP_ID . ':fix-encrypted-version') + ->setDescription('Fix the encrypted version if the encrypted file(s) are not downloadable.') + ->addArgument( + 'user', + InputArgument::OPTIONAL, + 'The id of the user whose files need fixing' + )->addOption( + 'path', + 'p', + InputOption::VALUE_REQUIRED, + 'Limit files to fix with path, e.g., --path="/Music/Artist". If path indicates a directory, all the files inside directory will be fixed.' + )->addOption( + 'all', + null, + InputOption::VALUE_NONE, + 'Run the fix for all users on the system, mutually exclusive with specifying a user id.' + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + \OC::$REQUESTEDAPP = Application::APP_ID; + $skipSignatureCheck = $this->config->getSystemValueBool('encryption_skip_signature_check', false); + $this->supportLegacy = $this->config->getSystemValueBool('encryption.legacy_format_support', false); + + if ($skipSignatureCheck) { + $output->writeln("Repairing is not possible when \"encryption_skip_signature_check\" is set. Please disable this flag in the configuration.\n"); + return 1; + } + + if (!$this->util->isMasterKeyEnabled()) { + $output->writeln("Repairing only works with master key encryption.\n"); + return 1; + } + + $user = $input->getArgument('user'); + $all = $input->getOption('all'); + $pathOption = \trim(($input->getOption('path') ?? ''), '/'); + + if ($user) { + if ($all) { + $output->writeln("Specifying a user id and --all are mutually exclusive"); + return 1; + } + + if ($this->userManager->get($user) === null) { + $output->writeln("User id $user does not exist. Please provide a valid user id"); + return 1; + } + + return $this->runForUser($user, $pathOption, $output); + } elseif ($all) { + $result = 0; + $this->userManager->callForSeenUsers(function (IUser $user) use ($pathOption, $output, &$result) { + $output->writeln("Processing files for " . $user->getUID()); + $result = $this->runForUser($user->getUID(), $pathOption, $output); + return $result === 0; + }); + return $result; + } else { + $output->writeln("Either a user id or --all needs to be provided"); + return 1; + } + } + + private function runForUser(string $user, string $pathOption, OutputInterface $output): int { + $pathToWalk = "/$user/files"; + if ($pathOption !== "") { + $pathToWalk = "$pathToWalk/$pathOption"; + } + return $this->walkPathOfUser($user, $pathToWalk, $output); + } + + /** + * @return int 0 for success, 1 for error + */ + private function walkPathOfUser(string $user, string $path, OutputInterface $output): int { + $this->setupUserFs($user); + if (!$this->view->file_exists($path)) { + $output->writeln("Path \"$path\" does not exist. Please provide a valid path."); + return 1; + } + + if ($this->view->is_file($path)) { + $output->writeln("Verifying the content of file \"$path\""); + $this->verifyFileContent($path, $output); + return 0; + } + $directories = []; + $directories[] = $path; + while ($root = \array_pop($directories)) { + $directoryContent = $this->view->getDirectoryContent($root); + foreach ($directoryContent as $file) { + $path = $root . '/' . $file['name']; + if ($this->view->is_dir($path)) { + $directories[] = $path; + } else { + $output->writeln("Verifying the content of file \"$path\""); + $this->verifyFileContent($path, $output); + } + } + } + return 0; + } + + /** + * @param bool $ignoreCorrectEncVersionCall, setting this variable to false avoids recursion + */ + private function verifyFileContent(string $path, OutputInterface $output, bool $ignoreCorrectEncVersionCall = true): bool { + try { + // since we're manually poking around the encrypted state we need to ensure that this isn't cached in the encryption wrapper + $mount = $this->view->getMount($path); + $storage = $mount->getStorage(); + if ($storage && $storage->instanceOfStorage(Encryption::class)) { + $storage->clearIsEncryptedCache(); + } + + /** + * In encryption, the files are read in a block size of 8192 bytes + * Read block size of 8192 and a bit more (808 bytes) + * If there is any problem, the first block should throw the signature + * mismatch error. Which as of now, is enough to proceed ahead to + * correct the encrypted version. + */ + $handle = $this->view->fopen($path, 'rb'); + + if ($handle === false) { + $output->writeln("Failed to open file: \"$path\" skipping"); + return true; + } + + if (\fread($handle, 9001) !== false) { + $fileInfo = $this->view->getFileInfo($path); + if (!$fileInfo) { + $output->writeln("File info not found for file: \"$path\""); + return true; + } + $encryptedVersion = $fileInfo->getEncryptedVersion(); + $stat = $this->view->stat($path); + if (($encryptedVersion == 0) && isset($stat['hasHeader']) && ($stat['hasHeader'] == true)) { + // The file has encrypted to false but has an encryption header + if ($ignoreCorrectEncVersionCall === true) { + // Lets rectify the file by correcting encrypted version + $output->writeln("Attempting to fix the path: \"$path\""); + return $this->correctEncryptedVersion($path, $output); + } + return false; + } + $output->writeln("The file \"$path\" is: OK"); + } + + \fclose($handle); + + return true; + } catch (ServerNotAvailableException $e) { + // not a "bad signature" error and likely "legacy cipher" exception + // this could mean that the file is maybe not encrypted but the encrypted version is set + if (!$this->supportLegacy && $ignoreCorrectEncVersionCall === true) { + $output->writeln("Attempting to fix the path: \"$path\""); + return $this->correctEncryptedVersion($path, $output, true); + } + return false; + } catch (HintException $e) { + $this->logger->warning("Issue: " . $e->getMessage()); + // If allowOnce is set to false, this becomes recursive. + if ($ignoreCorrectEncVersionCall === true) { + // Lets rectify the file by correcting encrypted version + $output->writeln("Attempting to fix the path: \"$path\""); + return $this->correctEncryptedVersion($path, $output); + } + return false; + } + } + + /** + * @param bool $includeZero whether to try zero version for unencrypted file + */ + private function correctEncryptedVersion(string $path, OutputInterface $output, bool $includeZero = false): bool { + $fileInfo = $this->view->getFileInfo($path); + if (!$fileInfo) { + $output->writeln("File info not found for file: \"$path\""); + return true; + } + $fileId = $fileInfo->getId(); + if ($fileId === null) { + $output->writeln("File info contains no id for file: \"$path\""); + return true; + } + $encryptedVersion = $fileInfo->getEncryptedVersion(); + $wrongEncryptedVersion = $encryptedVersion; + + $storage = $fileInfo->getStorage(); + + $cache = $storage->getCache(); + $fileCache = $cache->get($fileId); + if (!$fileCache) { + $output->writeln("File cache entry not found for file: \"$path\""); + return true; + } + + if ($storage->instanceOfStorage('OCA\Files_Sharing\ISharedStorage')) { + $output->writeln("The file: \"$path\" is a share. Please also run the script for the owner of the share"); + return true; + } + + // Save original encrypted version so we can restore it if decryption fails with all version + $originalEncryptedVersion = $encryptedVersion; + if ($encryptedVersion >= 0) { + if ($includeZero) { + // try with zero first + $cacheInfo = ['encryptedVersion' => 0, 'encrypted' => 0]; + $cache->put($fileCache->getPath(), $cacheInfo); + $output->writeln("Set the encrypted version to 0 (unencrypted)"); + if ($this->verifyFileContent($path, $output, false) === true) { + $output->writeln("Fixed the file: \"$path\" with version 0 (unencrypted)"); + return true; + } + } + + // Test by decrementing the value till 1 and if nothing works try incrementing + $encryptedVersion--; + while ($encryptedVersion > 0) { + $cacheInfo = ['encryptedVersion' => $encryptedVersion, 'encrypted' => $encryptedVersion]; + $cache->put($fileCache->getPath(), $cacheInfo); + $output->writeln("Decrement the encrypted version to $encryptedVersion"); + if ($this->verifyFileContent($path, $output, false) === true) { + $output->writeln("Fixed the file: \"$path\" with version " . $encryptedVersion . ""); + return true; + } + $encryptedVersion--; + } + + // So decrementing did not work. Now lets increment. Max increment is till 5 + $increment = 1; + while ($increment <= 5) { + /** + * The wrongEncryptedVersion would not be incremented so nothing to worry about here. + * Only the newEncryptedVersion is incremented. + * For example if the wrong encrypted version is 4 then + * cycle1 -> newEncryptedVersion = 5 ( 4 + 1) + * cycle2 -> newEncryptedVersion = 6 ( 4 + 2) + * cycle3 -> newEncryptedVersion = 7 ( 4 + 3) + */ + $newEncryptedVersion = $wrongEncryptedVersion + $increment; + + $cacheInfo = ['encryptedVersion' => $newEncryptedVersion, 'encrypted' => $newEncryptedVersion]; + $cache->put($fileCache->getPath(), $cacheInfo); + $output->writeln("Increment the encrypted version to $newEncryptedVersion"); + if ($this->verifyFileContent($path, $output, false) === true) { + $output->writeln("Fixed the file: \"$path\" with version " . $newEncryptedVersion . ""); + return true; + } + $increment++; + } + } + + $cacheInfo = ['encryptedVersion' => $originalEncryptedVersion, 'encrypted' => $originalEncryptedVersion]; + $cache->put($fileCache->getPath(), $cacheInfo); + $output->writeln("No fix found for \"$path\", restored version to original: $originalEncryptedVersion"); + + return false; + } + + /** + * Setup user file system + */ + private function setupUserFs(string $uid): void { + \OC_Util::tearDownFS(); + \OC_Util::setupFS($uid); + } +} -- GitLab From 07d9f5950ee4e84d100e7cdd6da77cf2aed14f03 Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Mon, 18 Nov 2024 12:18:55 +0600 Subject: [PATCH 10/18] rnd: change fileWrapper exception to SeviceUnavailable(503) --- lib/Filesystem/StorageWrapper.php | 42 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/Filesystem/StorageWrapper.php b/lib/Filesystem/StorageWrapper.php index f35a5510..1aa7d19c 100644 --- a/lib/Filesystem/StorageWrapper.php +++ b/lib/Filesystem/StorageWrapper.php @@ -7,7 +7,7 @@ namespace OCA\EcloudAccounts\Filesystem; use OC\Files\Cache\Cache; use OC\Files\Storage\Storage; use OC\Files\Storage\Wrapper\Wrapper; -use OCP\Files\ForbiddenException; +use OC\ServiceUnavailableException; use OCP\Files\Storage\IStorage; use OCP\Files\Storage\IWriteStreamStorage; @@ -20,10 +20,10 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { } /** - * @throws ForbiddenException + * @throws ServiceUnavailableException */ protected function checkFileAccess(string $path, bool $isDir = false): void { - throw new ForbiddenException('Access denied', false); + throw new ServiceUnavailableException('Access denied'); } /* @@ -35,7 +35,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return bool - * @throws ForbiddenException + * @throws ServiceUnavailableException */ public function mkdir($path) { $this->checkFileAccess($path, true); @@ -46,7 +46,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return bool - * @throws ForbiddenException + * @throws ServiceUnavailableException */ public function rmdir($path) { $this->checkFileAccess($path, true); @@ -61,7 +61,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function isCreatable($path) { try { $this->checkFileAccess($path); - } catch (ForbiddenException $e) { + } catch (ServiceUnavailableException $e) { return false; } } @@ -75,7 +75,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function isReadable($path) { try { $this->checkFileAccess($path); - } catch (ForbiddenException $e) { + } catch (ServiceUnavailableException $e) { return false; } } @@ -89,7 +89,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function isUpdatable($path) { try { $this->checkFileAccess($path); - } catch (ForbiddenException $e) { + } catch (ServiceUnavailableException $e) { return false; } } @@ -103,7 +103,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function isDeletable($path) { try { $this->checkFileAccess($path); - } catch (ForbiddenException $e) { + } catch (ServiceUnavailableException $e) { return false; } } @@ -111,7 +111,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function getPermissions($path) { try { $this->checkFileAccess($path); - } catch (ForbiddenException $e) { + } catch (ServiceUnavailableException $e) { return $this->mask; } } @@ -121,7 +121,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return string - * @throws ForbiddenException + * @throws ServiceUnavailableException */ public function file_get_contents($path) { $this->checkFileAccess($path); @@ -133,7 +133,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path * @param string $data * @return bool - * @throws ForbiddenException + * @throws ServiceUnavailableException */ public function file_put_contents($path, $data) { $this->checkFileAccess($path); @@ -144,7 +144,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return bool - * @throws ForbiddenException + * @throws ServiceUnavailableException */ public function unlink($path) { $this->checkFileAccess($path); @@ -156,7 +156,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path1 * @param string $path2 * @return bool - * @throws ForbiddenException + * @throws ServiceUnavailableException */ public function rename($path1, $path2) { $this->checkFileAccess($path1); @@ -169,7 +169,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path1 * @param string $path2 * @return bool - * @throws ForbiddenException + * @throws ServiceUnavailableException */ public function copy($path1, $path2) { $this->checkFileAccess($path1); @@ -182,7 +182,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path * @param string $mode * @return resource - * @throws ForbiddenException + * @throws ServiceUnavailableException */ public function fopen($path, $mode) { $this->checkFileAccess($path); @@ -195,7 +195,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path * @param int $mtime * @return bool - * @throws ForbiddenException + * @throws ServiceUnavailableException */ public function touch($path, $mtime = null) { $this->checkFileAccess($path); @@ -223,7 +223,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return array - * @throws ForbiddenException + * @throws ServiceUnavailableException */ public function getDirectDownload($path) { $this->checkFileAccess($path); @@ -234,7 +234,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $sourceInternalPath * @param string $targetInternalPath * @return bool - * @throws ForbiddenException + * @throws ServiceUnavailableException */ public function copyFromStorage(IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath) { $this->checkFileAccess($targetInternalPath); @@ -245,14 +245,14 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $sourceInternalPath * @param string $targetInternalPath * @return bool - * @throws ForbiddenException + * @throws ServiceUnavailableException */ public function moveFromStorage(IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath) { $this->checkFileAccess($targetInternalPath); } /** - * @throws ForbiddenException + * @throws ServiceUnavailableException */ public function writeStream(string $path, $stream, ?int $size = null): int { $this->checkFileAccess($path); -- GitLab From 534bb36d4a34d7faa96e0c5af715c6dae667d7be Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Mon, 18 Nov 2024 14:57:32 +0600 Subject: [PATCH 11/18] rnd: return Exception 503 from storageWrapper --- lib/Filesystem/StorageWrapper.php | 41 +++++++++++++++---------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/lib/Filesystem/StorageWrapper.php b/lib/Filesystem/StorageWrapper.php index 1aa7d19c..b50d063f 100644 --- a/lib/Filesystem/StorageWrapper.php +++ b/lib/Filesystem/StorageWrapper.php @@ -7,7 +7,6 @@ namespace OCA\EcloudAccounts\Filesystem; use OC\Files\Cache\Cache; use OC\Files\Storage\Storage; use OC\Files\Storage\Wrapper\Wrapper; -use OC\ServiceUnavailableException; use OCP\Files\Storage\IStorage; use OCP\Files\Storage\IWriteStreamStorage; @@ -20,10 +19,10 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { } /** - * @throws ServiceUnavailableException + * @throws \Exception */ protected function checkFileAccess(string $path, bool $isDir = false): void { - throw new ServiceUnavailableException('Access denied'); + throw new \Exception('Service unavailable', 503); } /* @@ -35,7 +34,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return bool - * @throws ServiceUnavailableException + * @throws \Exception */ public function mkdir($path) { $this->checkFileAccess($path, true); @@ -46,7 +45,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return bool - * @throws ServiceUnavailableException + * @throws \Exception */ public function rmdir($path) { $this->checkFileAccess($path, true); @@ -61,7 +60,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function isCreatable($path) { try { $this->checkFileAccess($path); - } catch (ServiceUnavailableException $e) { + } catch (\Exception $e) { return false; } } @@ -75,7 +74,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function isReadable($path) { try { $this->checkFileAccess($path); - } catch (ServiceUnavailableException $e) { + } catch (\Exception $e) { return false; } } @@ -89,7 +88,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function isUpdatable($path) { try { $this->checkFileAccess($path); - } catch (ServiceUnavailableException $e) { + } catch (\Exception $e) { return false; } } @@ -103,7 +102,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function isDeletable($path) { try { $this->checkFileAccess($path); - } catch (ServiceUnavailableException $e) { + } catch (\Exception $e) { return false; } } @@ -111,7 +110,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function getPermissions($path) { try { $this->checkFileAccess($path); - } catch (ServiceUnavailableException $e) { + } catch (\Exception $e) { return $this->mask; } } @@ -121,7 +120,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return string - * @throws ServiceUnavailableException + * @throws \Exception */ public function file_get_contents($path) { $this->checkFileAccess($path); @@ -133,7 +132,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path * @param string $data * @return bool - * @throws ServiceUnavailableException + * @throws \Exception */ public function file_put_contents($path, $data) { $this->checkFileAccess($path); @@ -144,7 +143,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return bool - * @throws ServiceUnavailableException + * @throws \Exception */ public function unlink($path) { $this->checkFileAccess($path); @@ -156,7 +155,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path1 * @param string $path2 * @return bool - * @throws ServiceUnavailableException + * @throws \Exception */ public function rename($path1, $path2) { $this->checkFileAccess($path1); @@ -169,7 +168,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path1 * @param string $path2 * @return bool - * @throws ServiceUnavailableException + * @throws \Exception */ public function copy($path1, $path2) { $this->checkFileAccess($path1); @@ -182,7 +181,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path * @param string $mode * @return resource - * @throws ServiceUnavailableException + * @throws \Exception */ public function fopen($path, $mode) { $this->checkFileAccess($path); @@ -195,7 +194,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path * @param int $mtime * @return bool - * @throws ServiceUnavailableException + * @throws \Exception */ public function touch($path, $mtime = null) { $this->checkFileAccess($path); @@ -223,7 +222,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return array - * @throws ServiceUnavailableException + * @throws \Exception */ public function getDirectDownload($path) { $this->checkFileAccess($path); @@ -234,7 +233,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $sourceInternalPath * @param string $targetInternalPath * @return bool - * @throws ServiceUnavailableException + * @throws \Exception */ public function copyFromStorage(IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath) { $this->checkFileAccess($targetInternalPath); @@ -245,14 +244,14 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $sourceInternalPath * @param string $targetInternalPath * @return bool - * @throws ServiceUnavailableException + * @throws \Exception */ public function moveFromStorage(IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath) { $this->checkFileAccess($targetInternalPath); } /** - * @throws ServiceUnavailableException + * @throws \Exception */ public function writeStream(string $path, $stream, ?int $size = null): int { $this->checkFileAccess($path); -- GitLab From 13d32a6bae6f6d892150ad661463a165368d616a Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Mon, 18 Nov 2024 15:40:52 +0600 Subject: [PATCH 12/18] rnd: return 503 from cacheWrapper --- lib/Filesystem/CacheWrapper.php | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/Filesystem/CacheWrapper.php b/lib/Filesystem/CacheWrapper.php index da6e3253..da7ba3af 100644 --- a/lib/Filesystem/CacheWrapper.php +++ b/lib/Filesystem/CacheWrapper.php @@ -8,7 +8,6 @@ use OC\Files\Cache\Wrapper\CacheWrapper as Wrapper; use OCP\Constants; use OCP\Files\Cache\ICache; use OCP\Files\Cache\ICacheEntry; -use OCP\Files\ForbiddenException; use OCP\Files\Search\ISearchQuery; class CacheWrapper extends Wrapper { @@ -27,8 +26,8 @@ class CacheWrapper extends Wrapper { protected function formatCacheEntry($entry) { if (isset($entry['path']) && isset($entry['permissions'])) { try { - throw new ForbiddenException('Access denied', false); - } catch (ForbiddenException) { + throw new \Exception('Access denied', 503); + } catch (\Exception) { $entry['permissions'] &= $this->mask; } } @@ -36,15 +35,15 @@ class CacheWrapper extends Wrapper { } public function insert($file, $data) { - throw new \Exception('User data cache insert is disabled.'); + throw new \Exception('User data cache insert is disabled.', 503); } public function update($id, $data) { - throw new \Exception('User data cache update is disabled.'); + throw new \Exception('User data cache update is disabled.', 503); } public function remove($fileId) { - throw new \Exception('User data cache removal is disabled.'); + throw new \Exception('User data cache removal is disabled.', 503); } public function searchQuery(ISearchQuery $searchQuery) { -- GitLab From 872527012907ead707d1c7fc8c283ec0bbac9285 Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Fri, 22 Nov 2024 17:45:09 +0600 Subject: [PATCH 13/18] rnd: return StorageNotAvailableException --- lib/Filesystem/StorageWrapper.php | 41 ++++++++++++++++--------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/lib/Filesystem/StorageWrapper.php b/lib/Filesystem/StorageWrapper.php index b50d063f..a68ce43d 100644 --- a/lib/Filesystem/StorageWrapper.php +++ b/lib/Filesystem/StorageWrapper.php @@ -9,6 +9,7 @@ use OC\Files\Storage\Storage; use OC\Files\Storage\Wrapper\Wrapper; use OCP\Files\Storage\IStorage; use OCP\Files\Storage\IWriteStreamStorage; +use OCP\Files\StorageNotAvailableException; class StorageWrapper extends Wrapper implements IWriteStreamStorage { /** @@ -19,10 +20,10 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { } /** - * @throws \Exception + * @throws StorageNotAvailableException */ protected function checkFileAccess(string $path, bool $isDir = false): void { - throw new \Exception('Service unavailable', 503); + throw new StorageNotAvailableException('Service unavailable'); } /* @@ -34,7 +35,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return bool - * @throws \Exception + * @throws StorageNotAvailableException */ public function mkdir($path) { $this->checkFileAccess($path, true); @@ -45,7 +46,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return bool - * @throws \Exception + * @throws StorageNotAvailableException */ public function rmdir($path) { $this->checkFileAccess($path, true); @@ -60,7 +61,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function isCreatable($path) { try { $this->checkFileAccess($path); - } catch (\Exception $e) { + } catch (StorageNotAvailableException $e) { return false; } } @@ -74,7 +75,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function isReadable($path) { try { $this->checkFileAccess($path); - } catch (\Exception $e) { + } catch (StorageNotAvailableException $e) { return false; } } @@ -88,7 +89,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function isUpdatable($path) { try { $this->checkFileAccess($path); - } catch (\Exception $e) { + } catch (StorageNotAvailableException $e) { return false; } } @@ -102,7 +103,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function isDeletable($path) { try { $this->checkFileAccess($path); - } catch (\Exception $e) { + } catch (StorageNotAvailableException $e) { return false; } } @@ -110,7 +111,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { public function getPermissions($path) { try { $this->checkFileAccess($path); - } catch (\Exception $e) { + } catch (StorageNotAvailableException $e) { return $this->mask; } } @@ -120,7 +121,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return string - * @throws \Exception + * @throws StorageNotAvailableException */ public function file_get_contents($path) { $this->checkFileAccess($path); @@ -132,7 +133,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path * @param string $data * @return bool - * @throws \Exception + * @throws StorageNotAvailableException */ public function file_put_contents($path, $data) { $this->checkFileAccess($path); @@ -143,7 +144,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return bool - * @throws \Exception + * @throws StorageNotAvailableException */ public function unlink($path) { $this->checkFileAccess($path); @@ -155,7 +156,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path1 * @param string $path2 * @return bool - * @throws \Exception + * @throws StorageNotAvailableException */ public function rename($path1, $path2) { $this->checkFileAccess($path1); @@ -168,7 +169,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path1 * @param string $path2 * @return bool - * @throws \Exception + * @throws StorageNotAvailableException */ public function copy($path1, $path2) { $this->checkFileAccess($path1); @@ -181,7 +182,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path * @param string $mode * @return resource - * @throws \Exception + * @throws StorageNotAvailableException */ public function fopen($path, $mode) { $this->checkFileAccess($path); @@ -194,7 +195,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $path * @param int $mtime * @return bool - * @throws \Exception + * @throws StorageNotAvailableException */ public function touch($path, $mtime = null) { $this->checkFileAccess($path); @@ -222,7 +223,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * * @param string $path * @return array - * @throws \Exception + * @throws StorageNotAvailableException */ public function getDirectDownload($path) { $this->checkFileAccess($path); @@ -233,7 +234,7 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $sourceInternalPath * @param string $targetInternalPath * @return bool - * @throws \Exception + * @throws StorageNotAvailableException */ public function copyFromStorage(IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath) { $this->checkFileAccess($targetInternalPath); @@ -244,14 +245,14 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @param string $sourceInternalPath * @param string $targetInternalPath * @return bool - * @throws \Exception + * @throws StorageNotAvailableException */ public function moveFromStorage(IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath) { $this->checkFileAccess($targetInternalPath); } /** - * @throws \Exception + * @throws StorageNotAvailableException */ public function writeStream(string $path, $stream, ?int $size = null): int { $this->checkFileAccess($path); -- GitLab From 8e77eccc3b04cb9c0fef151dcd1e2998b32fc993 Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Fri, 22 Nov 2024 17:49:18 +0600 Subject: [PATCH 14/18] rnd: remove try-catch block --- lib/Filesystem/StorageWrapper.php | 35 +++++++++---------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/lib/Filesystem/StorageWrapper.php b/lib/Filesystem/StorageWrapper.php index a68ce43d..4854aad1 100644 --- a/lib/Filesystem/StorageWrapper.php +++ b/lib/Filesystem/StorageWrapper.php @@ -59,11 +59,8 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @return bool */ public function isCreatable($path) { - try { - $this->checkFileAccess($path); - } catch (StorageNotAvailableException $e) { - return false; - } + $this->checkFileAccess($path); + return false; } /** @@ -73,11 +70,8 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @return bool */ public function isReadable($path) { - try { - $this->checkFileAccess($path); - } catch (StorageNotAvailableException $e) { - return false; - } + $this->checkFileAccess($path); + return false; } /** @@ -87,11 +81,8 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @return bool */ public function isUpdatable($path) { - try { - $this->checkFileAccess($path); - } catch (StorageNotAvailableException $e) { - return false; - } + $this->checkFileAccess($path); + return false; } /** @@ -101,19 +92,13 @@ class StorageWrapper extends Wrapper implements IWriteStreamStorage { * @return bool */ public function isDeletable($path) { - try { - $this->checkFileAccess($path); - } catch (StorageNotAvailableException $e) { - return false; - } + $this->checkFileAccess($path); + return false; } public function getPermissions($path) { - try { - $this->checkFileAccess($path); - } catch (StorageNotAvailableException $e) { - return $this->mask; - } + $this->checkFileAccess($path); + return $this->mask; } /** -- GitLab From 27c491af98fd7f94d6a95152f38dfd7a08c4be7c Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Mon, 25 Nov 2024 12:55:26 +0600 Subject: [PATCH 15/18] rnd: update cacheWrapper exception --- lib/Filesystem/CacheWrapper.php | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/Filesystem/CacheWrapper.php b/lib/Filesystem/CacheWrapper.php index da7ba3af..f26cced7 100644 --- a/lib/Filesystem/CacheWrapper.php +++ b/lib/Filesystem/CacheWrapper.php @@ -24,26 +24,30 @@ class CacheWrapper extends Wrapper { } protected function formatCacheEntry($entry) { - if (isset($entry['path']) && isset($entry['permissions'])) { - try { - throw new \Exception('Access denied', 503); - } catch (\Exception) { - $entry['permissions'] &= $this->mask; - } - } - return $entry; + throw new \OC\ServiceUnavailableException('Service unavailable'); + // if (isset($entry['path']) && isset($entry['permissions'])) { + // try { + // throw new \Exception('Access denied', 503); + // } catch (\Exception) { + // $entry['permissions'] &= $this->mask; + // } + // } + // return $entry; } public function insert($file, $data) { - throw new \Exception('User data cache insert is disabled.', 503); + throw new \OC\ServiceUnavailableException('Service unavailable'); + // throw new \Exception('User data cache insert is disabled.', 503); } public function update($id, $data) { - throw new \Exception('User data cache update is disabled.', 503); + throw new \OC\ServiceUnavailableException('Service unavailable'); + // throw new \Exception('User data cache update is disabled.', 503); } public function remove($fileId) { - throw new \Exception('User data cache removal is disabled.', 503); + throw new \OC\ServiceUnavailableException('Service unavailable'); + // throw new \Exception('User data cache removal is disabled.', 503); } public function searchQuery(ISearchQuery $searchQuery) { -- GitLab From 24d82146a57547fa1f7e1a6ea204a9dbf3152ea2 Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Mon, 25 Nov 2024 13:35:26 +0600 Subject: [PATCH 16/18] rnd: modify cacheWrapperException --- lib/Filesystem/CacheWrapper.php | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/Filesystem/CacheWrapper.php b/lib/Filesystem/CacheWrapper.php index f26cced7..630ab169 100644 --- a/lib/Filesystem/CacheWrapper.php +++ b/lib/Filesystem/CacheWrapper.php @@ -24,15 +24,14 @@ class CacheWrapper extends Wrapper { } protected function formatCacheEntry($entry) { - throw new \OC\ServiceUnavailableException('Service unavailable'); - // if (isset($entry['path']) && isset($entry['permissions'])) { - // try { - // throw new \Exception('Access denied', 503); - // } catch (\Exception) { - // $entry['permissions'] &= $this->mask; - // } - // } - // return $entry; + if (isset($entry['path']) && isset($entry['permissions'])) { + try { + throw new \Exception('Access denied', 503); + } catch (\Exception) { + $entry['permissions'] &= $this->mask; + } + } + return $entry; } public function insert($file, $data) { -- GitLab From f3c0fac1807cf5231d6ab08d57c740f8a91ab205 Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Wed, 27 Nov 2024 12:31:58 +0600 Subject: [PATCH 17/18] rnd: throw exception from cacheWrapper formatCacheEntry --- lib/Filesystem/CacheWrapper.php | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/Filesystem/CacheWrapper.php b/lib/Filesystem/CacheWrapper.php index 630ab169..f26cced7 100644 --- a/lib/Filesystem/CacheWrapper.php +++ b/lib/Filesystem/CacheWrapper.php @@ -24,14 +24,15 @@ class CacheWrapper extends Wrapper { } protected function formatCacheEntry($entry) { - if (isset($entry['path']) && isset($entry['permissions'])) { - try { - throw new \Exception('Access denied', 503); - } catch (\Exception) { - $entry['permissions'] &= $this->mask; - } - } - return $entry; + throw new \OC\ServiceUnavailableException('Service unavailable'); + // if (isset($entry['path']) && isset($entry['permissions'])) { + // try { + // throw new \Exception('Access denied', 503); + // } catch (\Exception) { + // $entry['permissions'] &= $this->mask; + // } + // } + // return $entry; } public function insert($file, $data) { -- GitLab From 4a9f3fc8d507ec552ea9ccc077b67b333268d33c Mon Sep 17 00:00:00 2001 From: Fahim Salam Chowdhury Date: Thu, 28 Nov 2024 13:10:43 +0600 Subject: [PATCH 18/18] rnd: only block eDrive storage requests --- appinfo/info.xml | 2 - lib/AppInfo/Application.php | 35 +-- lib/Command/FixEncryptedVersion.php | 334 -------------------------- lib/Command/Scan.php | 359 ---------------------------- 4 files changed, 10 insertions(+), 720 deletions(-) delete mode 100644 lib/Command/FixEncryptedVersion.php delete mode 100644 lib/Command/Scan.php diff --git a/appinfo/info.xml b/appinfo/info.xml index c5d99471..35a9b216 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -28,7 +28,5 @@ OCA\EcloudAccounts\Command\Migrate2FASecrets OCA\EcloudAccounts\Command\MigrateWebmailAddressbooks OCA\EcloudAccounts\Command\MapActiveAttributetoLDAP - OCA\EcloudAccounts\Command\Scan - OCA\EcloudAccounts\Command\FixEncryptedVersion diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index a19e8eca..ee84b503 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -42,9 +42,8 @@ use OCP\AppFramework\Bootstrap\IBootstrap; use OCP\AppFramework\Bootstrap\IRegistrationContext; use OCP\AppFramework\Http\Events\BeforeTemplateRenderedEvent; use OCP\Files\Storage\IStorage; -use OCP\IGroupManager; +use OCP\IRequest; use OCP\IUserManager; -use OCP\IUserSession; use OCP\User\Events\BeforeUserDeletedEvent; use OCP\User\Events\PasswordUpdatedEvent; use OCP\User\Events\UserChangedEvent; @@ -91,34 +90,20 @@ class Application extends App implements IBootstrap { * @return StorageWrapper|IStorage */ public function addStorageWrapperCallback($mountPoint, IStorage $storage) { - if (\OC::$CLI && \OC::$REQUESTEDAPP === self::APP_ID) { - return $storage; - } + $request = \OC::$server->get(IRequest::class); - $userSession = \OC::$server->get(IUserSession::class); - $currentUser = $userSession->getUser(); - if ($currentUser !== null) { - $groupManager = \OC::$server->get(IGroupManager::class); - $groups = $groupManager->getUserGroups($currentUser); + if ($request !== null) { + $userAgent = $request->getHeader('USER_AGENT'); + $userAgent = strtolower($userAgent); - if (!empty($groups)) { - foreach ($groups as $group) { - if ($group->getGID() === "recovery_done") { - return $storage; - } - } + if (strpos($userAgent, "eos") !== false || strpos($userAgent, "edrive") !== false) { + return new StorageWrapper([ + 'storage' => $storage, + 'mountPoint' => $mountPoint, + ]); } } - $instanceId = \OC::$server->getConfig()->getSystemValue('instanceid', ''); - $appdataFolder = 'appdata_' . $instanceId; - if ($mountPoint !== '/' && strpos($mountPoint, '/' . $appdataFolder) !== 0) { - return new StorageWrapper([ - 'storage' => $storage, - 'mountPoint' => $mountPoint, - ]); - } - return $storage; } } diff --git a/lib/Command/FixEncryptedVersion.php b/lib/Command/FixEncryptedVersion.php deleted file mode 100644 index bd6e75c7..00000000 --- a/lib/Command/FixEncryptedVersion.php +++ /dev/null @@ -1,334 +0,0 @@ - - * @author Ilja Neumann - * - * @copyright Copyright (c) 2019, ownCloud GmbH - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ - -declare(strict_types=1); - -namespace OCA\EcloudAccounts\Command; - -use OC\Files\Storage\Wrapper\Encryption; -use OC\Files\View; -use OC\ServerNotAvailableException; -use OCA\EcloudAccounts\AppInfo\Application; -use OCA\Encryption\Util; -use OCP\Files\IRootFolder; -use OCP\HintException; -use OCP\IConfig; -use OCP\IUser; -use OCP\IUserManager; -use Psr\Log\LoggerInterface; -use Symfony\Component\Console\Command\Command; -use Symfony\Component\Console\Input\InputArgument; -use Symfony\Component\Console\Input\InputInterface; -use Symfony\Component\Console\Input\InputOption; -use Symfony\Component\Console\Output\OutputInterface; - -class FixEncryptedVersion extends Command { - private bool $supportLegacy; - - public function __construct( - private IConfig $config, - private LoggerInterface $logger, - private IRootFolder $rootFolder, - private IUserManager $userManager, - private Util $util, - private View $view, - ) { - $this->supportLegacy = false; - - parent::__construct(); - } - - protected function configure(): void { - parent::configure(); - - $this - ->setName(Application::APP_ID . ':fix-encrypted-version') - ->setDescription('Fix the encrypted version if the encrypted file(s) are not downloadable.') - ->addArgument( - 'user', - InputArgument::OPTIONAL, - 'The id of the user whose files need fixing' - )->addOption( - 'path', - 'p', - InputOption::VALUE_REQUIRED, - 'Limit files to fix with path, e.g., --path="/Music/Artist". If path indicates a directory, all the files inside directory will be fixed.' - )->addOption( - 'all', - null, - InputOption::VALUE_NONE, - 'Run the fix for all users on the system, mutually exclusive with specifying a user id.' - ); - } - - protected function execute(InputInterface $input, OutputInterface $output): int { - \OC::$REQUESTEDAPP = Application::APP_ID; - $skipSignatureCheck = $this->config->getSystemValueBool('encryption_skip_signature_check', false); - $this->supportLegacy = $this->config->getSystemValueBool('encryption.legacy_format_support', false); - - if ($skipSignatureCheck) { - $output->writeln("Repairing is not possible when \"encryption_skip_signature_check\" is set. Please disable this flag in the configuration.\n"); - return 1; - } - - if (!$this->util->isMasterKeyEnabled()) { - $output->writeln("Repairing only works with master key encryption.\n"); - return 1; - } - - $user = $input->getArgument('user'); - $all = $input->getOption('all'); - $pathOption = \trim(($input->getOption('path') ?? ''), '/'); - - if ($user) { - if ($all) { - $output->writeln("Specifying a user id and --all are mutually exclusive"); - return 1; - } - - if ($this->userManager->get($user) === null) { - $output->writeln("User id $user does not exist. Please provide a valid user id"); - return 1; - } - - return $this->runForUser($user, $pathOption, $output); - } elseif ($all) { - $result = 0; - $this->userManager->callForSeenUsers(function (IUser $user) use ($pathOption, $output, &$result) { - $output->writeln("Processing files for " . $user->getUID()); - $result = $this->runForUser($user->getUID(), $pathOption, $output); - return $result === 0; - }); - return $result; - } else { - $output->writeln("Either a user id or --all needs to be provided"); - return 1; - } - } - - private function runForUser(string $user, string $pathOption, OutputInterface $output): int { - $pathToWalk = "/$user/files"; - if ($pathOption !== "") { - $pathToWalk = "$pathToWalk/$pathOption"; - } - return $this->walkPathOfUser($user, $pathToWalk, $output); - } - - /** - * @return int 0 for success, 1 for error - */ - private function walkPathOfUser(string $user, string $path, OutputInterface $output): int { - $this->setupUserFs($user); - if (!$this->view->file_exists($path)) { - $output->writeln("Path \"$path\" does not exist. Please provide a valid path."); - return 1; - } - - if ($this->view->is_file($path)) { - $output->writeln("Verifying the content of file \"$path\""); - $this->verifyFileContent($path, $output); - return 0; - } - $directories = []; - $directories[] = $path; - while ($root = \array_pop($directories)) { - $directoryContent = $this->view->getDirectoryContent($root); - foreach ($directoryContent as $file) { - $path = $root . '/' . $file['name']; - if ($this->view->is_dir($path)) { - $directories[] = $path; - } else { - $output->writeln("Verifying the content of file \"$path\""); - $this->verifyFileContent($path, $output); - } - } - } - return 0; - } - - /** - * @param bool $ignoreCorrectEncVersionCall, setting this variable to false avoids recursion - */ - private function verifyFileContent(string $path, OutputInterface $output, bool $ignoreCorrectEncVersionCall = true): bool { - try { - // since we're manually poking around the encrypted state we need to ensure that this isn't cached in the encryption wrapper - $mount = $this->view->getMount($path); - $storage = $mount->getStorage(); - if ($storage && $storage->instanceOfStorage(Encryption::class)) { - $storage->clearIsEncryptedCache(); - } - - /** - * In encryption, the files are read in a block size of 8192 bytes - * Read block size of 8192 and a bit more (808 bytes) - * If there is any problem, the first block should throw the signature - * mismatch error. Which as of now, is enough to proceed ahead to - * correct the encrypted version. - */ - $handle = $this->view->fopen($path, 'rb'); - - if ($handle === false) { - $output->writeln("Failed to open file: \"$path\" skipping"); - return true; - } - - if (\fread($handle, 9001) !== false) { - $fileInfo = $this->view->getFileInfo($path); - if (!$fileInfo) { - $output->writeln("File info not found for file: \"$path\""); - return true; - } - $encryptedVersion = $fileInfo->getEncryptedVersion(); - $stat = $this->view->stat($path); - if (($encryptedVersion == 0) && isset($stat['hasHeader']) && ($stat['hasHeader'] == true)) { - // The file has encrypted to false but has an encryption header - if ($ignoreCorrectEncVersionCall === true) { - // Lets rectify the file by correcting encrypted version - $output->writeln("Attempting to fix the path: \"$path\""); - return $this->correctEncryptedVersion($path, $output); - } - return false; - } - $output->writeln("The file \"$path\" is: OK"); - } - - \fclose($handle); - - return true; - } catch (ServerNotAvailableException $e) { - // not a "bad signature" error and likely "legacy cipher" exception - // this could mean that the file is maybe not encrypted but the encrypted version is set - if (!$this->supportLegacy && $ignoreCorrectEncVersionCall === true) { - $output->writeln("Attempting to fix the path: \"$path\""); - return $this->correctEncryptedVersion($path, $output, true); - } - return false; - } catch (HintException $e) { - $this->logger->warning("Issue: " . $e->getMessage()); - // If allowOnce is set to false, this becomes recursive. - if ($ignoreCorrectEncVersionCall === true) { - // Lets rectify the file by correcting encrypted version - $output->writeln("Attempting to fix the path: \"$path\""); - return $this->correctEncryptedVersion($path, $output); - } - return false; - } - } - - /** - * @param bool $includeZero whether to try zero version for unencrypted file - */ - private function correctEncryptedVersion(string $path, OutputInterface $output, bool $includeZero = false): bool { - $fileInfo = $this->view->getFileInfo($path); - if (!$fileInfo) { - $output->writeln("File info not found for file: \"$path\""); - return true; - } - $fileId = $fileInfo->getId(); - if ($fileId === null) { - $output->writeln("File info contains no id for file: \"$path\""); - return true; - } - $encryptedVersion = $fileInfo->getEncryptedVersion(); - $wrongEncryptedVersion = $encryptedVersion; - - $storage = $fileInfo->getStorage(); - - $cache = $storage->getCache(); - $fileCache = $cache->get($fileId); - if (!$fileCache) { - $output->writeln("File cache entry not found for file: \"$path\""); - return true; - } - - if ($storage->instanceOfStorage('OCA\Files_Sharing\ISharedStorage')) { - $output->writeln("The file: \"$path\" is a share. Please also run the script for the owner of the share"); - return true; - } - - // Save original encrypted version so we can restore it if decryption fails with all version - $originalEncryptedVersion = $encryptedVersion; - if ($encryptedVersion >= 0) { - if ($includeZero) { - // try with zero first - $cacheInfo = ['encryptedVersion' => 0, 'encrypted' => 0]; - $cache->put($fileCache->getPath(), $cacheInfo); - $output->writeln("Set the encrypted version to 0 (unencrypted)"); - if ($this->verifyFileContent($path, $output, false) === true) { - $output->writeln("Fixed the file: \"$path\" with version 0 (unencrypted)"); - return true; - } - } - - // Test by decrementing the value till 1 and if nothing works try incrementing - $encryptedVersion--; - while ($encryptedVersion > 0) { - $cacheInfo = ['encryptedVersion' => $encryptedVersion, 'encrypted' => $encryptedVersion]; - $cache->put($fileCache->getPath(), $cacheInfo); - $output->writeln("Decrement the encrypted version to $encryptedVersion"); - if ($this->verifyFileContent($path, $output, false) === true) { - $output->writeln("Fixed the file: \"$path\" with version " . $encryptedVersion . ""); - return true; - } - $encryptedVersion--; - } - - // So decrementing did not work. Now lets increment. Max increment is till 5 - $increment = 1; - while ($increment <= 5) { - /** - * The wrongEncryptedVersion would not be incremented so nothing to worry about here. - * Only the newEncryptedVersion is incremented. - * For example if the wrong encrypted version is 4 then - * cycle1 -> newEncryptedVersion = 5 ( 4 + 1) - * cycle2 -> newEncryptedVersion = 6 ( 4 + 2) - * cycle3 -> newEncryptedVersion = 7 ( 4 + 3) - */ - $newEncryptedVersion = $wrongEncryptedVersion + $increment; - - $cacheInfo = ['encryptedVersion' => $newEncryptedVersion, 'encrypted' => $newEncryptedVersion]; - $cache->put($fileCache->getPath(), $cacheInfo); - $output->writeln("Increment the encrypted version to $newEncryptedVersion"); - if ($this->verifyFileContent($path, $output, false) === true) { - $output->writeln("Fixed the file: \"$path\" with version " . $newEncryptedVersion . ""); - return true; - } - $increment++; - } - } - - $cacheInfo = ['encryptedVersion' => $originalEncryptedVersion, 'encrypted' => $originalEncryptedVersion]; - $cache->put($fileCache->getPath(), $cacheInfo); - $output->writeln("No fix found for \"$path\", restored version to original: $originalEncryptedVersion"); - - return false; - } - - /** - * Setup user file system - */ - private function setupUserFs(string $uid): void { - \OC_Util::tearDownFS(); - \OC_Util::setupFS($uid); - } -} diff --git a/lib/Command/Scan.php b/lib/Command/Scan.php deleted file mode 100644 index 3a4ea6be..00000000 --- a/lib/Command/Scan.php +++ /dev/null @@ -1,359 +0,0 @@ - - * @author Blaok - * @author Christoph Wurst - * @author Daniel Kesselberg - * @author J0WI - * @author Joas Schilling - * @author Joel S - * @author Jörn Friedrich Dreyer - * @author martin.mattel@diemattels.at - * @author Maxence Lange - * @author Robin Appelman - * @author Roeland Jago Douma - * @author Thomas Müller - * @author Vincent Petry - * - * @license AGPL-3.0 - * - * This code is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License, version 3, - * along with this program. If not, see - * - */ -declare(strict_types=1); - -namespace OCA\EcloudAccounts\Command; - -use OC\Core\Command\Base; -use OC\Core\Command\InterruptedException; -use OC\DB\Connection; -use OC\DB\ConnectionAdapter; -use OC\FilesMetadata\FilesMetadataManager; -use OC\ForbiddenException; -use OCA\EcloudAccounts\AppInfo\Application; -use OCP\EventDispatcher\IEventDispatcher; -use OCP\Files\Events\FileCacheUpdated; -use OCP\Files\Events\NodeAddedToCache; -use OCP\Files\Events\NodeRemovedFromCache; -use OCP\Files\IRootFolder; -use OCP\Files\Mount\IMountPoint; -use OCP\Files\NotFoundException; -use OCP\Files\StorageNotAvailableException; -use OCP\FilesMetadata\IFilesMetadataManager; -use OCP\IUserManager; -use Psr\Log\LoggerInterface; -use Symfony\Component\Console\Helper\Table; -use Symfony\Component\Console\Input\InputArgument; -use Symfony\Component\Console\Input\InputInterface; -use Symfony\Component\Console\Input\InputOption; -use Symfony\Component\Console\Output\OutputInterface; - -class Scan extends Base { - protected float $execTime = 0; - protected int $foldersCounter = 0; - protected int $filesCounter = 0; - protected int $errorsCounter = 0; - protected int $newCounter = 0; - protected int $updatedCounter = 0; - protected int $removedCounter = 0; - - public function __construct( - private IUserManager $userManager, - private IRootFolder $rootFolder, - private FilesMetadataManager $filesMetadataManager, - private IEventDispatcher $eventDispatcher, - private LoggerInterface $logger, - ) { - parent::__construct(); - } - - protected function configure(): void { - parent::configure(); - - $this - ->setName(Application::APP_ID.':scan') - ->setDescription('rescan filesystem') - ->addArgument( - 'user_id', - InputArgument::OPTIONAL | InputArgument::IS_ARRAY, - 'will rescan all files of the given user(s)' - ) - ->addOption( - 'path', - 'p', - InputArgument::OPTIONAL, - 'limit rescan to this path, eg. --path="/alice/files/Music", the user_id is determined by the path and the user_id parameter and --all are ignored' - ) - ->addOption( - 'generate-metadata', - null, - InputOption::VALUE_OPTIONAL, - 'Generate metadata for all scanned files; if specified only generate for named value', - '' - ) - ->addOption( - 'all', - null, - InputOption::VALUE_NONE, - 'will rescan all files of all known users' - )->addOption( - 'unscanned', - null, - InputOption::VALUE_NONE, - 'only scan files which are marked as not fully scanned' - )->addOption( - 'shallow', - null, - InputOption::VALUE_NONE, - 'do not scan folders recursively' - )->addOption( - 'home-only', - null, - InputOption::VALUE_NONE, - 'only scan the home storage, ignoring any mounted external storage or share' - ); - } - - protected function scanFiles(string $user, string $path, ?string $scanMetadata, OutputInterface $output, bool $backgroundScan = false, bool $recursive = true, bool $homeOnly = false): void { - $connection = $this->reconnectToDatabase($output); - $scanner = new \OC\Files\Utils\Scanner( - $user, - new ConnectionAdapter($connection), - \OC::$server->get(IEventDispatcher::class), - \OC::$server->get(LoggerInterface::class) - ); - - # check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function (string $path) use ($output, $scanMetadata) { - $output->writeln("\tFile\t$path", OutputInterface::VERBOSITY_VERBOSE); - ++$this->filesCounter; - $this->abortIfInterrupted(); - if ($scanMetadata !== null) { - $node = $this->rootFolder->get($path); - $this->filesMetadataManager->refreshMetadata( - $node, - ($scanMetadata !== '') ? IFilesMetadataManager::PROCESS_NAMED : IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND, - $scanMetadata - ); - } - }); - - $scanner->listen('\OC\Files\Utils\Scanner', 'scanFolder', function ($path) use ($output) { - $output->writeln("\tFolder\t$path", OutputInterface::VERBOSITY_VERBOSE); - ++$this->foldersCounter; - $this->abortIfInterrupted(); - }); - - $scanner->listen('\OC\Files\Utils\Scanner', 'StorageNotAvailable', function (StorageNotAvailableException $e) use ($output) { - $output->writeln('Error while scanning, storage not available (' . $e->getMessage() . ')', OutputInterface::VERBOSITY_VERBOSE); - ++$this->errorsCounter; - }); - - $scanner->listen('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', function ($fullPath) use ($output) { - $output->writeln("\tEntry \"" . $fullPath . '" will not be accessible due to incompatible encoding'); - ++$this->errorsCounter; - }); - - $this->eventDispatcher->addListener(NodeAddedToCache::class, function () { - ++$this->newCounter; - }); - $this->eventDispatcher->addListener(FileCacheUpdated::class, function () { - ++$this->updatedCounter; - }); - $this->eventDispatcher->addListener(NodeRemovedFromCache::class, function () { - ++$this->removedCounter; - }); - - try { - if ($backgroundScan) { - $scanner->backgroundScan($path); - } else { - $scanner->scan($path, $recursive, $homeOnly ? [$this, 'filterHomeMount'] : null); - } - } catch (ForbiddenException $e) { - $output->writeln("Home storage for user $user not writable or 'files' subdirectory missing"); - $output->writeln(' ' . $e->getMessage()); - $output->writeln('Make sure you\'re running the scan command only as the user the web server runs as'); - ++$this->errorsCounter; - } catch (InterruptedException $e) { - # exit the function if ctrl-c has been pressed - $output->writeln('Interrupted by user'); - } catch (NotFoundException $e) { - $output->writeln('Path not found: ' . $e->getMessage() . ''); - ++$this->errorsCounter; - } catch (\Exception $e) { - $output->writeln('Exception during scan: ' . $e->getMessage() . ''); - $output->writeln('' . $e->getTraceAsString() . ''); - ++$this->errorsCounter; - } - } - - public function filterHomeMount(IMountPoint $mountPoint): bool { - // any mountpoint inside '/$user/files/' - return substr_count($mountPoint->getMountPoint(), '/') <= 3; - } - - protected function execute(InputInterface $input, OutputInterface $output): int { - \OC::$REQUESTEDAPP = Application::APP_ID; - $inputPath = $input->getOption('path'); - if ($inputPath) { - $inputPath = '/' . trim($inputPath, '/'); - [, $user,] = explode('/', $inputPath, 3); - $users = [$user]; - } elseif ($input->getOption('all')) { - $users = $this->userManager->search(''); - } else { - $users = $input->getArgument('user_id'); - } - - # check quantity of users to be process and show it on the command line - $users_total = count($users); - if ($users_total === 0) { - $output->writeln('Please specify the user id to scan, --all to scan for all users or --path=...'); - return self::FAILURE; - } - - $this->initTools($output); - - // getOption() logic on VALUE_OPTIONAL - $metadata = null; // null if --generate-metadata is not set, empty if option have no value, value if set - if ($input->getOption('generate-metadata') !== '') { - $metadata = $input->getOption('generate-metadata') ?? ''; - } - - $user_count = 0; - foreach ($users as $user) { - if (is_object($user)) { - $user = $user->getUID(); - } - $path = $inputPath ?: '/' . $user; - ++$user_count; - if ($this->userManager->userExists($user)) { - $output->writeln("Starting scan for user $user_count out of $users_total ($user)"); - $this->scanFiles($user, $path, $metadata, $output, $input->getOption('unscanned'), !$input->getOption('shallow'), $input->getOption('home-only')); - $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); - } else { - $output->writeln("Unknown user $user_count $user"); - $output->writeln('', OutputInterface::VERBOSITY_VERBOSE); - } - - try { - $this->abortIfInterrupted(); - } catch (InterruptedException $e) { - break; - } - } - - $this->presentStats($output); - return self::SUCCESS; - } - - /** - * Initialises some useful tools for the Command - */ - protected function initTools(OutputInterface $output): void { - // Start the timer - $this->execTime = -microtime(true); - // Convert PHP errors to exceptions - set_error_handler( - fn (int $severity, string $message, string $file, int $line): bool => - $this->exceptionErrorHandler($output, $severity, $message, $file, $line), - E_ALL - ); - } - - /** - * Processes PHP errors in order to be able to show them in the output - * - * @see https://www.php.net/manual/en/function.set-error-handler.php - * - * @param int $severity the level of the error raised - * @param string $message - * @param string $file the filename that the error was raised in - * @param int $line the line number the error was raised - */ - public function exceptionErrorHandler(OutputInterface $output, int $severity, string $message, string $file, int $line): bool { - if (($severity === E_DEPRECATED) || ($severity === E_USER_DEPRECATED)) { - // Do not show deprecation warnings - return false; - } - $e = new \ErrorException($message, 0, $severity, $file, $line); - $output->writeln('Error during scan: ' . $e->getMessage() . ''); - $output->writeln('' . $e->getTraceAsString() . '', OutputInterface::VERBOSITY_VERY_VERBOSE); - ++$this->errorsCounter; - return true; - } - - protected function presentStats(OutputInterface $output): void { - // Stop the timer - $this->execTime += microtime(true); - - $this->logger->info("Completed scan of {$this->filesCounter} files in {$this->foldersCounter} folder. Found {$this->newCounter} new, {$this->updatedCounter} updated and {$this->removedCounter} removed items"); - - $headers = [ - 'Folders', - 'Files', - 'New', - 'Updated', - 'Removed', - 'Errors', - 'Elapsed time', - ]; - $niceDate = $this->formatExecTime(); - $rows = [ - $this->foldersCounter, - $this->filesCounter, - $this->newCounter, - $this->updatedCounter, - $this->removedCounter, - $this->errorsCounter, - $niceDate, - ]; - $table = new Table($output); - $table - ->setHeaders($headers) - ->setRows([$rows]); - $table->render(); - } - - - /** - * Formats microtime into a human-readable format - */ - protected function formatExecTime(): string { - $secs = (int)round($this->execTime); - # convert seconds into HH:MM:SS form - return sprintf('%02d:%02d:%02d', (int)($secs / 3600), ((int)($secs / 60) % 60), $secs % 60); - } - - protected function reconnectToDatabase(OutputInterface $output): Connection { - /** @var Connection $connection */ - $connection = \OC::$server->get(Connection::class); - try { - $connection->close(); - } catch (\Exception $ex) { - $output->writeln("Error while disconnecting from database: {$ex->getMessage()}"); - } - while (!$connection->isConnected()) { - try { - $connection->connect(); - } catch (\Exception $ex) { - $output->writeln("Error while re-connecting to database: {$ex->getMessage()}"); - sleep(60); - } - } - return $connection; - } -} -- GitLab