From c530d642b9d88711ea715bf81fe14052a8558743 Mon Sep 17 00:00:00 2001 From: Ilja Neumann Date: Fri, 1 Nov 2019 17:41:25 +0100 Subject: [PATCH] Flatten structure of file-layout inside the export dir. Before this PR all data-types (files, versions, trashbin...) were stored inside a single files directory which closely followed ownClouds home-folder layout. This allowed for fast iteration but also coupled the format to ownCloud which in turn introduced some design quirks (/files/files/) in files.jsonl (#111) All data type specific folders are now stored in the root directory of the export which allows simpler mapping from metadata (#118). This is also reflected in the architecture: The exporter traverses down from the specific directories instead from the home. A special "root folder" was introduced in files.jsonl to further decouple things from owncloud and to be able to carry the e-tag for the whole tree. Instead of "/files", "/" is now root in the export. Path class has been added to reduce path-merging boilerplate. --- lib/Exporter.php | 7 +- lib/Extractor/FilesExtractor.php | 2 +- .../FilesMetadataExtractor.php | 25 ++- lib/Importer.php | 6 +- lib/Importer/FilesImporter.php | 16 +- lib/Model/File.php | 1 + .../Nodes/RecursiveNodeIteratorFactory.php | 41 ---- lib/Utilities/Path.php | 60 ++++++ tests/unit/Extractor/FilesExtractorTest.php | 6 +- .../FilesMetadataExtractorTest.php | 24 ++- tests/unit/Importer/FilesImporterTest.php | 199 +++++++++--------- .../RecursiveNodeIteratorFactoryTest.php | 24 --- tests/unit/Utilities/PathTest.php | 57 +++++ 13 files changed, 277 insertions(+), 191 deletions(-) create mode 100644 lib/Utilities/Path.php create mode 100644 tests/unit/Utilities/PathTest.php diff --git a/lib/Exporter.php b/lib/Exporter.php index 9a075f7..cfe0884 100644 --- a/lib/Exporter.php +++ b/lib/Exporter.php @@ -24,6 +24,7 @@ use OCA\DataExporter\Extractor\FilesExtractor; use OCA\DataExporter\Extractor\MetadataExtractor; +use OCA\DataExporter\Utilities\Path; use Symfony\Component\Filesystem\Filesystem; class Exporter { @@ -56,15 +57,15 @@ public function __construct(Serializer $serializer, MetadataExtractor $metadataE * @return void */ public function export($uid, $exportDirectoryPath, $exportFiles = true) { - $exportPath = "$exportDirectoryPath/$uid"; + $exportPath = Path::join($exportDirectoryPath, $uid); $metaData = $this->metadataExtractor->extract($uid, $exportPath); $this->filesystem->dumpFile( - "$exportPath/user.json", + Path::join($exportPath, '/user.json'), $this->serializer->serialize($metaData) ); if ($exportFiles) { - $filesPath = \ltrim("$exportPath/files"); + $filesPath = Path::join($exportPath, 'files'); $this->filesExtractor->export($uid, $filesPath); } } diff --git a/lib/Extractor/FilesExtractor.php b/lib/Extractor/FilesExtractor.php index 9628339..396ef27 100644 --- a/lib/Extractor/FilesExtractor.php +++ b/lib/Extractor/FilesExtractor.php @@ -47,7 +47,7 @@ public function __construct(RecursiveNodeIteratorFactory $iteratorFactory, Files * @throws \OCP\Files\NotPermittedException */ public function export($userId, $exportPath) { - list($iterator, $baseFolder) = $this->iteratorFactory->getUserFolderParentRecursiveIterator($userId); + list($iterator, $baseFolder) = $this->iteratorFactory->getUserFolderRecursiveIterator($userId); /** @var \OCP\Files\Node $node */ foreach ($iterator as $node) { $nodePath = $node->getPath(); diff --git a/lib/Extractor/MetadataExtractor/FilesMetadataExtractor.php b/lib/Extractor/MetadataExtractor/FilesMetadataExtractor.php index b513c5f..bd8f09f 100644 --- a/lib/Extractor/MetadataExtractor/FilesMetadataExtractor.php +++ b/lib/Extractor/MetadataExtractor/FilesMetadataExtractor.php @@ -24,8 +24,9 @@ namespace OCA\DataExporter\Extractor\MetadataExtractor; use OC\User\NoUserException; -use OCA\DataExporter\Utilities\Iterators\Nodes\RecursiveNodeIteratorFactory; use OCA\DataExporter\Model\File; +use OCA\DataExporter\Utilities\Iterators\Nodes\RecursiveNodeIteratorFactory; +use OCA\DataExporter\Utilities\Path; use OCA\DataExporter\Utilities\StreamHelper; use OCP\Files\Node; @@ -56,17 +57,35 @@ public function __construct(RecursiveNodeIteratorFactory $iteratorFactory, Strea * @throws NoUserException */ public function extract($userId, $exportPath) { - list($iterator, $baseFolder) = $this->iteratorFactory->getUserFolderParentRecursiveIterator($userId); + list($iterator, $baseFolder) = $this->iteratorFactory->getUserFolderRecursiveIterator($userId); - $filename = $exportPath . '/' . $this::FILE_NAME; + $filename = Path::join($exportPath, $this::FILE_NAME); $this->streamFile = $this->streamHelper->initStream($filename, 'ab', true); + // Write root folder entry first to preserve it's metadata + $rootFolder = (new File()) + ->setType(File::TYPE_FOLDER) + ->setPath('/') + ->setETag($baseFolder->getEtag()) + ->setMtime($baseFolder->getMTime()) + ->setPermissions($baseFolder->getPermissions()); + + $this->streamHelper->writelnToStream($this->streamFile, $rootFolder); + foreach ($iterator as $node) { $nodePath = $node->getPath(); $relativePath = $baseFolder->getRelativePath($nodePath); $file = new File(); + if ("$relativePath/" === File::ROOT_FOLDER_PATH) { + $relativePath = '/'; + } + + if (\substr($relativePath, 0, \strlen(File::ROOT_FOLDER_PATH)) == File::ROOT_FOLDER_PATH) { + $relativePath = '/' . \substr($relativePath, \strlen(File::ROOT_FOLDER_PATH)); + } + $file->setPath($relativePath); $file->setETag($node->getEtag()); $file->setMtime($node->getMTime()); diff --git a/lib/Importer.php b/lib/Importer.php index 00666b5..3303887 100644 --- a/lib/Importer.php +++ b/lib/Importer.php @@ -26,6 +26,7 @@ use OCA\DataExporter\Importer\ImportException; use OCA\DataExporter\Importer\MetadataImporter; use OCA\DataExporter\Model\Metadata; +use OCA\DataExporter\Utilities\Path; use Symfony\Component\Filesystem\Filesystem; use OCA\DataExporter\Importer\FilesImporter; use OCA\DataExporter\Importer\MetadataImporter\ShareImporter; @@ -67,10 +68,11 @@ public function __construct( * @throws \OCP\PreConditionNotMetException */ public function import($pathToExportDir, $alias = null) { - $metaDataPath = "$pathToExportDir/user.json"; + $pathToExportDir =\rtrim($pathToExportDir, '\/'); + $metaDataPath = Path::join($pathToExportDir, 'user.json'); if (!$this->filesystem->exists($metaDataPath)) { - throw new ImportException("user.json not found in '$metaDataPath'"); + throw new ImportException("user.json not found in \'$metaDataPath\'"); } /** @var Metadata $metadata */ diff --git a/lib/Importer/FilesImporter.php b/lib/Importer/FilesImporter.php index ae4da95..b599440 100644 --- a/lib/Importer/FilesImporter.php +++ b/lib/Importer/FilesImporter.php @@ -27,6 +27,7 @@ use OCA\DataExporter\Utilities\StreamHelper; use OCP\Files\IRootFolder; use Symfony\Component\Filesystem\Filesystem; +use OCA\DataExporter\Utilities\Path; class FilesImporter { const FILE_NAME = 'files.jsonl'; @@ -69,10 +70,10 @@ public function import($userId, $exportPath) { /** * @var \OCP\Files\Folder $userFolder */ - $filename = $exportPath . '/' . $this::FILE_NAME; - $exportRootFilesPath = $exportPath . '/files'; + $filename = Path::join($exportPath, $this::FILE_NAME); + $exportRootFilesPath = Path::join($exportPath, '/files'); - $userFolder = $this->rootFolder->getUserFolder($userId)->getParent(); + $userFolder = $this->rootFolder->getUserFolder($userId); $this->streamFile = $this ->streamHelper ->initStream($filename, 'rb'); @@ -90,7 +91,7 @@ public function import($userId, $exportPath) { !== false ) { $fileCachePath = $fileMetadata->getPath(); - $pathToFileInExport = "$exportRootFilesPath/$fileCachePath"; + $pathToFileInExport = Path::join($exportRootFilesPath, $fileCachePath); if (!$this->filesystem->exists($pathToFileInExport)) { throw new ImportException("File '$pathToFileInExport' not found in export but exists in metadata.json"); @@ -98,7 +99,6 @@ public function import($userId, $exportPath) { if ($fileMetadata->getType() === File::TYPE_FILE) { $file = $userFolder->newFile($fileCachePath); - $src = \fopen($pathToFileInExport, "rb+"); if (!\is_resource($src)) { throw new \RuntimeException("Couldn't read file in export $pathToFileInExport"); @@ -124,7 +124,11 @@ public function import($userId, $exportPath) { } if ($fileMetadata->getType() === File::TYPE_FOLDER) { - $folder = $userFolder->newFolder($fileCachePath); + if ($fileMetadata->getPath() == '/') { + $folder = $userFolder; + } else { + $folder = $userFolder->newFolder($fileCachePath); + } $folder->getStorage()->getCache()->update($folder->getId(), [ 'etag' => $fileMetadata->getETag(), 'permissions' => $fileMetadata->getPermissions() diff --git a/lib/Model/File.php b/lib/Model/File.php index ee20de7..94cb765 100644 --- a/lib/Model/File.php +++ b/lib/Model/File.php @@ -32,6 +32,7 @@ class File { const TYPE_FOLDER = 'folder'; const TYPE_FILE = 'file'; + const ROOT_FOLDER_PATH = '/files/'; private $type; /** @var string */ diff --git a/lib/Utilities/Iterators/Nodes/RecursiveNodeIteratorFactory.php b/lib/Utilities/Iterators/Nodes/RecursiveNodeIteratorFactory.php index d6b6822..790bde5 100644 --- a/lib/Utilities/Iterators/Nodes/RecursiveNodeIteratorFactory.php +++ b/lib/Utilities/Iterators/Nodes/RecursiveNodeIteratorFactory.php @@ -64,45 +64,4 @@ public function getUserFolderRecursiveIterator($userId, $mode = \RecursiveIterat $nodeIterator->addSkipCondition($conditionDifferentStorage); return [new \RecursiveIteratorIterator($nodeIterator, $mode), $userFolder]; } - - /** - * Returns an array containing a recursive iterator to iterate over the files of the user as the first - * element of the array, and the base Folder node used in the iterator as the second element. Something like: - * [RecursiveIteratorIterator, Folder] - * If the getUserFolderRecursiveIterator method will return an iterator over the files - * of the user (//files/), this iterator will iterate over that parent folder - * (//) so you could get access to trashbin and versions and maybe other directories - * related the to user. - * It will use a RecursiveIteratorIterator class wrapping a RecursiveNodeIterator class. - * This RecursiveNodeIterator will return \OCP\Files\Node elements - * - * Note that a SkipNodeConditionDifferentStorage is already set in the iterator in order to traverse - * only the primary storage, and also a SkipNodeConditionIgnorePath to skip some folders containing - * temporary information - * - * Consider to use something like: - * ``` - * list($iterator, $baseFolder) = $factory->getUserFolderParentRecursiveIterator($userId); - * ``` - * - * You can traverse the iterator like: - * ``` - * foreach ($iterator as $key => $node) { .... } - * ``` - * Note that the $key will always be the path of the node, the same as $node->getPath() - * @param string $userId the id of the user - * @param int $mode one of the \RecursiveIteratorIterator constants - * @return array a RecursiveIteratorIterator wrapping a RecursiveNodeIterator and the base Folder node - * @throws \OC\User\NoUserException (unhandled exception) - */ - public function getUserFolderParentRecursiveIterator($userId, $mode = \RecursiveIteratorIterator::SELF_FIRST) { - $userFolder = $this->rootFolder->getUserFolder($userId); - $parentFolder = $userFolder->getParent(); - $nodeIterator = new RecursiveNodeIterator($parentFolder); - $conditionDifferentStorage = new SkipNodeConditionDifferentStorage($parentFolder->getStorage()->getId()); - $conditionIgnorePaths = new SkipNodeConditionIgnorePath($parentFolder, ['/cache', '/thumbnails', '/uploads']); - $nodeIterator->addSkipCondition($conditionDifferentStorage); - $nodeIterator->addSkipCondition($conditionIgnorePaths); - return [new \RecursiveIteratorIterator($nodeIterator, $mode), $parentFolder]; - } } diff --git a/lib/Utilities/Path.php b/lib/Utilities/Path.php new file mode 100644 index 0000000..df9e9ca --- /dev/null +++ b/lib/Utilities/Path.php @@ -0,0 +1,60 @@ + + * + * @copyright Copyright (c) 2019, ownCloud GmbH + * @license GPL-2.0 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General, + * Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + */ +namespace OCA\DataExporter\Utilities; + +class Path { + const REGEX = '#/+#'; + + /** + * Joins paths, removes duplicate and adds missing slashes. Preservers + * double slashes in the scheme part of the path e.g vfs://foo/bar + * + * @return string + */ + public static function join() { + $paths = []; + + foreach (\func_get_args() as $arg) { + if ($arg !== '') { + $paths[] = $arg; + } + } + + if (\count($paths) === 0) { + return ''; + } + + $firstPart = $paths[0]; + $path = \preg_replace(self::REGEX, '/', \join('/', $paths)); + $scheme = \parse_url($firstPart, PHP_URL_SCHEME); + $hasScheme = \substr($firstPart, 0, \strlen("$scheme://")) === "$scheme://"; + $slashWasRemoved = \substr($path, 0, \strlen("$scheme:/")) == "$scheme:/"; + + if ($hasScheme && $slashWasRemoved) { + $path = "$scheme://" . \substr($path, \strlen("$scheme:/")); + } + + return $path; + } +} diff --git a/tests/unit/Extractor/FilesExtractorTest.php b/tests/unit/Extractor/FilesExtractorTest.php index 01d2df5..2b46a3e 100644 --- a/tests/unit/Extractor/FilesExtractorTest.php +++ b/tests/unit/Extractor/FilesExtractorTest.php @@ -66,7 +66,7 @@ public function testExportFile() { })); // iterator can return an array because will just need to traverse it - $this->iteratorFactory->method('getUserFolderParentRecursiveIterator')->willReturn([[$mockFile], $userFolderParent]); + $this->iteratorFactory->method('getUserFolderRecursiveIterator')->willReturn([[$mockFile], $userFolderParent]); $this->filesExporter->export('usertest', '/tmp/randomF'); $content = \file_get_contents('/tmp/randomF/files/foo/bar.txt'); @@ -89,7 +89,7 @@ public function testExportFolder() { })); // iterator can return an array because will just need to traverse it - $this->iteratorFactory->method('getUserFolderParentRecursiveIterator')->willReturn([[$mockFolder], $userFolderParent]); + $this->iteratorFactory->method('getUserFolderRecursiveIterator')->willReturn([[$mockFolder], $userFolderParent]); $this->filesystem->expects($this->once()) ->method('mkdir') @@ -134,7 +134,7 @@ public function testExportFileAndFolder() { })); // iterator can return an array because will just need to traverse it - $this->iteratorFactory->method('getUserFolderParentRecursiveIterator') + $this->iteratorFactory->method('getUserFolderRecursiveIterator') ->willReturn([[$mockFolder1, $mockFolder2, $mockFile1, $mockFile2], $userFolderParent]); $this->filesystem->expects($this->exactly(2)) diff --git a/tests/unit/Extractor/MetadataExtractor/FilesMetadataExtractorTest.php b/tests/unit/Extractor/MetadataExtractor/FilesMetadataExtractorTest.php index 981691c..b0c16df 100644 --- a/tests/unit/Extractor/MetadataExtractor/FilesMetadataExtractorTest.php +++ b/tests/unit/Extractor/MetadataExtractor/FilesMetadataExtractorTest.php @@ -91,6 +91,10 @@ public function testExtract() { $mockFile2->method('getType')->willReturn(Node::TYPE_FILE); $userFolderParent = $this->createMock(Folder::class); + $userFolderParent->method('getEtag')->willReturn('123qweasdzxc'); + $userFolderParent->method('getMTime')->willReturn(1565074220); + $userFolderParent->method('getPermissions')->willReturn(31); + $userFolderParent->method('getType')->willReturn(Node::TYPE_FOLDER); $userFolderParent->method('getRelativePath') ->will($this->returnCallback(function ($path) { if (\strpos($path, '/usertest/') === 0) { @@ -101,41 +105,49 @@ public function testExtract() { })); // iterator can return an array because will just need to traverse it - $this->iteratorFactory->method('getUserFolderParentRecursiveIterator') + $this->iteratorFactory->method('getUserFolderRecursiveIterator') ->willReturn([[$mockFolder1, $mockFolder2, $mockFile1, $mockFile2], $userFolderParent]); + $expectedFolder0 = new File(); + $expectedFolder0->setPath('/') + ->setEtag('123qweasdzxc') + ->setMtime(1565074220) + ->setPermissions(31) + ->setType(File::TYPE_FOLDER); + $expectedFolder1 = new File(); - $expectedFolder1->setPath('/files/foo') + $expectedFolder1->setPath('/foo') ->setEtag('123qweasdzxc') ->setMtime(1565074220) ->setPermissions(31) ->setType(File::TYPE_FOLDER); $expectedFolder2 = new File(); - $expectedFolder2->setPath('/files/foo/courses') + $expectedFolder2->setPath('/foo/courses') ->setEtag('zaqxswcde') ->setMtime(1565074223) ->setPermissions(31) ->setType(File::TYPE_FOLDER); $expectedFile1 = new File(); - $expectedFile1->setPath('/files/foo/courses/awesome qwerty') + $expectedFile1->setPath('/foo/courses/awesome qwerty') ->setEtag('poiulkjhmnbv') ->setMtime(1565074221) ->setPermissions(1) ->setType(File::TYPE_FILE); $expectedFile2 = new File(); - $expectedFile2->setPath('/files/foo/bar.txt') + $expectedFile2->setPath('/foo/bar.txt') ->setEtag('123456789') ->setMtime(1565074120) ->setPermissions(9) ->setType(File::TYPE_FILE); $this->streamHelper - ->expects($this->exactly(4)) + ->expects($this->exactly(5)) ->method('writelnToStream') ->withConsecutive( + [$resource, $expectedFolder0], [$resource, $expectedFolder1], [$resource, $expectedFolder2], [$resource, $expectedFile1], diff --git a/tests/unit/Importer/FilesImporterTest.php b/tests/unit/Importer/FilesImporterTest.php index 8d772c0..15d30e9 100644 --- a/tests/unit/Importer/FilesImporterTest.php +++ b/tests/unit/Importer/FilesImporterTest.php @@ -85,45 +85,43 @@ public function setUp() { */ public function testFilesImporter() { // User Folder - $mockFolder1 = $this->createMock(Folder::class); - $mockFolder1->method('getPath')->willReturn('/testuser/files'); - $mockFolder1->method('getEtag')->willReturn('123qweasdzxc'); - $mockFolder1->method('getPermissions')->willReturn(31); - $mockFolder1->method('getType')->willReturn(Node::TYPE_FOLDER); + $userFolder = $this->createMock(Folder::class); + $userFolder->method('getPath')->willReturn('files/'); + $userFolder->method('getEtag')->willReturn('123qweasdzxc'); + $userFolder->method('getPermissions')->willReturn(31); + $userFolder->method('getType')->willReturn(Node::TYPE_FOLDER); - // User Folder Parent - $mockFolder2 = $this->createMock(Folder::class); - $mockFolder2->method('getPath')->willReturn('/testuser'); - $mockFolder2->method('getEtag')->willReturn('123qweasdzxc'); - $mockFolder2->method('getPermissions')->willReturn(31); - $mockFolder2->method('getType')->willReturn(Node::TYPE_FOLDER); + $cache = $this->createMock(Cache::class); + $storageUserFolder = $this->createMock(Storage::class); + $storageUserFolder->method('getCache')->willReturn($cache); + $userFolder->method('getStorage')->willReturn($storageUserFolder); // Test Folder - $mockFolder3 = $this->createMock(Folder::class); - $mockFolder3->method('getPath')->willReturn('files/AFolder'); - $mockFolder3->method('getId')->willReturn(1); - $mockFolder3->method('getEtag')->willReturn('5bc8867cc2375'); - $mockFolder3->method('getPermissions')->willReturn(31); - $mockFolder3->method('getType')->willReturn(Node::TYPE_FOLDER); - $storage3 = $this->createMock(Storage::class); + $mockFolder = $this->createMock(Folder::class); + $mockFolder->method('getPath')->willReturn('files/AFolder'); + $mockFolder->method('getId')->willReturn(1); + $mockFolder->method('getEtag')->willReturn('5bc8867cc2375'); + $mockFolder->method('getPermissions')->willReturn(31); + $mockFolder->method('getType')->willReturn(Node::TYPE_FOLDER); + $storageMockFolder = $this->createMock(Storage::class); $cache = $this->createMock(Cache::class); // Test that the File Cache is updated for this folder $cache->expects($this->once()) ->method('update') ->with(1, ['etag' => '5bc8867cc2375', 'permissions' => 31]); - $storage3 + $storageMockFolder ->expects($this->once()) ->method('getCache')->willReturn($cache); - $mockFolder3->method('getStorage')->willReturn($storage3); + $mockFolder->method('getStorage')->willReturn($storageMockFolder); // Test File - $mockFile1 = $this->createMock(FileNode::class); - $mockFile1->method('getPath')->willReturn('files/AFolder/afile.txt'); - $mockFile1->method('getId')->willReturn(2); - $mockFile1->method('getEtag')->willReturn('533c8d4b4c45b62e68cc09e810db7a23'); - $mockFile1->method('getPermissions')->willReturn(27); - $mockFile1->method('getType')->willReturn(Node::TYPE_FILE); - $mockFile1->method('fopen')->willReturn(\fopen('php://memory', 'wb+')); + $mockFile = $this->createMock(FileNode::class); + $mockFile->method('getPath')->willReturn('files/AFolder/afile.txt'); + $mockFile->method('getId')->willReturn(2); + $mockFile->method('getEtag')->willReturn('533c8d4b4c45b62e68cc09e810db7a23'); + $mockFile->method('getPermissions')->willReturn(27); + $mockFile->method('getType')->willReturn(Node::TYPE_FILE); + $mockFile->method('fopen')->willReturn(\fopen('php://memory', 'wb+')); $storageFile = $this->createMock(Storage::class); $cacheFile = $this->createMock(Cache::class); // Test that the File Cache is updated for this file @@ -132,28 +130,31 @@ public function testFilesImporter() { ->with(2, ['etag' => '533c8d4b4c45b62e68cc09e810db7a23', 'permissions' => 27]); $storageFile->expects($this->once()) ->method('getCache')->willReturn($cacheFile); - $mockFile1->method('getStorage')->willReturn($storageFile); - $mockFile1->method('putContent')->willReturn(true); + $mockFile->method('getStorage')->willReturn($storageFile); + $mockFile->method('putContent')->willReturn(true); vfsStreamWrapper::register(); $root = vfsStreamWrapper::setRoot(new vfsStreamDirectory('tmp')); - $vfile = new vfsStreamFile('testuser/files/files/AFolder/afile.txt'); + $vfile = new vfsStreamFile('testuser/files/AFolder/afile.txt'); $vfile->setContent('test'); $root->addChild($vfile); - $mockFolder2->method('newFolder') - ->willReturn($mockFolder3); - $mockFolder2->method('newFile') - ->willReturn($mockFile1); + $userFolder->method('newFolder') + ->willReturn($mockFolder); + $userFolder->method('newFile') + ->willReturn($mockFile); + $mockFolder->method('newFile') + ->willReturn($mockFile); - $mockFolder1->method('getParent')->willReturn($mockFolder2); - $this->rootFolder->method('getUserFolder')->willReturn($mockFolder1); + $mockFolder->method('getParent')->willReturn($userFolder); + $this->rootFolder->method('getUserFolder')->willReturn($userFolder); $this->filesystem ->method('exists') ->willReturnMap( [ - ['vfs://tmp/testuser/files/files/AFolder/afile.txt', true], - ['vfs://tmp/testuser/files/files/AFolder', true] + ['vfs://tmp/testuser/files/', true], + ['vfs://tmp/testuser/files/AFolder/afile.txt', true], + ['vfs://tmp/testuser/files/AFolder', true] ] ); $filesImporter = new FilesImporter( @@ -161,22 +162,31 @@ public function testFilesImporter() { $this->rootFolder, $this->streamHelper ); + + $rootMetadata = new File(); + $rootMetadata + ->setPath('/') + ->setType(File::TYPE_FOLDER) + ->setETag('533c8d4b4c45b62e68cc09e810db7a23') + ->setPermissions(27); + $fileMetadata = new File(); $fileMetadata - ->setPath('files/AFolder/afile.txt') + ->setPath('/AFolder/afile.txt') ->setType(File::TYPE_FILE) ->setETag('533c8d4b4c45b62e68cc09e810db7a23') ->setPermissions(27); + $folderMetadata = new File(); $folderMetadata - ->setPath('files/AFolder') + ->setPath('/AFolder') ->setType(File::TYPE_FOLDER) ->setETag('5bc8867cc2375') ->setPermissions(31); - $this->streamHelper->expects($this->exactly(3)) + $this->streamHelper->expects($this->exactly(4)) ->method('readlnFromStream') - ->willReturnOnConsecutiveCalls($folderMetadata, $fileMetadata, false); + ->willReturnOnConsecutiveCalls($rootMetadata, $folderMetadata, $fileMetadata, false); $filesImporter->import( 'testuser', @@ -236,7 +246,7 @@ public function testFilesImporterError() { $mockFile1->method('putContent')->willReturn(true); vfsStreamWrapper::register(); $root = vfsStreamWrapper::setRoot(new vfsStreamDirectory('tmp')); - $vfile = new vfsStreamFile('testuser/files/files/AFolder/afile.txt'); + $vfile = new vfsStreamFile('testuser/files/AFolder/afile.txt'); $vfile->setContent('test'); $root->addChild($vfile); @@ -285,92 +295,77 @@ public function testFilesImporterError() { */ public function testFilesImporterError2() { // User Folder - $mockFolder1 = $this->createMock(Folder::class); - $mockFolder1->method('getPath')->willReturn('/testuser/files'); - $mockFolder1->method('getEtag')->willReturn('123qweasdzxc'); - $mockFolder1->method('getPermissions')->willReturn(31); - $mockFolder1->method('getType')->willReturn(Node::TYPE_FOLDER); - - // User Folder Parent - $mockFolder2 = $this->createMock(Folder::class); - $mockFolder2->method('getPath')->willReturn('/testuser'); - $mockFolder2->method('getEtag')->willReturn('123qweasdzxc'); - $mockFolder2->method('getPermissions')->willReturn(31); - $mockFolder2->method('getType')->willReturn(Node::TYPE_FOLDER); + $userFolder = $this->createMock(Folder::class); + $userFolder->method('getPath')->willReturn('files/'); + $userFolder->method('getEtag')->willReturn('123qweasdzxc'); + $userFolder->method('getPermissions')->willReturn(31); + $userFolder->method('getType')->willReturn(Node::TYPE_FOLDER); - // Test Folder - $mockFolder3 = $this->createMock(Folder::class); - $mockFolder3->method('getPath')->willReturn('files/AFolder'); - $mockFolder3->method('getId')->willReturn(1); - $mockFolder3->method('getEtag')->willReturn('5bc8867cc2375'); - $mockFolder3->method('getPermissions')->willReturn(31); - $mockFolder3->method('getType')->willReturn(Node::TYPE_FOLDER); - $storage3 = $this->createMock(Storage::class); $cache = $this->createMock(Cache::class); - // Test that the File Cache is updated for this folder - $cache->method('update') - ->with(1, ['etag' => '5bc8867cc2375', 'permissions' => 31]); - $storage3->method('getCache')->willReturn($cache); - $mockFolder3->method('getStorage')->willReturn($storage3); + $storageUserFolder = $this->createMock(Storage::class); + $storageUserFolder->method('getCache')->willReturn($cache); + $userFolder->method('getStorage')->willReturn($storageUserFolder); // Test File - $mockFile1 = $this->createMock(FileNode::class); - $mockFile1->method('fopen')->willReturn(null); - $mockFile1->method('getPath')->willReturn('files/AFolder/afile.txt'); - $mockFile1->method('getId')->willReturn(2); - $mockFile1->method('getEtag')->willReturn('533c8d4b4c45b62e68cc09e810db7a23'); - $mockFile1->method('getPermissions')->willReturn(27); - $mockFile1->method('getType')->willReturn(Node::TYPE_FILE); + $mockFile = $this->createMock(FileNode::class); + $mockFile->method('getPath')->willReturn('files/afile.txt'); + $mockFile->method('getId')->willReturn(2); + $mockFile->method('getEtag')->willReturn('533c8d4b4c45b62e68cc09e810db7a23'); + $mockFile->method('getPermissions')->willReturn(27); + $mockFile->method('getType')->willReturn(Node::TYPE_FILE); + $mockFile->method('fopen')->willReturn(null); $storageFile = $this->createMock(Storage::class); $cacheFile = $this->createMock(Cache::class); - // Test that the File Cache is updated for this file - $cacheFile->method('update') - ->with(2, ['etag' => '533c8d4b4c45b62e68cc09e810db7a23', 'permissions' => 27]); + $storageFile->method('getCache')->willReturn($cacheFile); - $mockFile1->method('getStorage')->willReturn($storageFile); - $mockFile1->method('putContent')->willReturn(true); + $mockFile->method('getStorage')->willReturn($storageFile); + $mockFile->method('putContent')->willReturn(true); + vfsStreamWrapper::register(); $root = vfsStreamWrapper::setRoot(new vfsStreamDirectory('tmp')); - $vfile = new vfsStreamFile('testuser/files/files/AFolder/afile.txt'); + $vfile = new vfsStreamFile('testuser/files/afile.txt'); $vfile->setContent('test'); $root->addChild($vfile); - $mockFolder2->method('newFolder') - ->willReturn($mockFolder3); - $mockFolder2->method('newFile') - ->willReturn($mockFile1); + $userFolder->method('newFile') + ->willReturn($mockFile); - $mockFolder1->method('getParent')->willReturn($mockFolder2); - $this->rootFolder->method('getUserFolder')->willReturn($mockFolder1); + $this->rootFolder->method('getUserFolder') + ->willReturn($userFolder); $this->filesystem ->method('exists') - ->willReturn(true); + ->willReturnMap([ + ['vfs://tmp/testuser/files', true], + ['vfs://tmp/testuser/files/afile.txt', true], + ]); + $filesImporter = new FilesImporter( $this->filesystem, $this->rootFolder, $this->streamHelper ); - $fileMetadata = new File(); - $fileMetadata - ->setPath('files/AFolder/afile.txt') + + $rootMetadata = (new File()) + ->setPath('/') + ->setType(File::TYPE_FOLDER) + ->setETag('533c8d4b4c45b62e68cc09e810db7a23') + ->setPermissions(27); + + $fileMetadata = (new File()) + ->setPath('/afile.txt') ->setType(File::TYPE_FILE) ->setETag('533c8d4b4c45b62e68cc09e810db7a23') ->setPermissions(27); - $folderMetadata = new File(); - $folderMetadata - ->setPath('files/AFolder') - ->setType(File::TYPE_FOLDER) - ->setETag('5bc8867cc2375') - ->setPermissions(31); $this->streamHelper->method('readlnFromStream') - ->willReturnOnConsecutiveCalls($folderMetadata, $fileMetadata, false); + ->willReturnOnConsecutiveCalls( + $rootMetadata, + $fileMetadata, + false + ); - $filesImporter->import( - 'testuser', - 'vfs://tmp/testuser' - ); + $filesImporter->import('testuser', 'vfs://tmp/testuser'); vfsStreamWrapper::unregister(); } } diff --git a/tests/unit/Utilities/Iterators/Nodes/RecursiveNodeIteratorFactoryTest.php b/tests/unit/Utilities/Iterators/Nodes/RecursiveNodeIteratorFactoryTest.php index 0d19145..1f8f669 100644 --- a/tests/unit/Utilities/Iterators/Nodes/RecursiveNodeIteratorFactoryTest.php +++ b/tests/unit/Utilities/Iterators/Nodes/RecursiveNodeIteratorFactoryTest.php @@ -170,28 +170,4 @@ public function testGetUserFolderRecursiveIterator() { $this->assertEquals($expectedList, $currentList); $this->assertSame($this->userFolder, $userFolder); } - - public function testGetUserFolderParentRecursiveIterator() { - list($iterator, $parentUserFolder) = $this->factory->getUserFolderParentRecursiveIterator('usertest'); - - $fooFolder = "/usertest/files/foo"; - $expectedList = [ - "/usertest/files_versions" => "/usertest/files_versions", - "/usertest/files_versions/bar1.txt.v001" => "/usertest/files_versions/bar1.txt.v001", - "/usertest/files" => "/usertest/files", - "$fooFolder/bar1.txt" => "$fooFolder/bar1.txt", - "$fooFolder/bar2.txt" => "$fooFolder/bar2.txt", - "$fooFolder/bar" => "$fooFolder/bar", - "$fooFolder/bar/zzz1.png" => "$fooFolder/bar/zzz1.png", - "$fooFolder/bar/zzz2.png" => "$fooFolder/bar/zzz2.png", - "$fooFolder/pom" => "$fooFolder/pom", - "$fooFolder/pom/hue.txt" => "$fooFolder/pom/hue.txt", - ]; - $currentList = []; - foreach ($iterator as $key => $item) { - $currentList[$key] = $item->getPath(); - } - $this->assertEquals($expectedList, $currentList); - $this->assertSame($this->userFolder->getParent(), $parentUserFolder); - } } diff --git a/tests/unit/Utilities/PathTest.php b/tests/unit/Utilities/PathTest.php new file mode 100644 index 0000000..6d24b83 --- /dev/null +++ b/tests/unit/Utilities/PathTest.php @@ -0,0 +1,57 @@ + + * + * @copyright Copyright (c) 2019, ownCloud GmbH + * @license GPL-2.0 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General, + * Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + */ +namespace OCA\DataExporter\Tests\Unit\Utilities; + +use OCA\DataExporter\Utilities\Path; +use Test\TestCase; + +class PathTest extends TestCase { + public function joinProvider() { + return [ + [['/a', 'b', 'c/'], '/a/b/c/'], + [['/a', 'b', 'c'], '/a/b/c'], + [['a', 'b', 'c'], 'a/b/c'], + [['', 'a', '', 'b', '', '', 'c'], 'a/b/c'], + + [['//a/', '/b/', '/c/'], '/a/b/c/'], + [['/a', 'b', 'c//d'], '/a/b/c/d'], + [['a', 'b', 'c'], 'a/b/c'], + + [['vfs://a', '/b', '/c'], 'vfs://a/b/c'], + + [['vfs:a', '/b', '/c'], 'vfs:a/b/c'], + [['vfs:/a', '/b', '/c'], 'vfs:/a/b/c'], + + [['', '', ''], ''] + ]; + } + + /** + * @dataProvider joinProvider + */ + public function testJoin(array $input, $expectedResult) { + $actual = \call_user_func_array(Path::class . '::join', $input); + $this->assertEquals($expectedResult, $actual); + } +}