Merge pull request #89 from owncloud/feature/new-data-format

New data format
owncloud · Aug 28, 2019 · ba90718 · ba90718
2 parents 54a853c + 7eee03a
commit ba90718
Show file tree

Hide file tree

Showing 28 changed files with 1,439 additions and 667 deletions.
diff --git a/.drone.yml b/.drone.yml
@@ -20,7 +20,7 @@ pipeline:
   install-server:
     image: owncloudci/core
     pull: true
-    version: daily-stable10-qa
+    version: daily-master-qa
     db_type: ${DB_TYPE}
     db_name: ${DB_NAME}
     db_host: ${DB_HOST}
@@ -32,7 +32,7 @@ pipeline:
     image: owncloudci/php:${PHP_VERSION}
     pull: true
     commands:
-      - git clone -b stable10 --depth=1 https://github.com/owncloud/core.git /var/www/owncloud/testrunner
+      - git clone -b master --depth=1 https://github.com/owncloud/core.git /var/www/owncloud/testrunner
       - cp -r /var/www/owncloud/apps/data_exporter /var/www/owncloud/testrunner/apps/
       - cd /var/www/owncloud/testrunner/apps/data_exporter && make
       - cd /var/www/owncloud/testrunner
@@ -83,7 +83,7 @@ pipeline:
       - make test-php-style
     when:
       matrix:
-        TEST_SUITE: phpunit
+        TEST_SUITE: owncloud-coding-standard
 
   lint-test:
     image: owncloudci/php:${PHP_VERSION}
@@ -94,7 +94,7 @@ pipeline:
       - make test-php-lint
     when:
       matrix:
-        TEST_SUITE: phpunit
+        TEST_SUITE: lint-test
 
   phpstan:
     # phpstan requires php7.1
@@ -106,7 +106,7 @@ pipeline:
       - make test-php-phpstan
     when:
       matrix:
-        TEST_SUITE: phpunit
+        TEST_SUITE: php-stan
 
   phan:
     # phan requires a recent php-ast extension
@@ -118,7 +118,7 @@ pipeline:
       - make test-php-phan
     when:
       matrix:
-        TEST_SUITE: phpunit
+        TEST_SUITE: phan
 
   phpunit-unit-tests:
     image: owncloudci/php:${PHP_VERSION}
@@ -129,7 +129,6 @@ pipeline:
       matrix:
         TEST_SUITE: phpunit
 
-
   phpunit-integration-tests:
     image: owncloudci/php:${PHP_VERSION}
     pull: true
@@ -219,7 +218,6 @@ services:
       - MYSQL_ROOT_PASSWORD=secret
     when:
       matrix:
-        DB_TYPE: mysql
         DB_HOST: mysql
 
   mysqlmb4:
@@ -231,7 +229,6 @@ services:
       - MYSQL_ROOT_PASSWORD=secret
     when:
       matrix:
-        DB_TYPE: mysql
         DB_HOST: mysqlmb4
 
   pgsql:
@@ -256,27 +253,46 @@ services:
 
 matrix:
   include:
-    # unit tests
+     # owncloud-coding-standard
+     - PHP_VERSION: 7.2
+       TEST_SUITE: owncloud-coding-standard
+
+     # owncloud-coding-standard
+     - PHP_VERSION: 7.2
+       TEST_SUITE: lint-test
+
+     # php-stan
+     - PHP_VERSION: 7.1
+       TEST_SUITE: php-stan
+       NEED_SERVER: true
+       NEED_INSTALL_APP: true
+
+     # phan
+     - PHP_VERSION: 7.2
+       TEST_SUITE: phan
+       NEED_SERVER: true
+
+     # unit tests
      - PHP_VERSION: 7.0
-       OC_VERSION: daily-stable10-qa
+       OC_VERSION: daily-master-qa
        DB_TYPE: sqlite
        REBUILT_CACHE: true
        FLUSH_CACHE: true
        TEST_SUITE: phpunit
 
      - PHP_VERSION: 7.1
-       OC_VERSION: daily-stable10-qa
+       OC_VERSION: daily-master-qa
        DB_TYPE: sqlite
        COVERAGE: true
        TEST_SUITE: phpunit
 
      - PHP_VERSION: 7.2
-       OC_VERSION: daily-stable10-qa
+       OC_VERSION: daily-master-qa
        DB_TYPE: sqlite
        TEST_SUITE: phpunit
 
      - PHP_VERSION: 7.1
-       OC_VERSION: daily-stable10-qa
+       OC_VERSION: daily-master-qa
        DB_TYPE: pgsql
        DB_HOST: pgsql
        DB_NAME: oc_db
@@ -285,15 +301,15 @@ matrix:
        TEST_SUITE: phpunit
 
      - PHP_VERSION: 7.1
-       OC_VERSION: daily-stable10-qa
+       OC_VERSION: daily-master-qa
        DB_TYPE: oci
        DB_HOST: oci
        DB_NAME: XE
        DB_USERNAME: autotest
        TEST_SUITE: phpunit
 
      - PHP_VERSION: 7.1
-       OC_VERSION: daily-stable10-qa
+       OC_VERSION: daily-master-qa
        DB_TYPE: mysql
        DB_HOST: mysqlmb4
        DB_NAME: oc_db
@@ -302,7 +318,7 @@ matrix:
        TEST_SUITE: phpunit
 
      - PHP_VERSION: 7.1
-       OC_VERSION: daily-stable10-qa
+       OC_VERSION: daily-master-qa
        DB_TYPE: mysql
        DB_HOST: mysql
        DB_NAME: oc_db
@@ -313,54 +329,7 @@ matrix:
      # acceptance tests
 
      - PHP_VERSION: 7.0
-       OC_VERSION: daily-stable10-qa
-       TEST_SUITE: cli-acceptance
-       BEHAT_SUITE: cliDataExporter
-       DB_TYPE: mysql
-       DB_HOST: mysql
-       DB_NAME: oc_db
-       DB_USERNAME: admin
-       DB_PASSWORD: secret
-       NEED_SERVER: true
-       NEED_INSTALL_APP: true
-
-     - PHP_VERSION: 7.1
-       OC_VERSION: daily-stable10-qa
-       TEST_SUITE: cli-acceptance
-       BEHAT_SUITE: cliDataExporter
-       DB_TYPE: mysql
-       DB_HOST: mysql
-       DB_NAME: oc_db
-       DB_USERNAME: admin
-       DB_PASSWORD: secret
-       NEED_SERVER: true
-       NEED_INSTALL_APP: true
-
-     - PHP_VERSION: 7.1
-       OC_VERSION: daily-stable10-qa
-       TEST_SUITE: cli-acceptance
-       BEHAT_SUITE: cliDataExporter
-       DB_HOST: oci
-       DB_TYPE: oci
-       DB_NAME: XE
-       DB_USERNAME: autotest
-       NEED_SERVER: true
-       NEED_INSTALL_APP: true
-
-     - PHP_VERSION: 7.1
-       OC_VERSION: daily-stable10-qa
-       TEST_SUITE: cli-acceptance
-       BEHAT_SUITE: cliDataExporter
-       DB_TYPE: pgsql
-       DB_HOST: pgsql
-       DB_NAME: oc_db
-       DB_USERNAME: admin
-       DB_PASSWORD: secret
-       NEED_SERVER: true
-       NEED_INSTALL_APP: true
-
-     - PHP_VERSION: 7.2
-       OC_VERSION: daily-stable10-qa
+       OC_VERSION: daily-master-qa
        TEST_SUITE: cli-acceptance
        BEHAT_SUITE: cliDataExporter
        DB_TYPE: mysql

diff --git a/lib/Exporter.php b/lib/Exporter.php
@@ -44,11 +44,21 @@ public function __construct(Serializer $serializer, MetadataExtractor $metadataE
 		$this->filesystem = $filesystem;
 	}
 
+	/**
+	 * @param string $uid
+	 * @param string $exportDirectoryPath
+	 *
+	 * @throws \OCP\Files\NotFoundException
+	 * @throws \OCP\Files\NotPermittedException
+	 * @throws \Exception
+	 *
+	 * @return void
+	 */
 	public function export($uid, $exportDirectoryPath) {
 		$exportPath = "$exportDirectoryPath/$uid";
-		$metaData = $this->metadataExtractor->extract($uid);
+		$metaData = $this->metadataExtractor->extract($uid, $exportPath);
 		$this->filesystem->dumpFile(
-			"$exportPath/metadata.json",
+			"$exportPath/user.json",
 			$this->serializer->serialize($metaData)
 		);
 

diff --git a/lib/Extractor/MetadataExtractor.php b/lib/Extractor/MetadataExtractor.php
@@ -80,17 +80,17 @@ public function __construct(
 	 * @throws \Exception
 	 * @throws \RuntimeException if user can not be read
 	 */
-	public function extract($uid) {
+	public function extract($uid, $exportPath) {
 		$user = $this->userExtractor->extract($uid);
-		$user->setPreferences($this->preferencesExtractor->extract($uid))
-			->setShares($this->sharesExtractor->extract($uid));
-
+		$user->setPreferences($this->preferencesExtractor->extract($uid));
 		$metadata = new Metadata();
 		$metadata->setDate(new \DateTimeImmutable())
 			->setUser($user)
-			->setFiles($this->filesMetadataExtractor->extract($uid))
 			->setOriginServer($this->urlGenerator->getAbsoluteURL('/'));
 
+		$this->filesMetadataExtractor->extract($uid, $exportPath);
+		$this->sharesExtractor->extract($uid, $exportPath);
+
 		return $metadata;
 	}
 }
diff --git a/lib/Extractor/MetadataExtractor/FilesMetadataExtractor.php b/lib/Extractor/MetadataExtractor/FilesMetadataExtractor.php
@@ -23,27 +23,43 @@
  */
 namespace OCA\DataExporter\Extractor\MetadataExtractor;
 
+use OC\User\NoUserException;
 use OCA\DataExporter\Utilities\Iterators\Nodes\RecursiveNodeIteratorFactory;
 use OCA\DataExporter\Model\File;
+use OCA\DataExporter\Utilities\StreamHelper;
 use OCP\Files\Node;
 
 class FilesMetadataExtractor {
+	const FILE_NAME = 'files.jsonl';
 	/** @var RecursiveNodeIteratorFactory  */
 	private $iteratorFactory;
+	/**
+	 * @var StreamHelper
+	 */
+	private $streamHelper;
+	/**
+	 * @var resource
+	 */
+	private $streamFile;
 
-	public function __construct(RecursiveNodeIteratorFactory $iteratorFactory) {
+	public function __construct(RecursiveNodeIteratorFactory $iteratorFactory, StreamHelper $streamHelper) {
 		$this->iteratorFactory = $iteratorFactory;
+		$this->streamHelper = $streamHelper;
 	}
 
 	/**
 	 * @param string $userId
-	 * @return File[]
-	 * @throws \OCP\Files\InvalidPathException
-	 * @throws \OCP\Files\NotFoundException
+	 * @param string $exportPath
+	 *
+	 * @return void
+	 *
+	 * @throws NoUserException
 	 */
-	public function extract($userId) {
+	public function extract($userId, $exportPath) {
 		list($iterator, $baseFolder) = $this->iteratorFactory->getUserFolderParentRecursiveIterator($userId);
-		$files = [];
+
+		$filename = $exportPath . '/' . $this::FILE_NAME;
+		$this->streamFile = $this->streamHelper->initStream($filename, 'ab', true);
 
 		foreach ($iterator as $node) {
 			$nodePath = $node->getPath();
@@ -53,6 +69,7 @@ public function extract($userId) {
 
 			$file->setPath($relativePath);
 			$file->setETag($node->getEtag());
+			$file->setMtime($node->getMTime());
 			$file->setPermissions($node->getPermissions());
 
 			if ($node->getType() === Node::TYPE_FILE) {
@@ -61,9 +78,8 @@ public function extract($userId) {
 				$file->setType(File::TYPE_FOLDER);
 			}
 
-			$files[] = $file;
+			$this->streamHelper->writelnToStream($this->streamFile, $file);
 		}
-
-		return $files;
+		$this->streamHelper->closeStream($this->streamFile);
 	}
 }