Merge pull request #1044 from nextcloud/fix/webdav-and-tests

test(dav): Add tests for dav interface to full-run-test.yml

.github/workflows/cluster-faces-test.yml

@@ -17,7 +17,7 @@ env:
 
 jobs:
   php:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     strategy:
       # do not stop on another job's failure
@@ -132,21 +132,36 @@ jobs:
         run: |
           ./occ app:enable -vvv ${{ env.APP_NAME }}
 
+      - name: Remove unnecessary models to make space
+        run: |
+          rm -rf apps/recognize/models
+
+      - uses: actions/cache/restore@v3
+        id: photos-cache
+        with:
+          path: data/admin/files/
+          key: https://cloud.marcelklehr.de/s/PkNYbmKnwMiQMFD/download/IMDb-Face.zip
+
       - name: Upload photos
+        if: steps.photos-cache.outputs.cache-hit != 'true'
         run: |
           mkdir -p data/admin/files/
           cd data/admin/files
           wget https://cloud.marcelklehr.de/s/PkNYbmKnwMiQMFD/download/IMDb-Face.zip
           unzip IMDb-Face.zip
           rm IMDb-Face.zip
 
-      - name: Set pure-js mode
-        run: |
-          ./occ config:app:set --value ${{ matrix.pure-js-mode }} recognize tensorflow.purejs
+      - uses: actions/cache/save@v3
+        with:
+          path: data/admin/files/
+          key: https://cloud.marcelklehr.de/s/PkNYbmKnwMiQMFD/download/IMDb-Face.zip
 
-      - name: Set faces.enabled
+      - name: Set config
         run: |
+          ./occ config:app:set --value ${{ matrix.pure-js-mode }} recognize tensorflow.purejs
           ./occ config:app:set --value true recognize faces.enabled
+          # only use one core. GH actions has 2
+          ./occ config:app:set --value 1 recognize tensorflow.cores
 
       - uses: actions/cache/restore@v3
         id: db-cache
@@ -183,7 +198,7 @@ jobs:
         id: clustering-cache
         with:
           path: out.json
-          key: ${{ runner.os }}-${{ hashFiles('out.txt', 'apps/recognize/src/classifier_faces.js', 'apps/recognize/lib/Classifiers/Classifier.php', 'apps/recognize/lib/Classifiers/Images/ClusteringFaceClassifier.php', 'apps/recognize/lib/Clustering/**', 'apps/recognize/lib/Service/FaceClusterAnalyzer.php', 'apps/recognize/lib/Command/ClusterFaces.php') }}-${{ matrix.pure-js-mode }}
+          key: ${{ runner.os }}-${{ hashFiles('out.txt', 'apps/recognize/src/classifier_faces.js', 'apps/recognize/lib/Classifiers/Classifier.php', 'apps/recognize/lib/Classifiers/Images/ClusteringFaceClassifier.php', 'apps/recognize/lib/Clustering/**', 'apps/recognize/lib/Dav/**', 'apps/recognize/lib/Service/FaceClusterAnalyzer.php', 'apps/recognize/lib/Command/ClusterFaces.php') }}-${{ matrix.pure-js-mode }}
 
       - name: Run clustering
         if: steps.clustering-cache.outputs.cache-hit != 'true'

.github/workflows/full-run-test.yml

@@ -181,3 +181,28 @@ jobs:
         run: |
           ./occ recognize:classify
           ./occ recognize:cluster-faces
+
+      - name: Install cadaver
+        if: ${{ matrix.faces-enabled }}
+        run: |
+          sudo apt -y install cadaver
+
+      - name: Test webdav access
+        if: ${{ matrix.faces-enabled }}
+        run: |
+          cat > ~/.netrc <<EOF
+          machine localhost
+          login admin
+          password password
+          EOF
+          cadaver http://localhost:8080/remote.php/dav/recognize/admin/ <<EOF
+          mv unassigned-faces/* faces/0/
+          mkdir faces/toot
+          mv faces/0/* faces/toot/
+          mv faces/toot/ faces/blah/
+          EOF
+
+      - name: Print logs
+        if: always()
+        run: |
+          cat data/nextcloud.log

lib/BackgroundJobs/ClusterFacesJob.php

@@ -19,7 +19,6 @@ class ClusterFacesJob extends QueuedJob {
 	private FaceClusterAnalyzer $clusterAnalyzer;
 	private IJobList $jobList;
 	private LoggerInterface $logger;
-
 	public const BATCH_SIZE = 10000;
 	private SettingsService $settingsService;
 

lib/Classifiers/Classifier.php

@@ -231,6 +231,7 @@ public function classifyFiles(string $model, array $queueFiles, int $timeout): \
 					$i++;
 				}
 			}
+			$proc->stop();
 			$this->cleanUpTmpFiles();
 			if ($i !== count($paths)) {
 				$this->logger->warning('Classifier process output: '.$errOut);
@@ -340,8 +341,10 @@ public function generatePreviewWithProvider(File $file): string {
 			$previewImage = imagecreatefromstring(file_get_contents($tmpname));
 			$use_gd_quality = (int)$this->config->getSystemValue('recognize.preview.quality', '100');
 			if (imagejpeg($previewImage, $tmpname, $use_gd_quality) === false) {
+				imagedestroy($previewImage);
 				throw new \OCA\Recognize\Exception\Exception('Could not copy preview file to temp folder');
 			}
+			imagedestroy($previewImage);
 		}
 
 		return $tmpname;
@@ -379,8 +382,12 @@ public function generatePreviewWithGD(string $path): string {
 
 		$use_gd_quality = (int)$this->config->getSystemValue('recognize.preview.quality', '100');
 		if (imagejpeg($previewImage, $tmpname, $use_gd_quality) === false) {
+			imagedestroy($image);
+			imagedestroy($previewImage);
 			throw new \OCA\Recognize\Exception\Exception('Could not copy preview file to temp folder');
 		}
+		imagedestroy($image);
+		imagedestroy($previewImage);
 
 		return $tmpname;
 	}

lib/Classifiers/Images/ClusteringFaceClassifier.php

@@ -115,14 +115,14 @@ public function classify(array $queueFiles): void {
 						$this->logger->error('Could not store face detection in database', ['exception' => $e]);
 						continue;
 					}
-					$usersToCluster[] = $userId;
+					$usersToCluster[$userId] = true;
 				}
 				$this->config->setAppValue('recognize', self::MODEL_NAME.'.status', 'true');
 				$this->config->setAppValue('recognize', self::MODEL_NAME.'.lastFile', time());
 			}
 		}
 
-		$usersToCluster = array_unique($usersToCluster);
+		$usersToCluster = array_keys($usersToCluster);
 		foreach ($usersToCluster as $userId) {
 			if (!$this->jobList->has(ClusterFacesJob::class, ['userId' => $userId])) {
 				$this->logger->debug('scheduling ClusterFacesJob for user '.$userId);

lib/Command/Classify.php

@@ -141,6 +141,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 					}
 				}
 			} while ($i > 0);
+			\OC_Util::tearDownFS();
 		}
 		return 0;
 	}

lib/Dav/Faces/PropFindPlugin.php

@@ -50,9 +50,10 @@ public function propFind(PropFind $propFind, INode $node) {
 					)
 				);
 			});
+			$propFind->handle(self::FILE_NAME_PROPERTYNAME, fn () => $node->getFile()->getName());
+			$propFind->handle(self::REALPATH_PROPERTYNAME, fn () => $node->getFile()->getPath());
 			$propFind->handle(FilesPlugin::INTERNAL_FILEID_PROPERTYNAME, fn () => $node->getFile()->getId());
 			$propFind->handle(FilesPlugin::GETETAG_PROPERTYNAME, fn () => $node->getETag());
-			$propFind->handle(self::FILE_NAME_PROPERTYNAME, fn () => $node->getFile()->getName());
 			$propFind->handle(TagsPlugin::FAVORITE_PROPERTYNAME, fn () => $node->isFavorite() ? 1 : 0);
 			$propFind->handle(FilesPlugin::HAS_PREVIEW_PROPERTYNAME, fn () => json_encode($this->previewManager->isAvailable($node->getFile()->getFileInfo())));
 			$propFind->handle(FilesPlugin::PERMISSIONS_PROPERTYNAME, function () use ($node): string {

lib/Dav/Faces/UnassignedFacesHome.php

@@ -30,7 +30,6 @@ public function __construct(IUser $user, FaceDetectionMapper $faceDetectionMappe
 		$this->faceDetectionMapper = $faceDetectionMapper;
 		$this->rootFolder = $rootFolder;
 		$this->tagManager = $tagManager;
-		$this->metadataManager = $metadataManager;
 		$this->previewManager = $previewManager;
 	}
 
@@ -82,7 +81,7 @@ public function getChild($name): FacePhoto {
 			throw new NotFound();
 		}
 
-		return new UnassignedFacePhoto($this->faceDetectionMapper, $detection, $this->rootFolder->getUserFolder($this->user->getUID()), $this->tagManager, $this->metadataManager, $this->previewManager);
+		return new UnassignedFacePhoto($this->faceDetectionMapper, $detection, $this->rootFolder->getUserFolder($this->user->getUID()), $this->tagManager, $this->previewManager);
 	}
 
 	public function childExists($name): bool {

lib/Dav/RecognizeHome.php

@@ -34,7 +34,6 @@ public function __construct(array $principalInfo, FaceClusterMapper $faceCluster
 		$this->faceDetectionMapper = $faceDetectionMapper;
 		$this->rootFolder = $rootFolder;
 		$this->tagManager = $tagManager;
-		$this->metadataManager = $metadataManager;
 		$this->previewManager = $previewManager;
 	}
 
@@ -60,11 +59,11 @@ public function createDirectory($name) {
 	}
 
 	private function getFacesHome() {
-		return new FacesHome($this->faceClusterMapper, $this->user, $this->faceDetectionMapper, $this->rootFolder, $this->tagManager, $this->metadataManager, $this->previewManager);
+		return new FacesHome($this->faceClusterMapper, $this->user, $this->faceDetectionMapper, $this->rootFolder, $this->tagManager, $this->previewManager);
 	}
 
 	private function getUnassignedFacesHome() {
-		return new UnassignedFacesHome($this->user, $this->faceDetectionMapper, $this->rootFolder, $this->tagManager, $this->metadataManager, $this->previewManager);
+		return new UnassignedFacesHome($this->user, $this->faceDetectionMapper, $this->rootFolder, $this->tagManager, $this->previewManager);
 	}
 
 	public function getChild($name) {

lib/Service/StorageService.php

@@ -66,11 +66,13 @@ public function getMounts(): \Generator {
 				// Only crawl files, not cache or trashbin
 				$qb = new CacheQueryBuilder($this->db, $this->systemConfig, $this->logger, $this->metadataManager);
 				try {
-					/** @var array|false $root */
-					$root = $qb->selectFileCache()
+					$res = $qb->selectFileCache()
 						->andWhere($qb->expr()->eq('filecache.storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
 						->andWhere($qb->expr()->eq('filecache.path', $qb->createNamedParameter('files')))
-						->executeQuery()->fetch();
+						->executeQuery();
+					/** @var array|false $root */
+					$root = $res->fetch();
+					$res->closeCursor();
 					if ($root !== false) {
 						$overrideRoot = intval($root['fileid']);
 					}

src/components/ViewAdmin.vue

@@ -221,15 +221,15 @@
 					<span>{{ t('recognize', 'Number of CPU Cores (0 for no limit)') }}</span>
 				</label>
 			</p>
-      <p>&nbsp;</p>
-      <p>{{ t('recognize', 'By default, recognize will only ever run one classifier process at a time. If you have a lot of resources available and want to run as many processes in parallel as possible, you can turn on concurrency here.') }}</p>
-      <p>
-        <NcCheckboxRadioSwitch :checked.sync="settings['concurrency.enabled']"
-                               type="switch"
-                               @update:checked="onChange">
-          {{ t('recognize', 'Enable unlimited concurrency of classifier processes') }}
-        </NcCheckboxRadioSwitch>
-      </p>
+			<p>&nbsp;</p>
+			<p>{{ t('recognize', 'By default, recognize will only ever run one classifier process at a time. If you have a lot of resources available and want to run as many processes in parallel as possible, you can turn on concurrency here.') }}</p>
+			<p>
+				<NcCheckboxRadioSwitch :checked.sync="settings['concurrency.enabled']"
+					type="switch"
+					@update:checked="onChange">
+					{{ t('recognize', 'Enable unlimited concurrency of classifier processes') }}
+				</NcCheckboxRadioSwitch>
+			</p>
 		</NcSettingsSection>
 		<NcSettingsSection :title="t('recognize', 'Tensorflow WASM mode')">
 			<p v-if="avx === undefined || platform === undefined || musl === undefined">
@@ -262,7 +262,10 @@
 		</NcSettingsSection>
 		<NcSettingsSection :title="t('recognize', 'Tensorflow GPU mode')">
 			<p>
-				<NcCheckboxRadioSwitch :checked.sync="settings['tensorflow.gpu']" type="switch" @update:checked="onChange" :disabled="settings['tensorflow.purejs']">
+				<NcCheckboxRadioSwitch :checked.sync="settings['tensorflow.gpu']"
+					type="switch"
+					:disabled="settings['tensorflow.purejs']"
+					@update:checked="onChange">
 					{{ t('recognize', 'Enable GPU mode') }}
 				</NcCheckboxRadioSwitch>
 			</p>