Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[stable8] test(FileListener): More tests with folders #1206

Merged
merged 4 commits into from
Oct 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/cluster-faces-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
matrix:
php-versions: ['8.2']
databases: ['sqlite']
server-versions: ['master']
server-versions: ['stable30']
pure-js-mode: ['false']

name: Test cluster-faces command on ${{ matrix.server-versions }} wasm:${{ matrix.pure-js-mode }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/files-scan-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
matrix:
php-versions: ['8.2']
databases: ['sqlite', 'mysql', 'pgsql']
server-versions: ['master']
server-versions: ['stable30']

name: Test files:scan command on ${{ matrix.databases }}-${{ matrix.server-versions }}

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/full-run-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
matrix:
php-versions: ['8.2']
databases: ['sqlite']
server-versions: ['master']
server-versions: ['stable30']
pure-js-mode: ['false']
imagenet-enabled: ['true']
faces-enabled: ['true']
Expand All @@ -39,7 +39,7 @@ jobs:
movinet-enabled: ['true']
include:
# test pure-js once
- server-versions: master
- server-versions: stable30
databases: sqlite
php-versions: 8.2
pure-js-mode: true
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/phpunit-mysql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ jobs:
strategy:
matrix:
php-versions: ['8.1', '8.2', '8.3']
server-versions: ['master']
server-versions: ['stable30']

services:
mysql:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/phpunit-pgsql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ jobs:
fail-fast: false
matrix:
php-versions: ['8.2']
server-versions: ['master']
server-versions: ['stable30']

services:
postgres:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/phpunit-sqlite.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
fail-fast: false
matrix:
php-versions: ['8.2']
server-versions: ['master']
server-versions: ['stable30']

steps:
- name: Set app env
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/psalm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ jobs:
strategy:
matrix:
php-versions: [ '8.1', '8.2', '8.3' ]
server-versions: [ 'dev-master' ]
server-versions: [ 'dev-stable30' ]
fail-fast: false

name: Nextcloud
Expand Down
2 changes: 1 addition & 1 deletion composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
}
},
"require-dev": {
"nextcloud/ocp": "dev-master",
"nextcloud/ocp": "dev-stable30",
"symfony/console": "^5.4",
"symfony/process": "^5.2"
},
Expand Down
115 changes: 91 additions & 24 deletions lib/Hooks/FileListener.php
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
*/
class FileListener implements IEventListener {
private ?bool $movingFromIgnoredTerritory;
private ?array $movingDirFromIgnoredTerritory;
/** @var list<string> */
private array $sourceUserIds;
private ?Node $source = null;
Expand All @@ -64,6 +65,7 @@ public function __construct(
private IJobList $jobList,
) {
$this->movingFromIgnoredTerritory = null;
$this->movingDirFromIgnoredTerritory = null;
$this->sourceUserIds = [];
}

Expand Down Expand Up @@ -139,16 +141,22 @@ public function handle(Event $event): void {
}
}
if ($event instanceof BeforeNodeRenamedEvent) {
$this->movingFromIgnoredTerritory = null;
$this->movingDirFromIgnoredTerritory = [];
if (in_array($event->getSource()->getName(), [...Constants::IGNORE_MARKERS_ALL, ...Constants::IGNORE_MARKERS_IMAGE, ...Constants::IGNORE_MARKERS_AUDIO, ...Constants::IGNORE_MARKERS_VIDEO], true)) {
$this->resetIgnoreCache($event->getSource());
return;
}
// We try to remember whether the source node is in ignored territory
// because after moving isIgnored doesn't work anymore :(
if ($this->isIgnored($event->getSource())) {
$this->movingFromIgnoredTerritory = true;
if ($event->getSource()->getType() !== FileInfo::TYPE_FOLDER) {
if ($this->isFileIgnored($event->getSource())) {
$this->movingFromIgnoredTerritory = true;
} else {
$this->movingFromIgnoredTerritory = false;
}
} else {
$this->movingFromIgnoredTerritory = false;
$this->movingDirFromIgnoredTerritory = $this->getDirIgnores($event->getSource());
}
$this->sourceUserIds = $this->getUsersWithFileAccess($event->getSource());
$this->source = $event->getSource();
Expand All @@ -175,23 +183,45 @@ public function handle(Event $event): void {
$this->postDelete($event->getTarget()->getParent());
return;
}

if ($this->movingFromIgnoredTerritory) {
if ($this->isIgnored($event->getTarget())) {
if ($event->getTarget()->getType() !== FileInfo::TYPE_FOLDER) {
if ($this->movingFromIgnoredTerritory) {
if ($this->isFileIgnored($event->getTarget())) {
return;
}
$this->postInsert($event->getTarget());
return;
}
if ($this->isFileIgnored($event->getTarget())) {
$this->postDelete($event->getTarget());
return;
}
} else {
if ($this->movingDirFromIgnoredTerritory !== null && count($this->movingDirFromIgnoredTerritory) !== 0) {
$oldIgnores = $this->movingDirFromIgnoredTerritory;
$newIgnores = $this->getDirIgnores($event->getTarget());
$diff1 = array_diff($newIgnores, $oldIgnores);
$diff2 = array_diff($oldIgnores, $newIgnores);
if (count($diff1) !== 0 || count($diff2) !== 0) {
if (count($diff1) !== 0) {
$this->postDelete($event->getTarget(), true, $diff1);
}
if (count($diff2) !== 0) {
$this->postInsert($event->getTarget(), true, $diff2);
}
}
return;
}
$ignoredMimeTypes = $this->getDirIgnores($event->getTarget());
if (!empty($ignoredMimeTypes)) {
$this->postDelete($event->getTarget(), true, $ignoredMimeTypes);
return;
}
$this->postInsert($event->getTarget());
return;
}
if ($this->isIgnored($event->getTarget())) {
$this->postDelete($event->getTarget());
return;
}
$this->postRename($this->source ?? $event->getSource(), $event->getTarget());
return;
}
if ($event instanceof BeforeNodeDeletedEvent) {
$this->postDelete($event->getNode(), false);
$this->postDelete($event->getNode());
return;
}
if ($event instanceof NodeDeletedEvent) {
Expand Down Expand Up @@ -246,22 +276,26 @@ public function handle(Event $event): void {
}
}

public function postDelete(Node $node, bool $recurse = true): void {
public function postDelete(Node $node, bool $recurse = true, ?array $mimeTypes = null): void {
if ($node->getType() === FileInfo::TYPE_FOLDER) {
if (!$recurse) {
return;
}
try {
/** @var Folder $node */
foreach ($node->getDirectoryListing() as $child) {
$this->postDelete($child);
$this->postDelete($child, true, $mimeTypes);
}
} catch (NotFoundException $e) {
$this->logger->debug($e->getMessage(), ['exception' => $e]);
}
return;
}

if ($mimeTypes !== null && !in_array($node->getMimetype(), $mimeTypes)) {
return;
}

// Try Deleting possibly existing face detections
try {
/**
Expand Down Expand Up @@ -291,7 +325,7 @@ public function postDelete(Node $node, bool $recurse = true): void {
/**
* @throws \OCP\Files\InvalidPathException
*/
public function postInsert(Node $node, bool $recurse = true): void {
public function postInsert(Node $node, bool $recurse = true, ?array $mimeTypes = null): void {
if ($node->getType() === FileInfo::TYPE_FOLDER) {
if (!$recurse) {
return;
Expand All @@ -309,14 +343,18 @@ public function postInsert(Node $node, bool $recurse = true): void {
return;
}

if ($mimeTypes !== null && !in_array($node->getMimetype(), $mimeTypes)) {
return;
}

$queueFile = new QueueFile();
if ($node->getMountPoint()->getNumericStorageId() === null) {
return;
}
$queueFile->setStorageId($node->getMountPoint()->getNumericStorageId());
$queueFile->setRootId($node->getMountPoint()->getStorageRootId());

if ($this->isIgnored($node)) {
if ($this->isFileIgnored($node)) {
return;
}

Expand Down Expand Up @@ -401,7 +439,7 @@ private function copyFaceDetectionsForNode(string $ownerId, array $usersToAdd, a
* @throws \OCP\Files\InvalidPathException
* @throws \OCP\Files\NotFoundException
*/
public function isIgnored(Node $node) : bool {
public function isFileIgnored(Node $node) : bool {
$ignoreMarkers = [];
$mimeType = $node->getMimetype();
$storageId = $node->getMountPoint()->getNumericStorageId();
Expand All @@ -419,24 +457,53 @@ public function isIgnored(Node $node) : bool {
if (in_array($mimeType, Constants::AUDIO_FORMATS)) {
$ignoreMarkers = array_merge($ignoreMarkers, Constants::IGNORE_MARKERS_AUDIO);
}

if (count($ignoreMarkers) === 0) {
return true;
}

$ignoreMarkers = array_merge($ignoreMarkers, Constants::IGNORE_MARKERS_ALL);
$ignoredPaths = $this->ignoreService->getIgnoredDirectories($storageId, $ignoreMarkers);

$relevantIgnorePaths = array_filter($ignoredPaths, static function (string $ignoredPath) use ($node) {
return stripos($node->getInternalPath(), $ignoredPath ? $ignoredPath . '/' : $ignoredPath) === 0;
});

if (count($relevantIgnorePaths) > 0) {
return true;
foreach ($ignoredPaths as $ignoredPath) {
if (stripos($node->getInternalPath(), $ignoredPath ? $ignoredPath . '/' : $ignoredPath) === 0) {
return true;
}
}

return false;
}

/**
* @param \OCP\Files\Node $node
* @return array
* @throws Exception
*/
public function getDirIgnores(Node $node) : array {
$storageId = $node->getMountPoint()->getNumericStorageId();
if ($storageId === null) {
return [];
}

$ignoredMimeTypes = [];
foreach ([
[Constants::IGNORE_MARKERS_IMAGE, Constants::IMAGE_FORMATS],
[Constants::IGNORE_MARKERS_VIDEO, Constants::VIDEO_FORMATS],
[Constants::IGNORE_MARKERS_AUDIO, Constants::AUDIO_FORMATS],
[Constants::IGNORE_MARKERS_ALL, array_merge(Constants::IMAGE_FORMATS, Constants::VIDEO_FORMATS, Constants::AUDIO_FORMATS)],
] as $iteration) {
[$ignoreMarkers, $mimeTypes] = $iteration;
$ignoredPaths = $this->ignoreService->getIgnoredDirectories($storageId, $ignoreMarkers);
foreach ($ignoredPaths as $ignoredPath) {
if (stripos($node->getInternalPath(), $ignoredPath ? $ignoredPath . '/' : $ignoredPath) === 0) {
$ignoredMimeTypes = array_unique(array_merge($ignoredMimeTypes, $mimeTypes));
}
}
}

return $ignoredMimeTypes;
}

private function resetIgnoreCache(Node $node) : void {
$storageId = $node->getMountPoint()->getNumericStorageId();
if ($storageId === null) {
Expand Down
53 changes: 53 additions & 0 deletions tests/ClassifierTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,59 @@ public function testFileListener(string $ignoreFileName) : void {
self::assertCount(0, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'entry should have been removed from imagenet queue after moving it into ignored territory');
}

/**
* @dataProvider ignoreImageFilesProvider
* @return void
* @throws \OCP\DB\Exception
* @throws \OCP\Files\InvalidPathException
* @throws \OCP\Files\NotFoundException
* @throws \OCP\Files\NotPermittedException
*/
public function testFileListenerWithFolder(string $ignoreFileName) : void {
$this->config->setAppValueString('imagenet.enabled', 'true');
$this->queue->clearQueue(ImagenetClassifier::MODEL_NAME);

$folder = $this->userFolder->newFolder('/folder/');
$this->testFile = $folder->newFile('/alpine.jpg', file_get_contents(__DIR__.'/res/alpine.JPG'));
$ignoreFolder = $this->userFolder->newFolder('/test/ignore/');
$ignoredFolder = $ignoreFolder->newFolder('/folder/');
$ignoreFile = $this->userFolder->newFile('/test/' . $ignoreFileName, '');
$this->ignoredFile = $ignoredFolder->newFile('/alpine-2.jpg', file_get_contents(__DIR__.'/res/alpine.JPG'));

$storageId = $this->testFile->getMountPoint()->getNumericStorageId();
$rootId = $this->testFile->getMountPoint()->getStorageRootId();

self::assertCount(1, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'one element should have been added to imagenet queue');

$folder->delete();

self::assertCount(0, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'entry should have been removed from imagenet queue');

$ignoreFile->delete();

self::assertCount(1, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'one element should have been added to imagenet queue after deleting ignore file');

$ignoreFile = $this->userFolder->newFile('/test/' . $ignoreFileName, '');

self::assertCount(0, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'entry should have been removed from imagenet queue after creating ignore file');

$ignoredFolder->move($this->userFolder->getPath() . '/folder2');

self::assertCount(1, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'one element should have been added to imagenet queue after moving it out of ignored territory');

$ignoreFile->move($this->userFolder->getPath() . '/' . $ignoreFileName);

self::assertCount(0, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'entry should have been removed from imagenet queue after moving ignore file');

$ignoreFile->move($this->userFolder->getPath() . '/test/' . $ignoreFileName);

self::assertCount(1, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'one element should have been added to imagenet queue after moving ignore file');

$ignoredFolder->move($this->userFolder->getPath() . '/test/ignore/folder');

self::assertCount(0, $this->queue->getFromQueue(ImagenetClassifier::MODEL_NAME, $storageId, $rootId, 100), 'entry should have been removed from imagenet queue after moving it into ignored territory');
}

/**
* @dataProvider ignoreImageFilesProvider
* @return void
Expand Down
Loading