From 7996ba35cc6135d553b899fbd2fd6d646bf76ba4 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 6 Nov 2024 14:14:36 +0100 Subject: [PATCH 1/7] Disallow creating volume from pending volume twice --- app/Http/Requests/UpdatePendingVolume.php | 4 ++++ .../Api/PendingVolumeControllerTest.php | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/app/Http/Requests/UpdatePendingVolume.php b/app/Http/Requests/UpdatePendingVolume.php index 1736e0aa1..f5542b988 100644 --- a/app/Http/Requests/UpdatePendingVolume.php +++ b/app/Http/Requests/UpdatePendingVolume.php @@ -60,6 +60,10 @@ public function withValidator($validator) if (!$rule->passes('files', $files)) { $validator->errors()->add('files', $rule->message()); } + + if (!is_null($this->pendingVolume->volume_id)) { + $validator->errors()->add('id', 'A volume was already created for this pending volume.'); + } }); } diff --git a/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php b/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php index 9c36a4919..8b1e10a39 100644 --- a/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php +++ b/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php @@ -844,6 +844,26 @@ public function testUpdateAuthorizeDisk() ])->assertSuccessful(); } + public function testUpdatVolumeExists() + { + config(['volumes.editor_storage_disks' => ['test']]); + $disk = Storage::fake('test'); + $disk->put('images/1.jpg', 'abc'); + + $id = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'volume_id' => $this->volume()->id, + ])->id; + $this->beAdmin(); + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.jpg'], + ])->assertStatus(422); + } + public function testDestroy() { $pv = PendingVolume::factory()->create([ From b5ba0b18f1752e5c4b99714664c87f49f396656d Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 6 Nov 2024 15:20:48 +0100 Subject: [PATCH 2/7] Implement API to start annotation/file label import for volume --- .../Api/PendingVolumeController.php | 49 +++++++ .../Requests/StorePendingVolumeFromVolume.php | 67 ++++++++++ routes/api.php | 4 + .../Api/PendingVolumeControllerTest.php | 124 +++++++++++++++++- 4 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 app/Http/Requests/StorePendingVolumeFromVolume.php diff --git a/app/Http/Controllers/Api/PendingVolumeController.php b/app/Http/Controllers/Api/PendingVolumeController.php index c04a28809..d695407b6 100644 --- a/app/Http/Controllers/Api/PendingVolumeController.php +++ b/app/Http/Controllers/Api/PendingVolumeController.php @@ -3,10 +3,13 @@ namespace Biigle\Http\Controllers\Api; use Biigle\Http\Requests\StorePendingVolume; +use Biigle\Http\Requests\StorePendingVolumeFromVolume; use Biigle\Http\Requests\UpdatePendingVolume; use Biigle\Http\Requests\UpdatePendingVolumeAnnotationLabels; use Biigle\Jobs\CreateNewImagesOrVideos; use Biigle\PendingVolume; +use Biigle\Project; +use Biigle\Role; use Biigle\Volume; use DB; use Illuminate\Http\Request; @@ -75,6 +78,52 @@ public function store(StorePendingVolume $request) return redirect()->route('pending-volume', $pv->id); } + /** + * Creates a new pending volume based on an existing volume. + * + * @api {post} volumes/:id/pending-volumes Create a new pending volume from an existing volume + * @apiDescription This initiates the annotation or file label import for an existing volume that already has a metadata file. This only works if the metadata file contains annotation and/or file label information. + * @apiGroup Volumes + * @apiName StoreVolumePendingVolumes + * @apiPermission projectAdmin + * + * @apiParam {Number} id The volume ID. + * + * @apiParam (Attributes) {Bool} import_annotations Whether to import annotations from the metadata file. + * @apiParam (Attributes) {Bool} import_file_labels Whether to import file labels from the metadata file. + */ + public function storeVolume(StorePendingVolumeFromVolume $request) + { + $project = Project::inCommon($request->user(), $request->volume->id, [Role::adminId()])->first(); + + $pv = $project->pendingVolumes()->create([ + 'volume_id' => $request->volume->id, + 'media_type_id' => $request->volume->media_type_id, + 'user_id' => $request->user()->id, + 'metadata_parser' => $request->volume->metadata_parser, + 'import_annotations' => $request->input('import_annotations', false), + 'import_file_labels' => $request->input('import_file_labels', false), + ]); + + $pv->update([ + 'metadata_file_path' => $pv->id.'.'.pathinfo($request->volume->metadata_file_path, PATHINFO_EXTENSION), + ]); + $stream = Storage::disk(config('volumes.metadata_storage_disk')) + ->readStream($request->volume->metadata_file_path); + Storage::disk(config('volumes.pending_metadata_storage_disk')) + ->writeStream($pv->metadata_file_path, $stream); + + if ($this->isAutomatedRequest()) { + return $pv; + } + + if ($pv->import_annotations) { + return redirect()->route('pending-volume-annotation-labels', $pv->id); + } + + return redirect()->route('pending-volume-file-labels', $pv->id); + } + /** * Update a pending volume to create an actual volume * diff --git a/app/Http/Requests/StorePendingVolumeFromVolume.php b/app/Http/Requests/StorePendingVolumeFromVolume.php new file mode 100644 index 000000000..cb727449e --- /dev/null +++ b/app/Http/Requests/StorePendingVolumeFromVolume.php @@ -0,0 +1,67 @@ +volume = Volume::findOrFail($this->route('id')); + + return $this->user()->can('update', $this->volume); + } + + /** + * Get the validation rules that apply to the request. + * + * @return array|string> + */ + public function rules(): array + { + return [ + 'import_annotations' => 'bool|required_without:import_file_labels', + 'import_file_labels' => 'bool|required_without:import_annotations', + ]; + } + + /** + * Configure the validator instance. + * + * @param \Illuminate\Validation\Validator $validator + * @return void + */ + public function withValidator($validator) + { + $validator->after(function ($validator) { + if (!$this->input('import_annotations') && !$this->input('import_file_labels')) { + $validator->errors()->add('id', 'Either import_annotations or import_file_labels must be set to true.'); + } + + if ($validator->errors()->isNotEmpty()) { + return; + } + + $metadata = $this->volume->getMetadata(); + + if (is_null($metadata)) { + $validator->errors()->add('id', 'The volume has no metadata file.'); + return; + } + + if ($this->input('import_annotations') && !$metadata->hasAnnotations()) { + $validator->errors()->add('import_annotations', 'The volume metadata has no annotation information.'); + } + + if ($this->input('import_file_labels') && !$metadata->hasFileLabels()) { + $validator->errors()->add('import_file_labels', 'The volume metadata has no file label information.'); + } + + }); + } +} diff --git a/routes/api.php b/routes/api.php index 1eae8ebf9..0d0226e07 100644 --- a/routes/api.php +++ b/routes/api.php @@ -317,6 +317,10 @@ 'parameters' => ['volumes' => 'id'], ]); +$router->post( + 'volumes/{id}/pending-volumes', 'PendingVolumeController@storeVolume' +); + $router->group([ 'prefix' => 'volumes', 'namespace' => 'Volumes', diff --git a/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php b/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php index 8b1e10a39..e9cfc8814 100644 --- a/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php +++ b/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php @@ -6,14 +6,22 @@ use Biigle\Jobs\CreateNewImagesOrVideos; use Biigle\MediaType; use Biigle\PendingVolume; +use Biigle\Services\MetadataParsing\ImageAnnotation; use Biigle\Services\MetadataParsing\ImageCsvParser; +use Biigle\Services\MetadataParsing\ImageMetadata; +use Biigle\Services\MetadataParsing\Label; +use Biigle\Services\MetadataParsing\LabelAndUser; +use Biigle\Services\MetadataParsing\User; use Biigle\Services\MetadataParsing\VideoCsvParser; +use Biigle\Services\MetadataParsing\VolumeMetadata; +use Biigle\Shape; use Biigle\Volume; use Exception; use FileCache; use Illuminate\Http\UploadedFile; -use Queue; -use Storage; +use Illuminate\Support\Facades\Cache; +use Illuminate\Support\Facades\Queue; +use Illuminate\Support\Facades\Storage; class PendingVolumeControllerTest extends ApiTestCase { @@ -209,6 +217,118 @@ public function testStoreFromUI() $response->assertRedirectToRoute('pending-volume', $pv->id); } + public function testStoreVolumeAnnotations() + { + config([ + 'volumes.metadata_storage_disk' => 'metadata', + 'volumes.pending_metadata_storage_disk' => 'pending-metadata', + ]); + $metaDisk = Storage::fake('metadata'); + $metaDisk->put('metadata.csv', 'abc'); + $pendingDisk = Storage::fake('pending-metadata'); + + $id = $this->volume()->id; + $this->doTestApiRoute('POST', "/api/v1/volumes/{$id}/pending-volumes"); + + $this->beEditor(); + $this->post("/api/v1/volumes/{$id}/pending-volumes")->assertStatus(403); + + $this->beAdmin(); + // Missing arguments. + $this->json('POST', "/api/v1/volumes/{$id}/pending-volumes")->assertStatus(422); + + // Needs metadata file. + $this->json('POST', "/api/v1/volumes/{$id}/pending-volumes", [ + 'import_annotations' => true, + ])->assertStatus(422); + + $this->volume()->update([ + 'metadata_file_path' => 'metadata.csv', + 'metadata_parser' => ImageCsvParser::class, + ]); + + // Metadata has no annotations. + $this->json('POST', "/api/v1/volumes/{$id}/pending-volumes", [ + 'import_annotations' => true, + ])->assertStatus(422); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-metadata-metadata.csv', $metadata); + + $this->json('POST', "/api/v1/volumes/{$id}/pending-volumes", [ + 'import_annotations' => true, + ])->assertStatus(201); + + $pv = PendingVolume::where('volume_id', $id)->first(); + $this->assertSame($this->volume()->media_type_id, $pv->media_type_id); + $this->assertSame($this->admin()->id, $pv->user_id); + $this->assertSame("{$pv->id}.csv", $pv->metadata_file_path); + $this->assertSame(ImageCsvParser::class, $pv->metadata_parser); + $this->assertSame($this->project()->id, $pv->project_id); + $this->assertSame($id, $pv->volume_id); + $this->assertTrue($pv->import_annotations); + $this->assertFalse($pv->import_file_labels); + $pendingDisk->assertExists("{$pv->id}.csv"); + } + + public function testStoreVolumeFileLabels() + { + config([ + 'volumes.metadata_storage_disk' => 'metadata', + 'volumes.pending_metadata_storage_disk' => 'pending-metadata', + ]); + $metaDisk = Storage::fake('metadata'); + $metaDisk->put('metadata.csv', 'abc'); + $pendingDisk = Storage::fake('pending-metadata'); + + $id = $this->volume()->id; + $this->beAdmin(); + + // Needs metadata file. + $this->json('POST', "/api/v1/volumes/{$id}/pending-volumes", [ + 'import_file_labels' => true, + ])->assertStatus(422); + + $this->volume()->update([ + 'metadata_file_path' => 'metadata.csv', + 'metadata_parser' => ImageCsvParser::class, + ]); + + // Metadata has no file labels. + $this->json('POST', "/api/v1/volumes/{$id}/pending-volumes", [ + 'import_file_labels' => true, + ])->assertStatus(422); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-metadata-metadata.csv', $metadata); + + $this->json('POST', "/api/v1/volumes/{$id}/pending-volumes", [ + 'import_file_labels' => true, + ])->assertStatus(201); + + $pv = PendingVolume::where('volume_id', $id)->first(); + $this->assertTrue($pv->import_file_labels); + } + public function testUpdateImages() { config(['volumes.editor_storage_disks' => ['test']]); From 037053d08fa18d9075f8dd2cc34431e71d67490c Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 6 Nov 2024 15:45:49 +0100 Subject: [PATCH 3/7] Implement deletion of previous pending volumes if new import starts --- .../Api/PendingVolumeController.php | 4 ++ .../Api/PendingVolumeControllerTest.php | 47 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/app/Http/Controllers/Api/PendingVolumeController.php b/app/Http/Controllers/Api/PendingVolumeController.php index d695407b6..79ba29ce2 100644 --- a/app/Http/Controllers/Api/PendingVolumeController.php +++ b/app/Http/Controllers/Api/PendingVolumeController.php @@ -96,6 +96,10 @@ public function storeVolume(StorePendingVolumeFromVolume $request) { $project = Project::inCommon($request->user(), $request->volume->id, [Role::adminId()])->first(); + // Delete individually to trigger deletion of metadata files. + PendingVolume::where('volume_id', $request->volume->id) + ->eachById(fn ($pv) => $pv->delete()); + $pv = $project->pendingVolumes()->create([ 'volume_id' => $request->volume->id, 'media_type_id' => $request->volume->media_type_id, diff --git a/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php b/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php index e9cfc8814..54b225388 100644 --- a/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php +++ b/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php @@ -329,6 +329,53 @@ public function testStoreVolumeFileLabels() $this->assertTrue($pv->import_file_labels); } + public function testStoreVolumeTwice() + { + config([ + 'volumes.metadata_storage_disk' => 'metadata', + 'volumes.pending_metadata_storage_disk' => 'pending-metadata', + ]); + $metaDisk = Storage::fake('metadata'); + $metaDisk->put('metadata.csv', 'abc'); + $pendingDisk = Storage::fake('pending-metadata'); + + $id = $this->volume()->id; + + $old = PendingVolume::factory()->create([ + 'volume_id' => $id, + ]); + + $this->beAdmin(); + + $this->volume()->update([ + 'metadata_file_path' => 'metadata.csv', + 'metadata_parser' => ImageCsvParser::class, + ]); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-metadata-metadata.csv', $metadata); + + $this->json('POST', "/api/v1/volumes/{$id}/pending-volumes", [ + 'import_annotations' => true, + ])->assertStatus(201); + + $pv = PendingVolume::where('volume_id', $id)->first(); + $this->assertNotSame($old->id, $pv->id); + $this->assertNull($old->fresh()); + } + public function testUpdateImages() { config(['volumes.editor_storage_disks' => ['test']]); From d99bb7530c9c14f662e4f6d8f99ff9b9b0f8447e Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 6 Nov 2024 16:18:57 +0100 Subject: [PATCH 4/7] Implement basic UI to import annotations/file labels in a volume --- .../assets/js/volumes/metadataUpload.vue | 14 +++++++++++ .../views/volumes/edit/metadata.blade.php | 23 ++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/resources/assets/js/volumes/metadataUpload.vue b/resources/assets/js/volumes/metadataUpload.vue index 15d183354..c52b67376 100644 --- a/resources/assets/js/volumes/metadataUpload.vue +++ b/resources/assets/js/volumes/metadataUpload.vue @@ -21,6 +21,10 @@ export default { hasMetadata: false, parsers: [], selectedParser: null, + importForm: { + importAnnotations: 0, + importFileLabels: 0, + }, }; }, methods: { @@ -68,6 +72,16 @@ export default { // filter from the selected parser. this.$nextTick(() => this.$refs.fileInput.click()); }, + importAnnotations() { + this.importForm.importAnnotations = 1; + this.importForm.importFileLabels = 0; + this.$nextTick(() => this.$refs.importForm.submit()); + }, + importFileLabels() { + this.importForm.importAnnotations = 0; + this.importForm.importFileLabels = 1; + this.$nextTick(() => this.$refs.importForm.submit()); + }, }, created() { this.volumeId = biigle.$require('volumes.id'); diff --git a/resources/views/volumes/edit/metadata.blade.php b/resources/views/volumes/edit/metadata.blade.php index 37d83d876..0506a7842 100644 --- a/resources/views/volumes/edit/metadata.blade.php +++ b/resources/views/volumes/edit/metadata.blade.php @@ -7,6 +7,17 @@ @endif + + + +