Merge pull request #965 from biigle/issue-922

Annotation import for existing volumes

app/Http/Controllers/Api/PendingVolumeController.php

@@ -3,10 +3,13 @@
 namespace Biigle\Http\Controllers\Api;
 
 use Biigle\Http\Requests\StorePendingVolume;
+use Biigle\Http\Requests\StorePendingVolumeFromVolume;
 use Biigle\Http\Requests\UpdatePendingVolume;
 use Biigle\Http\Requests\UpdatePendingVolumeAnnotationLabels;
 use Biigle\Jobs\CreateNewImagesOrVideos;
 use Biigle\PendingVolume;
+use Biigle\Project;
+use Biigle\Role;
 use Biigle\Volume;
 use DB;
 use Illuminate\Http\Request;
@@ -75,6 +78,56 @@ public function store(StorePendingVolume $request)
         return redirect()->route('pending-volume', $pv->id);
     }
 
+    /**
+     * Creates a new pending volume based on an existing volume.
+     *
+     * @api {post} volumes/:id/pending-volumes Create a new pending volume from an existing volume
+     * @apiDescription This initiates the annotation or file label import for an existing volume that already has a metadata file. This only works if the metadata file contains annotation and/or file label information.
+     * @apiGroup Volumes
+     * @apiName StoreVolumePendingVolumes
+     * @apiPermission projectAdmin
+     *
+     * @apiParam {Number} id The volume ID.
+     *
+     * @apiParam (Attributes) {Bool} import_annotations Whether to import annotations from the metadata file.
+     * @apiParam (Attributes) {Bool} import_file_labels Whether to import file labels from the metadata file.
+     */
+    public function storeVolume(StorePendingVolumeFromVolume $request)
+    {
+        $project = Project::inCommon($request->user(), $request->volume->id, [Role::adminId()])->first();
+
+        // Delete individually to trigger deletion of metadata files.
+        PendingVolume::where('volume_id', $request->volume->id)
+            ->eachById(fn ($pv) => $pv->delete());
+
+        $pv = $project->pendingVolumes()->create([
+            'volume_id' => $request->volume->id,
+            'media_type_id' => $request->volume->media_type_id,
+            'user_id' => $request->user()->id,
+            'metadata_parser' => $request->volume->metadata_parser,
+            'import_annotations' => $request->input('import_annotations', false),
+            'import_file_labels' => $request->input('import_file_labels', false),
+        ]);
+
+        $pv->update([
+            'metadata_file_path' => $pv->id.'.'.pathinfo($request->volume->metadata_file_path, PATHINFO_EXTENSION),
+        ]);
+        $stream = Storage::disk(config('volumes.metadata_storage_disk'))
+            ->readStream($request->volume->metadata_file_path);
+        Storage::disk(config('volumes.pending_metadata_storage_disk'))
+            ->writeStream($pv->metadata_file_path, $stream);
+
+        if ($this->isAutomatedRequest()) {
+            return $pv;
+        }
+
+        if ($pv->import_annotations) {
+            return redirect()->route('pending-volume-annotation-labels', $pv->id);
+        }
+
+        return redirect()->route('pending-volume-file-labels', $pv->id);
+    }
+
     /**
      * Update a pending volume to create an actual volume
      *

app/Http/Controllers/Api/Volumes/MetadataController.php

@@ -56,7 +56,8 @@ public function show($id)
      * @apiPermission projectAdmin
      * @apiDescription This endpoint allows adding or updating metadata such as geo
      * coordinates for volume files. The uploaded metadata file replaces any previously
-     * uploaded file.
+     * uploaded file. The JSON response indicates whether the uploaded metadata file
+     * contained annotations or file labels.
      *
      * @apiParam {Number} id The volume ID.
      *
@@ -73,7 +74,7 @@ public function show($id)
      *
      * @param StoreVolumeMetadata $request
      *
-     * @return void
+     * @return array<string, bool>|void
      */
     public function store(StoreVolumeMetadata $request)
     {
@@ -83,6 +84,15 @@ public function store(StoreVolumeMetadata $request)
         $request->volume->saveMetadata($request->file('file'));
         $request->volume->update(['metadata_parser' => $request->input('parser')]);
         Queue::push(new UpdateVolumeMetadata($request->volume));
+
+        if ($this->isAutomatedRequest()) {
+            $metadata = $request->volume->getMetadata();
+
+            return [
+                'has_annotations' => $metadata->hasAnnotations(),
+                'has_file_labels' => $metadata->hasFileLabels(),
+            ];
+        }
     }
 
     /**

app/Http/Requests/StorePendingVolumeFromVolume.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace Biigle\Http\Requests;
+
+use Biigle\Volume;
+use Illuminate\Foundation\Http\FormRequest;
+
+class StorePendingVolumeFromVolume extends FormRequest
+{
+    /**
+     * Determine if the user is authorized to make this request.
+     */
+    public function authorize(): bool
+    {
+        $this->volume = Volume::findOrFail($this->route('id'));
+
+        return $this->user()->can('update', $this->volume);
+    }
+
+    /**
+     * Get the validation rules that apply to the request.
+     *
+     * @return array<string, \Illuminate\Contracts\Validation\ValidationRule|array<mixed>|string>
+     */
+    public function rules(): array
+    {
+        return [
+            'import_annotations' => 'bool|required_without:import_file_labels',
+            'import_file_labels' => 'bool|required_without:import_annotations',
+        ];
+    }
+
+    /**
+     * Configure the validator instance.
+     *
+     * @param  \Illuminate\Validation\Validator  $validator
+     * @return void
+     */
+    public function withValidator($validator)
+    {
+        $validator->after(function ($validator) {
+            if (!$this->input('import_annotations') && !$this->input('import_file_labels')) {
+                $validator->errors()->add('id', 'Either import_annotations or import_file_labels must be set to true.');
+            }
+
+            if ($validator->errors()->isNotEmpty()) {
+                return;
+            }
+
+            $metadata = $this->volume->getMetadata();
+
+            if (is_null($metadata)) {
+                $validator->errors()->add('id', 'The volume has no metadata file.');
+                return;
+            }
+
+            if ($this->input('import_annotations') && !$metadata->hasAnnotations()) {
+                $validator->errors()->add('import_annotations', 'The volume metadata has no annotation information.');
+            }
+
+            if ($this->input('import_file_labels') && !$metadata->hasFileLabels()) {
+                $validator->errors()->add('import_file_labels', 'The volume metadata has no file label information.');
+            }
+
+        });
+    }
+}

app/Http/Requests/UpdatePendingVolume.php

@@ -60,6 +60,10 @@ public function withValidator($validator)
             if (!$rule->passes('files', $files)) {
                 $validator->errors()->add('files', $rule->message());
             }
+
+            if (!is_null($this->pendingVolume->volume_id)) {
+                $validator->errors()->add('id', 'A volume was already created for this pending volume.');
+            }
         });
     }
 

app/Traits/HasMetadataFile.php

@@ -68,6 +68,8 @@ public function deleteMetadata($noUpdate = false): void
     {
         if ($this->hasMetadata()) {
             Storage::disk($this->getMetadataFileDisk())->delete($this->metadata_file_path);
+            $disk = $this->getMetadataFileDisk();
+            Cache::store('array')->forget("metadata-{$disk}-{$this->metadata_file_path}");
             if (!$noUpdate) {
                 $this->update(['metadata_file_path' => null]);
             }

resources/assets/js/volumes/metadataUpload.vue

@@ -19,10 +19,27 @@ export default {
             success: false,
             message: undefined,
             hasMetadata: false,
+            hasMetadataAnnotations: false,
+            hasMetadataFileLabels: false,
             parsers: [],
             selectedParser: null,
+            importForm: {
+                importAnnotations: 0,
+                importFileLabels: 0,
+            },
         };
     },
+    computed: {
+        showImportDropdown() {
+            return this.hasMetadataAnnotations && this.hasMetadataFileLabels;
+        },
+        showImportAnnotations() {
+            return this.hasMetadataAnnotations && !this.hasMetadataFileLabels;
+        },
+        showImportFileLabels() {
+            return !this.hasMetadataAnnotations && this.hasMetadataFileLabels;
+        },
+    },
     methods: {
         handleSuccess() {
             this.error = false;
@@ -47,7 +64,11 @@ export default {
             data.append('file', event.target.files[0]);
             data.append('parser', this.selectedParser.parserClass);
             MetadataApi.save({id: this.volumeId}, data)
-                .then(() => this.hasMetadata = true)
+                .then((response) => {
+                    this.hasMetadata = true
+                    this.hasMetadataAnnotations = response.body.has_annotations;
+                    this.hasMetadataFileLabels = response.body.has_file_labels;
+                })
                 .then(this.handleSuccess)
                 .catch(this.handleError)
                 .finally(this.finishLoading);
@@ -60,6 +81,8 @@ export default {
         },
         handleFileDeleted() {
             this.hasMetadata = false;
+            this.hasMetadataAnnotations = false;
+            this.hasMetadataFileLabels = false;
             MessageStore.success('The metadata file was deleted.');
         },
         selectFile(parser) {
@@ -68,10 +91,22 @@ export default {
             // filter from the selected parser.
             this.$nextTick(() => this.$refs.fileInput.click());
         },
+        importAnnotations() {
+            this.importForm.importAnnotations = 1;
+            this.importForm.importFileLabels = 0;
+            this.$nextTick(() => this.$refs.importForm.submit());
+        },
+        importFileLabels() {
+            this.importForm.importAnnotations = 0;
+            this.importForm.importFileLabels = 1;
+            this.$nextTick(() => this.$refs.importForm.submit());
+        },
     },
     created() {
         this.volumeId = biigle.$require('volumes.id');
         this.hasMetadata = biigle.$require('volumes.hasMetadata');
+        this.hasMetadataAnnotations = biigle.$require('volumes.hasMetadataAnnotations');
+        this.hasMetadataFileLabels = biigle.$require('volumes.hasMetadataFileLabels');
         this.parsers = biigle.$require('volumes.parsers');
     },
 };

resources/views/manual/tutorials/volumes/file-metadata.blade.php

@@ -120,7 +120,7 @@
 image_2.png,2016-12-19 17:09:31,52.215,28.501,-1502.5,28.25,2.1
 </pre>
         <p>
-            The metadata CSV file can be uploaded when a new volume is created. For existing volumes, metadata can be uploaded by volume admins on the volume edit page that you can reach with the <button class="btn btn-default btn-xs"><span class="fa fa-pencil-alt" aria-hidden="true"></span></button> button of the volume overview. This will replace any previously imported metadata.
+            The metadata CSV file can be uploaded when a new volume is created. For existing volumes, metadata can be uploaded by volume admins on the volume edit page that you can reach with the <button class="btn btn-default btn-xs"><span class="fa fa-pencil-alt" aria-hidden="true"></span></button> button of the volume overview. This will replace any previously imported metadata. Some metadata formats also contain annotations or file labels. In this case, BIIGLE will show buttons to initiate the import as well.
         </p>
     </div>
     <div class="row">

resources/views/volumes/edit.blade.php

@@ -6,7 +6,9 @@
         biigle.$declare('volumes.id', {!! $volume->id !!});
         biigle.$declare('volumes.annotationSessions', {!! $annotationSessions !!});
         biigle.$declare('volumes.type', '{!! $type !!}');
-        biigle.$declare('volumes.hasMetadata', '{!! $volume->hasMetadata() !!}');
+        biigle.$declare('volumes.hasMetadata', {!! $volume->hasMetadata() ? 'true' : 'false' !!});
+        biigle.$declare('volumes.hasMetadataAnnotations', {!! ($volume->hasMetadata() && $volume->getMetadata()->hasAnnotations()) ? 'true' : 'false' !!});
+        biigle.$declare('volumes.hasMetadataFileLabels', {!! ($volume->hasMetadata() && $volume->getMetadata()->hasFileLabels()) ? 'true' : 'false' !!});
         biigle.$declare('volumes.parsers', {!! $parsers !!});
     </script>
     @mixin('volumesEditScripts')

resources/views/volumes/edit/metadata.blade.php

@@ -7,6 +7,19 @@
         @endif
         <span class="pull-right">
             <span class="loader" v-bind:class="{'loader--active':loading}"></span>
+            <dropdown tag="span" v-if="showImportDropdown" v-cloak>
+                <button class="btn btn-default btn-xs dropdown-toggle" type="button" title="Import annotations or file labels from the metadata file attached to this volume"><i class="fa fa-file-import"></i> Import <span class="caret"></span></button>
+                <template slot="dropdown">
+                    <li>
+                        <a href="#" v-on:click.prevent="importAnnotations" title="Import annotations from the metadata file attached to this volume">Annotations</a>
+                    </li>
+                    <li>
+                        <a href="#" v-on:click.prevent="importFileLabels" title="Import file labels from the metadata file attached to this volume">File labels</a>
+                    </li>
+                </template>
+            </dropdown>
+            <button v-cloak v-if="showImportAnnotations" v-on:click.prevent="importAnnotations" class="btn btn-default btn-xs" type="button" title="Import annotations from the metadata file attached to this volume"><i class="fa fa-file-import"></i> Import annotations</button>
+            <button v-cloak v-if="showImportFileLabels" v-on:click.prevent="importFileLabels" class="btn btn-default btn-xs" type="button" title="Import file labels from the metadata file attached to this volume"><i class="fa fa-file-import"></i> Import annotations</button>
             <dropdown tag="span" v-if="hasMetadata" v-cloak>
                 <button class="btn btn-default btn-xs dropdown-toggle" type="button" title="Manage the metadata file attached to this volume" :disabled="loading"><i class="fa fa-file-alt"></i> Manage file <span class="caret"></span></button>
                 <template slot="dropdown">
@@ -21,12 +34,22 @@
         </span>
     </div>
     <div class="panel-body">
+        <form ref="importForm" action="{{url("api/v1/volumes/{$volume->id}/pending-volumes")}}" method="post">
+            <input type="hidden" name="_token" value="{{ csrf_token() }}">
+            <input type="hidden" name="import_annotations" v-model="importForm.importAnnotations">
+            <input type="hidden" name="import_file_labels" v-model="importForm.importFileLabels">
+        </form>
+        @if ($errors->hasAny(['import_annotations', 'import_file_labels']))
+            <p class="text-danger">
+                {{ $errors->first() }}
+            </p>
+        @endif
         <p>
             Upload a metadata file to attach it to the volume and update the @if ($volume->isImageVolume()) image @else video @endif metadata.
         </p>
         <p class="text-center">
             <dropdown tag="span">
-                <button class="btn btn-default dropdown-toggle" type="button" :disabled="loading"><i class="fa fa-file-alt"></i> Upload file <span class="caret"></span></button>
+                <button class="btn btn-default dropdown-toggle" type="button" :disabled="loading"><i class="fa fa-upload"></i> Upload file <span class="caret"></span></button>
                 <template slot="dropdown">
                     <li v-for="parser in parsers">
                         <a href="#" v-on:click.prevent="selectFile(parser)" v-text="parser.name"></a>

routes/api.php

@@ -317,6 +317,10 @@
     'parameters' => ['volumes' => 'id'],
 ]);
 
+$router->post(
+    'volumes/{id}/pending-volumes', 'PendingVolumeController@storeVolume'
+);
+
 $router->group([
     'prefix' => 'volumes',
     'namespace' => 'Volumes',