diff --git a/src/js/collections/DataPackage.js b/src/js/collections/DataPackage.js index e0911e30d..cd2c47524 100644 --- a/src/js/collections/DataPackage.js +++ b/src/js/collections/DataPackage.js @@ -415,6 +415,72 @@ define([ } }, + + /** + * Fetches member models in batches to avoid fetching all members simultaneously. + * + * @param {Array} models - The array of member models to fetch. + * @param {number} [index=0] - The current index of the model being fetched. + * @param {number} [batchSize=10] - The number of models to fetch in each batch. + */ + fetchMemberModels(models, index = 0, batchSize = 0) { + // Update the number of file metadata items being loaded + this.packageModel.set("numLoadingFileMetadata", models.length - index); + // If the index is greater than or equal to the length of the models array, stop fetching + if (index >= models.length) { + return; + } + if (batchSize == 0) batchSize = models.length; + + const collection = this; + // Slice the models array to get the current batch + const batch = models.slice(index, index + batchSize); + + // Create an array of promises for fetching each model in the batch + const fetchPromises = batch.map((memberModel) => { + return new Promise((resolve) => { + // Fetch the member model + memberModel.fetch(); + // Once the model is synced, handle the response + memberModel.once("sync", (oldModel) => { + const newModel = collection.getMember(oldModel); + + // If the type of the old model is different from the new model + if (oldModel.type != newModel.type) { + if (newModel.type == "DataPackage") { + // If the new model is a DataPackage, replace the old model with the new one + oldModel.trigger("replace", newModel); + resolve(); + } else { + // Otherwise, fetch the new model and replace the old model with the new one + newModel.set("synced", false); + newModel.fetch(); + newModel.once("sync", (fetchedModel) => { + fetchedModel.set("synced", true); + collection.remove(oldModel); + collection.add(fetchedModel); + oldModel.trigger("replace", newModel); + if (newModel.type == "EML") collection.trigger("add:EML"); + resolve(); + }); + } + } else { + // If the type of the old model is the same as the new model, merge the new model into the collection + newModel.set("synced", true); + collection.add(newModel, { merge: true }); + if (newModel.type == "EML") collection.trigger("add:EML"); + resolve(); + } + }); + }); + }); + + // Once all fetch promises are resolved, fetch the next batch + Promise.all(fetchPromises).then(() => { + this.fetchMemberModels.call(collection, models, index + batchSize, batchSize); + }); + }, + /** * Overload fetch calls for a DataPackage * @param {object} [options] - Optional options for this fetch that get sent with the XHR request @@ -688,56 +754,8 @@ define([ // Don't fetch each member model if the fetchModels property on this Collection is set to false if (this.fetchModels !== false) { - // Add the models to the collection now, silently - // this.add(models, {silent: true}); - - // Retrieve the model for each member - _.each( - models, - function (memberModel) { - const collection = this; - - memberModel.fetch(); - memberModel.once("sync", (oldModel) => { - // Get the right model type based on the model values - const newModel = collection.getMember(oldModel); - - // If the model type has changed, then mark the model as unsynced, since there may be custom fetch() options for the new model - if (oldModel.type != newModel.type) { - // DataPackages shouldn't be fetched until we support nested packages better in the UI - if (newModel.type == "DataPackage") { - // Trigger a replace event so other parts of the app know when a model has been replaced with a different type - oldModel.trigger("replace", newModel); - } else { - newModel.set("synced", false); - - newModel.fetch(); - newModel.once("sync", (fetchedModel) => { - fetchedModel.set("synced", true); - - // Remove the model from the collection and add it back - collection.remove(oldModel); - collection.add(fetchedModel); - - // Trigger a replace event so other parts of the app know when a model has been replaced with a different type - oldModel.trigger("replace", newModel); - - if (newModel.type == "EML") - collection.trigger("add:EML"); - }); - } - } else { - newModel.set("synced", true); - collection.add(newModel, { - merge: true, - }); - - if (newModel.type == "EML") collection.trigger("add:EML"); - } - }); - }, - this, - ); + // Start fetching member models + this.fetchMemberModels.call(this, models, 0, MetacatUI.appModel.get("batchModeValue")); } } catch (error) { console.log(error); @@ -3661,6 +3679,7 @@ define([ this.packageModel.updateSysMeta(); }, + /** * Tracks the upload status of DataONEObject models in this collection. If they are * `loading` into the DOM or `in progress` of an upload to the server, they will be considered as "loading" files. diff --git a/src/js/models/AppModel.js b/src/js/models/AppModel.js index d5af3c6c3..4f81fd210 100644 --- a/src/js/models/AppModel.js +++ b/src/js/models/AppModel.js @@ -2413,6 +2413,22 @@ define(["jquery", "underscore", "backbone"], function ($, _, Backbone) { * @example application%2Fbagit-097 */ packageFormat: "application%2Fbagit-1.0", + /** + * Whether to batch requests to the DataONE API. This is an experimental feature + * and should be used with caution. If set to a number greater than 0, MetacatUI will + * batch requests to the DataONE API and send them in groups of this size. This can + * improve performance when making many requests to the DataONE API, but can also + * cause issues if the requests are too large or if the DataONE API is not able to + * handle the batched requests. + * + * Currently, this feature is only used in the DataPackageModel when fetching the + * list of DataONE member models. + * + * @type {number} + * @default 0 + * @example 20 + */ + batchModeValue: 0, }, MetacatUI.AppConfig, ), diff --git a/src/js/views/metadata/EML211EditorView.js b/src/js/views/metadata/EML211EditorView.js index 30e4e66e7..95cda8c4c 100644 --- a/src/js/views/metadata/EML211EditorView.js +++ b/src/js/views/metadata/EML211EditorView.js @@ -552,6 +552,12 @@ define([ "change:numLoadingFiles", this.toggleEnableControls, ); + + this.listenTo( + MetacatUI.rootDataPackage.packageModel, + "change:numLoadingFileMetadata", + this.toggleEnableControls, + ); }, renderChildren: function (model, options) {}, @@ -1215,15 +1221,19 @@ define([ toggleEnableControls: function () { if (MetacatUI.rootDataPackage.packageModel.get("isLoadingFiles")) { let noun = - MetacatUI.rootDataPackage.packageModel.get("numLoadingFiles") > 1 - ? " files" - : " file"; + MetacatUI.rootDataPackage.packageModel.get("numLoadingFiles") > 1 + ? " files" + : " file"; this.disableControls( - "Waiting for " + + "Waiting for " + MetacatUI.rootDataPackage.packageModel.get("numLoadingFiles") + noun + " to upload...", ); + } else if (MetacatUI.rootDataPackage.packageModel.get("numLoadingFileMetadata") >0) { + this.disableControls("Waiting for " + + MetacatUI.rootDataPackage.packageModel.get("numLoadingFileMetadata") + + " file metadata to load..."); } else { this.enableControls(); }