From 0f34fda37228791f5d90e13eeaf383c3dd8b3cda Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 21 Oct 2024 17:54:15 +0200 Subject: [PATCH 01/17] stream upload unknown size files behind a new option streamingUploadSizeless COMPANION_STREAMING_UPLOAD_SIZELESS for tus --- docs/companion.md | 8 ++++++ .../@uppy/companion/src/config/companion.js | 1 + .../@uppy/companion/src/server/Uploader.js | 25 ++++++++++++++----- .../@uppy/companion/src/standalone/helper.js | 1 + 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/docs/companion.md b/docs/companion.md index e58c00a6ee..e3c6a41360 100644 --- a/docs/companion.md +++ b/docs/companion.md @@ -327,6 +327,7 @@ const options = { logClientVersion: true, periodicPingUrls: [], streamingUpload: true, + streamingUploadSizeless: false, clientSocketConnectTimeout: 60000, metrics: true, }; @@ -638,6 +639,13 @@ enabled, it will lead to _faster uploads_ because companion will start uploading at the same time as downloading using `stream.pipe`. If `false`, files will be fully downloaded first, then uploaded. Defaults to `true`. +#### `streamingUploadSizeless` `COMPANION_STREAMING_UPLOAD_SIZELESS` + +A boolean flag to tell Companion whether to also upload files that have an +unknown size. Currently this is only supported for Tus uploads. Note that this +requires an optional extension on the Tus server. Default is `false`. If set to +`true`, `streamingUpload` has to be also set to `true`. + #### `maxFileSize` `COMPANION_MAX_FILE_SIZE` If this value is set, companion will limit the maximum file size to process. If diff --git a/packages/@uppy/companion/src/config/companion.js b/packages/@uppy/companion/src/config/companion.js index 801280b763..43edeba343 100644 --- a/packages/@uppy/companion/src/config/companion.js +++ b/packages/@uppy/companion/src/config/companion.js @@ -20,6 +20,7 @@ const defaultOptions = { allowLocalUrls: false, periodicPingUrls: [], streamingUpload: true, + streamingUploadSizeless: false, clientSocketConnectTimeout: 60000, metrics: true, } diff --git a/packages/@uppy/companion/src/server/Uploader.js b/packages/@uppy/companion/src/server/Uploader.js index 25febdb309..773ff41cbd 100644 --- a/packages/@uppy/companion/src/server/Uploader.js +++ b/packages/@uppy/companion/src/server/Uploader.js @@ -220,10 +220,14 @@ class Uploader { if (this.readStream) this.readStream.destroy(err) } - async _uploadByProtocol(req) { + _getUploadProtocol() { // todo a default protocol should not be set. We should ensure that the user specifies their protocol. // after we drop old versions of uppy client we can remove this - const protocol = this.options.protocol || PROTOCOLS.multipart + return this.options.protocol || PROTOCOLS.multipart + } + + async _uploadByProtocol(req) { + const protocol = this._getUploadProtocol() switch (protocol) { case PROTOCOLS.multipart: @@ -264,8 +268,16 @@ class Uploader { this.readStream = fileStream } - _needDownloadFirst() { - return !this.options.size || !this.options.companionOptions.streamingUpload + _canStream() { + const protocol = this._getUploadProtocol() + + return this.options.companionOptions.streamingUpload && ( + this.options.size + // only tus uploads can be streamed without size, TODO: add also others + || (this.options.companionOptions.streamingUploadSizeless && ( + protocol === PROTOCOLS.tus + )) + ) } /** @@ -281,7 +293,8 @@ class Uploader { this.#uploadState = states.uploading this.readStream = stream - if (this._needDownloadFirst()) { + + if (!this._canStream()) { logger.debug('need to download the whole file first', 'controller.get.provider.size', this.shortToken) // Some streams need to be downloaded entirely first, because we don't know their size from the provider // This is true for zoom and drive (exported files) or some URL downloads. @@ -429,7 +442,7 @@ class Uploader { // If fully downloading before uploading, combine downloaded and uploaded bytes // This will make sure that the user sees half of the progress before upload starts (while downloading) let combinedBytes = bytesUploaded - if (this._needDownloadFirst()) { + if (!this._canStream()) { combinedBytes = Math.floor((combinedBytes + (this.downloadedBytes || 0)) / 2) } diff --git a/packages/@uppy/companion/src/standalone/helper.js b/packages/@uppy/companion/src/standalone/helper.js index e6ccd9887b..f631b1f1ea 100644 --- a/packages/@uppy/companion/src/standalone/helper.js +++ b/packages/@uppy/companion/src/standalone/helper.js @@ -182,6 +182,7 @@ const getConfigFromEnv = () => { // cookieDomain is kind of a hack to support distributed systems. This should be improved but we never got so far. cookieDomain: process.env.COMPANION_COOKIE_DOMAIN, streamingUpload: process.env.COMPANION_STREAMING_UPLOAD ? process.env.COMPANION_STREAMING_UPLOAD === 'true' : undefined, + streamingUploadSizeless: process.env.COMPANION_STREAMING_UPLOAD_SIZELESS ? process.env.COMPANION_STREAMING_UPLOAD_SIZELESS === 'true' : undefined, maxFileSize: process.env.COMPANION_MAX_FILE_SIZE ? parseInt(process.env.COMPANION_MAX_FILE_SIZE, 10) : undefined, chunkSize: process.env.COMPANION_CHUNK_SIZE ? parseInt(process.env.COMPANION_CHUNK_SIZE, 10) : undefined, clientSocketConnectTimeout: process.env.COMPANION_CLIENT_SOCKET_CONNECT_TIMEOUT From dd3a6d294139a0a243d625b629530fe2a0f5c262 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 21 Oct 2024 17:58:41 +0200 Subject: [PATCH 02/17] allow for all upload protocols seems to be working closes #5305 --- docs/companion.md | 6 ++++-- packages/@uppy/companion/src/server/Uploader.js | 8 ++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/companion.md b/docs/companion.md index e3c6a41360..6468b4f652 100644 --- a/docs/companion.md +++ b/docs/companion.md @@ -643,8 +643,10 @@ fully downloaded first, then uploaded. Defaults to `true`. A boolean flag to tell Companion whether to also upload files that have an unknown size. Currently this is only supported for Tus uploads. Note that this -requires an optional extension on the Tus server. Default is `false`. If set to -`true`, `streamingUpload` has to be also set to `true`. +requires an optional extension on the Tus server if using Tus uploads. For form +multipart uploads it requres a server that can handle +`transfer-encoding: chunked`. Default is `false`. If set to `true`, +`streamingUpload` also has to be set to `true`. #### `maxFileSize` `COMPANION_MAX_FILE_SIZE` diff --git a/packages/@uppy/companion/src/server/Uploader.js b/packages/@uppy/companion/src/server/Uploader.js index 773ff41cbd..c1bf7ba250 100644 --- a/packages/@uppy/companion/src/server/Uploader.js +++ b/packages/@uppy/companion/src/server/Uploader.js @@ -269,14 +269,10 @@ class Uploader { } _canStream() { - const protocol = this._getUploadProtocol() - return this.options.companionOptions.streamingUpload && ( this.options.size // only tus uploads can be streamed without size, TODO: add also others - || (this.options.companionOptions.streamingUploadSizeless && ( - protocol === PROTOCOLS.tus - )) + || this.options.companionOptions.streamingUploadSizeless ) } @@ -619,7 +615,7 @@ class Uploader { const response = await runRequest(url, reqOptions) - if (bytesUploaded !== this.size) { + if (this.size != null && bytesUploaded !== this.size) { const errMsg = `uploaded only ${bytesUploaded} of ${this.size} with status: ${response.statusCode}` logger.error(errMsg, 'upload.multipart.mismatch.error') throw new Error(errMsg) From 503856c4346ad9a4cd2f891ed78be1ff87d55a94 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 10 Nov 2024 16:56:30 +0800 Subject: [PATCH 03/17] refactor and fix bug where progress was not always emitted --- packages/@uppy/core/src/Uppy.test.ts | 12 +++++++---- packages/@uppy/core/src/Uppy.ts | 31 +++++++++++++++++----------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/packages/@uppy/core/src/Uppy.test.ts b/packages/@uppy/core/src/Uppy.test.ts index 0a27e16e52..270a3f0faf 100644 --- a/packages/@uppy/core/src/Uppy.test.ts +++ b/packages/@uppy/core/src/Uppy.test.ts @@ -1762,7 +1762,8 @@ describe('src/Core', () => { data: {}, }) - core.calculateTotalProgress() + // @ts-ignore + core[Symbol.for('uppy test: updateTotalProgress')]() const uploadPromise = core.upload() await Promise.all([ @@ -1844,7 +1845,8 @@ describe('src/Core', () => { data: {}, }) - core.calculateTotalProgress() + // @ts-ignore + core[Symbol.for('uppy test: updateTotalProgress')]() // foo.jpg at 35%, bar.jpg at 0% expect(core.getState().totalProgress).toBe(18) @@ -1893,7 +1895,8 @@ describe('src/Core', () => { bytesTotal: 17175, }) - core.calculateTotalProgress() + // @ts-ignore + core[Symbol.for('uppy test: updateTotalProgress')]() core.calculateProgress.flush() expect(core.getState().totalProgress).toEqual(66) @@ -1937,7 +1940,8 @@ describe('src/Core', () => { bytesTotal: 17175, }) - core.calculateTotalProgress() + // @ts-ignore + core[Symbol.for('uppy test: updateTotalProgress')]() core.calculateProgress.flush() expect(core.getState().totalProgress).toEqual(66) diff --git a/packages/@uppy/core/src/Uppy.ts b/packages/@uppy/core/src/Uppy.ts index f8e25e538e..0bd8757844 100644 --- a/packages/@uppy/core/src/Uppy.ts +++ b/packages/@uppy/core/src/Uppy.ts @@ -1270,7 +1270,7 @@ export class Uppy< } this.setState(stateUpdate) - this.calculateTotalProgress() + this.#updateTotalProgress() const removedFileIDs = Object.keys(removedFiles) removedFileIDs.forEach((fileID) => { @@ -1457,13 +1457,24 @@ export class Uppy< }, }) - this.calculateTotalProgress() + this.#updateTotalProgress() }, 500, { leading: true, trailing: true }, ) - calculateTotalProgress(): void { + #updateTotalProgress() { + const totalProgress = this.#calculateTotalProgress() + this.emit('progress', totalProgress) + this.setState({ totalProgress }) + } + + // eslint-disable-next-line class-methods-use-this, @typescript-eslint/explicit-module-boundary-types + private [Symbol.for('uppy test: updateTotalProgress')]() { + return this.#updateTotalProgress() + } + + #calculateTotalProgress() { // calculate total progress, using the number of files currently uploading, // multiplied by 100 and the summ of individual progress of each file const files = this.getFiles() @@ -1477,9 +1488,7 @@ export class Uppy< }) if (inProgress.length === 0) { - this.emit('progress', 0) - this.setState({ totalProgress: 0 }) - return + return 0 } const sizedFiles = inProgress.filter( @@ -1495,8 +1504,7 @@ export class Uppy< return acc + (file.progress.percentage as number) }, 0) const totalProgress = Math.round((currentProgress / progressMax) * 100) - this.setState({ totalProgress }) - return + return totalProgress } let totalSize = sizedFiles.reduce((acc, file) => { @@ -1522,8 +1530,7 @@ export class Uppy< totalProgress = 100 } - this.setState({ totalProgress }) - this.emit('progress', totalProgress) + return totalProgress } /** @@ -1659,7 +1666,7 @@ export class Uppy< }) } - this.calculateTotalProgress() + this.#updateTotalProgress() }) this.on('preprocess-progress', (file, progress) => { @@ -1729,7 +1736,7 @@ export class Uppy< this.on('restored', () => { // Files may have changed--ensure progress is still accurate. - this.calculateTotalProgress() + this.#updateTotalProgress() }) // @ts-expect-error should fix itself when dashboard it typed (also this doesn't belong here) From 3cd20a19fd904df53f3c288d3985ebed24bb4eb0 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 10 Nov 2024 16:59:38 +0800 Subject: [PATCH 04/17] fix type --- packages/@uppy/utils/src/emitSocketProgress.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/@uppy/utils/src/emitSocketProgress.ts b/packages/@uppy/utils/src/emitSocketProgress.ts index 9caf3718e5..e6abae1dd8 100644 --- a/packages/@uppy/utils/src/emitSocketProgress.ts +++ b/packages/@uppy/utils/src/emitSocketProgress.ts @@ -1,9 +1,9 @@ import throttle from 'lodash/throttle.js' +import type { Uppy } from '@uppy/core' import type { UppyFile } from './UppyFile.ts' -import type { FileProgress } from './FileProgress.ts' function emitSocketProgress( - uploader: any, + uploader: { uppy: Uppy }, progressData: { progress: string // pre-formatted percentage bytesTotal: number @@ -18,7 +18,7 @@ function emitSocketProgress( uploadStarted: file.progress.uploadStarted ?? 0, bytesUploaded, bytesTotal, - } satisfies FileProgress) + }) } } From c2e74d6d2bf6138006319bb637f16a959383e7b2 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sun, 10 Nov 2024 21:22:04 +0800 Subject: [PATCH 05/17] fix progress throttling only do it on total progress --- packages/@uppy/core/src/Uppy.test.ts | 13 +- packages/@uppy/core/src/Uppy.ts | 150 ++++++++++-------- .../@uppy/utils/src/emitSocketProgress.ts | 6 +- 3 files changed, 88 insertions(+), 81 deletions(-) diff --git a/packages/@uppy/core/src/Uppy.test.ts b/packages/@uppy/core/src/Uppy.test.ts index 270a3f0faf..9b47b2263e 100644 --- a/packages/@uppy/core/src/Uppy.test.ts +++ b/packages/@uppy/core/src/Uppy.test.ts @@ -1187,7 +1187,7 @@ describe('src/Core', () => { core.addUploader((fileIDs) => { fileIDs.forEach((fileID) => { const file = core.getFile(fileID) - if (/bar/.test(file.name)) { + if (file.name != null && /bar/.test(file.name)) { // @ts-ignore core.emit( 'upload-error', @@ -1701,6 +1701,9 @@ describe('src/Core', () => { const fileId = Object.keys(core.getState().files)[0] const file = core.getFile(fileId) + + core.emit('upload-start', [core.getFile(fileId)]) + // @ts-ignore core.emit('upload-progress', file, { bytesUploaded: 12345, @@ -1711,7 +1714,7 @@ describe('src/Core', () => { bytesUploaded: 12345, bytesTotal: 17175, uploadComplete: false, - uploadStarted: null, + uploadStarted: expect.any(Number), }) // @ts-ignore @@ -1720,14 +1723,12 @@ describe('src/Core', () => { bytesTotal: 17175, }) - core.calculateProgress.flush() - expect(core.getFile(fileId).progress).toEqual({ percentage: 100, bytesUploaded: 17175, bytesTotal: 17175, uploadComplete: false, - uploadStarted: null, + uploadStarted: expect.any(Number), }) }) @@ -1897,7 +1898,6 @@ describe('src/Core', () => { // @ts-ignore core[Symbol.for('uppy test: updateTotalProgress')]() - core.calculateProgress.flush() expect(core.getState().totalProgress).toEqual(66) }) @@ -1942,7 +1942,6 @@ describe('src/Core', () => { // @ts-ignore core[Symbol.for('uppy test: updateTotalProgress')]() - core.calculateProgress.flush() expect(core.getState().totalProgress).toEqual(66) expect(core.getState().allowNewUpload).toEqual(true) diff --git a/packages/@uppy/core/src/Uppy.ts b/packages/@uppy/core/src/Uppy.ts index 0bd8757844..2717894c02 100644 --- a/packages/@uppy/core/src/Uppy.ts +++ b/packages/@uppy/core/src/Uppy.ts @@ -1270,7 +1270,7 @@ export class Uppy< } this.setState(stateUpdate) - this.#updateTotalProgress() + this.#updateTotalProgressThrottled() const removedFileIDs = Object.keys(removedFiles) removedFileIDs.forEach((fileID) => { @@ -1416,59 +1416,80 @@ export class Uppy< }) } - // ___Why throttle at 500ms? - // - We must throttle at >250ms for superfocus in Dashboard to work well - // (because animation takes 0.25s, and we want to wait for all animations to be over before refocusing). - // [Practical Check]: if thottle is at 100ms, then if you are uploading a file, - // and click 'ADD MORE FILES', - focus won't activate in Firefox. - // - We must throttle at around >500ms to avoid performance lags. - // [Practical Check] Firefox, try to upload a big file for a prolonged period of time. Laptop will start to heat up. - // todo when uploading multiple files, this will cause problems because they share the same throttle, - // meaning some files might never get their progress reported (eaten up by progress events from other files) - calculateProgress = throttle( - (file, data) => { - const fileInState = this.getFile(file?.id) - if (file == null || !fileInState) { - this.log( - `Not setting progress for a file that has been removed: ${file?.id}`, - ) - return - } + #handleUploadProgress = ( + file: UppyFile | undefined, + progress: FileProgressStarted, + ) => { + const fileInState = file ? this.getFile(file.id) : undefined + if (file == null || !fileInState) { + this.log( + `Not setting progress for a file that has been removed: ${file?.id}`, + ) + return + } - if (fileInState.progress.percentage === 100) { - this.log( - `Not setting progress for a file that has been already uploaded: ${file.id}`, - ) - return - } + if (fileInState.progress.percentage === 100) { + this.log( + `Not setting progress for a file that has been already uploaded: ${file.id}`, + ) + return + } + const newProgress = { + bytesTotal: progress.bytesTotal, // bytesTotal may be null or zero; in that case we can't divide by it - const canHavePercentage = - Number.isFinite(data.bytesTotal) && data.bytesTotal > 0 + percentage: + ( + progress.bytesTotal != null && + Number.isFinite(progress.bytesTotal) && + progress.bytesTotal > 0 + ) ? + Math.round((progress.bytesUploaded / progress.bytesTotal) * 100) + : 0, + } + + if (fileInState.progress.uploadStarted != null) { this.setFileState(file.id, { progress: { ...fileInState.progress, - bytesUploaded: data.bytesUploaded, - bytesTotal: data.bytesTotal, - percentage: - canHavePercentage ? - Math.round((data.bytesUploaded / data.bytesTotal) * 100) - : 0, + bytesUploaded: progress.bytesUploaded, + ...newProgress, }, }) + } else { + this.setFileState(file.id, { + progress: { + ...fileInState.progress, + ...newProgress, + }, + }) + } - this.#updateTotalProgress() - }, - 500, - { leading: true, trailing: true }, - ) + this.#updateTotalProgressThrottled() + } #updateTotalProgress() { - const totalProgress = this.#calculateTotalProgress() + let totalProgress = Math.round(this.#calculateTotalProgress() * 100) + if (totalProgress > 100) totalProgress = 100 + else if (totalProgress < 0) totalProgress = 0 + this.emit('progress', totalProgress) this.setState({ totalProgress }) } + // ___Why throttle at 500ms? + // - We must throttle at >250ms for superfocus in Dashboard to work well + // (because animation takes 0.25s, and we want to wait for all animations to be over before refocusing). + // [Practical Check]: if thottle is at 100ms, then if you are uploading a file, + // and click 'ADD MORE FILES', - focus won't activate in Firefox. + // - We must throttle at around >500ms to avoid performance lags. + // [Practical Check] Firefox, try to upload a big file for a prolonged period of time. Laptop will start to heat up. + #updateTotalProgressThrottled = throttle( + () => this.#updateTotalProgress(), + 500, + { leading: true, trailing: true }, + ) + // eslint-disable-next-line class-methods-use-this, @typescript-eslint/explicit-module-boundary-types private [Symbol.for('uppy test: updateTotalProgress')]() { return this.#updateTotalProgress() @@ -1476,10 +1497,10 @@ export class Uppy< #calculateTotalProgress() { // calculate total progress, using the number of files currently uploading, - // multiplied by 100 and the summ of individual progress of each file + // between 0 and 1 and sum of individual progress of each file const files = this.getFiles() - const inProgress = files.filter((file) => { + const filesInProgress = files.filter((file) => { return ( file.progress.uploadStarted || file.progress.preprocess || @@ -1487,50 +1508,41 @@ export class Uppy< ) }) - if (inProgress.length === 0) { + if (filesInProgress.length === 0) { return 0 } - const sizedFiles = inProgress.filter( + const sizedFiles = filesInProgress.filter( (file) => file.progress.bytesTotal != null, ) - const unsizedFiles = inProgress.filter( + const unsizedFiles = filesInProgress.filter( (file) => file.progress.bytesTotal == null, ) if (sizedFiles.length === 0) { - const progressMax = inProgress.length * 100 - const currentProgress = unsizedFiles.reduce((acc, file) => { - return acc + (file.progress.percentage as number) - }, 0) - const totalProgress = Math.round((currentProgress / progressMax) * 100) - return totalProgress + const totalUnsizedProgress = unsizedFiles.reduce( + (acc, file) => acc + (file.progress.percentage ?? 0) / 100, + 0, + ) + + return totalUnsizedProgress / unsizedFiles.length } - let totalSize = sizedFiles.reduce((acc, file) => { + let totalFilesSize = sizedFiles.reduce((acc, file) => { return (acc + (file.progress.bytesTotal ?? 0)) as number }, 0) - const averageSize = totalSize / sizedFiles.length - totalSize += averageSize * unsizedFiles.length + const averageSize = totalFilesSize / sizedFiles.length + totalFilesSize += averageSize * unsizedFiles.length - let uploadedSize = 0 + let totalUploadedSize = 0 sizedFiles.forEach((file) => { - uploadedSize += file.progress.bytesUploaded as number + totalUploadedSize += file.progress.bytesUploaded || 0 }) unsizedFiles.forEach((file) => { - uploadedSize += (averageSize * (file.progress.percentage || 0)) / 100 + totalUploadedSize += averageSize * ((file.progress.percentage ?? 0) / 100) }) - let totalProgress = - totalSize === 0 ? 0 : Math.round((uploadedSize / totalSize) * 100) - - // hot fix, because: - // uploadedSize ended up larger than totalSize, resulting in 1325% total - if (totalProgress > 100) { - totalProgress = 100 - } - - return totalProgress + return totalFilesSize === 0 ? 0 : totalUploadedSize / totalFilesSize } /** @@ -1629,7 +1641,7 @@ export class Uppy< this.on('upload-start', onUploadStarted) - this.on('upload-progress', this.calculateProgress) + this.on('upload-progress', this.#handleUploadProgress) this.on('upload-success', (file, uploadResp) => { if (file == null || !this.getFile(file.id)) { @@ -1666,7 +1678,7 @@ export class Uppy< }) } - this.#updateTotalProgress() + this.#updateTotalProgressThrottled() }) this.on('preprocess-progress', (file, progress) => { @@ -1736,7 +1748,7 @@ export class Uppy< this.on('restored', () => { // Files may have changed--ensure progress is still accurate. - this.#updateTotalProgress() + this.#updateTotalProgressThrottled() }) // @ts-expect-error should fix itself when dashboard it typed (also this doesn't belong here) diff --git a/packages/@uppy/utils/src/emitSocketProgress.ts b/packages/@uppy/utils/src/emitSocketProgress.ts index e6abae1dd8..d5687b08ed 100644 --- a/packages/@uppy/utils/src/emitSocketProgress.ts +++ b/packages/@uppy/utils/src/emitSocketProgress.ts @@ -1,4 +1,3 @@ -import throttle from 'lodash/throttle.js' import type { Uppy } from '@uppy/core' import type { UppyFile } from './UppyFile.ts' @@ -22,7 +21,4 @@ function emitSocketProgress( } } -export default throttle(emitSocketProgress, 300, { - leading: true, - trailing: true, -}) +export default emitSocketProgress From a152913cb4d2850f569b7ed9c5d5cde6b8db82a5 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 11 Nov 2024 17:53:14 +0800 Subject: [PATCH 06/17] Improve progress in UI - only show progress percent and total bytes for files that we know the size of. (but all files will still be included in number of files) - use `null` as an unknown value for progress and ETA, allowing us to remove ETA from UI when unknown - `percentage` make use of `undefined` when progress is not yet known - don't show percentage in UI when unknown - add a new state field `progress` that's the same as `totalProgress` but can also be `null` --- packages/@uppy/core/src/Uppy.test.ts | 10 ++- packages/@uppy/core/src/Uppy.ts | 70 ++++++++++--------- packages/@uppy/dashboard/src/Dashboard.tsx | 2 +- .../dashboard/src/components/Dashboard.tsx | 2 +- .../@uppy/progress-bar/src/ProgressBar.tsx | 4 +- packages/@uppy/status-bar/src/Components.tsx | 19 ++--- packages/@uppy/status-bar/src/StatusBar.tsx | 29 +++++--- packages/@uppy/status-bar/src/StatusBarUI.tsx | 4 +- packages/@uppy/utils/src/FileProgress.ts | 4 ++ .../@uppy/utils/src/emitSocketProgress.ts | 4 +- 10 files changed, 87 insertions(+), 61 deletions(-) diff --git a/packages/@uppy/core/src/Uppy.test.ts b/packages/@uppy/core/src/Uppy.test.ts index 9b47b2263e..88bcb12184 100644 --- a/packages/@uppy/core/src/Uppy.test.ts +++ b/packages/@uppy/core/src/Uppy.test.ts @@ -282,6 +282,7 @@ describe('src/Core', () => { meta: {}, plugins: {}, totalProgress: 0, + progress: null, recoveredState: null, } @@ -316,6 +317,7 @@ describe('src/Core', () => { meta: {}, plugins: {}, totalProgress: 0, + progress: null, recoveredState: null, }) // new state @@ -335,6 +337,7 @@ describe('src/Core', () => { meta: {}, plugins: {}, totalProgress: 0, + progress: null, recoveredState: null, }) }) @@ -386,6 +389,7 @@ describe('src/Core', () => { meta: {}, plugins: {}, totalProgress: 0, + progress: null, recoveredState: null, }) }) @@ -577,6 +581,7 @@ describe('src/Core', () => { meta: {}, plugins: {}, totalProgress: 0, + progress: null, recoveredState: null, }) expect(plugin.mocks.uninstall.mock.calls.length).toEqual(1) @@ -1776,7 +1781,6 @@ describe('src/Core', () => { bytesUploaded: 0, // null indicates unsized bytesTotal: null, - percentage: 0, }) // @ts-ignore @@ -1849,8 +1853,8 @@ describe('src/Core', () => { // @ts-ignore core[Symbol.for('uppy test: updateTotalProgress')]() - // foo.jpg at 35%, bar.jpg at 0% - expect(core.getState().totalProgress).toBe(18) + // foo.jpg at 35%, bar.jpg has unknown size and will not be counted + expect(core.getState().totalProgress).toBe(36) core.destroy() }) diff --git a/packages/@uppy/core/src/Uppy.ts b/packages/@uppy/core/src/Uppy.ts index 2717894c02..2c05ccf122 100644 --- a/packages/@uppy/core/src/Uppy.ts +++ b/packages/@uppy/core/src/Uppy.ts @@ -232,7 +232,8 @@ export interface State details?: string | Record | null }> plugins: Plugins - totalProgress: number + totalProgress: number // todo remove backward compat + progress: number | null companion?: Record } @@ -361,6 +362,7 @@ type OmitFirstArg = T extends [any, ...infer U] ? U : never const defaultUploadState = { totalProgress: 0, + progress: null, allowNewUpload: true, error: null, recoveredState: null, @@ -758,7 +760,7 @@ export class Uppy< isUploadInProgress: boolean isSomeGhost: boolean } { - const { files: filesObject, totalProgress, error } = this.getState() + const { files: filesObject, progress: totalProgress, error } = this.getState() const files = Object.values(filesObject) const inProgressFiles: UppyFile[] = [] @@ -1445,15 +1447,15 @@ export class Uppy< progress.bytesTotal > 0 ) ? Math.round((progress.bytesUploaded / progress.bytesTotal) * 100) - : 0, + : undefined, } if (fileInState.progress.uploadStarted != null) { this.setFileState(file.id, { progress: { ...fileInState.progress, - bytesUploaded: progress.bytesUploaded, ...newProgress, + bytesUploaded: progress.bytesUploaded, }, }) } else { @@ -1469,12 +1471,19 @@ export class Uppy< } #updateTotalProgress() { - let totalProgress = Math.round(this.#calculateTotalProgress() * 100) - if (totalProgress > 100) totalProgress = 100 - else if (totalProgress < 0) totalProgress = 0 + const totalProgress = this.#calculateTotalProgress() + let totalProgressPercent: number | null = null; + if (totalProgress != null) { + totalProgressPercent = Math.round(totalProgress * 100) + if (totalProgressPercent > 100) totalProgressPercent = 100 + else if (totalProgressPercent < 0) totalProgressPercent = 0 + } - this.emit('progress', totalProgress) - this.setState({ totalProgress }) + this.emit('progress', totalProgressPercent ?? 0) // todo remove `?? 0` in next major + this.setState({ + totalProgress: totalProgressPercent ?? 0, // todo remove backward compat in next major + progress: totalProgressPercent, + }) } // ___Why throttle at 500ms? @@ -1512,35 +1521,31 @@ export class Uppy< return 0 } - const sizedFiles = filesInProgress.filter( - (file) => file.progress.bytesTotal != null, - ) - const unsizedFiles = filesInProgress.filter( - (file) => file.progress.bytesTotal == null, + const sizedFilesInProgress = filesInProgress.filter( + (file) => file.progress.bytesTotal != null && file.progress.bytesTotal !== 0, ) - if (sizedFiles.length === 0) { - const totalUnsizedProgress = unsizedFiles.reduce( - (acc, file) => acc + (file.progress.percentage ?? 0) / 100, - 0, - ) + if (sizedFilesInProgress.length === 0) { + return null // we don't have any files that we can know the percentage progress of + } - return totalUnsizedProgress / unsizedFiles.length + if (sizedFilesInProgress.every((file) => file.progress.uploadComplete)) { + // If every uploading file is complete, and we're still getting progress, it means either + // 1. there's a bug somewhere in some progress reporting code (maybe not even ours) + // and we're still getting progress, so let's just ignore it + // 2. there are files with unknown size (bytesTotal == null), still uploading, + // and we cannot say anything about their progress + // In any case, return null because it doesn't make any sense to show a progress + return null } - let totalFilesSize = sizedFiles.reduce((acc, file) => { - return (acc + (file.progress.bytesTotal ?? 0)) as number - }, 0) - const averageSize = totalFilesSize / sizedFiles.length - totalFilesSize += averageSize * unsizedFiles.length + const totalFilesSize = sizedFilesInProgress.reduce((acc, file) => ( + acc + (file.progress.bytesTotal ?? 0) + ), 0) - let totalUploadedSize = 0 - sizedFiles.forEach((file) => { - totalUploadedSize += file.progress.bytesUploaded || 0 - }) - unsizedFiles.forEach((file) => { - totalUploadedSize += averageSize * ((file.progress.percentage ?? 0) / 100) - }) + const totalUploadedSize = sizedFilesInProgress.reduce((acc, file) => ( + acc + (file.progress.bytesUploaded || 0) + ), 0) return totalFilesSize === 0 ? 0 : totalUploadedSize / totalFilesSize } @@ -1628,7 +1633,6 @@ export class Uppy< progress: { uploadStarted: Date.now(), uploadComplete: false, - percentage: 0, bytesUploaded: 0, bytesTotal: file.size, } as FileProgressStarted, diff --git a/packages/@uppy/dashboard/src/Dashboard.tsx b/packages/@uppy/dashboard/src/Dashboard.tsx index dc9a7f5768..137cd7af1a 100644 --- a/packages/@uppy/dashboard/src/Dashboard.tsx +++ b/packages/@uppy/dashboard/src/Dashboard.tsx @@ -1202,7 +1202,7 @@ export default class Dashboard extends UIPlugin< isAllComplete, isAllPaused, totalFileCount: Object.keys(files).length, - totalProgress: state.totalProgress, + totalProgress: state.progress, allowNewUpload, acquirers, theme, diff --git a/packages/@uppy/dashboard/src/components/Dashboard.tsx b/packages/@uppy/dashboard/src/components/Dashboard.tsx index 36d185e3f9..379247a618 100644 --- a/packages/@uppy/dashboard/src/components/Dashboard.tsx +++ b/packages/@uppy/dashboard/src/components/Dashboard.tsx @@ -44,7 +44,7 @@ type DashboardUIProps = { isAllComplete: boolean isAllPaused: boolean totalFileCount: number - totalProgress: number + totalProgress: number | null allowNewUpload: boolean acquirers: TargetWithRender[] theme: string diff --git a/packages/@uppy/progress-bar/src/ProgressBar.tsx b/packages/@uppy/progress-bar/src/ProgressBar.tsx index f4a5d9ac69..bae4f06ba0 100644 --- a/packages/@uppy/progress-bar/src/ProgressBar.tsx +++ b/packages/@uppy/progress-bar/src/ProgressBar.tsx @@ -40,10 +40,10 @@ export default class ProgressBar< } render(state: State): ComponentChild { - const progress = state.totalProgress || 0 + const { progress } = state // before starting and after finish should be hidden if specified in the options const isHidden = - (progress === 0 || progress === 100) && this.opts.hideAfterFinish + (progress == null || progress === 0 || progress === 100) && this.opts.hideAfterFinish return (
1 + const totalUploadedSizeStr = prettierBytes(totalUploadedSize) + + return (
{ifShowFilesUploadedOfTotal && @@ -289,14 +292,14 @@ function ProgressDetails(props: ProgressDetailsProps) { */} {ifShowFilesUploadedOfTotal && renderDot()} - {i18n('dataUploadedOfTotal', { - complete: prettierBytes(totalUploadedSize), + {totalSize !== 0 ? i18n('dataUploadedOfTotal', { + complete: totalUploadedSizeStr, total: prettierBytes(totalSize), - })} + }) : totalUploadedSizeStr} {renderDot()} - {i18n('xTimeLeft', { + {totalETA != null && i18n('xTimeLeft', { time: prettyETA(totalETA), })} @@ -355,7 +358,7 @@ function UploadNewlyAddedFiles(props: UploadNewlyAddedFilesProps) { interface ProgressBarUploadingProps { i18n: I18n supportsUploadProgress: boolean - totalProgress: number + totalProgress: number | null showProgressDetails: boolean | undefined isUploadStarted: boolean isAllComplete: boolean @@ -365,7 +368,7 @@ interface ProgressBarUploadingProps { complete: number totalUploadedSize: number totalSize: number - totalETA: number + totalETA: number | null startUpload: () => void } @@ -427,7 +430,7 @@ function ProgressBarUploading(props: ProgressBarUploadingProps) { : null}
- {supportsUploadProgress ? `${title}: ${totalProgress}%` : title} + {supportsUploadProgress && totalProgress != null ? `${title}: ${totalProgress}%` : title}
{renderProgressDetails()} diff --git a/packages/@uppy/status-bar/src/StatusBar.tsx b/packages/@uppy/status-bar/src/StatusBar.tsx index 9083288378..9012e1ec5a 100644 --- a/packages/@uppy/status-bar/src/StatusBar.tsx +++ b/packages/@uppy/status-bar/src/StatusBar.tsx @@ -102,11 +102,15 @@ export default class StatusBar extends UIPlugin< #computeSmoothETA(totalBytes: { uploaded: number - total: number - remaining: number - }): number { - if (totalBytes.total === 0 || totalBytes.remaining === 0) { - return 0 + total: number | null // null means indeterminate + }) { + if (totalBytes.total == null || totalBytes.total === 0) { + return null + } + + const remaining = totalBytes.total - totalBytes.uploaded + if (remaining <= 0) { + return null } // When state is restored, lastUpdateTime is still nullish at this point. @@ -131,7 +135,7 @@ export default class StatusBar extends UIPlugin< currentSpeed : emaFilter(currentSpeed, this.#previousSpeed, speedFilterHalfLife, dt) this.#previousSpeed = filteredSpeed - const instantETA = totalBytes.remaining / filteredSpeed + const instantETA = remaining / filteredSpeed const updatedPreviousETA = Math.max(this.#previousETA! - dt, 0) const filteredETA = @@ -155,7 +159,7 @@ export default class StatusBar extends UIPlugin< capabilities, files, allowNewUpload, - totalProgress, + progress: totalProgress, error, recoveredState, } = state @@ -182,14 +186,21 @@ export default class StatusBar extends UIPlugin< let totalSize = 0 let totalUploadedSize = 0 + // If at least one file has an unknown size, it doesn't make sense to display a total size + if (startedFiles.every((f) => f.progress.bytesTotal != null && f.progress.bytesTotal !== 0)) { + startedFiles.forEach((file) => { + totalSize += file.progress.bytesTotal || 0 + totalUploadedSize += file.progress.bytesUploaded || 0 + }) + } + startedFiles.forEach((file) => { - totalSize += file.progress.bytesTotal || 0 totalUploadedSize += file.progress.bytesUploaded || 0 }) + const totalETA = this.#computeSmoothETA({ uploaded: totalUploadedSize, total: totalSize, - remaining: totalSize - totalUploadedSize, }) return StatusBarUI({ diff --git a/packages/@uppy/status-bar/src/StatusBarUI.tsx b/packages/@uppy/status-bar/src/StatusBarUI.tsx index 6f5cc4480f..f3cbf1b76f 100644 --- a/packages/@uppy/status-bar/src/StatusBarUI.tsx +++ b/packages/@uppy/status-bar/src/StatusBarUI.tsx @@ -40,7 +40,7 @@ export interface StatusBarUIProps { hideRetryButton?: boolean recoveredState: State['recoveredState'] uploadState: (typeof statusBarStates)[keyof typeof statusBarStates] - totalProgress: number + totalProgress: number | null files: Record> supportsUploadProgress: boolean hideAfterFinish?: boolean @@ -55,7 +55,7 @@ export interface StatusBarUIProps { numUploads: number complete: number totalSize: number - totalETA: number + totalETA: number | null totalUploadedSize: number } diff --git a/packages/@uppy/utils/src/FileProgress.ts b/packages/@uppy/utils/src/FileProgress.ts index e493abb201..00ce4c8c8f 100644 --- a/packages/@uppy/utils/src/FileProgress.ts +++ b/packages/@uppy/utils/src/FileProgress.ts @@ -16,6 +16,10 @@ export type FileProcessingInfo = interface FileProgressBase { uploadComplete?: boolean percentage?: number + // note that Companion will send `bytesTotal` 0 if unknown size (not `null`). + // this is not perfect because some files can actually have a size of 0, + // and then we might think those files have an unknown size + // todo we should change this in companion bytesTotal: number | null preprocess?: FileProcessingInfo postprocess?: FileProcessingInfo diff --git a/packages/@uppy/utils/src/emitSocketProgress.ts b/packages/@uppy/utils/src/emitSocketProgress.ts index d5687b08ed..3a9212fa60 100644 --- a/packages/@uppy/utils/src/emitSocketProgress.ts +++ b/packages/@uppy/utils/src/emitSocketProgress.ts @@ -1,10 +1,10 @@ -import type { Uppy } from '@uppy/core' +import type { Uppy } from '@uppy/core/src/Uppy.js' import type { UppyFile } from './UppyFile.ts' function emitSocketProgress( uploader: { uppy: Uppy }, progressData: { - progress: string // pre-formatted percentage + progress: string // pre-formatted percentage number as a string bytesTotal: number bytesUploaded: number }, From 633e4014a94f4893b7baafed1cc457f206d48613 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 11 Nov 2024 18:06:59 +0800 Subject: [PATCH 07/17] fix build error --- .../companion-client/src/RequestClient.ts | 22 ++++++++++++++++- packages/@uppy/utils/package.json | 1 - .../@uppy/utils/src/emitSocketProgress.ts | 24 ------------------- 3 files changed, 21 insertions(+), 26 deletions(-) delete mode 100644 packages/@uppy/utils/src/emitSocketProgress.ts diff --git a/packages/@uppy/companion-client/src/RequestClient.ts b/packages/@uppy/companion-client/src/RequestClient.ts index 236ce62527..b80251c3c1 100644 --- a/packages/@uppy/companion-client/src/RequestClient.ts +++ b/packages/@uppy/companion-client/src/RequestClient.ts @@ -4,7 +4,6 @@ import pRetry, { AbortError } from 'p-retry' import fetchWithNetworkError from '@uppy/utils/lib/fetchWithNetworkError' import ErrorWithCause from '@uppy/utils/lib/ErrorWithCause' -import emitSocketProgress from '@uppy/utils/lib/emitSocketProgress' import getSocketHost from '@uppy/utils/lib/getSocketHost' import type Uppy from '@uppy/core' @@ -81,6 +80,27 @@ async function handleJSONResponse(res: Response): Promise { throw new HttpError({ statusCode: res.status, message: errMsg }) } +function emitSocketProgress( + uploader: { uppy: Uppy }, + progressData: { + progress: string // pre-formatted percentage number as a string + bytesTotal: number + bytesUploaded: number + }, + file: UppyFile, +): void { + const { progress, bytesUploaded, bytesTotal } = progressData + if (progress) { + uploader.uppy.log(`Upload progress: ${progress}`) + uploader.uppy.emit('upload-progress', file, { + uploadStarted: file.progress.uploadStarted ?? 0, + bytesUploaded, + bytesTotal, + }) + } +} + + export default class RequestClient { static VERSION = packageJson.version diff --git a/packages/@uppy/utils/package.json b/packages/@uppy/utils/package.json index 57bde2ea7a..4306a76b42 100644 --- a/packages/@uppy/utils/package.json +++ b/packages/@uppy/utils/package.json @@ -27,7 +27,6 @@ "./lib/dataURItoBlob": "./lib/dataURItoBlob.js", "./lib/dataURItoFile": "./lib/dataURItoFile.js", "./lib/emaFilter": "./lib/emaFilter.js", - "./lib/emitSocketProgress": "./lib/emitSocketProgress.js", "./lib/findAllDOMElements": "./lib/findAllDOMElements.js", "./lib/findDOMElement": "./lib/findDOMElement.js", "./lib/generateFileID": "./lib/generateFileID.js", diff --git a/packages/@uppy/utils/src/emitSocketProgress.ts b/packages/@uppy/utils/src/emitSocketProgress.ts deleted file mode 100644 index 3a9212fa60..0000000000 --- a/packages/@uppy/utils/src/emitSocketProgress.ts +++ /dev/null @@ -1,24 +0,0 @@ -import type { Uppy } from '@uppy/core/src/Uppy.js' -import type { UppyFile } from './UppyFile.ts' - -function emitSocketProgress( - uploader: { uppy: Uppy }, - progressData: { - progress: string // pre-formatted percentage number as a string - bytesTotal: number - bytesUploaded: number - }, - file: UppyFile, -): void { - const { progress, bytesUploaded, bytesTotal } = progressData - if (progress) { - uploader.uppy.log(`Upload progress: ${progress}`) - uploader.uppy.emit('upload-progress', file, { - uploadStarted: file.progress.uploadStarted ?? 0, - bytesUploaded, - bytesTotal, - }) - } -} - -export default emitSocketProgress From e3b2a1f699e4fc92a37dc0b1acbd771c77e0e673 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 11 Nov 2024 18:15:10 +0800 Subject: [PATCH 08/17] format --- .../companion-client/src/RequestClient.ts | 1 - packages/@uppy/core/src/Uppy.ts | 25 ++++++++++++------- .../@uppy/progress-bar/src/ProgressBar.tsx | 3 ++- packages/@uppy/status-bar/src/Components.tsx | 22 +++++++++------- packages/@uppy/status-bar/src/StatusBar.tsx | 6 ++++- 5 files changed, 36 insertions(+), 21 deletions(-) diff --git a/packages/@uppy/companion-client/src/RequestClient.ts b/packages/@uppy/companion-client/src/RequestClient.ts index b80251c3c1..a6d5a2fee2 100644 --- a/packages/@uppy/companion-client/src/RequestClient.ts +++ b/packages/@uppy/companion-client/src/RequestClient.ts @@ -100,7 +100,6 @@ function emitSocketProgress( } } - export default class RequestClient { static VERSION = packageJson.version diff --git a/packages/@uppy/core/src/Uppy.ts b/packages/@uppy/core/src/Uppy.ts index 2c05ccf122..d5ace85fc5 100644 --- a/packages/@uppy/core/src/Uppy.ts +++ b/packages/@uppy/core/src/Uppy.ts @@ -760,7 +760,11 @@ export class Uppy< isUploadInProgress: boolean isSomeGhost: boolean } { - const { files: filesObject, progress: totalProgress, error } = this.getState() + const { + files: filesObject, + progress: totalProgress, + error, + } = this.getState() const files = Object.values(filesObject) const inProgressFiles: UppyFile[] = [] @@ -1472,7 +1476,7 @@ export class Uppy< #updateTotalProgress() { const totalProgress = this.#calculateTotalProgress() - let totalProgressPercent: number | null = null; + let totalProgressPercent: number | null = null if (totalProgress != null) { totalProgressPercent = Math.round(totalProgress * 100) if (totalProgressPercent > 100) totalProgressPercent = 100 @@ -1522,7 +1526,8 @@ export class Uppy< } const sizedFilesInProgress = filesInProgress.filter( - (file) => file.progress.bytesTotal != null && file.progress.bytesTotal !== 0, + (file) => + file.progress.bytesTotal != null && file.progress.bytesTotal !== 0, ) if (sizedFilesInProgress.length === 0) { @@ -1539,13 +1544,15 @@ export class Uppy< return null } - const totalFilesSize = sizedFilesInProgress.reduce((acc, file) => ( - acc + (file.progress.bytesTotal ?? 0) - ), 0) + const totalFilesSize = sizedFilesInProgress.reduce( + (acc, file) => acc + (file.progress.bytesTotal ?? 0), + 0, + ) - const totalUploadedSize = sizedFilesInProgress.reduce((acc, file) => ( - acc + (file.progress.bytesUploaded || 0) - ), 0) + const totalUploadedSize = sizedFilesInProgress.reduce( + (acc, file) => acc + (file.progress.bytesUploaded || 0), + 0, + ) return totalFilesSize === 0 ? 0 : totalUploadedSize / totalFilesSize } diff --git a/packages/@uppy/progress-bar/src/ProgressBar.tsx b/packages/@uppy/progress-bar/src/ProgressBar.tsx index bae4f06ba0..ebc3e28984 100644 --- a/packages/@uppy/progress-bar/src/ProgressBar.tsx +++ b/packages/@uppy/progress-bar/src/ProgressBar.tsx @@ -43,7 +43,8 @@ export default class ProgressBar< const { progress } = state // before starting and after finish should be hidden if specified in the options const isHidden = - (progress == null || progress === 0 || progress === 100) && this.opts.hideAfterFinish + (progress == null || progress === 0 || progress === 100) && + this.opts.hideAfterFinish return (
{ifShowFilesUploadedOfTotal && @@ -292,16 +291,19 @@ function ProgressDetails(props: ProgressDetailsProps) { */} {ifShowFilesUploadedOfTotal && renderDot()} - {totalSize !== 0 ? i18n('dataUploadedOfTotal', { - complete: totalUploadedSizeStr, - total: prettierBytes(totalSize), - }) : totalUploadedSizeStr} + {totalSize !== 0 ? + i18n('dataUploadedOfTotal', { + complete: totalUploadedSizeStr, + total: prettierBytes(totalSize), + }) + : totalUploadedSizeStr} {renderDot()} - {totalETA != null && i18n('xTimeLeft', { - time: prettyETA(totalETA), - })} + {totalETA != null && + i18n('xTimeLeft', { + time: prettyETA(totalETA), + })}
) @@ -430,7 +432,9 @@ function ProgressBarUploading(props: ProgressBarUploadingProps) { : null}
- {supportsUploadProgress && totalProgress != null ? `${title}: ${totalProgress}%` : title} + {supportsUploadProgress && totalProgress != null ? + `${title}: ${totalProgress}%` + : title}
{renderProgressDetails()} diff --git a/packages/@uppy/status-bar/src/StatusBar.tsx b/packages/@uppy/status-bar/src/StatusBar.tsx index 9012e1ec5a..9996518b87 100644 --- a/packages/@uppy/status-bar/src/StatusBar.tsx +++ b/packages/@uppy/status-bar/src/StatusBar.tsx @@ -187,7 +187,11 @@ export default class StatusBar extends UIPlugin< let totalUploadedSize = 0 // If at least one file has an unknown size, it doesn't make sense to display a total size - if (startedFiles.every((f) => f.progress.bytesTotal != null && f.progress.bytesTotal !== 0)) { + if ( + startedFiles.every( + (f) => f.progress.bytesTotal != null && f.progress.bytesTotal !== 0, + ) + ) { startedFiles.forEach((file) => { totalSize += file.progress.bytesTotal || 0 totalUploadedSize += file.progress.bytesUploaded || 0 From 96832c452fbc18b53828906759eef9e48535774a Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Tue, 12 Nov 2024 17:47:33 +0800 Subject: [PATCH 09/17] fix progress when upload complete --- packages/@uppy/core/src/Uppy.ts | 33 ++++++++++++------- packages/@uppy/status-bar/src/Components.tsx | 6 ++-- packages/@uppy/status-bar/src/StatusBar.tsx | 8 +++-- packages/@uppy/status-bar/src/StatusBarUI.tsx | 2 +- packages/@uppy/utils/src/FileProgress.ts | 2 +- 5 files changed, 31 insertions(+), 20 deletions(-) diff --git a/packages/@uppy/core/src/Uppy.ts b/packages/@uppy/core/src/Uppy.ts index d5ace85fc5..9376155525 100644 --- a/packages/@uppy/core/src/Uppy.ts +++ b/packages/@uppy/core/src/Uppy.ts @@ -1513,6 +1513,7 @@ export class Uppy< // between 0 and 1 and sum of individual progress of each file const files = this.getFiles() + // note: also includes files that have completed uploading: const filesInProgress = files.filter((file) => { return ( file.progress.uploadStarted || @@ -1525,20 +1526,28 @@ export class Uppy< return 0 } - const sizedFilesInProgress = filesInProgress.filter( - (file) => - file.progress.bytesTotal != null && file.progress.bytesTotal !== 0, - ) - - if (sizedFilesInProgress.length === 0) { - return null // we don't have any files that we can know the percentage progress of + if (filesInProgress.every((file) => file.progress.uploadComplete)) { + // If every uploading file is complete, and we're still getting progress, it probably means + // there's a bug somewhere in some progress reporting code (maybe not even our code) + // and we're still getting progress, so let's just assume it means a 100% progress + return 1 } - if (sizedFilesInProgress.every((file) => file.progress.uploadComplete)) { - // If every uploading file is complete, and we're still getting progress, it means either - // 1. there's a bug somewhere in some progress reporting code (maybe not even ours) - // and we're still getting progress, so let's just ignore it - // 2. there are files with unknown size (bytesTotal == null), still uploading, + const isSizedFile = (file: UppyFile) => + file.progress.bytesTotal != null && file.progress.bytesTotal !== 0 + + const sizedFilesInProgress = filesInProgress.filter(isSizedFile) + const unsizedFilesInProgress = filesInProgress.filter( + (file) => !isSizedFile(file), + ) + + if ( + sizedFilesInProgress.every((file) => file.progress.uploadComplete) && + unsizedFilesInProgress.length > 0 && + !unsizedFilesInProgress.every((file) => file.progress.uploadComplete) + ) { + // we are done with uploading all files of known size, however + // there is at least one file with unknown size still uploading, // and we cannot say anything about their progress // In any case, return null because it doesn't make any sense to show a progress return null diff --git a/packages/@uppy/status-bar/src/Components.tsx b/packages/@uppy/status-bar/src/Components.tsx index 8d21b2923d..680b78cce1 100644 --- a/packages/@uppy/status-bar/src/Components.tsx +++ b/packages/@uppy/status-bar/src/Components.tsx @@ -265,7 +265,7 @@ interface ProgressDetailsProps { numUploads: number complete: number totalUploadedSize: number - totalSize: number + totalSize: number | null totalETA: number | null } @@ -291,7 +291,7 @@ function ProgressDetails(props: ProgressDetailsProps) { */} {ifShowFilesUploadedOfTotal && renderDot()} - {totalSize !== 0 ? + {totalSize != null ? i18n('dataUploadedOfTotal', { complete: totalUploadedSizeStr, total: prettierBytes(totalSize), @@ -369,7 +369,7 @@ interface ProgressBarUploadingProps { numUploads: number complete: number totalUploadedSize: number - totalSize: number + totalSize: number | null totalETA: number | null startUpload: () => void } diff --git a/packages/@uppy/status-bar/src/StatusBar.tsx b/packages/@uppy/status-bar/src/StatusBar.tsx index 9996518b87..5c124eefdd 100644 --- a/packages/@uppy/status-bar/src/StatusBar.tsx +++ b/packages/@uppy/status-bar/src/StatusBar.tsx @@ -183,21 +183,23 @@ export default class StatusBar extends UIPlugin< const resumableUploads = !!capabilities.resumableUploads const supportsUploadProgress = capabilities.uploadProgress !== false - let totalSize = 0 + let totalSize: number | null = null let totalUploadedSize = 0 - // If at least one file has an unknown size, it doesn't make sense to display a total size + // Only if all files have a known size, does it make sense to display a total size if ( startedFiles.every( (f) => f.progress.bytesTotal != null && f.progress.bytesTotal !== 0, ) ) { + totalSize = 0 startedFiles.forEach((file) => { - totalSize += file.progress.bytesTotal || 0 + totalSize! += file.progress.bytesTotal || 0 totalUploadedSize += file.progress.bytesUploaded || 0 }) } + // however uploaded size we will always have startedFiles.forEach((file) => { totalUploadedSize += file.progress.bytesUploaded || 0 }) diff --git a/packages/@uppy/status-bar/src/StatusBarUI.tsx b/packages/@uppy/status-bar/src/StatusBarUI.tsx index f3cbf1b76f..2d1ca2687d 100644 --- a/packages/@uppy/status-bar/src/StatusBarUI.tsx +++ b/packages/@uppy/status-bar/src/StatusBarUI.tsx @@ -54,7 +54,7 @@ export interface StatusBarUIProps { showProgressDetails?: boolean numUploads: number complete: number - totalSize: number + totalSize: number | null totalETA: number | null totalUploadedSize: number } diff --git a/packages/@uppy/utils/src/FileProgress.ts b/packages/@uppy/utils/src/FileProgress.ts index 00ce4c8c8f..0437e400d9 100644 --- a/packages/@uppy/utils/src/FileProgress.ts +++ b/packages/@uppy/utils/src/FileProgress.ts @@ -15,7 +15,7 @@ export type FileProcessingInfo = // TODO explore whether all of these properties need to be optional interface FileProgressBase { uploadComplete?: boolean - percentage?: number + percentage?: number // undefined if we don't know the percentage (e.g. for files with `bytesTotal` null) // note that Companion will send `bytesTotal` 0 if unknown size (not `null`). // this is not perfect because some files can actually have a size of 0, // and then we might think those files have an unknown size From a19f68edc9522dce1b72897e5c51447dc3a79d3f Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Tue, 12 Nov 2024 22:38:28 +0800 Subject: [PATCH 10/17] use execa for companion load balancer if not, then it leaves zombie companion instances running in the background when e2e stops have to be manually killed before running e2e again --- e2e/start-companion-with-load-balancer.mjs | 53 ++++----- package.json | 1 + yarn.lock | 124 ++++++++++++++++++++- 3 files changed, 144 insertions(+), 34 deletions(-) diff --git a/e2e/start-companion-with-load-balancer.mjs b/e2e/start-companion-with-load-balancer.mjs index c2d33d508b..eccec38f67 100755 --- a/e2e/start-companion-with-load-balancer.mjs +++ b/e2e/start-companion-with-load-balancer.mjs @@ -1,9 +1,10 @@ #!/usr/bin/env node -import { spawn } from 'node:child_process' import http from 'node:http' import httpProxy from 'http-proxy' import process from 'node:process' +import { execaNode } from 'execa'; + const numInstances = 3 const lbPort = 3020 @@ -49,41 +50,27 @@ function createLoadBalancer (baseUrls) { const isWindows = process.platform === 'win32' const isOSX = process.platform === 'darwin' -const startCompanion = ({ name, port }) => { - const cp = spawn(process.execPath, [ +const startCompanion = ({ name, port }) => execaNode('packages/@uppy/companion/src/standalone/start-server.js', { + nodeOptions: [ '-r', 'dotenv/config', // Watch mode support is limited to Windows and macOS at the time of writing. ...(isWindows || isOSX ? ['--watch-path', 'packages/@uppy/companion/src', '--watch'] : []), - './packages/@uppy/companion/src/standalone/start-server.js', - ], { - cwd: new URL('../', import.meta.url), - stdio: 'inherit', - env: { - // Note: these env variables will override anything set in .env - ...process.env, - COMPANION_PORT: port, - COMPANION_SECRET: 'development', // multi instance will not work without secret set - COMPANION_PREAUTH_SECRET: 'development', // multi instance will not work without secret set - COMPANION_ALLOW_LOCAL_URLS: 'true', - COMPANION_ENABLE_URL_ENDPOINT: 'true', - COMPANION_LOGGER_PROCESS_NAME: name, - COMPANION_CLIENT_ORIGINS: 'true', - }, - }) - // Adding a `then` property so the return value is awaitable: - return Object.defineProperty(cp, 'then', { - __proto__: null, - writable: true, - configurable: true, - value: Promise.prototype.then.bind(new Promise((resolve, reject) => { - cp.on('exit', (code) => { - if (code === 0) resolve(cp) - else reject(new Error(`Non-zero exit code: ${code}`)) - }) - cp.on('error', reject) - })), - }) -} + ], + cwd: new URL('../', import.meta.url), + stdio: 'inherit', + env: { + // Note: these env variables will override anything set in .env + ...process.env, + COMPANION_PORT: port, + COMPANION_SECRET: 'development', // multi instance will not work without secret set + COMPANION_PREAUTH_SECRET: 'development', // multi instance will not work without secret set + COMPANION_ALLOW_LOCAL_URLS: 'true', + COMPANION_ENABLE_URL_ENDPOINT: 'true', + COMPANION_LOGGER_PROCESS_NAME: name, + COMPANION_CLIENT_ORIGINS: 'true', + }, +}) + const hosts = Array.from({ length: numInstances }, (_, index) => { const port = companionStartPort + index diff --git a/package.json b/package.json index b1fa51fec6..6cdfd8e627 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,7 @@ "eslint-plugin-react": "^7.22.0", "eslint-plugin-react-hooks": "^4.2.0", "eslint-plugin-unicorn": "^53.0.0", + "execa": "^9.5.1", "github-contributors-list": "^1.2.4", "glob": "^8.0.0", "jsdom": "^24.0.0", diff --git a/yarn.lock b/yarn.lock index 426223c54b..5226905b7b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6195,6 +6195,13 @@ __metadata: languageName: node linkType: hard +"@sec-ant/readable-stream@npm:^0.4.1": + version: 0.4.1 + resolution: "@sec-ant/readable-stream@npm:0.4.1" + checksum: 10/aac89581652ac85debe7c5303451c2ebf8bf25ca25db680e4b9b73168f6940616d9a4bbe3348981827b1159b14e2f2e6af4b7bd5735cac898c12d5c51909c102 + languageName: node + linkType: hard + "@sideway/address@npm:^4.1.5": version: 4.1.5 resolution: "@sideway/address@npm:4.1.5" @@ -6290,6 +6297,13 @@ __metadata: languageName: node linkType: hard +"@sindresorhus/merge-streams@npm:^4.0.0": + version: 4.0.0 + resolution: "@sindresorhus/merge-streams@npm:4.0.0" + checksum: 10/16551c787f5328c8ef05fd9831ade64369ccc992df78deb635ec6c44af217d2f1b43f8728c348cdc4e00585ff2fad6e00d8155199cbf6b154acc45fe65cbf0aa + languageName: node + linkType: hard + "@sinonjs/commons@npm:^3.0.0": version: 3.0.1 resolution: "@sinonjs/commons@npm:3.0.1" @@ -8129,6 +8143,7 @@ __metadata: eslint-plugin-react: "npm:^7.22.0" eslint-plugin-react-hooks: "npm:^4.2.0" eslint-plugin-unicorn: "npm:^53.0.0" + execa: "npm:^9.5.1" github-contributors-list: "npm:^1.2.4" glob: "npm:^8.0.0" jsdom: "npm:^24.0.0" @@ -14974,6 +14989,26 @@ __metadata: languageName: node linkType: hard +"execa@npm:^9.5.1": + version: 9.5.1 + resolution: "execa@npm:9.5.1" + dependencies: + "@sindresorhus/merge-streams": "npm:^4.0.0" + cross-spawn: "npm:^7.0.3" + figures: "npm:^6.1.0" + get-stream: "npm:^9.0.0" + human-signals: "npm:^8.0.0" + is-plain-obj: "npm:^4.1.0" + is-stream: "npm:^4.0.1" + npm-run-path: "npm:^6.0.0" + pretty-ms: "npm:^9.0.0" + signal-exit: "npm:^4.1.0" + strip-final-newline: "npm:^4.0.0" + yoctocolors: "npm:^2.0.0" + checksum: 10/aa030cdd43ffbf6a8825c16eec1515729553ce3655a8fa5165f0ddab2320957a9783effbeff37662e238e6f5d979d9732e3baa4bcaaeba4360856e627a214177 + languageName: node + linkType: hard + "executable@npm:^4.1.1": version: 4.1.1 resolution: "executable@npm:4.1.1" @@ -15541,6 +15576,15 @@ __metadata: languageName: node linkType: hard +"figures@npm:^6.1.0": + version: 6.1.0 + resolution: "figures@npm:6.1.0" + dependencies: + is-unicode-supported: "npm:^2.0.0" + checksum: 10/9822d13630bee8e6a9f2da866713adf13854b07e0bfde042defa8bba32d47a1c0b2afa627ce73837c674cf9a5e3edce7e879ea72cb9ea7960b2390432d8e1167 + languageName: node + linkType: hard + "file-entry-cache@npm:^6.0.1": version: 6.0.1 resolution: "file-entry-cache@npm:6.0.1" @@ -16172,6 +16216,16 @@ __metadata: languageName: node linkType: hard +"get-stream@npm:^9.0.0": + version: 9.0.1 + resolution: "get-stream@npm:9.0.1" + dependencies: + "@sec-ant/readable-stream": "npm:^0.4.1" + is-stream: "npm:^4.0.1" + checksum: 10/ce56e6db6bcd29ca9027b0546af035c3e93dcd154ca456b54c298901eb0e5b2ce799c5d727341a100c99e14c523f267f1205f46f153f7b75b1f4da6d98a21c5e + languageName: node + linkType: hard + "get-symbol-description@npm:^1.0.2": version: 1.0.2 resolution: "get-symbol-description@npm:1.0.2" @@ -17017,6 +17071,13 @@ __metadata: languageName: node linkType: hard +"human-signals@npm:^8.0.0": + version: 8.0.0 + resolution: "human-signals@npm:8.0.0" + checksum: 10/89acdc7081ac2a065e41cca7351c4b0fe2382e213b7372f90df6a554e340f31b49388a307adc1d6f4c60b2b4fe81eeff0bc1f44be6f5d844311cd92ccc7831c6 + languageName: node + linkType: hard + "humanize-ms@npm:^1.2.1": version: 1.2.1 resolution: "humanize-ms@npm:1.2.1" @@ -17873,7 +17934,7 @@ __metadata: languageName: node linkType: hard -"is-plain-obj@npm:^4.0.0": +"is-plain-obj@npm:^4.0.0, is-plain-obj@npm:^4.1.0": version: 4.1.0 resolution: "is-plain-obj@npm:4.1.0" checksum: 10/6dc45da70d04a81f35c9310971e78a6a3c7a63547ef782e3a07ee3674695081b6ca4e977fbb8efc48dae3375e0b34558d2bcd722aec9bddfa2d7db5b041be8ce @@ -17959,6 +18020,13 @@ __metadata: languageName: node linkType: hard +"is-stream@npm:^4.0.1": + version: 4.0.1 + resolution: "is-stream@npm:4.0.1" + checksum: 10/cbea3f1fc271b21ceb228819d0c12a0965a02b57f39423925f99530b4eb86935235f258f06310b67cd02b2d10b49e9a0998f5ececf110ab7d3760bae4055ad23 + languageName: node + linkType: hard + "is-string@npm:^1.0.5, is-string@npm:^1.0.7": version: 1.0.7 resolution: "is-string@npm:1.0.7" @@ -18000,6 +18068,13 @@ __metadata: languageName: node linkType: hard +"is-unicode-supported@npm:^2.0.0": + version: 2.1.0 + resolution: "is-unicode-supported@npm:2.1.0" + checksum: 10/f254e3da6b0ab1a57a94f7273a7798dd35d1d45b227759f600d0fa9d5649f9c07fa8d3c8a6360b0e376adf916d151ec24fc9a50c5295c58bae7ca54a76a063f9 + languageName: node + linkType: hard + "is-weakmap@npm:^2.0.2": version: 2.0.2 resolution: "is-weakmap@npm:2.0.2" @@ -22914,6 +22989,16 @@ __metadata: languageName: node linkType: hard +"npm-run-path@npm:^6.0.0": + version: 6.0.0 + resolution: "npm-run-path@npm:6.0.0" + dependencies: + path-key: "npm:^4.0.0" + unicorn-magic: "npm:^0.3.0" + checksum: 10/1a1b50aba6e6af7fd34a860ba2e252e245c4a59b316571a990356417c0cdf0414cabf735f7f52d9c330899cb56f0ab804a8e21fb12a66d53d7843e39ada4a3b6 + languageName: node + linkType: hard + "npmlog@npm:^6.0.0": version: 6.0.2 resolution: "npmlog@npm:6.0.2" @@ -23643,6 +23728,13 @@ __metadata: languageName: node linkType: hard +"parse-ms@npm:^4.0.0": + version: 4.0.0 + resolution: "parse-ms@npm:4.0.0" + checksum: 10/673c801d9f957ff79962d71ed5a24850163f4181a90dd30c4e3666b3a804f53b77f1f0556792e8b2adbb5d58757907d1aa51d7d7dc75997c2a56d72937cbc8b7 + languageName: node + linkType: hard + "parse-node-version@npm:^1.0.1": version: 1.0.1 resolution: "parse-node-version@npm:1.0.1" @@ -24652,6 +24744,15 @@ __metadata: languageName: node linkType: hard +"pretty-ms@npm:^9.0.0": + version: 9.1.0 + resolution: "pretty-ms@npm:9.1.0" + dependencies: + parse-ms: "npm:^4.0.0" + checksum: 10/3622a8999e4b2aa05ff64bf48c7e58143b3ede6e3434f8ce5588def90ebcf6af98edf79532344c4c9e14d5ad25deb3f0f5ca9f9b91e5d2d1ac26dad9cf428fc0 + languageName: node + linkType: hard + "proc-log@npm:^2.0.0, proc-log@npm:^2.0.1": version: 2.0.1 resolution: "proc-log@npm:2.0.1" @@ -27890,6 +27991,13 @@ __metadata: languageName: node linkType: hard +"strip-final-newline@npm:^4.0.0": + version: 4.0.0 + resolution: "strip-final-newline@npm:4.0.0" + checksum: 10/b5fe48f695d74863153a3b3155220e6e9bf51f4447832998c8edec38e6559b3af87a9fe5ac0df95570a78a26f5fa91701358842eab3c15480e27980b154a145f + languageName: node + linkType: hard + "strip-indent@npm:^3.0.0": version: 3.0.0 resolution: "strip-indent@npm:3.0.0" @@ -29219,6 +29327,13 @@ __metadata: languageName: node linkType: hard +"unicorn-magic@npm:^0.3.0": + version: 0.3.0 + resolution: "unicorn-magic@npm:0.3.0" + checksum: 10/bdd7d7c522f9456f32a0b77af23f8854f9a7db846088c3868ec213f9550683ab6a2bdf3803577eacbafddb4e06900974385841ccb75338d17346ccef45f9cb01 + languageName: node + linkType: hard + "unified-args@npm:^11.0.0": version: 11.0.1 resolution: "unified-args@npm:11.0.1" @@ -31009,6 +31124,13 @@ __metadata: languageName: node linkType: hard +"yoctocolors@npm:^2.0.0": + version: 2.1.1 + resolution: "yoctocolors@npm:2.1.1" + checksum: 10/563fbec88bce9716d1044bc98c96c329e1d7a7c503e6f1af68f1ff914adc3ba55ce953c871395e2efecad329f85f1632f51a99c362032940321ff80c42a6f74d + languageName: node + linkType: hard + "zone.js@npm:~0.14.3": version: 0.14.7 resolution: "zone.js@npm:0.14.7" From c55cdf04ad5abedf3ce4ae383ae372084a3baec3 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Tue, 12 Nov 2024 22:39:09 +0800 Subject: [PATCH 11/17] update docs and tests for new state.progress --- docs/framework-integrations/react.mdx | 2 +- docs/uppy-core.mdx | 2 +- examples/react-example/App.tsx | 2 +- examples/react-native-expo/App.js | 2 +- packages/@uppy/core/src/Uppy.test.ts | 9 ++++++++- packages/@uppy/react/src/useUppyState.test.tsx | 10 ++++++++++ 6 files changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/framework-integrations/react.mdx b/docs/framework-integrations/react.mdx index 5d2267ea33..d596366b3f 100644 --- a/docs/framework-integrations/react.mdx +++ b/docs/framework-integrations/react.mdx @@ -74,7 +74,7 @@ times, this is needed if you are building a custom UI for Uppy in React. const [uppy] = useState(() => new Uppy()); const files = useUppyState(uppy, (state) => state.files); -const totalProgress = useUppyState(uppy, (state) => state.totalProgress); +const totalProgress = useUppyState(uppy, (state) => state.progress); // We can also get specific plugin state. // Note that the value on `plugins` depends on the `id` of the plugin. const metaFields = useUppyState( diff --git a/docs/uppy-core.mdx b/docs/uppy-core.mdx index d00a7160bd..4685d18d29 100644 --- a/docs/uppy-core.mdx +++ b/docs/uppy-core.mdx @@ -794,7 +794,7 @@ const state = { capabilities: { resumableUploads: false, }, - totalProgress: 0, + progress: null, meta: { ...this.opts.meta }, info: { isHidden: true, diff --git a/examples/react-example/App.tsx b/examples/react-example/App.tsx index 3387e8045c..014687a14a 100644 --- a/examples/react-example/App.tsx +++ b/examples/react-example/App.tsx @@ -77,7 +77,7 @@ export default function App() { uppy, (state) => Object.keys(state.files).length, ) - const totalProgress = useUppyState(uppy, (state) => state.totalProgress) + const totalProgress = useUppyState(uppy, (state) => state.progress) // Also possible to get the state of all plugins. const plugins = useUppyState(uppy, (state) => state.plugins) diff --git a/examples/react-native-expo/App.js b/examples/react-native-expo/App.js index f380b4251a..3454bb9a2a 100644 --- a/examples/react-native-expo/App.js +++ b/examples/react-native-expo/App.js @@ -39,7 +39,7 @@ export default function App () { setState({ progress: progress.bytesUploaded, total: progress.bytesTotal, - totalProgress: uppy.state.totalProgress, + totalProgress: uppy.state.progress, uploadStarted: true, }) }) diff --git a/packages/@uppy/core/src/Uppy.test.ts b/packages/@uppy/core/src/Uppy.test.ts index 88bcb12184..653c8230cd 100644 --- a/packages/@uppy/core/src/Uppy.test.ts +++ b/packages/@uppy/core/src/Uppy.test.ts @@ -361,7 +361,7 @@ describe('src/Core', () => { const coreStateUpdateEventMock = vi.fn() core.on('cancel-all', coreCancelEventMock) core.on('state-update', coreStateUpdateEventMock) - core.setState({ foo: 'bar', totalProgress: 30 }) + core.setState({ foo: 'bar', totalProgress: 30, progress: 30 }) core.cancelAll() @@ -1382,6 +1382,7 @@ describe('src/Core', () => { expect(core.getFiles().length).toEqual(1) core.setState({ totalProgress: 50, + progress: 30, }) const file = core.getFile(fileId) @@ -1390,6 +1391,7 @@ describe('src/Core', () => { expect(core.getFiles().length).toEqual(0) expect(fileRemovedEventMock.mock.calls[0][0]).toEqual(file) expect(core.getState().totalProgress).toEqual(0) + expect(core.getState().progress).toEqual(0) }) }) @@ -1796,6 +1798,7 @@ describe('src/Core', () => { }) expect(core.getState().totalProgress).toBe(36) + expect(core.getState().progress).toBe(36) // @ts-ignore finishUpload() @@ -1855,6 +1858,7 @@ describe('src/Core', () => { // foo.jpg at 35%, bar.jpg has unknown size and will not be counted expect(core.getState().totalProgress).toBe(36) + expect(core.getState().progress).toBe(36) core.destroy() }) @@ -1904,6 +1908,7 @@ describe('src/Core', () => { core[Symbol.for('uppy test: updateTotalProgress')]() expect(core.getState().totalProgress).toEqual(66) + expect(core.getState().progress).toBe(66) }) it('should emit the progress', () => { @@ -1948,6 +1953,7 @@ describe('src/Core', () => { core[Symbol.for('uppy test: updateTotalProgress')]() expect(core.getState().totalProgress).toEqual(66) + expect(core.getState().progress).toEqual(66) expect(core.getState().allowNewUpload).toEqual(true) expect(core.getState().error).toEqual(null) expect(core.getState().recoveredState).toEqual(null) @@ -1972,6 +1978,7 @@ describe('src/Core', () => { core.clear() expect(core.getState()).toMatchObject({ totalProgress: 0, + progress: null, allowNewUpload: true, error: null, recoveredState: null, diff --git a/packages/@uppy/react/src/useUppyState.test.tsx b/packages/@uppy/react/src/useUppyState.test.tsx index 7a8d376319..dd46815cdd 100644 --- a/packages/@uppy/react/src/useUppyState.test.tsx +++ b/packages/@uppy/react/src/useUppyState.test.tsx @@ -13,13 +13,23 @@ describe('useUppyState', () => { const { result, rerender } = renderHook(() => useUppyState(uppy, (state) => state.totalProgress), ) + const { result: result2, rerender: rerender2 } = renderHook(() => + useUppyState(uppy, (state) => state.progress), + ) expectTypeOf(result.current).toEqualTypeOf() + expectTypeOf(result2.current).toEqualTypeOf() expect(result.current).toBe(0) + expect(result2.current).toBe(null) + act(() => uppy.setState({ progress: 50 })) act(() => uppy.setState({ totalProgress: 50 })) rerender() + rerender2() expect(result.current).toBe(50) + expect(result2.current).toBe(50) rerender() + rerender2() expect(result.current).toBe(50) + expect(result2.current).toBe(50) }) it('does not re-render unnecessarily', () => { From 3997639c9cf66b99067a99642d36814193482fa1 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 25 Nov 2024 23:52:58 +0800 Subject: [PATCH 12/17] revert progress/totalProgress --- docs/framework-integrations/react.mdx | 2 +- docs/uppy-core.mdx | 2 +- examples/react-native-expo/App.js | 2 +- packages/@uppy/core/src/Uppy.test.ts | 14 +------------- packages/@uppy/core/src/Uppy.ts | 15 ++++----------- packages/@uppy/dashboard/src/Dashboard.tsx | 2 +- .../@uppy/dashboard/src/components/Dashboard.tsx | 2 +- packages/@uppy/progress-bar/src/ProgressBar.tsx | 8 ++++---- packages/@uppy/react/src/useUppyState.test.tsx | 10 ---------- packages/@uppy/status-bar/src/Components.tsx | 4 ++-- packages/@uppy/status-bar/src/StatusBar.tsx | 2 +- packages/@uppy/status-bar/src/StatusBarUI.tsx | 2 +- 12 files changed, 18 insertions(+), 47 deletions(-) diff --git a/docs/framework-integrations/react.mdx b/docs/framework-integrations/react.mdx index d596366b3f..5d2267ea33 100644 --- a/docs/framework-integrations/react.mdx +++ b/docs/framework-integrations/react.mdx @@ -74,7 +74,7 @@ times, this is needed if you are building a custom UI for Uppy in React. const [uppy] = useState(() => new Uppy()); const files = useUppyState(uppy, (state) => state.files); -const totalProgress = useUppyState(uppy, (state) => state.progress); +const totalProgress = useUppyState(uppy, (state) => state.totalProgress); // We can also get specific plugin state. // Note that the value on `plugins` depends on the `id` of the plugin. const metaFields = useUppyState( diff --git a/docs/uppy-core.mdx b/docs/uppy-core.mdx index 4685d18d29..d00a7160bd 100644 --- a/docs/uppy-core.mdx +++ b/docs/uppy-core.mdx @@ -794,7 +794,7 @@ const state = { capabilities: { resumableUploads: false, }, - progress: null, + totalProgress: 0, meta: { ...this.opts.meta }, info: { isHidden: true, diff --git a/examples/react-native-expo/App.js b/examples/react-native-expo/App.js index 3454bb9a2a..f380b4251a 100644 --- a/examples/react-native-expo/App.js +++ b/examples/react-native-expo/App.js @@ -39,7 +39,7 @@ export default function App () { setState({ progress: progress.bytesUploaded, total: progress.bytesTotal, - totalProgress: uppy.state.progress, + totalProgress: uppy.state.totalProgress, uploadStarted: true, }) }) diff --git a/packages/@uppy/core/src/Uppy.test.ts b/packages/@uppy/core/src/Uppy.test.ts index 653c8230cd..cb6b985a84 100644 --- a/packages/@uppy/core/src/Uppy.test.ts +++ b/packages/@uppy/core/src/Uppy.test.ts @@ -282,7 +282,6 @@ describe('src/Core', () => { meta: {}, plugins: {}, totalProgress: 0, - progress: null, recoveredState: null, } @@ -317,7 +316,6 @@ describe('src/Core', () => { meta: {}, plugins: {}, totalProgress: 0, - progress: null, recoveredState: null, }) // new state @@ -337,7 +335,6 @@ describe('src/Core', () => { meta: {}, plugins: {}, totalProgress: 0, - progress: null, recoveredState: null, }) }) @@ -361,7 +358,7 @@ describe('src/Core', () => { const coreStateUpdateEventMock = vi.fn() core.on('cancel-all', coreCancelEventMock) core.on('state-update', coreStateUpdateEventMock) - core.setState({ foo: 'bar', totalProgress: 30, progress: 30 }) + core.setState({ foo: 'bar', totalProgress: 30 }) core.cancelAll() @@ -389,7 +386,6 @@ describe('src/Core', () => { meta: {}, plugins: {}, totalProgress: 0, - progress: null, recoveredState: null, }) }) @@ -581,7 +577,6 @@ describe('src/Core', () => { meta: {}, plugins: {}, totalProgress: 0, - progress: null, recoveredState: null, }) expect(plugin.mocks.uninstall.mock.calls.length).toEqual(1) @@ -1382,7 +1377,6 @@ describe('src/Core', () => { expect(core.getFiles().length).toEqual(1) core.setState({ totalProgress: 50, - progress: 30, }) const file = core.getFile(fileId) @@ -1391,7 +1385,6 @@ describe('src/Core', () => { expect(core.getFiles().length).toEqual(0) expect(fileRemovedEventMock.mock.calls[0][0]).toEqual(file) expect(core.getState().totalProgress).toEqual(0) - expect(core.getState().progress).toEqual(0) }) }) @@ -1798,7 +1791,6 @@ describe('src/Core', () => { }) expect(core.getState().totalProgress).toBe(36) - expect(core.getState().progress).toBe(36) // @ts-ignore finishUpload() @@ -1858,7 +1850,6 @@ describe('src/Core', () => { // foo.jpg at 35%, bar.jpg has unknown size and will not be counted expect(core.getState().totalProgress).toBe(36) - expect(core.getState().progress).toBe(36) core.destroy() }) @@ -1908,7 +1899,6 @@ describe('src/Core', () => { core[Symbol.for('uppy test: updateTotalProgress')]() expect(core.getState().totalProgress).toEqual(66) - expect(core.getState().progress).toBe(66) }) it('should emit the progress', () => { @@ -1953,7 +1943,6 @@ describe('src/Core', () => { core[Symbol.for('uppy test: updateTotalProgress')]() expect(core.getState().totalProgress).toEqual(66) - expect(core.getState().progress).toEqual(66) expect(core.getState().allowNewUpload).toEqual(true) expect(core.getState().error).toEqual(null) expect(core.getState().recoveredState).toEqual(null) @@ -1978,7 +1967,6 @@ describe('src/Core', () => { core.clear() expect(core.getState()).toMatchObject({ totalProgress: 0, - progress: null, allowNewUpload: true, error: null, recoveredState: null, diff --git a/packages/@uppy/core/src/Uppy.ts b/packages/@uppy/core/src/Uppy.ts index 9376155525..3d65a7c1e1 100644 --- a/packages/@uppy/core/src/Uppy.ts +++ b/packages/@uppy/core/src/Uppy.ts @@ -232,8 +232,7 @@ export interface State details?: string | Record | null }> plugins: Plugins - totalProgress: number // todo remove backward compat - progress: number | null + totalProgress: number companion?: Record } @@ -362,7 +361,6 @@ type OmitFirstArg = T extends [any, ...infer U] ? U : never const defaultUploadState = { totalProgress: 0, - progress: null, allowNewUpload: true, error: null, recoveredState: null, @@ -760,11 +758,7 @@ export class Uppy< isUploadInProgress: boolean isSomeGhost: boolean } { - const { - files: filesObject, - progress: totalProgress, - error, - } = this.getState() + const { files: filesObject, totalProgress, error } = this.getState() const files = Object.values(filesObject) const inProgressFiles: UppyFile[] = [] @@ -1483,10 +1477,9 @@ export class Uppy< else if (totalProgressPercent < 0) totalProgressPercent = 0 } - this.emit('progress', totalProgressPercent ?? 0) // todo remove `?? 0` in next major + this.emit('progress', totalProgressPercent ?? 0) this.setState({ - totalProgress: totalProgressPercent ?? 0, // todo remove backward compat in next major - progress: totalProgressPercent, + totalProgress: totalProgressPercent ?? 0, }) } diff --git a/packages/@uppy/dashboard/src/Dashboard.tsx b/packages/@uppy/dashboard/src/Dashboard.tsx index 137cd7af1a..dc9a7f5768 100644 --- a/packages/@uppy/dashboard/src/Dashboard.tsx +++ b/packages/@uppy/dashboard/src/Dashboard.tsx @@ -1202,7 +1202,7 @@ export default class Dashboard extends UIPlugin< isAllComplete, isAllPaused, totalFileCount: Object.keys(files).length, - totalProgress: state.progress, + totalProgress: state.totalProgress, allowNewUpload, acquirers, theme, diff --git a/packages/@uppy/dashboard/src/components/Dashboard.tsx b/packages/@uppy/dashboard/src/components/Dashboard.tsx index 379247a618..36d185e3f9 100644 --- a/packages/@uppy/dashboard/src/components/Dashboard.tsx +++ b/packages/@uppy/dashboard/src/components/Dashboard.tsx @@ -44,7 +44,7 @@ type DashboardUIProps = { isAllComplete: boolean isAllPaused: boolean totalFileCount: number - totalProgress: number | null + totalProgress: number allowNewUpload: boolean acquirers: TargetWithRender[] theme: string diff --git a/packages/@uppy/progress-bar/src/ProgressBar.tsx b/packages/@uppy/progress-bar/src/ProgressBar.tsx index ebc3e28984..42a2036ca7 100644 --- a/packages/@uppy/progress-bar/src/ProgressBar.tsx +++ b/packages/@uppy/progress-bar/src/ProgressBar.tsx @@ -40,10 +40,10 @@ export default class ProgressBar< } render(state: State): ComponentChild { - const { progress } = state + const { totalProgress } = state // before starting and after finish should be hidden if specified in the options const isHidden = - (progress == null || progress === 0 || progress === 100) && + (totalProgress === 0 || totalProgress === 100) && this.opts.hideAfterFinish return (
-
{progress}
+
{totalProgress}
) } diff --git a/packages/@uppy/react/src/useUppyState.test.tsx b/packages/@uppy/react/src/useUppyState.test.tsx index dd46815cdd..7a8d376319 100644 --- a/packages/@uppy/react/src/useUppyState.test.tsx +++ b/packages/@uppy/react/src/useUppyState.test.tsx @@ -13,23 +13,13 @@ describe('useUppyState', () => { const { result, rerender } = renderHook(() => useUppyState(uppy, (state) => state.totalProgress), ) - const { result: result2, rerender: rerender2 } = renderHook(() => - useUppyState(uppy, (state) => state.progress), - ) expectTypeOf(result.current).toEqualTypeOf() - expectTypeOf(result2.current).toEqualTypeOf() expect(result.current).toBe(0) - expect(result2.current).toBe(null) - act(() => uppy.setState({ progress: 50 })) act(() => uppy.setState({ totalProgress: 50 })) rerender() - rerender2() expect(result.current).toBe(50) - expect(result2.current).toBe(50) rerender() - rerender2() expect(result.current).toBe(50) - expect(result2.current).toBe(50) }) it('does not re-render unnecessarily', () => { diff --git a/packages/@uppy/status-bar/src/Components.tsx b/packages/@uppy/status-bar/src/Components.tsx index 680b78cce1..45e0004fd9 100644 --- a/packages/@uppy/status-bar/src/Components.tsx +++ b/packages/@uppy/status-bar/src/Components.tsx @@ -360,7 +360,7 @@ function UploadNewlyAddedFiles(props: UploadNewlyAddedFilesProps) { interface ProgressBarUploadingProps { i18n: I18n supportsUploadProgress: boolean - totalProgress: number | null + totalProgress: number showProgressDetails: boolean | undefined isUploadStarted: boolean isAllComplete: boolean @@ -432,7 +432,7 @@ function ProgressBarUploading(props: ProgressBarUploadingProps) { : null}
- {supportsUploadProgress && totalProgress != null ? + {supportsUploadProgress && totalProgress !== 0 ? `${title}: ${totalProgress}%` : title}
diff --git a/packages/@uppy/status-bar/src/StatusBar.tsx b/packages/@uppy/status-bar/src/StatusBar.tsx index 5c124eefdd..2027abcd06 100644 --- a/packages/@uppy/status-bar/src/StatusBar.tsx +++ b/packages/@uppy/status-bar/src/StatusBar.tsx @@ -159,7 +159,7 @@ export default class StatusBar extends UIPlugin< capabilities, files, allowNewUpload, - progress: totalProgress, + totalProgress, error, recoveredState, } = state diff --git a/packages/@uppy/status-bar/src/StatusBarUI.tsx b/packages/@uppy/status-bar/src/StatusBarUI.tsx index 2d1ca2687d..cbefede30b 100644 --- a/packages/@uppy/status-bar/src/StatusBarUI.tsx +++ b/packages/@uppy/status-bar/src/StatusBarUI.tsx @@ -40,7 +40,7 @@ export interface StatusBarUIProps { hideRetryButton?: boolean recoveredState: State['recoveredState'] uploadState: (typeof statusBarStates)[keyof typeof statusBarStates] - totalProgress: number | null + totalProgress: number files: Record> supportsUploadProgress: boolean hideAfterFinish?: boolean From a6864cf383198ff26d40b8376197b75aa253b9e4 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 25 Nov 2024 23:54:21 +0800 Subject: [PATCH 13/17] improve doc --- docs/companion.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/companion.md b/docs/companion.md index 6e8f5537e1..37f3d8ec54 100644 --- a/docs/companion.md +++ b/docs/companion.md @@ -644,11 +644,11 @@ fully downloaded first, then uploaded. Defaults to `true`. #### `streamingUploadSizeless` `COMPANION_STREAMING_UPLOAD_SIZELESS` A boolean flag to tell Companion whether to also upload files that have an -unknown size. Currently this is only supported for Tus uploads. Note that this -requires an optional extension on the Tus server if using Tus uploads. For form -multipart uploads it requres a server that can handle -`transfer-encoding: chunked`. Default is `false`. If set to `true`, -`streamingUpload` also has to be set to `true`. +unknown size using streaming. If disabled, files of unknown size will be fully +downloaded first. Note that for Tus, this requires an optional extension on the +Tus server if using Tus uploads. For form multipart uploads it requres a server +that can handle `transfer-encoding: chunked`. Default is `false`. If set to +`true`, `streamingUpload` also has to be set to `true`. #### `maxFileSize` `COMPANION_MAX_FILE_SIZE` From 927db353d4e15c0b9659b9be70228a3ee4c087e3 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 25 Nov 2024 23:58:01 +0800 Subject: [PATCH 14/17] remove option streamingUploadSizeless we agreed that this can be considered not a breaking change --- docs/companion.md | 15 ++++----------- packages/@uppy/companion/src/config/companion.js | 1 - packages/@uppy/companion/src/server/Uploader.js | 6 +----- packages/@uppy/companion/src/standalone/helper.js | 1 - 4 files changed, 5 insertions(+), 18 deletions(-) diff --git a/docs/companion.md b/docs/companion.md index 37f3d8ec54..a2e563a870 100644 --- a/docs/companion.md +++ b/docs/companion.md @@ -329,7 +329,6 @@ const options = { logClientVersion: true, periodicPingUrls: [], streamingUpload: true, - streamingUploadSizeless: false, clientSocketConnectTimeout: 60000, metrics: true, }; @@ -639,16 +638,10 @@ Prometheus metrics (by default metrics are enabled.) A boolean flag to tell Companion whether to enable streaming uploads. If enabled, it will lead to _faster uploads_ because companion will start uploading at the same time as downloading using `stream.pipe`. If `false`, files will be -fully downloaded first, then uploaded. Defaults to `true`. - -#### `streamingUploadSizeless` `COMPANION_STREAMING_UPLOAD_SIZELESS` - -A boolean flag to tell Companion whether to also upload files that have an -unknown size using streaming. If disabled, files of unknown size will be fully -downloaded first. Note that for Tus, this requires an optional extension on the -Tus server if using Tus uploads. For form multipart uploads it requres a server -that can handle `transfer-encoding: chunked`. Default is `false`. If set to -`true`, `streamingUpload` also has to be set to `true`. +fully downloaded first, then uploaded. Note that for Tus, this requires an +optional extension on the Tus server if using Tus uploads. For form multipart +uploads it requres a server that can handle `transfer-encoding: chunked`. +Defaults to `true`. #### `maxFileSize` `COMPANION_MAX_FILE_SIZE` diff --git a/packages/@uppy/companion/src/config/companion.js b/packages/@uppy/companion/src/config/companion.js index cae8c77ed7..b847f3df7a 100644 --- a/packages/@uppy/companion/src/config/companion.js +++ b/packages/@uppy/companion/src/config/companion.js @@ -20,7 +20,6 @@ const defaultOptions = { allowLocalUrls: false, periodicPingUrls: [], streamingUpload: true, - streamingUploadSizeless: false, clientSocketConnectTimeout: 60000, metrics: true, } diff --git a/packages/@uppy/companion/src/server/Uploader.js b/packages/@uppy/companion/src/server/Uploader.js index c1bf7ba250..9c2e3dd64b 100644 --- a/packages/@uppy/companion/src/server/Uploader.js +++ b/packages/@uppy/companion/src/server/Uploader.js @@ -269,11 +269,7 @@ class Uploader { } _canStream() { - return this.options.companionOptions.streamingUpload && ( - this.options.size - // only tus uploads can be streamed without size, TODO: add also others - || this.options.companionOptions.streamingUploadSizeless - ) + return this.options.companionOptions.streamingUpload } /** diff --git a/packages/@uppy/companion/src/standalone/helper.js b/packages/@uppy/companion/src/standalone/helper.js index f631b1f1ea..e6ccd9887b 100644 --- a/packages/@uppy/companion/src/standalone/helper.js +++ b/packages/@uppy/companion/src/standalone/helper.js @@ -182,7 +182,6 @@ const getConfigFromEnv = () => { // cookieDomain is kind of a hack to support distributed systems. This should be improved but we never got so far. cookieDomain: process.env.COMPANION_COOKIE_DOMAIN, streamingUpload: process.env.COMPANION_STREAMING_UPLOAD ? process.env.COMPANION_STREAMING_UPLOAD === 'true' : undefined, - streamingUploadSizeless: process.env.COMPANION_STREAMING_UPLOAD_SIZELESS ? process.env.COMPANION_STREAMING_UPLOAD_SIZELESS === 'true' : undefined, maxFileSize: process.env.COMPANION_MAX_FILE_SIZE ? parseInt(process.env.COMPANION_MAX_FILE_SIZE, 10) : undefined, chunkSize: process.env.COMPANION_CHUNK_SIZE ? parseInt(process.env.COMPANION_CHUNK_SIZE, 10) : undefined, clientSocketConnectTimeout: process.env.COMPANION_CLIENT_SOCKET_CONNECT_TIMEOUT From ae08b3cbed2b277ce7c1bc4ddbde8087328b7415 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 25 Nov 2024 23:58:26 +0800 Subject: [PATCH 15/17] change progress the to "of unknown" --- packages/@uppy/status-bar/src/Components.tsx | 2 +- packages/@uppy/status-bar/src/locale.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/@uppy/status-bar/src/Components.tsx b/packages/@uppy/status-bar/src/Components.tsx index 45e0004fd9..747e7c74a1 100644 --- a/packages/@uppy/status-bar/src/Components.tsx +++ b/packages/@uppy/status-bar/src/Components.tsx @@ -296,7 +296,7 @@ function ProgressDetails(props: ProgressDetailsProps) { complete: totalUploadedSizeStr, total: prettierBytes(totalSize), }) - : totalUploadedSizeStr} + : i18n('dataUploadedOfUnknown', { complete: totalUploadedSizeStr })} {renderDot()} diff --git a/packages/@uppy/status-bar/src/locale.ts b/packages/@uppy/status-bar/src/locale.ts index 33aadfd909..d37d89bd6f 100644 --- a/packages/@uppy/status-bar/src/locale.ts +++ b/packages/@uppy/status-bar/src/locale.ts @@ -27,6 +27,7 @@ export default { }, // When `showProgressDetails` is set, shows the amount of bytes that have been uploaded so far. dataUploadedOfTotal: '%{complete} of %{total}', + dataUploadedOfUnknown: '%{complete} of unknown', // When `showProgressDetails` is set, shows an estimation of how long the upload will take to complete. xTimeLeft: '%{time} left', // Used as the label for the button that starts an upload. From 2491d206743454659c236659999c02292ffff9fc Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Tue, 26 Nov 2024 00:06:15 +0800 Subject: [PATCH 16/17] revert --- examples/react-example/App.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/react-example/App.tsx b/examples/react-example/App.tsx index 014687a14a..3387e8045c 100644 --- a/examples/react-example/App.tsx +++ b/examples/react-example/App.tsx @@ -77,7 +77,7 @@ export default function App() { uppy, (state) => Object.keys(state.files).length, ) - const totalProgress = useUppyState(uppy, (state) => state.progress) + const totalProgress = useUppyState(uppy, (state) => state.totalProgress) // Also possible to get the state of all plugins. const plugins = useUppyState(uppy, (state) => state.plugins) From ed279b5b5d3bd7bbdda9cee7a71077e883a8fe59 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Tue, 26 Nov 2024 19:15:44 +0800 Subject: [PATCH 17/17] remove companion doc --- docs/companion.md | 994 ---------------------------------------------- 1 file changed, 994 deletions(-) delete mode 100644 docs/companion.md diff --git a/docs/companion.md b/docs/companion.md deleted file mode 100644 index a2e563a870..0000000000 --- a/docs/companion.md +++ /dev/null @@ -1,994 +0,0 @@ ---- -sidebar_position: 4 ---- - -# Companion - -Companion is an open source server application which **takes away the complexity -of authentication and the cost of downloading files from remote sources**, such -as Instagram, Google Drive, and others. Companion is a server-to-server -orchestrator that streams files from a source to a destination, and files are -never stored in Companion. Companion can run either as a standalone -(self-hosted) application, [Transloadit-hosted](#hosted), or plugged in as an -Express middleware into an existing application. The Uppy client requests remote -files from Companion, which it will download and simultaneously upload to your -[Tus server](/docs/tus), [AWS bucket](/docs/aws-s3), or any server that supports -[PUT, POST or Multipart uploads](/docs/xhr-upload). - -This means a user uploading a 5GB video from Google Drive from their phone isn’t -eating into their data plans and you don’t have to worry about implementing -OAuth. - -## When should I use it? - -If you want to let users download files from [Box][], [Dropbox][], [Facebook][], -[Google Drive][googledrive], [Google Photos][googlephotos], [Instagram][], -[OneDrive][], [Unsplash][], [Import from URL][url], or [Zoom][] — you need -Companion. - -Companion supports the same [uploaders](/docs/guides/choosing-uploader) as Uppy: -[Tus](/docs/tus), [AWS S3](/docs/aws-s3), and [regular multipart](/docs/tus). -But instead of manually setting a plugin, Uppy sends along a header with the -uploader and Companion will use the same on the server. This means if you are -using [Tus](/docs/tus) for your local uploads, you can send your remote uploads -to the same Tus server (and likewise for your AWS S3 bucket). - -:::note - -Companion only deals with _remote_ files, _local_ files are still uploaded from -the client with your upload plugin. - -::: - -## Hosted - -Using [Transloadit][] services comes with a hosted version of Companion so you -don’t have to worry about hosting your own server. Whether you are on a free or -paid Transloadit [plan](https://transloadit.com/pricing/), you can use -Companion. It’s not possible to rent a Companion server without a Transloadit -plan. - -[**Sign-up for a (free) plan**](https://transloadit.com/pricing/). - -:::tip - -Choosing Transloadit for your file services also comes with credentials for all -remote providers. This means you don’t have to waste time going through the -approval process of every app. You can still add your own credentials in the -Transloadit admin page if you want. - -::: - -:::info - -Downloading and uploading files through Companion doesn’t count towards your -[monthly quota](https://transloadit.com/docs/faq/1gb-worth/), it’s a way for -files to arrive at Transloadit servers, much like Uppy. - -::: - -To do so each provider plugin must be configured with Transloadit’s Companion -URLs: - -```js -import { COMPANION_URL, COMPANION_ALLOWED_HOSTS } from '@uppy/transloadit'; -import Dropbox from '@uppy/dropbox'; - -uppy.use(Dropbox, { - companionUrl: COMPANION_URL, - companionAllowedHosts: COMPANION_ALLOWED_HOSTS, -}); -``` - -You may also hit rate limits, because the OAuth application is shared between -everyone using Transloadit. - -To solve that, you can use your own OAuth keys with Transloadit’s hosted -Companion servers by using Transloadit Template Credentials. [Create a Template -Credential][template-credentials] on the Transloadit site. Select “Companion -OAuth” for the service, and enter the key and secret for the provider you want -to use. Then you can pass the name of the new credentials to that provider: - -```js -import { COMPANION_URL, COMPANION_ALLOWED_HOSTS } from '@uppy/transloadit'; -import Dropbox from '@uppy/dropbox'; - -uppy.use(Dropbox, { - companionUrl: COMPANION_URL, - companionAllowedHosts: COMPANION_ALLOWED_HOSTS, - companionKeysParams: { - key: 'YOUR_TRANSLOADIT_API_KEY', - credentialsName: 'my_companion_dropbox_creds', - }, -}); -``` - -## Installation & use - -Companion is installed from npm. Depending on how you want to run Companion, the -install process is slightly different. Companion can be integrated as middleware -into your [Express](https://expressjs.com/) app or as a standalone server. Most -people probably want to run it as a standalone server, while the middleware -could be used to further customise Companion or integrate it into your own HTTP -server code. - -:::note - -Since v2, you need to be running `node.js >= v10.20.1` to use Companion. More -information in the -[migrating to 2.0](/docs/guides/migration-guides/#migrate-from-uppy-1x-to-2x) -guide. - -Windows is not a supported platform right now. It may work, and we’re happy to -accept improvements in this area, but we can’t provide support. - -::: - -### Standalone mode - -You can use the standalone version if you want to run Companion as it’s own -Node.js process. It’s a configured Express server with sessions, logging, and -security best practices. First you’ll typically want to install it globally: - -```bash -npm install -g @uppy/companion -``` - -Standalone Companion will always serve HTTP (not HTTPS) and expects a reverse -proxy with SSL termination in front of it when running in production. See -[`COMPANION_PROTOCOL`](#server) for more information. - -Companion ships with an executable file (`bin/companion`) which is the -standalone server. Unlike the middleware version, options are set via -environment variables. - -:::info - -Checkout [options](#options) for the available options in JS and environment -variable formats. - -::: - -You need at least these three to get started: - -```bash -export COMPANION_SECRET="shh!Issa Secret!" -export COMPANION_DOMAIN="YOUR SERVER DOMAIN" -export COMPANION_DATADIR="PATH/TO/DOWNLOAD/DIRECTORY" -``` - -Then run: - -```bash -companion -``` - -You can also pass in the path to your JSON config file, like so: - -```bash -companion --config /path/to/companion.json -``` - -You may also want to run Companion in a process manager like -[PM2](https://pm2.keymetrics.io/) to make sure it gets restarted on upon -crashing as well as allowing scaling to many instances. - -### Express middleware mode - -First install it into your Node.js project with your favorite package manager: - -```bash -npm install @uppy/companion -``` - -To plug Companion into an existing server, call its `.app` method, passing in an -[options](#options) object as a parameter. This returns a server instance that -you can mount on a route in your Express app. Note: do **not** use the `cors` -module in your project, because Companion already includes it. Use the -`corsOrigins` Companion option to customise CORS behavior. - -```js -import express from 'express'; -import bodyParser from 'body-parser'; -import session from 'express-session'; -import companion from '@uppy/companion'; - -const app = express(); - -// Companion requires body-parser and express-session middleware. -// You can add it like this if you use those throughout your app. -// -// If you are using something else in your app, you can add these -// middlewares in the same subpath as Companion instead. -app.use(bodyParser.json()); -app.use(session({ secret: 'some secrety secret' })); - -const companionOptions = { - providerOptions: { - drive: { - key: 'GOOGLE_DRIVE_KEY', - secret: 'GOOGLE_DRIVE_SECRET', - }, - }, - server: { - host: 'localhost:3020', - protocol: 'http', - // Default installations normally don't need a path. - // However if you specify a `path`, you MUST specify - // the same path in `app.use()` below, - // e.g. app.use('/companion', companionApp) - // path: '/companion', - }, - filePath: '/path/to/folder/', -}; - -const { app: companionApp } = companion.app(companionOptions); -app.use(companionApp); -``` - -Companion uses WebSockets to communicate progress, errors, and successes to the -client. This is what Uppy listens to to update it’s internal state and UI. - -Add the Companion WebSocket server using the `companion.socket` function: - -```js -const server = app.listen(PORT); - -companion.socket(server); -``` - -If WebSockets fail for some reason Uppy and Companion will fallback to HTTP -polling. - -### Running many instances - -We recommend running at least two instances in production, so that if the -Node.js event loop gets blocked by one or more requests (due to a bug or spike -in traffic), it doesn’t also block or slow down all other requests as well (as -Node.js is single threaded). - -As an example for scale, one enterprise customer of Transloadit, who self-hosts -Companion to power an education service that is used by many universities -globally, deploys 7 Companion instances. Their earlier solution ran on 35 -instances. In our general experience Companion will saturate network interface -cards before other resources on commodity virtual servers (`c5d.2xlarge` for -instance). - -Your mileage may vary, so we recommend to add observability. You can let -Prometheus crawl the `/metrics` endpoint and graph that with Grafana for -instance. - -#### Using unique endpoints - -One option is to run many instances with each instance having its own unique -endpoint. This could be on separate ports, (sub)domain names, or IPs. With this -setup, you can either: - -1. Implement your own logic that will direct each upload to a specific Companion - endpoint by setting the `companionUrl` option -2. Setting the Companion option `COMPANION_SELF_ENDPOINT`. This option will - cause Companion to respond with a `i-am` HTTP header containing the value - from `COMPANION_SELF_ENDPOINT`. When Uppy’s sees this header, it will pin all - requests for the upload to this endpoint. - -In either case, you would then also typically configure a single Companion -instance (one endpoint) to handle all OAuth authentication requests, so that you -only need to specify a single OAuth callback URL. See also `oauthDomain` and -`validHosts`. - -#### Using a load balancer - -The other option is to set up a load balancer in front of many Companion -instances. Then Uppy will only see a single endpoint and send all requests to -the associated load balancer, which will then distribute them between Companion -instances. The companion instances coordinate their messages and events over -Redis so that any instance can serve the client’s requests. Note that sticky -sessions are **not** needed with this setup. Here are the requirements for this -setup: - -- The instances need to be connected to the same Redis server. -- You need to set `COMPANION_SECRET` to the same value on both servers. -- if you use the `companionKeysParams` feature (Transloadit), you also need - `COMPANION_PREAUTH_SECRET` to be the same on each instance. -- All other configuration needs to be the same, except if you’re running many - instances on the same machine, then `COMPANION_PORT` should be different for - each instance. - -## API - -### Options - -:::tip - -The headings display the JS and environment variable options (`option` -`ENV_OPTION`). When integrating Companion into your own server, you pass the -options to `companion.app()`. If you are using the standalone version, you -configure Companion using environment variables. Some options only exist as -environment variables or only as a JS option. - -::: - -
- Default configuration - -```javascript -const options = { - server: { - protocol: 'http', - path: '', - }, - providerOptions: {}, - s3: { - endpoint: 'https://{service}.{region}.amazonaws.com', - conditions: [], - useAccelerateEndpoint: false, - getKey: ({ filename }) => `${crypto.randomUUID()}-${filename}`, - expires: 800, // seconds - }, - allowLocalUrls: false, - logClientVersion: true, - periodicPingUrls: [], - streamingUpload: true, - clientSocketConnectTimeout: 60000, - metrics: true, -}; -``` - -
- -#### `filePath` `COMPANION_DATADIR` - -Full path to the directory to which provider files will be downloaded -temporarily. - -#### `secret` `COMPANION_SECRET` `COMPANION_SECRET_FILE` - -A secret string which Companion uses to generate authorization tokens. You -should generate a long random string for this. For example: - -```js -const crypto = require('node:crypto'); - -const secret = crypto.randomBytes(64).toString('hex'); -``` - -:::caution - -Omitting the `secret` in the standalone version will generate a secret for you, -using the above `crypto` string. But when integrating with Express you must -provide it yourself. This is an essential security measure. - -::: - -:::note - -Using a secret file means passing an absolute path to a file with any extension, -which has only the secret, nothing else. - -::: - -#### `preAuthSecret` `COMPANION_PREAUTH_SECRET` `COMPANION_PREAUTH_SECRET_FILE` - -If you are using the [Transloadit](/docs/transloadit) `companionKeysParams` -feature (Transloadit-hosted Companion using your own custom OAuth credentials), -set this variable to a strong randomly generated secret. See also -`COMPANION_SECRET` (but do not use the same secret!) - -:::note - -Using a secret file means passing an absolute path to a file with any extension, -which has only the secret, nothing else. - -::: - -#### `uploadUrls` `COMPANION_UPLOAD_URLS` - -An allowlist (array) of strings (exact URLs) or regular expressions. Companion -will only accept uploads to these URLs. This ensures that your Companion -instance is only allowed to upload to your trusted servers and prevents -[SSRF](https://en.wikipedia.org/wiki/Server-side_request_forgery) attacks. - -#### `COMPANION_PORT` - -The port on which to start the standalone server, defaults to 3020. This is a -standalone-only option. - -#### `COMPANION_COOKIE_DOMAIN` - -Allows you to customize the domain of the cookies created for Express sessions. -This is a standalone-only option. - -#### `COMPANION_HIDE_WELCOME` - -Setting this to `true` disables the welcome message shown at `/`. This is a -standalone-only option. - -#### `redisUrl` `COMPANION_REDIS_URL` - -URL to running Redis server. This can be used to scale Companion horizontally -using many instances. See [How to scale Companion](#how-to-scale-companion). - -#### `COMPANION_REDIS_EXPRESS_SESSION_PREFIX` - -Set a custom prefix for redis keys created by -[connect-redis](https://github.com/tj/connect-redis). Defaults to -`companion-session:`. Sessions are used for storing authentication state and for -allowing thumbnails to be loaded by the browser via Companion and for OAuth2. -See also `COMPANION_REDIS_PUBSUB_SCOPE`. - -#### `redisOptions` `COMPANION_REDIS_OPTIONS` - -An object of -[options supported by the `ioredis` client](https://github.com/redis/ioredis). -See also -[`RedisOptions`](https://github.com/redis/ioredis/blob/af832752040e616daf51621681bcb40cab965a9b/lib/redis/RedisOptions.ts#L8). - -#### `redisPubSubScope` `COMPANION_REDIS_PUBSUB_SCOPE` - -Use a scope for the companion events at the Redis server. Setting this option -will prefix all events with the name provided and a colon. See also -`COMPANION_REDIS_EXPRESS_SESSION_PREFIX`. - -#### `server` - -Configuration options for the underlying server. - -| Key / Environment variable | Value | Description | -| ---------------------------------------- | ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `protocol` `COMPANION_PROTOCOL` | `http` or `https` | Used to build a URL to reference the Companion instance itself, which is used for headers and cookies. Companion itself always runs as a HTTP server, so locally you should use `http`. You must to set this to `https` once you enabled SSL/HTTPS for your domain in production by running a reverse https-proxy in front of Companion, or with a built-in HTTPS feature of your hosting service. | -| `host` `COMPANION_DOMAIN` | `String` | Your server’s publicly facing hostname (for example `example.com`). | -| `oauthDomain` `COMPANION_OAUTH_DOMAIN` | `String` | If you have several instances of Companion with different (and perhaps dynamic) subdomains, you can set a single fixed subdomain and server (such as `sub1.example.com`) to handle your OAuth authentication for you. This would then redirect back to the correct instance with the required credentials on completion. This way you only need to configure a single callback URL for OAuth providers. | -| `path` `COMPANION_PATH` | `String` | The server path to where the Companion app is sitting. For instance, if Companion is at `example.com/companion`, then the path would be `/companion`). | -| `implicitPath` `COMPANION_IMPLICIT_PATH` | `String` | If the URL’s path in your reverse proxy is different from your Companion path in your express app, then you need to set this path as `implicitPath`. For instance, if your Companion URL is `example.com/mypath/companion`. Where the path `/mypath` is defined in your NGINX server, while `/companion` is set in your express app. Then you need to set the option `implicitPath` to `/mypath`, and set the `path` option to `/companion`. | -| `validHosts` `COMPANION_DOMAINS` | `Array` | If you are setting an `oauthDomain`, you need to set a list of valid hosts, so the oauth handler can validate the host of the Uppy instance requesting the authentication. This is essentially a list of valid domains running your Companion instances. The list may also contain regex patterns. e.g `['sub2.example.com', 'sub3.example.com', '(\\w+).example.com']` | - -#### `sendSelfEndpoint` `COMPANION_SELF_ENDPOINT` - -This is essentially the same as the `server.host + server.path` attributes. The -major reason for this attribute is that, when set, it adds the value as the -`i-am` header of every request response. - -#### `providerOptions` - -Object to enable providers with their keys and secrets. For example: - -```json -{ - "drive": { - "key": "***", - "secret": "***" - } -} -``` - -When using the standalone version you use the corresponding environment -variables or point to a secret file (such as `COMPANION_GOOGLE_SECRET_FILE`). - -:::note - -Secret files need an absolute path to a file with any extension which only has -the secret, nothing else. - -::: - -| Service | Key | Environment variables | -| ------------- | -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Box | `box` | `COMPANION_BOX_KEY`, `COMPANION_BOX_SECRET`, `COMPANION_BOX_SECRET_FILE` | -| Dropbox | `dropbox` | `COMPANION_DROPBOX_KEY`, `COMPANION_DROPBOX_SECRET`, `COMPANION_DROPBOX_SECRET_FILE` | -| Facebook | `facebook` | `COMPANION_FACEBOOK_KEY`, `COMPANION_FACEBOOK_SECRET`, `COMPANION_FACEBOOK_SECRET_FILE` | -| Google Drive | `drive` | `COMPANION_GOOGLE_KEY`, `COMPANION_GOOGLE_SECRET`, `COMPANION_GOOGLE_SECRET_FILE` | -| Google Photos | `googlephotos` | `COMPANION_GOOGLE_KEY`, `COMPANION_GOOGLE_SECRET`, `COMPANION_GOOGLE_SECRET_FILE` | -| Instagram | `instagram` | `COMPANION_INSTAGRAM_KEY`, `COMPANION_INSTAGRAM_SECRET`, `COMPANION_INSTAGRAM_SECRET_FILE` | -| OneDrive | `onedrive` | `COMPANION_ONEDRIVE_KEY`, `COMPANION_ONEDRIVE_SECRET`, `COMPANION_ONEDRIVE_SECRET_FILE`, `COMPANION_ONEDRIVE_DOMAIN_VALIDATION` (Settings this variable to `true` enables a route that can be used to validate your app with OneDrive) | -| Zoom | `zoom` | `COMPANION_ZOOM_KEY`, `COMPANION_ZOOM_SECRET`, `COMPANION_ZOOM_SECRET_FILE`, `COMPANION_ZOOM_VERIFICATION_TOKEN` | - -#### `s3` - -Companion comes with signature endpoints for AWS S3. These can be used by the -Uppy client to sign requests to upload files directly to S3, without exposing -secret S3 keys in the browser. Companion also supports uploading files from -providers like Dropbox and Instagram directly into S3. - -##### `s3.key` `COMPANION_AWS_KEY` - -The S3 access key ID. - -##### `s3.secret` `COMPANION_AWS_SECRET` `COMPANION_AWS_SECRET_FILE` - -The S3 secret access key. - -:::note - -Using a secret file means passing an absolute path to a file with any extension, -which has only the secret, nothing else. - -::: - -##### `s3.endpoint` `COMPANION_AWS_ENDPOINT` - -Optional URL to a custom S3 (compatible) service. Otherwise uses the default -from the AWS SDK. - -##### `s3.bucket` `COMPANION_AWS_BUCKET` - -The name of the bucket to store uploaded files in. - -A `string` or function that returns the name of the bucket as a `string` and -takes one argument which is an object with the following properties: - -- `filename`, the original name of the uploaded file; -- `metadata` provided by the user for the file (will only be provided during the - initial calls for each uploaded files, otherwise it will be `undefined`). -- `req`, Express.js `Request` object. Do not use any Companion internals from - the req object, as these might change in any minor version of Companion. - -#### `s3.forcePathStyle` `COMPANION_AWS_FORCE_PATH_STYLE` - -This adds support for setting the S3 client’s `forcePathStyle` option. That is -necessary to use Uppy/Companion alongside localstack in development -environments. **Default**: `false`. - -##### `s3.region` `COMPANION_AWS_REGION` - -The datacenter region where the target bucket is located. - -##### `COMPANION_AWS_PREFIX` - -An optional prefix for all uploaded keys. This is a standalone-only option. The -same can be achieved by the `getKey` option when using the express middleware. - -##### `s3.awsClientOptions` - -You can supply any -[S3 option supported by the AWS SDK](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property) -in the `providerOptions.s3.awsClientOptions` object, _except for_ the below: - -- `accessKeyId`. Instead, use the `providerOptions.s3.key` property. This is to - make configuration names consistent between different Companion features. -- `secretAccessKey`. Instead, use the `providerOptions.s3.secret` property. This - is to make configuration names consistent between different Companion - features. - -Be aware that some options may cause wrong behaviour if they conflict with -Companion’s assumptions. If you find that a particular option does not work as -expected, please -[open an issue on the Uppy repository](https://github.com/transloadit/uppy/issues/new) -so we can document it here. - -##### `s3.getKey({ filename, metadata, req })` - -Get the key name for a file. The key is the file path to which the file will be -uploaded in your bucket. This option should be a function receiving three -arguments: - -- `filename`, the original name of the uploaded file; -- `metadata`, user-provided metadata for the file. -- `req`, Express.js `Request` object. Do not use any Companion internals from - the req object, as these might change in any minor version of Companion. - -This function should return a string `key`. The `req` parameter can be used to -upload to a user-specific folder in your bucket, for example: - -```js -app.use(authenticationMiddleware); -app.use( - uppy.app({ - providerOptions: { - s3: { - getKey: ({ req, filename, metadata }) => `${req.user.id}/${filename}`, - /* auth options */ - }, - }, - }), -); -``` - -The default implementation returns the `filename`, so all files will be uploaded -to the root of the bucket as their original file name. - -```js -app.use( - uppy.app({ - providerOptions: { - s3: { - getKey: ({ filename, metadata }) => filename, - }, - }, - }), -); -``` - -When signing on the client, this function will only be called for multipart -uploads. - -#### `COMPANION_AWS_USE_ACCELERATE_ENDPOINT` - -Enable S3 -[Transfer Acceleration](https://docs.aws.amazon.com/AmazonS3/latest/userguide/transfer-acceleration.html). -This is a standalone-only option. - -#### `COMPANION_AWS_EXPIRES` - -Set `X-Amz-Expires` query parameter in the presigned urls (in seconds, default: -300\). This is a standalone-only option. - -#### `COMPANION_AWS_ACL` - -Set a -[Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl) -for uploaded objects. This is a standalone-only option. - -#### `customProviders` - -This option enables you to add custom providers along with the already supported -providers. See [adding custom providers](#how-to-add-custom-providers) for more -information. - -#### `logClientVersion` - -A boolean flag to tell Companion whether to log its version upon startup. - -#### `metrics` `COMPANION_HIDE_METRICS` - -A boolean flag to tell Companion whether to provide an endpoint `/metrics` with -Prometheus metrics (by default metrics are enabled.) - -#### `streamingUpload` `COMPANION_STREAMING_UPLOAD` - -A boolean flag to tell Companion whether to enable streaming uploads. If -enabled, it will lead to _faster uploads_ because companion will start uploading -at the same time as downloading using `stream.pipe`. If `false`, files will be -fully downloaded first, then uploaded. Note that for Tus, this requires an -optional extension on the Tus server if using Tus uploads. For form multipart -uploads it requres a server that can handle `transfer-encoding: chunked`. -Defaults to `true`. - -#### `maxFileSize` `COMPANION_MAX_FILE_SIZE` - -If this value is set, companion will limit the maximum file size to process. If -unset, it will process files without any size limit (this is the default). - -#### `periodicPingUrls` `COMPANION_PERIODIC_PING_URLS` - -If this value is set, companion will periodically send POST requests to the -specified URLs. Useful for keeping track of companion instances as a keep-alive. - -#### `periodicPingInterval` `COMPANION_PERIODIC_PING_INTERVAL` - -Interval for periodic ping requests (in ms). - -#### `periodicPingStaticPayload` `COMPANION_PERIODIC_PING_STATIC_JSON_PAYLOAD` - -A `JSON.stringify`-able JavaScript Object that will be sent as part of the JSON -body in the period ping requests. - -#### `allowLocalUrls` `COMPANION_ALLOW_LOCAL_URLS` - -A boolean flag to tell Companion whether to allow requesting local URLs -(non-internet IPs). - -:::caution - -Only enable this in development. **Enabling it in production is a security -risk.** - -::: - -#### `corsOrigins` (required) - -Allowed CORS Origins. Passed as the `origin` option in -[cors](https://github.com/expressjs/cors#configuration-options). - -Note this is used for both CORS’ `Access-Control-Allow-Origin` header, and for -the -[`targetOrigin`](https://developer.mozilla.org/en-US/docs/Web/API/Window/postMessage#targetorigin) -for `postMessage` calls in the context of OAuth. - -Setting it to `true` treats any origin as a trusted one, making it easier to -impersonate your brand. Setting it to `false` disables cross-origin support, use -this if you’re serving Companion and Uppy from the same domain name. - -##### `COMPANION_CLIENT_ORIGINS` - -Stand-alone alternative to the `corsOrigins` option. A comma-separated string of -origins, or `'true'` (which will be interpreted as the boolean value `true`), or -`'false'` (which will be interpreted as the boolean value `false`). -`COMPANION_CLIENT_ORIGINS_REGEX` will be ignored if this option is used. - -##### `COMPANION_CLIENT_ORIGINS_REGEX` - -:::note - -In most cases, you should not be using a regex, and instead provide the list of -accepted origins to `COMPANION_CLIENT_ORIGINS`. If you have to use this option, -have in mind that this regex will be used to parse unfiltered user input, so -make sure you’re validating the entirety of the string. - -::: - -Stand-alone alternative to the `corsOrigins` option. Like -`COMPANION_CLIENT_ORIGINS`, but allows a single regex instead. - -#### `chunkSize` `COMPANION_CHUNK_SIZE` - -Controls how big the uploaded chunks are for AWS S3 Multipart and Tus. Smaller -values lead to more overhead, but larger values lead to slower retries in case -of bad network connections. Passed to tus-js-client -[`chunkSize`](https://github.com/tus/tus-js-client/blob/master/docs/api.md#chunksize) -as well as -[AWS S3 Multipart](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html) -`partSize`. - -#### `enableUrlEndpoint` `COMPANION_ENABLE_URL_ENDPOINT` - -Set this to `true` to enable the [URL functionalily](https://uppy.io/docs/url/). -Default: `false`. - -### Events - -The object returned by `companion.app()` also has a property `companionEmitter` -which is an `EventEmitter` that emits the following events: - -- `upload-start` - When an upload starts, this event is emitted with an object - containing the property `token`, which is a unique ID for the upload. -- **token** - The event name is the token from `upload-start`. The event has an - object with the following properties: - - `action` - One of the following strings: - - `success` - When the upload succeeds. - - `error` - When the upload fails with an error. - - `payload` - the error or success payload. - -Example code for using the `EventEmitter` to handle a finished file upload: - -```js -const companionApp = companion.app(options); -const { companionEmitter: emitter } = companionApp; - -emitter.on('upload-start', ({ token }) => { - console.log('Upload started', token); - - function onUploadEvent({ action, payload }) { - if (action === 'success') { - emitter.off(token, onUploadEvent); // avoid listener leak - console.log('Upload finished', token, payload.url); - } else if (action === 'error') { - emitter.off(token, onUploadEvent); // avoid listener leak - console.error('Upload failed', payload); - } - } - emitter.on(token, onUploadEvent); -}); -``` - - - -## Frequently asked questions - -### Do you have a live example? - -An example server is running at . - -### How does the Authentication and Token mechanism work? - -This section describes how Authentication works between Companion and Providers. -While this behaviour is the same for all Providers (Dropbox, Instagram, Google -Drive, etc.), we are going to be referring to Dropbox in place of any Provider -throughout this section. - -The following steps describe the actions that take place when a user -Authenticates and Uploads from Dropbox through Companion: - -- The visitor to a website with Uppy clicks `Connect to Dropbox`. -- Uppy sends a request to Companion, which in turn sends an OAuth request to - Dropbox (Requires that OAuth credentials from Dropbox have been added to - Companion). -- Dropbox asks the visitor to log in, and whether the Website should be allowed - to access your files -- If the visitor agrees, Companion will receive a token from Dropbox, with which - we can temporarily download files. -- Companion encrypts the token with a secret key and sends the encrypted token - to Uppy (client) -- Every time the visitor clicks on a folder in Uppy, it asks Companion for the - new list of files, with this question, the token (still encrypted by - Companion) is sent along. -- Companion decrypts the token, requests the list of files from Dropbox and - sends it to Uppy. -- When a file is selected for upload, Companion receives the token again - according to this procedure, decrypts it again, and thereby downloads the file - from Dropbox. -- As the bytes arrive, Companion uploads the bytes to the final destination - (depending on the configuration: Apache, a Tus server, S3 bucket, etc). -- Companion reports progress to Uppy, as if it were a local upload. -- Completed! - -### How to use provider redirect URIs? - -When generating your provider API keys on their corresponding developer -platforms (e.g -[Google Developer Console](https://console.developers.google.com/)), you’d need -to provide a `redirect URI` for the OAuth authorization process. In general the -redirect URI for each provider takes the format: - -`http(s)://$YOUR_COMPANION_HOST_NAME/$PROVIDER_NAME/redirect` - -For example, if your Companion server is hosted on -`https://my.companion.server.com`, then the redirect URI you would supply for -your OneDrive provider would be: - -`https://my.companion.server.com/onedrive/redirect` - -Please see -[Supported Providers](https://uppy.io/docs/companion/#Supported-providers) for a -list of all Providers and their corresponding names. - -### How to use Companion with Kubernetes? - -We have a detailed -[guide](https://github.com/transloadit/uppy/blob/main/packages/%40uppy/companion/KUBERNETES.md) -on running Companion in Kubernetes. - -### How to add custom providers? - -As of now, Companion supports the -[providers listed here](https://uppy.io/docs/companion/#Supported-providers) out -of the box, but you may also choose to add your own custom providers. You can do -this by passing the `customProviders` option when calling the Uppy `app` method. -The custom provider is expected to support Oauth 1 or 2 for -authentication/authorization. - -```javascript -import providerModule from './path/to/provider/module'; - -const options = { - customProviders: { - myprovidername: { - config: { - authorize_url: 'https://mywebsite.com/authorize', - access_url: 'https://mywebsite.com/token', - oauth: 2, - key: '***', - secret: '***', - scope: ['read', 'write'], - }, - module: providerModule, - }, - }, -}; - -uppy.app(options); -``` - -The `customProviders` option should be an object containing each custom -provider. Each custom provider would, in turn, be an object with two keys, -`config` and `module`. The `config` option would contain Oauth API settings, -while the `module` would point to the provider module. - -To work well with Companion, the **module** must be a class with the following -methods. Note that the methods must be `async`, return a `Promise` or reject -with an `Error`): - -1. `async list ({ token, directory, query })` - Returns a object containing a - list of user files (such as a list of all the files in a particular - directory). See [example returned list data structure](#list-data). `token` - - authorization token (retrieved from oauth process) to send along with your - request - - `directory` - the id/name of the directory from which data is to be - retrieved. This may be ignored if it doesn’t apply to your provider - - `query` - expressjs query params object received by the server (in case - some data you need in there). -2. `async download ({ token, id, query })` - Downloads a particular file from - the provider. Returns an object with a single property `{ stream }` - a - [`stream.Readable`](https://nodejs.org/api/stream.html#stream_class_stream_readable), - which will be read from and uploaded to the destination. To prevent memory - leaks, make sure you release your stream if you reject this method with an - error. - - `token` - authorization token (retrieved from oauth process) to send along - with your request. - - `id` - ID of the file being downloaded. - - `query` - expressjs query params object received by the server (in case - some data you need in there). -3. `async size ({ token, id, query })` - Returns the byte size of the file that - needs to be downloaded as a `Number`. If the size of the object is not known, - `null` may be returned. - - `token` - authorization token (retrieved from oauth process) to send along - with your request. - - `id` - ID of the file being downloaded. - - `query` - expressjs query params object received by the server (in case - some data you need in there). - -The class must also have: - -- A unique `static authProvider` string property - a lowercased value which - indicates name of the [`grant`](https://github.com/simov/grant) OAuth2 - provider to use (e.g `google` for Google). If your provider doesn’t use - OAuth2, you can omit this property. -- A `static` property `static version = 2`, which is the current version of the - Companion Provider API. - -See also -[example code with a custom provider](https://github.com/transloadit/uppy/blob/main/examples/custom-provider/server). - -#### list data - -```json -{ - // username or email of the user whose provider account is being accessed - "username": "johndoe", - // list of files and folders in the directory. An item is considered a folder - // if it mainly exists as a collection to contain sub-items - "items": [ - { - // boolean value of whether or NOT it's a folder - "isFolder": false, - // icon image URL - "icon": "https://random-api.url.com/fileicon.jpg", - // name of the item - "name": "myfile.jpg", - // the mime type of the item. Only relevant if the item is NOT a folder - "mimeType": "image/jpg", - // the id (in string) of the item - "id": "uniqueitemid", - // thumbnail image URL. Only relevant if the item is NOT a folder - "thumbnail": "https://random-api.url.com/filethumbnail.jpg", - // for folders this is typically the value that will be passed as "directory" in the list(...) method. - // For files, this is the value that will be passed as id in the download(...) method. - "requestPath": "file-or-folder-requestpath", - // datetime string (in ISO 8601 format) of when this item was last modified - "modifiedDate": "2020-06-29T19:59:58Z", - // the size in bytes of the item. Only relevant if the item is NOT a folder - "size": 278940, - "custom": { - // an object that may contain some more custom fields that you may need to send to the client. Only add this object if you have a need for it. - "customData1": "the value", - "customData2": "the value" - } - // more items here - } - ], - // if the "items" list is paginated, this is the request path needed to fetch the next page. - "nextPagePath": "directory-name?cursor=cursor-to-next-page" -} -``` - -### How to run Companion locally? - -1. To set up Companion for local development, please clone the Uppy repo and - install, like so: - - ```bash - git clone https://github.com/transloadit/uppy - cd uppy - yarn install - ``` - -2. Configure your environment variables by copying the `env.example.sh` file to - `env.sh` and edit it to its correct values. - - ```bash - cp .env.example .env - $EDITOR .env - ``` - -3. To start the server, run: - - ```bash - yarn run start:companion - ``` - -This would get the Companion instance running on `http://localhost:3020`. It -uses [`node --watch`](https://nodejs.org/api/cli.html#--watch) so it will -automatically restart when files are changed. - -[box]: /docs/box -[dropbox]: /docs/dropbox -[facebook]: /docs/facebook -[googledrive]: /docs/google-drive -[googlephotos]: /docs/google-photos -[instagram]: /docs/instagram -[onedrive]: /docs/onedrive -[unsplash]: /docs/unsplash -[url]: /docs/url -[zoom]: /docs/zoom -[transloadit]: https://transloadit.com -[template-credentials]: - https://transloadit.com/docs/#how-to-create-template-credentials