From 22b183a437ee0e2f17ec31c3416f1b059cc71be1 Mon Sep 17 00:00:00 2001 From: Murderlon Date: Thu, 19 Dec 2024 15:01:40 +0100 Subject: [PATCH 1/3] Allow Transloadit-hosted Companion with other uploaders --- packages/@uppy/aws-s3/src/index.ts | 16 ++++++++++- packages/@uppy/core/src/Uppy.ts | 1 + packages/@uppy/transloadit/src/index.ts | 35 ++++++++++++++++++------- packages/@uppy/xhr-upload/src/index.ts | 12 ++++++++- private/dev/Dashboard.js | 18 ++++++++++++- 5 files changed, 70 insertions(+), 12 deletions(-) diff --git a/packages/@uppy/aws-s3/src/index.ts b/packages/@uppy/aws-s3/src/index.ts index 153a7399f1..8892f4c278 100644 --- a/packages/@uppy/aws-s3/src/index.ts +++ b/packages/@uppy/aws-s3/src/index.ts @@ -923,10 +923,24 @@ export default class AwsS3Multipart< // eslint-disable-next-line class-methods-use-this #getCompanionClientArgs(file: UppyFile) { + const opts = { ...this.opts } + + // When you .use(AwsS3) with .use(Transloadit, { companionOnly: true }), + // local files are uploaded with this plugin and remote files with the Transloadit plugin. + // Since the Transloadit plugin uses the tus plugin underneath, it's possible to have file.tus + // even though we are in this plugin. + // @ts-expect-error typed in @uppy/tus + if (file.tus) { + // @ts-expect-error typed in @uppy/tus + Object.assign(opts, file.tus) + } + return { ...file.remote?.body, - protocol: 's3-multipart', + endpoint: opts.endpoint, + protocol: this.uppy.getState().remoteUploader || 's3-multipart', size: file.data.size, + headers: opts.headers, metadata: file.meta, } } diff --git a/packages/@uppy/core/src/Uppy.ts b/packages/@uppy/core/src/Uppy.ts index ed9e6bd9e8..09367bc1af 100644 --- a/packages/@uppy/core/src/Uppy.ts +++ b/packages/@uppy/core/src/Uppy.ts @@ -231,6 +231,7 @@ export interface State currentUploads: Record> allowNewUpload: boolean recoveredState: null | Required, 'files' | 'currentUploads'>> + remoteUploader?: 'tus' | 's3-multipart' | 'multipart' error: string | null files: { [key: string]: UppyFile diff --git a/packages/@uppy/transloadit/src/index.ts b/packages/@uppy/transloadit/src/index.ts index 6ec52bbc2d..d9bdccd73a 100644 --- a/packages/@uppy/transloadit/src/index.ts +++ b/packages/@uppy/transloadit/src/index.ts @@ -130,6 +130,12 @@ export interface TransloaditOptions waitForMetadata?: boolean importFromUploadURLs?: boolean alwaysRunAssembly?: boolean + /** + * Only use Transloadit for remote file uploads (such as from Google Drive). + * Enabling this means you have to install another plugin for local files, + * such as @uppy/aws-s3 or @uppy/xhr-upload. + */ + companionOnly?: boolean limit?: number clientName?: string | null retryDelays?: number[] @@ -145,6 +151,7 @@ const defaultOptions = { waitForMetadata: false, alwaysRunAssembly: false, importFromUploadURLs: false, + companionOnly: false, limit: 20, retryDelays: [7_000, 10_000, 15_000, 20_000], clientName: null, @@ -295,6 +302,13 @@ export default class Transloadit< this.i18nInit() + if (this.opts.companionOnly) { + // Transloadit is only a pre and post processor. + // To let Transloadit hosted Companion download the file, + // we instruct any other upload plugin to use tus for remote uploads. + this.uppy.setState({ remoteUploader: 'tus' }) + } + this.client = new Client({ service: this.opts.service, client: this.#getClientVersion(), @@ -802,26 +816,30 @@ export default class Transloadit< assemblyOptions.fields ??= {} validateParams(assemblyOptions.params) + const ids = + this.opts.companionOnly ? + fileIDs.filter((id) => this.uppy.getFile(id).isRemote) + : fileIDs try { const assembly = // this.assembly can already be defined if we recovered files with Golden Retriever (this.#onRestored) - this.assembly ?? (await this.#createAssembly(fileIDs, assemblyOptions)) + this.assembly ?? (await this.#createAssembly(ids, assemblyOptions)) if (assembly == null) throw new Error('All files were canceled after assembly was created') if (this.opts.importFromUploadURLs) { - await this.#reserveFiles(assembly, fileIDs) + await this.#reserveFiles(assembly, ids) } - fileIDs.forEach((fileID) => { + ids.forEach((fileID) => { const file = this.uppy.getFile(fileID) this.uppy.emit('preprocess-complete', file) }) this.#createAssemblyWatcher(assembly.status.assembly_id) - this.#connectAssembly(assembly, fileIDs) + this.#connectAssembly(assembly, ids) } catch (err) { - fileIDs.forEach((fileID) => { + ids.forEach((fileID) => { const file = this.uppy.getFile(fileID) // Clear preprocessing state when the Assembly could not be created, // otherwise the UI gets confused about the lingering progress keys @@ -937,9 +955,9 @@ export default class Transloadit< if (this.opts.importFromUploadURLs) { // No uploader needed when importing; instead we take the upload URL from an existing uploader. this.uppy.on('upload-success', this.#onFileUploadURLAvailable) - } else { - // we don't need it here. - // the regional endpoint from the Transloadit API before we can set it. + // If companionOnly is true, another uploader plugin is installed for local uploads + // and we only use Transloadit to create an assembly for the remote files. + } else if (!this.opts.companionOnly) { this.uppy.use(Tus, { // Disable tus-js-client fingerprinting, otherwise uploading the same file at different times // will upload to an outdated Assembly, and we won't get socket events for it. @@ -954,7 +972,6 @@ export default class Transloadit< // Send all metadata to Transloadit. Metadata set by the user // ends up as in the template as `file.user_meta` allowedMetaFields: true, - // Pass the limit option to @uppy/tus limit: this.opts.limit, rateLimitedQueue: this.#rateLimitedQueue, retryDelays: this.opts.retryDelays, diff --git a/packages/@uppy/xhr-upload/src/index.ts b/packages/@uppy/xhr-upload/src/index.ts index dcab608307..6750aee146 100644 --- a/packages/@uppy/xhr-upload/src/index.ts +++ b/packages/@uppy/xhr-upload/src/index.ts @@ -438,9 +438,19 @@ export default class XHRUpload< opts.allowedMetaFields, file.meta, ) + // When you .use(AwsS3) with .use(Transloadit, { companionOnly: true }), + // local files are uploaded with this plugin and remote files with the Transloadit plugin. + // Since the Transloadit plugin uses the tus plugin underneath, it's possible to have file.tus + // even though we are in this plugin. + // @ts-expect-error typed in @uppy/tus + if (file.tus) { + // @ts-expect-error typed in @uppy/tus + Object.assign(opts, file.tus) + } + return { ...file.remote?.body, - protocol: 'multipart', + protocol: this.uppy.getState().remoteUploader || 'multipart', endpoint: opts.endpoint, size: file.data.size, fieldname: opts.fieldName, diff --git a/private/dev/Dashboard.js b/private/dev/Dashboard.js index 726dc1ed07..3b16662e26 100644 --- a/private/dev/Dashboard.js +++ b/private/dev/Dashboard.js @@ -168,7 +168,7 @@ export default () => { .use(Webdav, { target: Dashboard, companionUrl: COMPANION_URL, - companionAllowedHosts + companionAllowedHosts, }) .use(Audio, { target: Dashboard, @@ -195,6 +195,15 @@ export default () => { shouldUseMultipart: false, }) break + case 's3-with-transloadit-companion': + uppyDashboard.use(AwsS3, { endpoint: COMPANION_URL }) + uppyDashboard.use(Transloadit, { + service: TRANSLOADIT_SERVICE_URL, + waitForEncoding: true, + assemblyOptions, + companionOnly: true, + }) + break case 's3-multipart': uppyDashboard.use(AwsS3, { endpoint: COMPANION_URL, @@ -207,6 +216,12 @@ export default () => { limit: 6, bundle: false, }) + uppyDashboard.use(Transloadit, { + service: TRANSLOADIT_SERVICE_URL, + waitForEncoding: true, + assemblyOptions, + companionOnly: true, + }) break case 'transloadit': uppyDashboard.use(Transloadit, { @@ -221,6 +236,7 @@ export default () => { waitForEncoding: true, importFromUploadURLs: true, assemblyOptions, + companionOnly: true, }) break case 'transloadit-xhr': From 9fa3f7d54b11d9ae2d7ae0dad4a7080bdd3affbc Mon Sep 17 00:00:00 2001 From: Murderlon Date: Thu, 23 Jan 2025 14:08:21 +0100 Subject: [PATCH 2/3] Set tus properties more explicitly --- packages/@uppy/aws-s3/src/index.ts | 9 +++------ packages/@uppy/xhr-upload/src/index.ts | 11 ++++------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/packages/@uppy/aws-s3/src/index.ts b/packages/@uppy/aws-s3/src/index.ts index 3f7f0fcbe9..5912398175 100644 --- a/packages/@uppy/aws-s3/src/index.ts +++ b/packages/@uppy/aws-s3/src/index.ts @@ -929,17 +929,14 @@ export default class AwsS3Multipart< // Since the Transloadit plugin uses the tus plugin underneath, it's possible to have file.tus // even though we are in this plugin. // @ts-expect-error typed in @uppy/tus - if (file.tus) { - // @ts-expect-error typed in @uppy/tus - Object.assign(opts, file.tus) - } + const tusOpts = file.tus return { ...file.remote?.body, - endpoint: opts.endpoint, + endpoint: tusOpts.endpoint ?? opts.endpoint, protocol: this.uppy.getState().remoteUploader || 's3-multipart', size: file.data.size, - headers: opts.headers, + headers: { ...opts.headers, ...tusOpts.headers }, metadata: file.meta, } } diff --git a/packages/@uppy/xhr-upload/src/index.ts b/packages/@uppy/xhr-upload/src/index.ts index 99aacb9526..29cfca8006 100644 --- a/packages/@uppy/xhr-upload/src/index.ts +++ b/packages/@uppy/xhr-upload/src/index.ts @@ -440,20 +440,18 @@ export default class XHRUpload< opts.allowedMetaFields, file.meta, ) - // When you .use(AwsS3) with .use(Transloadit, { companionOnly: true }), + // When you .use(XHR) with .use(Transloadit, { companionOnly: true }), // local files are uploaded with this plugin and remote files with the Transloadit plugin. // Since the Transloadit plugin uses the tus plugin underneath, it's possible to have file.tus // even though we are in this plugin. // @ts-expect-error typed in @uppy/tus - if (file.tus) { - // @ts-expect-error typed in @uppy/tus - Object.assign(opts, file.tus) - } + const tusOpts = file.tus return { ...file.remote?.body, protocol: this.uppy.getState().remoteUploader || 'multipart', - endpoint: opts.endpoint, + endpoint: tusOpts.endpoint ?? opts.endpoint, + headers: { ...opts.headers, ...tusOpts.headers }, size: file.data.size, fieldname: opts.fieldName, metadata: Object.fromEntries( @@ -461,7 +459,6 @@ export default class XHRUpload< ), httpMethod: opts.method, useFormData: opts.formData, - headers: opts.headers, } } From 1bcd04c1dc91f52117b72529e0f8d86a23e042ca Mon Sep 17 00:00:00 2001 From: Murderlon Date: Thu, 23 Jan 2025 14:14:39 +0100 Subject: [PATCH 3/3] Rename companionOnly -> onlyRemoteFiles --- packages/@uppy/aws-s3/src/index.ts | 6 +++--- packages/@uppy/transloadit/src/index.ts | 12 ++++++------ packages/@uppy/xhr-upload/src/index.ts | 2 +- private/dev/Dashboard.js | 10 ++-------- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/packages/@uppy/aws-s3/src/index.ts b/packages/@uppy/aws-s3/src/index.ts index 5912398175..111840481b 100644 --- a/packages/@uppy/aws-s3/src/index.ts +++ b/packages/@uppy/aws-s3/src/index.ts @@ -924,7 +924,7 @@ export default class AwsS3Multipart< #getCompanionClientArgs(file: UppyFile) { const opts = { ...this.opts } - // When you .use(AwsS3) with .use(Transloadit, { companionOnly: true }), + // When you .use(AwsS3) with .use(Transloadit, { onlyRemoteFiles: true }), // local files are uploaded with this plugin and remote files with the Transloadit plugin. // Since the Transloadit plugin uses the tus plugin underneath, it's possible to have file.tus // even though we are in this plugin. @@ -933,10 +933,10 @@ export default class AwsS3Multipart< return { ...file.remote?.body, - endpoint: tusOpts.endpoint ?? opts.endpoint, protocol: this.uppy.getState().remoteUploader || 's3-multipart', - size: file.data.size, + endpoint: tusOpts.endpoint ?? opts.endpoint, headers: { ...opts.headers, ...tusOpts.headers }, + size: file.data.size, metadata: file.meta, } } diff --git a/packages/@uppy/transloadit/src/index.ts b/packages/@uppy/transloadit/src/index.ts index 20ad36f64a..90f68b3aea 100644 --- a/packages/@uppy/transloadit/src/index.ts +++ b/packages/@uppy/transloadit/src/index.ts @@ -140,7 +140,7 @@ export interface TransloaditOptions * Enabling this means you have to install another plugin for local files, * such as @uppy/aws-s3 or @uppy/xhr-upload. */ - companionOnly?: boolean + onlyRemoteFiles?: boolean limit?: number clientName?: string | null retryDelays?: number[] @@ -156,7 +156,7 @@ const defaultOptions = { waitForMetadata: false, alwaysRunAssembly: false, importFromUploadURLs: false, - companionOnly: false, + onlyRemoteFiles: false, limit: 20, retryDelays: [7_000, 10_000, 15_000, 20_000], clientName: null, @@ -307,7 +307,7 @@ export default class Transloadit< this.i18nInit() - if (this.opts.companionOnly) { + if (this.opts.onlyRemoteFiles) { // Transloadit is only a pre and post processor. // To let Transloadit hosted Companion download the file, // we instruct any other upload plugin to use tus for remote uploads. @@ -822,7 +822,7 @@ export default class Transloadit< assemblyOptions.fields ??= {} validateParams(assemblyOptions.params) const ids = - this.opts.companionOnly ? + this.opts.onlyRemoteFiles ? fileIDs.filter((id) => this.uppy.getFile(id).isRemote) : fileIDs @@ -960,9 +960,9 @@ export default class Transloadit< if (this.opts.importFromUploadURLs) { // No uploader needed when importing; instead we take the upload URL from an existing uploader. this.uppy.on('upload-success', this.#onFileUploadURLAvailable) - // If companionOnly is true, another uploader plugin is installed for local uploads + // If onlyRemoteFiles is true, another uploader plugin is installed for local uploads // and we only use Transloadit to create an assembly for the remote files. - } else if (!this.opts.companionOnly) { + } else if (!this.opts.onlyRemoteFiles) { this.uppy.use(Tus, { // Disable tus-js-client fingerprinting, otherwise uploading the same file at different times // will upload to an outdated Assembly, and we won't get socket events for it. diff --git a/packages/@uppy/xhr-upload/src/index.ts b/packages/@uppy/xhr-upload/src/index.ts index 29cfca8006..c2944e66d0 100644 --- a/packages/@uppy/xhr-upload/src/index.ts +++ b/packages/@uppy/xhr-upload/src/index.ts @@ -440,7 +440,7 @@ export default class XHRUpload< opts.allowedMetaFields, file.meta, ) - // When you .use(XHR) with .use(Transloadit, { companionOnly: true }), + // When you .use(XHR) with .use(Transloadit, { onlyRemoteFiles: true }), // local files are uploaded with this plugin and remote files with the Transloadit plugin. // Since the Transloadit plugin uses the tus plugin underneath, it's possible to have file.tus // even though we are in this plugin. diff --git a/private/dev/Dashboard.js b/private/dev/Dashboard.js index 3b16662e26..cd8bb5bbd7 100644 --- a/private/dev/Dashboard.js +++ b/private/dev/Dashboard.js @@ -201,7 +201,7 @@ export default () => { service: TRANSLOADIT_SERVICE_URL, waitForEncoding: true, assemblyOptions, - companionOnly: true, + onlyRemoteFiles: true, }) break case 's3-multipart': @@ -216,12 +216,6 @@ export default () => { limit: 6, bundle: false, }) - uppyDashboard.use(Transloadit, { - service: TRANSLOADIT_SERVICE_URL, - waitForEncoding: true, - assemblyOptions, - companionOnly: true, - }) break case 'transloadit': uppyDashboard.use(Transloadit, { @@ -236,7 +230,7 @@ export default () => { waitForEncoding: true, importFromUploadURLs: true, assemblyOptions, - companionOnly: true, + onlyRemoteFiles: true, }) break case 'transloadit-xhr':