From a145e3bc7cbc833bb0ebc1267a0041b4835b9bba Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Wed, 10 Apr 2024 15:41:36 +0200 Subject: [PATCH 1/3] @uppy/aws-s3: remove legacy plugin --- packages/@uppy/aws-s3-multipart/CHANGELOG.md | 236 ---- packages/@uppy/aws-s3-multipart/README.md | 40 +- packages/@uppy/aws-s3-multipart/package.json | 13 +- packages/@uppy/aws-s3-multipart/src/index.ts | 1012 +---------------- packages/@uppy/aws-s3/CHANGELOG.md | 181 ++- packages/@uppy/aws-s3/package.json | 11 +- .../src/HTTPCommunicationQueue.ts | 0 packages/@uppy/aws-s3/src/MiniXHRUpload.js | 235 ---- .../src/MultipartUploader.ts | 0 .../src/createSignedURL.test.ts | 0 .../src/createSignedURL.ts | 0 packages/@uppy/aws-s3/src/index.js | 367 ------ packages/@uppy/aws-s3/src/index.test.js | 69 -- .../src/index.test.ts | 0 packages/@uppy/aws-s3/src/index.ts | 1010 ++++++++++++++++ packages/@uppy/aws-s3/src/isXml.js | 35 - packages/@uppy/aws-s3/src/isXml.test.js | 65 -- packages/@uppy/aws-s3/src/locale.js | 5 - .../{aws-s3-multipart => aws-s3}/src/utils.ts | 0 .../tsconfig.build.json | 0 .../tsconfig.json | 0 21 files changed, 1143 insertions(+), 2136 deletions(-) delete mode 100644 packages/@uppy/aws-s3-multipart/CHANGELOG.md rename packages/@uppy/{aws-s3-multipart => aws-s3}/src/HTTPCommunicationQueue.ts (100%) delete mode 100644 packages/@uppy/aws-s3/src/MiniXHRUpload.js rename packages/@uppy/{aws-s3-multipart => aws-s3}/src/MultipartUploader.ts (100%) rename packages/@uppy/{aws-s3-multipart => aws-s3}/src/createSignedURL.test.ts (100%) rename packages/@uppy/{aws-s3-multipart => aws-s3}/src/createSignedURL.ts (100%) delete mode 100644 packages/@uppy/aws-s3/src/index.js delete mode 100644 packages/@uppy/aws-s3/src/index.test.js rename packages/@uppy/{aws-s3-multipart => aws-s3}/src/index.test.ts (100%) create mode 100644 packages/@uppy/aws-s3/src/index.ts delete mode 100644 packages/@uppy/aws-s3/src/isXml.js delete mode 100644 packages/@uppy/aws-s3/src/isXml.test.js delete mode 100644 packages/@uppy/aws-s3/src/locale.js rename packages/@uppy/{aws-s3-multipart => aws-s3}/src/utils.ts (100%) rename packages/@uppy/{aws-s3-multipart => aws-s3}/tsconfig.build.json (100%) rename packages/@uppy/{aws-s3-multipart => aws-s3}/tsconfig.json (100%) diff --git a/packages/@uppy/aws-s3-multipart/CHANGELOG.md b/packages/@uppy/aws-s3-multipart/CHANGELOG.md deleted file mode 100644 index c5fbb4b69b..0000000000 --- a/packages/@uppy/aws-s3-multipart/CHANGELOG.md +++ /dev/null @@ -1,236 +0,0 @@ -# @uppy/aws-s3-multipart - -## 4.0.0-beta.1 - -Released: 2024-03-28 -Included in: Uppy v4.0.0-beta.1 - -- @uppy/aws-s3-multipart: mark `opts` as optional (Antoine du Hamel / #5039) -- @uppy/aws-s3-multipart,@uppy/tus,@uppy/utils,@uppy/xhr-upload: Make `allowedMetaFields` consistent (Merlijn Vos / #5011) -- @uppy/aws-s3-multipart: refactor to TS (Antoine du Hamel / #4902) -- @uppy/aws-s3-multipart: fix escaping issue with client signed request (Hiroki Shimizu / #5006) - -## 3.11.0 - -Released: 2024-03-27 -Included in: Uppy v3.24.0 - -- @uppy/aws-s3-multipart: mark `opts` as optional (Antoine du Hamel / #5039) -- @uppy/aws-s3-multipart: refactor to TS (Antoine du Hamel / #4902) -- @uppy/aws-s3-multipart: fix escaping issue with client signed request (Hiroki Shimizu / #5006) - -## 3.8.0 - -Released: 2023-10-20 -Included in: Uppy v3.18.0 - -- @uppy/aws-s3-multipart: fix `TypeError` (Antoine du Hamel / #4748) -- @uppy/aws-s3-multipart: pass `signal` as separate arg for backward compat (Antoine du Hamel / #4746) -- @uppy/aws-s3-multipart: fix `uploadURL` when using `PUT` (Antoine du Hamel / #4701) - -## 3.7.0 - -Released: 2023-09-29 -Included in: Uppy v3.17.0 - -- @uppy/aws-s3-multipart: retry signature request (Merlijn Vos / #4691) -- @uppy/aws-s3-multipart: aws-s3-multipart - call `#setCompanionHeaders` in `setOptions` (jur-ng / #4687) - -## 3.6.0 - -Released: 2023-09-05 -Included in: Uppy v3.15.0 - -- @uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/companion-client,@uppy/core,@uppy/tus,@uppy/utils,@uppy/xhr-upload: Move remote file upload logic into companion-client (Merlijn Vos / #4573) - -## 3.5.4 - -Released: 2023-08-23 -Included in: Uppy v3.14.1 - -- @uppy/aws-s3-multipart: fix types when using deprecated option (Antoine du Hamel / #4634) -- @uppy/aws-s3-multipart,@uppy/aws-s3: allow empty objects for `fields` types (Antoine du Hamel / #4631) - -## 3.5.3 - -Released: 2023-08-15 -Included in: Uppy v3.14.0 - -- @uppy/aws-s3-multipart: pass the `uploadURL` back to the caller (Antoine du Hamel / #4614) -- @uppy/aws-s3,@uppy/aws-s3-multipart: update types (Antoine du Hamel / #4611) -- @uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/companion,@uppy/transloadit,@uppy/xhr-upload: use uppercase HTTP method names (Antoine du Hamel / #4612) -- @uppy/aws-s3,@uppy/aws-s3-multipart: update types (bdirito / #4576) - -## 3.5.2 - -Released: 2023-07-24 -Included in: Uppy v3.13.1 - -- @uppy/aws-s3-multipart: refresh file before calling user-defined functions (mjlumetta / #4557) - -## 3.5.1 - -Released: 2023-07-20 -Included in: Uppy v3.13.0 - -- @uppy/aws-s3-multipart: fix crash on pause/resume (Merlijn Vos / #4581) -- @uppy/aws-s3-multipart: do not access `globalThis.crypto` on the top-level (Bryan J Swift / #4584) - -## 3.5.0 - -Released: 2023-07-13 -Included in: Uppy v3.12.0 - -- @uppy/aws-s3-multipart: add support for signing on the client (Antoine du Hamel / #4519) -- @uppy/aws-s3-multipart: fix lint warning (Antoine du Hamel / #4569) -- @uppy/aws-s3-multipart: fix support for non-multipart PUT upload (Antoine du Hamel / #4568) - -## 3.4.1 - -Released: 2023-07-06 -Included in: Uppy v3.11.0 - -- @uppy/aws-s3-multipart: increase priority of abort and complete (Stefan Schonert / #4542) -- @uppy/aws-s3-multipart: fix upload retry using an outdated ID (Antoine du Hamel / #4544) -- @uppy/aws-s3-multipart: fix Golden Retriever integration (Antoine du Hamel / #4526) -- @uppy/aws-s3-multipart: add types to internal fields (Antoine du Hamel / #4535) -- @uppy/aws-s3-multipart: fix pause/resume (Antoine du Hamel / #4523) -- @uppy/aws-s3-multipart: fix resume single-chunk multipart uploads (Antoine du Hamel / #4528) -- @uppy/aws-s3-multipart: disable pause/resume for remote uploads in the UI (Artur Paikin / #4500) - -## 3.4.0 - -Released: 2023-06-19 -Included in: Uppy v3.10.0 - -- @uppy/aws-s3-multipart: fix the chunk size calculation (Antoine du Hamel / #4508) -- @uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/tus,@uppy/utils,@uppy/xhr-upload: When file is removed (or all are canceled), controller.abort queued requests (Artur Paikin / #4504) -- @uppy/aws-s3-multipart,@uppy/tus,@uppy/xhr-upload: Don't close socket while upload is still in progress (Artur Paikin / #4479) -- @uppy/aws-s3-multipart: fix `getUploadParameters` option (Antoine du Hamel / #4465) - -## 3.3.0 - -Released: 2023-05-02 -Included in: Uppy v3.9.0 - -- @uppy/aws-s3-multipart: allowedMetaFields: null means “include all” (Artur Paikin / #4437) -- @uppy/aws-s3-multipart: add `shouldUseMultipart ` option (Antoine du Hamel / #4205) -- @uppy/aws-s3-multipart: make retries more robust (Antoine du Hamel / #4424) - -## 3.1.3 - -Released: 2023-04-04 -Included in: Uppy v3.7.0 - -- @uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/tus,@uppy/xhr-upload: make sure that we reset serverToken when an upload fails (Mikael Finstad / #4376) -- @uppy/aws-s3-multipart: do not auto-open sockets, clean them up on abort (Antoine du Hamel) - -## 3.1.2 - -Released: 2023-01-26 -Included in: Uppy v3.4.0 - -- @uppy/aws-s3-multipart: fix metadata shape (Antoine du Hamel / #4267) -- @uppy/aws-s3-multipart: add support for `allowedMetaFields` option (Antoine du Hamel / #4215) -- @uppy/aws-s3-multipart: fix singPart type (Stefan Schonert / #4224) - -## 3.1.1 - -Released: 2022-11-16 -Included in: Uppy v3.3.1 - -- @uppy/aws-s3-multipart: handle slow connections better (Antoine du Hamel / #4213) -- @uppy/aws-s3-multipart: Fix typo in url check (Christian Franke / #4211) - -## 3.1.0 - -Released: 2022-11-10 -Included in: Uppy v3.3.0 - -- @uppy/aws-s3-multipart: empty the queue when pausing (Antoine du Hamel / #4203) -- @uppy/aws-s3-multipart: refactor rate limiting approach (Antoine du Hamel / #4187) -- @uppy/aws-s3-multipart: change limit to 6 (Antoine du Hamel / #4199) -- @uppy/aws-s3-multipart: remove unused `timeout` option (Antoine du Hamel / #4186) -- @uppy/aws-s3-multipart,@uppy/tus: fix `Timed out waiting for socket` (Antoine du Hamel / #4177) - -## 3.0.2 - -Released: 2022-09-25 -Included in: Uppy v3.1.0 - -- @uppy/audio,@uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/box,@uppy/companion-client,@uppy/companion,@uppy/compressor,@uppy/core,@uppy/dashboard,@uppy/drag-drop,@uppy/drop-target,@uppy/dropbox,@uppy/facebook,@uppy/file-input,@uppy/form,@uppy/golden-retriever,@uppy/google-drive,@uppy/image-editor,@uppy/informer,@uppy/instagram,@uppy/locales,@uppy/onedrive,@uppy/progress-bar,@uppy/provider-views,@uppy/react,@uppy/redux-dev-tools,@uppy/remote-sources,@uppy/screen-capture,@uppy/status-bar,@uppy/store-default,@uppy/store-redux,@uppy/svelte,@uppy/thumbnail-generator,@uppy/transloadit,@uppy/tus,@uppy/unsplash,@uppy/url,@uppy/utils,@uppy/vue,@uppy/webcam,@uppy/xhr-upload,@uppy/zoom: add missing entries to changelog for individual packages (Antoine du Hamel / #4092) - -## 3.0.0 - -Released: 2022-08-22 -Included in: Uppy v3.0.0 - -- Switch to ESM - -## 3.0.0-beta.4 - -Released: 2022-08-16 -Included in: Uppy v3.0.0-beta.5 - -- @uppy/aws-s3-multipart: Fix when using Companion (Merlijn Vos / #3969) -- @uppy/aws-s3-multipart: Fix race condition in `#uploadParts` (Morgan Zolob / #3955) -- @uppy/aws-s3-multipart: ignore exception inside `abortMultipartUpload` (Antoine du Hamel / #3950) - -## 3.0.0-beta.3 - -Released: 2022-08-03 -Included in: Uppy v3.0.0-beta.4 - -- @uppy/aws-s3-multipart: Correctly handle errors for `prepareUploadParts` (Merlijn Vos / #3912) - -## 3.0.0-beta.2 - -Released: 2022-07-27 -Included in: Uppy v3.0.0-beta.3 - -- @uppy/aws-s3-multipart: make `headers` part indexed too in `prepareUploadParts` (Merlijn Vos / #3895) - -## 2.4.1 - -Released: 2022-06-07 -Included in: Uppy v2.12.0 - -- @uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/tus: queue socket token requests for remote files (Merlijn Vos / #3797) -- @uppy/aws-s3-multipart: allow `companionHeaders` to be modified with `setOptions` (Paulo Lemos Neto / #3770) - -## 2.4.0 - -Released: 2022-05-30 -Included in: Uppy v2.11.0 - -- @uppy/angular,@uppy/audio,@uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/box,@uppy/core,@uppy/dashboard,@uppy/drag-drop,@uppy/dropbox,@uppy/facebook,@uppy/file-input,@uppy/form,@uppy/golden-retriever,@uppy/google-drive,@uppy/image-editor,@uppy/informer,@uppy/instagram,@uppy/onedrive,@uppy/progress-bar,@uppy/react,@uppy/redux-dev-tools,@uppy/robodog,@uppy/screen-capture,@uppy/status-bar,@uppy/store-default,@uppy/store-redux,@uppy/thumbnail-generator,@uppy/transloadit,@uppy/tus,@uppy/unsplash,@uppy/url,@uppy/vue,@uppy/webcam,@uppy/xhr-upload,@uppy/zoom: doc: update bundler recommendation (Antoine du Hamel / #3763) -- @uppy/aws-s3-multipart: refactor to ESM (Antoine du Hamel / #3672) - -## 2.3.0 - -Released: 2022-05-14 -Included in: Uppy v2.10.0 - -- @uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/core,@uppy/react,@uppy/transloadit,@uppy/tus,@uppy/xhr-upload: proposal: Cancel assemblies optional (Mikael Finstad / #3575) -- @uppy/aws-s3-multipart: export interface AwsS3MultipartOptions (Matteo Padovano / #3709) - -## 2.2.2 - -Released: 2022-04-27 -Included in: Uppy v2.9.4 - -- @uppy/aws-s3-multipart: Add `companionCookiesRule` type to @uppy/aws-s3-multipart (Mauricio Ribeiro / #3623) - -## 2.2.1 - -Released: 2022-03-02 -Included in: Uppy v2.7.0 - -- @uppy/aws-s3-multipart: Add chunks back to prepareUploadParts, indexed by partNumber (Kevin West / #3520) - -## 2.2.0 - -Released: 2021-12-07 -Included in: Uppy v2.3.0 - -- @uppy/aws-s3-multipart: Drop `lockedCandidatesForBatch` and mark chunks as 'busy' when preparing (Yegor Yarko / #3342) diff --git a/packages/@uppy/aws-s3-multipart/README.md b/packages/@uppy/aws-s3-multipart/README.md index 02e189b551..0fbee68830 100644 --- a/packages/@uppy/aws-s3-multipart/README.md +++ b/packages/@uppy/aws-s3-multipart/README.md @@ -1,41 +1,3 @@ # @uppy/aws-s3-multipart -Uppy logo: a smiling puppy above a pink upwards arrow - -[![npm version](https://img.shields.io/npm/v/@uppy/aws-s3-multipart.svg?style=flat-square)](https://www.npmjs.com/package/@uppy/aws-s3-multipart) -![CI status for Uppy tests](https://github.com/transloadit/uppy/workflows/Tests/badge.svg) -![CI status for Companion tests](https://github.com/transloadit/uppy/workflows/Companion/badge.svg) -![CI status for browser tests](https://github.com/transloadit/uppy/workflows/End-to-end%20tests/badge.svg) - -The AwsS3Multipart plugin can be used to upload files directly to an S3 bucket using S3’s Multipart upload strategy. With this strategy, files are chopped up in parts of 5MB+ each, so they can be uploaded concurrently. It’s also reliable: if a single part fails to upload, only that 5MB has to be retried. - -Uppy is being developed by the folks at [Transloadit](https://transloadit.com), a versatile file encoding service. - -## Example - -```js -import Uppy from '@uppy/core' -import AwsS3Multipart from '@uppy/aws-s3-multipart' - -const uppy = new Uppy() -uppy.use(AwsS3Multipart, { - limit: 2, - companionUrl: 'https://companion.myapp.com/', -}) -``` - -## Installation - -```bash -$ npm install @uppy/aws-s3-multipart -``` - -Alternatively, you can also use this plugin in a pre-built bundle from Transloadit’s CDN: Edgly. In that case `Uppy` will attach itself to the global `window.Uppy` object. See the [main Uppy documentation](https://uppy.io/docs/#Installation) for instructions. - -## Documentation - -Documentation for this plugin can be found on the [Uppy website](https://uppy.io/docs/aws-s3-multipart). - -## License - -[The MIT License](./LICENSE). +This package is deprecated. Use [`@uppy/aws-s3`](https://npmjs.org/package/@uppy/aws-s3) instead. diff --git a/packages/@uppy/aws-s3-multipart/package.json b/packages/@uppy/aws-s3-multipart/package.json index 3afc9f4e1c..1be50b340c 100644 --- a/packages/@uppy/aws-s3-multipart/package.json +++ b/packages/@uppy/aws-s3-multipart/package.json @@ -23,18 +23,7 @@ "url": "git+https://github.com/transloadit/uppy.git" }, "dependencies": { - "@uppy/companion-client": "workspace:^", - "@uppy/utils": "workspace:^" - }, - "devDependencies": { - "@aws-sdk/client-s3": "^3.362.0", - "@aws-sdk/s3-request-presigner": "^3.362.0", - "nock": "^13.1.0", - "vitest": "^1.2.1", - "whatwg-fetch": "3.6.2" - }, - "peerDependencies": { - "@uppy/core": "workspace:^" + "@uppy/aws-s3": "workspace:^" }, "stableVersion": "3.11.0" } diff --git a/packages/@uppy/aws-s3-multipart/src/index.ts b/packages/@uppy/aws-s3-multipart/src/index.ts index 6e0aa32f70..4296902192 100644 --- a/packages/@uppy/aws-s3-multipart/src/index.ts +++ b/packages/@uppy/aws-s3-multipart/src/index.ts @@ -1,1010 +1,2 @@ -import BasePlugin, { - type DefinePluginOpts, - type PluginOpts, -} from '@uppy/core/lib/BasePlugin.js' -import { RequestClient } from '@uppy/companion-client' -import type { RequestOptions } from '@uppy/utils/lib/CompanionClientProvider' -import type { Body as _Body, Meta, UppyFile } from '@uppy/utils/lib/UppyFile' -import type { Uppy } from '@uppy/core' -import EventManager from '@uppy/core/lib/EventManager.js' -import { RateLimitedQueue } from '@uppy/utils/lib/RateLimitedQueue' -import { - filterNonFailedFiles, - filterFilesToEmitUploadStarted, -} from '@uppy/utils/lib/fileFilters' -import { createAbortError } from '@uppy/utils/lib/AbortController' -import getAllowedMetaFields from '@uppy/utils/lib/getAllowedMetaFields' -import MultipartUploader from './MultipartUploader.ts' -import { throwIfAborted } from './utils.ts' -import type { - UploadResult, - UploadResultWithSignal, - MultipartUploadResultWithSignal, - UploadPartBytesResult, - Body, -} from './utils.ts' -import createSignedURL from './createSignedURL.ts' -import { HTTPCommunicationQueue } from './HTTPCommunicationQueue.ts' -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-ignore We don't want TS to generate types for the package.json -import packageJson from '../package.json' - -interface MultipartFile extends UppyFile { - s3Multipart: UploadResult -} - -type PartUploadedCallback = ( - file: UppyFile, - part: { PartNumber: number; ETag: string }, -) => void - -declare module '@uppy/core' { - export interface UppyEventMap { - 's3-multipart:part-uploaded': PartUploadedCallback - } -} - -function assertServerError(res: T): T { - if ((res as any)?.error) { - const error = new Error((res as any).message) - Object.assign(error, (res as any).error) - throw error - } - return res -} - -export interface AwsS3STSResponse { - credentials: { - AccessKeyId: string - SecretAccessKey: string - SessionToken: string - Expiration?: string - } - bucket: string - region: string -} - -/** - * Computes the expiry time for a request signed with temporary credentials. If - * no expiration was provided, or an invalid value (e.g. in the past) is - * provided, undefined is returned. This function assumes the client clock is in - * sync with the remote server, which is a requirement for the signature to be - * validated for AWS anyway. - */ -function getExpiry( - credentials: AwsS3STSResponse['credentials'], -): number | undefined { - const expirationDate = credentials.Expiration - if (expirationDate) { - const timeUntilExpiry = Math.floor( - ((new Date(expirationDate) as any as number) - Date.now()) / 1000, - ) - if (timeUntilExpiry > 9) { - return timeUntilExpiry - } - } - return undefined -} - -function getAllowedMetadata>({ - meta, - allowedMetaFields, - querify = false, -}: { - meta: M - allowedMetaFields?: string[] | null - querify?: boolean -}) { - const metaFields = allowedMetaFields ?? Object.keys(meta) - - if (!meta) return {} - - return Object.fromEntries( - metaFields - .filter((key) => meta[key] != null) - .map((key) => { - const realKey = querify ? `metadata[${key}]` : key - const value = String(meta[key]) - return [realKey, value] - }), - ) -} - -type MaybePromise = T | Promise - -type SignPartOptions = { - uploadId: string - key: string - partNumber: number - body: Blob - signal?: AbortSignal -} - -export type AwsS3UploadParameters = - | { - method: 'POST' - url: string - fields: Record - expires?: number - headers?: Record - } - | { - method?: 'PUT' - url: string - fields?: Record - expires?: number - headers?: Record - } - -export interface AwsS3Part { - PartNumber?: number - Size?: number - ETag?: string -} - -type AWSS3WithCompanion = { - companionUrl: string - companionHeaders?: Record - companionCookiesRule?: string - getTemporarySecurityCredentials?: true -} -type AWSS3WithoutCompanion = { - getTemporarySecurityCredentials?: (options?: { - signal?: AbortSignal - }) => MaybePromise - uploadPartBytes?: (options: { - signature: AwsS3UploadParameters - body: FormData | Blob - size?: number - onProgress: any - onComplete: any - signal?: AbortSignal - }) => Promise -} - -type AWSS3NonMultipartWithCompanionMandatory = { - // No related options -} - -type AWSS3NonMultipartWithoutCompanionMandatory< - M extends Meta, - B extends Body, -> = { - getUploadParameters: ( - file: UppyFile, - options: RequestOptions, - ) => MaybePromise -} -type AWSS3NonMultipartWithCompanion = AWSS3WithCompanion & - AWSS3NonMultipartWithCompanionMandatory & { - shouldUseMultipart: false - } - -type AWSS3NonMultipartWithoutCompanion< - M extends Meta, - B extends Body, -> = AWSS3WithoutCompanion & - AWSS3NonMultipartWithoutCompanionMandatory & { - shouldUseMultipart: false - } - -type AWSS3MultipartWithoutCompanionMandatorySignPart< - M extends Meta, - B extends Body, -> = { - signPart: ( - file: UppyFile, - opts: SignPartOptions, - ) => MaybePromise -} -/** @deprecated Use signPart instead */ -type AWSS3MultipartWithoutCompanionMandatoryPrepareUploadParts< - M extends Meta, - B extends Body, -> = { - /** @deprecated Use signPart instead */ - prepareUploadParts: ( - file: UppyFile, - partData: { - uploadId: string - key: string - parts: [{ number: number; chunk: Blob }] - signal?: AbortSignal - }, - ) => MaybePromise<{ - presignedUrls: Record - headers?: Record> - }> -} -type AWSS3MultipartWithoutCompanionMandatory = { - getChunkSize?: (file: UppyFile) => number - createMultipartUpload: (file: UppyFile) => MaybePromise - listParts: ( - file: UppyFile, - opts: UploadResultWithSignal, - ) => MaybePromise - abortMultipartUpload: ( - file: UppyFile, - opts: UploadResultWithSignal, - ) => MaybePromise - completeMultipartUpload: ( - file: UppyFile, - opts: { - uploadId: string - key: string - parts: AwsS3Part[] - signal: AbortSignal - }, - ) => MaybePromise<{ location?: string }> -} & ( - | AWSS3MultipartWithoutCompanionMandatorySignPart - | AWSS3MultipartWithoutCompanionMandatoryPrepareUploadParts -) - -type AWSS3MultipartWithoutCompanion< - M extends Meta, - B extends Body, -> = AWSS3WithoutCompanion & - AWSS3MultipartWithoutCompanionMandatory & { - shouldUseMultipart?: true - } - -type AWSS3MultipartWithCompanion< - M extends Meta, - B extends Body, -> = AWSS3WithCompanion & - Partial> & { - shouldUseMultipart?: true - } - -type AWSS3MaybeMultipartWithCompanion< - M extends Meta, - B extends Body, -> = AWSS3WithCompanion & - Partial> & - AWSS3NonMultipartWithCompanionMandatory & { - shouldUseMultipart: (file: UppyFile) => boolean - } - -type AWSS3MaybeMultipartWithoutCompanion< - M extends Meta, - B extends Body, -> = AWSS3WithoutCompanion & - AWSS3MultipartWithoutCompanionMandatory & - AWSS3NonMultipartWithoutCompanionMandatory & { - shouldUseMultipart: (file: UppyFile) => boolean - } - -type RequestClientOptions = Partial< - ConstructorParameters>[1] -> - -interface _AwsS3MultipartOptions extends PluginOpts, RequestClientOptions { - allowedMetaFields?: string[] | boolean - limit?: number - retryDelays?: number[] | null -} - -export type AwsS3MultipartOptions< - M extends Meta, - B extends Body, -> = _AwsS3MultipartOptions & - ( - | AWSS3NonMultipartWithCompanion - | AWSS3NonMultipartWithoutCompanion - | AWSS3MultipartWithCompanion - | AWSS3MultipartWithoutCompanion - | AWSS3MaybeMultipartWithCompanion - | AWSS3MaybeMultipartWithoutCompanion - ) - -const defaultOptions = { - allowedMetaFields: true, - limit: 6, - getTemporarySecurityCredentials: false as any, - shouldUseMultipart: ((file: UppyFile) => - file.size !== 0) as any as true, // TODO: Switch default to: - // eslint-disable-next-line no-bitwise - // shouldUseMultipart: (file) => file.size >> 10 >> 10 > 100, - retryDelays: [0, 1000, 3000, 5000], - companionHeaders: {}, -} satisfies Partial> - -export default class AwsS3Multipart< - M extends Meta, - B extends Body, -> extends BasePlugin< - DefinePluginOpts, keyof typeof defaultOptions> & - // We also have a few dynamic options defined below: - Pick< - AWSS3MultipartWithoutCompanionMandatory, - | 'getChunkSize' - | 'createMultipartUpload' - | 'listParts' - | 'abortMultipartUpload' - | 'completeMultipartUpload' - > & - Required> & - AWSS3MultipartWithoutCompanionMandatorySignPart & - AWSS3NonMultipartWithoutCompanionMandatory, - M, - B -> { - static VERSION = packageJson.version - - #companionCommunicationQueue - - #client: RequestClient - - protected requests: any - - protected uploaderEvents: Record | null> - - protected uploaders: Record | null> - - protected uploaderSockets: Record - - constructor(uppy: Uppy, opts?: AwsS3MultipartOptions) { - super(uppy, { - ...defaultOptions, - uploadPartBytes: AwsS3Multipart.uploadPartBytes, - createMultipartUpload: null as any, - listParts: null as any, - abortMultipartUpload: null as any, - completeMultipartUpload: null as any, - signPart: null as any, - getUploadParameters: null as any, - ...opts, - }) - // We need the `as any` here because of the dynamic default options. - this.type = 'uploader' - this.id = this.opts.id || 'AwsS3Multipart' - // @ts-expect-error TODO: remove unused - this.title = 'AWS S3 Multipart' - // TODO: only initiate `RequestClient` is `companionUrl` is defined. - this.#client = new RequestClient(uppy, opts as any) - - const dynamicDefaultOptions = { - createMultipartUpload: this.createMultipartUpload, - listParts: this.listParts, - abortMultipartUpload: this.abortMultipartUpload, - completeMultipartUpload: this.completeMultipartUpload, - signPart: - opts?.getTemporarySecurityCredentials ? - this.createSignedURL - : this.signPart, - getUploadParameters: - opts?.getTemporarySecurityCredentials ? - (this.createSignedURL as any) - : this.getUploadParameters, - } satisfies Partial> - - for (const key of Object.keys(dynamicDefaultOptions)) { - if (this.opts[key as keyof typeof dynamicDefaultOptions] == null) { - this.opts[key as keyof typeof dynamicDefaultOptions] = - dynamicDefaultOptions[key as keyof typeof dynamicDefaultOptions].bind( - this, - ) - } - } - if ( - (opts as AWSS3MultipartWithoutCompanionMandatoryPrepareUploadParts) - ?.prepareUploadParts != null && - (opts as AWSS3MultipartWithoutCompanionMandatorySignPart) - .signPart == null - ) { - this.opts.signPart = async ( - file: UppyFile, - { uploadId, key, partNumber, body, signal }: SignPartOptions, - ) => { - const { presignedUrls, headers } = await ( - opts as AWSS3MultipartWithoutCompanionMandatoryPrepareUploadParts< - M, - B - > - ).prepareUploadParts(file, { - uploadId, - key, - parts: [{ number: partNumber, chunk: body }], - signal, - }) - return { - url: presignedUrls?.[partNumber], - headers: headers?.[partNumber], - } - } - } - - /** - * Simultaneous upload limiting is shared across all uploads with this plugin. - * - * @type {RateLimitedQueue} - */ - this.requests = - (this.opts as any).rateLimitedQueue ?? - new RateLimitedQueue(this.opts.limit) - this.#companionCommunicationQueue = new HTTPCommunicationQueue( - this.requests, - this.opts, - this.#setS3MultipartState, - this.#getFile, - ) - - this.uploaders = Object.create(null) - this.uploaderEvents = Object.create(null) - this.uploaderSockets = Object.create(null) - } - - private [Symbol.for('uppy test: getClient')]() { - return this.#client - } - - setOptions(newOptions: Partial>): void { - this.#companionCommunicationQueue.setOptions(newOptions) - super.setOptions(newOptions) - this.#setCompanionHeaders() - } - - /** - * Clean up all references for a file's upload: the MultipartUploader instance, - * any events related to the file, and the Companion WebSocket connection. - * - * Set `opts.abort` to tell S3 that the multipart upload is cancelled and must be removed. - * This should be done when the user cancels the upload, not when the upload is completed or errored. - */ - resetUploaderReferences(fileID: string, opts?: { abort: boolean }): void { - if (this.uploaders[fileID]) { - this.uploaders[fileID]!.abort({ really: opts?.abort || false }) - this.uploaders[fileID] = null - } - if (this.uploaderEvents[fileID]) { - this.uploaderEvents[fileID]!.remove() - this.uploaderEvents[fileID] = null - } - if (this.uploaderSockets[fileID]) { - // @ts-expect-error TODO: remove this block in the next major - this.uploaderSockets[fileID].close() - // @ts-expect-error TODO: remove this block in the next major - this.uploaderSockets[fileID] = null - } - } - - // TODO: make this a private method in the next major - assertHost(method: string): void { - if (!this.opts.companionUrl) { - throw new Error( - `Expected a \`companionUrl\` option containing a Companion address, or if you are not using Companion, a custom \`${method}\` implementation.`, - ) - } - } - - createMultipartUpload( - file: UppyFile, - signal?: AbortSignal, - ): Promise { - this.assertHost('createMultipartUpload') - throwIfAborted(signal) - - const allowedMetaFields = getAllowedMetaFields( - this.opts.allowedMetaFields, - file.meta, - ) - const metadata = getAllowedMetadata({ meta: file.meta, allowedMetaFields }) - - return this.#client - .post( - 's3/multipart', - { - filename: file.name, - type: file.type, - metadata, - }, - { signal }, - ) - .then(assertServerError) - } - - listParts( - file: UppyFile, - { key, uploadId, signal }: UploadResultWithSignal, - oldSignal?: AbortSignal, - ): Promise { - signal ??= oldSignal // eslint-disable-line no-param-reassign - this.assertHost('listParts') - throwIfAborted(signal) - - const filename = encodeURIComponent(key) - return this.#client - .get(`s3/multipart/${uploadId}?key=${filename}`, { signal }) - .then(assertServerError) - } - - completeMultipartUpload( - file: UppyFile, - { key, uploadId, parts, signal }: MultipartUploadResultWithSignal, - oldSignal?: AbortSignal, - ): Promise { - signal ??= oldSignal // eslint-disable-line no-param-reassign - this.assertHost('completeMultipartUpload') - throwIfAborted(signal) - - const filename = encodeURIComponent(key) - const uploadIdEnc = encodeURIComponent(uploadId) - return this.#client - .post( - `s3/multipart/${uploadIdEnc}/complete?key=${filename}`, - { parts }, - { signal }, - ) - .then(assertServerError) - } - - #cachedTemporaryCredentials: MaybePromise - - async #getTemporarySecurityCredentials(options?: RequestOptions) { - throwIfAborted(options?.signal) - - if (this.#cachedTemporaryCredentials == null) { - // We do not await it just yet, so concurrent calls do not try to override it: - if (this.opts.getTemporarySecurityCredentials === true) { - this.assertHost('getTemporarySecurityCredentials') - this.#cachedTemporaryCredentials = this.#client - .get('s3/sts', options) - .then(assertServerError) - } else { - this.#cachedTemporaryCredentials = - this.opts.getTemporarySecurityCredentials(options) - } - this.#cachedTemporaryCredentials = await this.#cachedTemporaryCredentials - setTimeout( - () => { - // At half the time left before expiration, we clear the cache. That's - // an arbitrary tradeoff to limit the number of requests made to the - // remote while limiting the risk of using an expired token in case the - // clocks are not exactly synced. - // The HTTP cache should be configured to ensure a client doesn't request - // more tokens than it needs, but this timeout provides a second layer of - // security in case the HTTP cache is disabled or misconfigured. - this.#cachedTemporaryCredentials = null as any - }, - (getExpiry(this.#cachedTemporaryCredentials.credentials) || 0) * 500, - ) - } - - return this.#cachedTemporaryCredentials - } - - async createSignedURL( - file: UppyFile, - options: SignPartOptions, - ): Promise { - const data = await this.#getTemporarySecurityCredentials(options) - const expires = getExpiry(data.credentials) || 604_800 // 604 800 is the max value accepted by AWS. - - const { uploadId, key, partNumber } = options - - // Return an object in the correct shape. - return { - method: 'PUT', - expires, - fields: {}, - url: `${await createSignedURL({ - accountKey: data.credentials.AccessKeyId, - accountSecret: data.credentials.SecretAccessKey, - sessionToken: data.credentials.SessionToken, - expires, - bucketName: data.bucket, - Region: data.region, - Key: key ?? `${crypto.randomUUID()}-${file.name}`, - uploadId, - partNumber, - })}`, - // Provide content type header required by S3 - headers: { - 'Content-Type': file.type as string, - }, - } - } - - signPart( - file: UppyFile, - { uploadId, key, partNumber, signal }: SignPartOptions, - ): Promise { - this.assertHost('signPart') - throwIfAborted(signal) - - if (uploadId == null || key == null || partNumber == null) { - throw new Error( - 'Cannot sign without a key, an uploadId, and a partNumber', - ) - } - - const filename = encodeURIComponent(key) - return this.#client - .get( - `s3/multipart/${uploadId}/${partNumber}?key=${filename}`, - { signal }, - ) - .then(assertServerError) - } - - abortMultipartUpload( - file: UppyFile, - { key, uploadId, signal }: UploadResultWithSignal, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - oldSignal?: AbortSignal, // TODO: remove in next major - ): Promise { - signal ??= oldSignal // eslint-disable-line no-param-reassign - this.assertHost('abortMultipartUpload') - - const filename = encodeURIComponent(key) - const uploadIdEnc = encodeURIComponent(uploadId) - return this.#client - .delete(`s3/multipart/${uploadIdEnc}?key=${filename}`, undefined, { - signal, - }) - .then(assertServerError) - } - - getUploadParameters( - file: UppyFile, - options: RequestOptions, - ): Promise { - const { meta } = file - const { type, name: filename } = meta - const allowedMetaFields = getAllowedMetaFields( - this.opts.allowedMetaFields, - file.meta, - ) - const metadata = getAllowedMetadata({ - meta, - allowedMetaFields, - querify: true, - }) - - const query = new URLSearchParams({ filename, type, ...metadata } as Record< - string, - string - >) - - return this.#client.get(`s3/params?${query}`, options) - } - - static async uploadPartBytes({ - signature: { url, expires, headers, method = 'PUT' }, - body, - size = (body as Blob).size, - onProgress, - onComplete, - signal, - }: { - signature: AwsS3UploadParameters - body: FormData | Blob - size?: number - onProgress: any - onComplete: any - signal?: AbortSignal - }): Promise { - throwIfAborted(signal) - - if (url == null) { - throw new Error('Cannot upload to an undefined URL') - } - - return new Promise((resolve, reject) => { - const xhr = new XMLHttpRequest() - xhr.open(method, url, true) - if (headers) { - Object.keys(headers).forEach((key) => { - xhr.setRequestHeader(key, headers[key]) - }) - } - xhr.responseType = 'text' - if (typeof expires === 'number') { - xhr.timeout = expires * 1000 - } - - function onabort() { - xhr.abort() - } - function cleanup() { - signal?.removeEventListener('abort', onabort) - } - signal?.addEventListener('abort', onabort) - - xhr.upload.addEventListener('progress', (ev) => { - onProgress(ev) - }) - - xhr.addEventListener('abort', () => { - cleanup() - - reject(createAbortError()) - }) - - xhr.addEventListener('timeout', () => { - cleanup() - - const error = new Error('Request has expired') - ;(error as any).source = { status: 403 } - reject(error) - }) - xhr.addEventListener('load', (ev) => { - cleanup() - - if ( - xhr.status === 403 && - xhr.responseText.includes('Request has expired') - ) { - const error = new Error('Request has expired') - ;(error as any).source = xhr - reject(error) - return - } - if (xhr.status < 200 || xhr.status >= 300) { - const error = new Error('Non 2xx') - ;(error as any).source = xhr - reject(error) - return - } - - // todo make a proper onProgress API (breaking change) - onProgress?.({ loaded: size, lengthComputable: true }) - - // NOTE This must be allowed by CORS. - const etag = xhr.getResponseHeader('ETag') - const location = xhr.getResponseHeader('Location') - - if (method.toUpperCase() === 'POST' && location === null) { - // Not being able to read the Location header is not a fatal error. - // eslint-disable-next-line no-console - console.warn( - 'AwsS3/Multipart: Could not read the Location header. This likely means CORS is not configured correctly on the S3 Bucket. See https://uppy.io/docs/aws-s3-multipart#S3-Bucket-Configuration for instructions.', - ) - } - if (etag === null) { - reject( - new Error( - 'AwsS3/Multipart: Could not read the ETag header. This likely means CORS is not configured correctly on the S3 Bucket. See https://uppy.io/docs/aws-s3-multipart#S3-Bucket-Configuration for instructions.', - ), - ) - return - } - - onComplete?.(etag) - resolve({ - ETag: etag, - ...(location ? { location } : undefined), - }) - }) - - xhr.addEventListener('error', (ev) => { - cleanup() - - const error = new Error('Unknown error') - ;(error as any).source = ev.target - reject(error) - }) - - xhr.send(body) - }) - } - - #setS3MultipartState = ( - file: UppyFile, - { key, uploadId }: UploadResult, - ) => { - const cFile = this.uppy.getFile(file.id) - if (cFile == null) { - // file was removed from store - return - } - - this.uppy.setFileState(file.id, { - s3Multipart: { - ...(cFile as MultipartFile).s3Multipart, - key, - uploadId, - }, - } as Partial>) - } - - #getFile = (file: UppyFile) => { - return this.uppy.getFile(file.id) || file - } - - #uploadLocalFile(file: UppyFile) { - return new Promise((resolve, reject) => { - const onProgress = (bytesUploaded: number, bytesTotal: number) => { - this.uppy.emit('upload-progress', this.uppy.getFile(file.id), { - // @ts-expect-error TODO: figure out if we need this - uploader: this, - bytesUploaded, - bytesTotal, - }) - } - - const onError = (err: unknown) => { - this.uppy.log(err as Error) - this.uppy.emit('upload-error', file, err as Error) - - this.resetUploaderReferences(file.id) - reject(err) - } - - const onSuccess = (result: B) => { - const uploadResp = { - body: { - ...result, - }, - status: 200, - uploadURL: result.location, - } - - this.resetUploaderReferences(file.id) - - this.uppy.emit('upload-success', this.#getFile(file), uploadResp) - - if (result.location) { - this.uppy.log(`Download ${file.name} from ${result.location}`) - } - - resolve() - } - - const upload = new MultipartUploader(file.data, { - // .bind to pass the file object to each handler. - companionComm: this.#companionCommunicationQueue, - - log: (...args: Parameters['log']>) => this.uppy.log(...args), - getChunkSize: - this.opts.getChunkSize ? this.opts.getChunkSize.bind(this) : null, - - onProgress, - onError, - onSuccess, - onPartComplete: (part) => { - this.uppy.emit( - 's3-multipart:part-uploaded', - this.#getFile(file), - part, - ) - }, - - file, - shouldUseMultipart: this.opts.shouldUseMultipart, - - ...(file as MultipartFile).s3Multipart, - }) - - this.uploaders[file.id] = upload - const eventManager = new EventManager(this.uppy) - this.uploaderEvents[file.id] = eventManager - - eventManager.onFileRemove(file.id, (removed) => { - upload.abort() - this.resetUploaderReferences(file.id, { abort: true }) - resolve(`upload ${removed} was removed`) - }) - - eventManager.onCancelAll(file.id, (options) => { - if (options?.reason === 'user') { - upload.abort() - this.resetUploaderReferences(file.id, { abort: true }) - } - resolve(`upload ${file.id} was canceled`) - }) - - eventManager.onFilePause(file.id, (isPaused) => { - if (isPaused) { - upload.pause() - } else { - upload.start() - } - }) - - eventManager.onPauseAll(file.id, () => { - upload.pause() - }) - - eventManager.onResumeAll(file.id, () => { - upload.start() - }) - - upload.start() - }) - } - - // eslint-disable-next-line class-methods-use-this - #getCompanionClientArgs(file: UppyFile) { - return { - ...file.remote?.body, - protocol: 's3-multipart', - size: file.data.size, - metadata: file.meta, - } - } - - #upload = async (fileIDs: string[]) => { - if (fileIDs.length === 0) return undefined - - const files = this.uppy.getFilesByIds(fileIDs) - const filesFiltered = filterNonFailedFiles(files) - const filesToEmit = filterFilesToEmitUploadStarted(filesFiltered) - - this.uppy.emit('upload-start', filesToEmit) - - const promises = filesFiltered.map((file) => { - if (file.isRemote) { - const getQueue = () => this.requests - this.#setResumableUploadsCapability(false) - const controller = new AbortController() - - const removedHandler = (removedFile: UppyFile) => { - if (removedFile.id === file.id) controller.abort() - } - this.uppy.on('file-removed', removedHandler) - - const uploadPromise = this.uppy - .getRequestClientForFile>(file) - .uploadRemoteFile(file, this.#getCompanionClientArgs(file), { - signal: controller.signal, - getQueue, - }) - - this.requests.wrapSyncFunction( - () => { - this.uppy.off('file-removed', removedHandler) - }, - { priority: -1 }, - )() - - return uploadPromise - } - - return this.#uploadLocalFile(file) - }) - - const upload = await Promise.all(promises) - // After the upload is done, another upload may happen with only local files. - // We reset the capability so that the next upload can use resumable uploads. - this.#setResumableUploadsCapability(true) - return upload - } - - #setCompanionHeaders = () => { - this.#client.setCompanionHeaders(this.opts.companionHeaders) - } - - #setResumableUploadsCapability = (boolean: boolean) => { - const { capabilities } = this.uppy.getState() - this.uppy.setState({ - capabilities: { - ...capabilities, - resumableUploads: boolean, - }, - }) - } - - #resetResumableCapability = () => { - this.#setResumableUploadsCapability(true) - } - - install(): void { - this.#setResumableUploadsCapability(true) - this.uppy.addPreProcessor(this.#setCompanionHeaders) - this.uppy.addUploader(this.#upload) - this.uppy.on('cancel-all', this.#resetResumableCapability) - } - - uninstall(): void { - this.uppy.removePreProcessor(this.#setCompanionHeaders) - this.uppy.removeUploader(this.#upload) - this.uppy.off('cancel-all', this.#resetResumableCapability) - } -} - -export type uploadPartBytes = (typeof AwsS3Multipart< - any, - any ->)['uploadPartBytes'] +export * from '@uppy/aws-s3' +export { default } from '@uppy/aws-s3' diff --git a/packages/@uppy/aws-s3/CHANGELOG.md b/packages/@uppy/aws-s3/CHANGELOG.md index 2e06eec98b..c5fbb4b69b 100644 --- a/packages/@uppy/aws-s3/CHANGELOG.md +++ b/packages/@uppy/aws-s3/CHANGELOG.md @@ -1,93 +1,157 @@ -# @uppy/aws-s3 +# @uppy/aws-s3-multipart -## 3.6.1 +## 4.0.0-beta.1 -Released: 2024-02-19 -Included in: Uppy v3.22.0 +Released: 2024-03-28 +Included in: Uppy v4.0.0-beta.1 -- @uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/companion-client,@uppy/tus,@uppy/xhr-upload: update `uppyfile` objects before emitting events (antoine du hamel / #4928) +- @uppy/aws-s3-multipart: mark `opts` as optional (Antoine du Hamel / #5039) +- @uppy/aws-s3-multipart,@uppy/tus,@uppy/utils,@uppy/xhr-upload: Make `allowedMetaFields` consistent (Merlijn Vos / #5011) +- @uppy/aws-s3-multipart: refactor to TS (Antoine du Hamel / #4902) +- @uppy/aws-s3-multipart: fix escaping issue with client signed request (Hiroki Shimizu / #5006) -## 3.6.0 +## 3.11.0 -Released: 2023-12-12 -Included in: Uppy v3.21.0 +Released: 2024-03-27 +Included in: Uppy v3.24.0 -- @uppy/aws-s3: change Companion URL in tests (Antoine du Hamel) +- @uppy/aws-s3-multipart: mark `opts` as optional (Antoine du Hamel / #5039) +- @uppy/aws-s3-multipart: refactor to TS (Antoine du Hamel / #4902) +- @uppy/aws-s3-multipart: fix escaping issue with client signed request (Hiroki Shimizu / #5006) -## 3.3.0 +## 3.8.0 + +Released: 2023-10-20 +Included in: Uppy v3.18.0 + +- @uppy/aws-s3-multipart: fix `TypeError` (Antoine du Hamel / #4748) +- @uppy/aws-s3-multipart: pass `signal` as separate arg for backward compat (Antoine du Hamel / #4746) +- @uppy/aws-s3-multipart: fix `uploadURL` when using `PUT` (Antoine du Hamel / #4701) + +## 3.7.0 + +Released: 2023-09-29 +Included in: Uppy v3.17.0 + +- @uppy/aws-s3-multipart: retry signature request (Merlijn Vos / #4691) +- @uppy/aws-s3-multipart: aws-s3-multipart - call `#setCompanionHeaders` in `setOptions` (jur-ng / #4687) + +## 3.6.0 Released: 2023-09-05 Included in: Uppy v3.15.0 - @uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/companion-client,@uppy/core,@uppy/tus,@uppy/utils,@uppy/xhr-upload: Move remote file upload logic into companion-client (Merlijn Vos / #4573) -## 3.2.3 +## 3.5.4 Released: 2023-08-23 Included in: Uppy v3.14.1 +- @uppy/aws-s3-multipart: fix types when using deprecated option (Antoine du Hamel / #4634) - @uppy/aws-s3-multipart,@uppy/aws-s3: allow empty objects for `fields` types (Antoine du Hamel / #4631) -## 3.2.2 +## 3.5.3 Released: 2023-08-15 Included in: Uppy v3.14.0 +- @uppy/aws-s3-multipart: pass the `uploadURL` back to the caller (Antoine du Hamel / #4614) - @uppy/aws-s3,@uppy/aws-s3-multipart: update types (Antoine du Hamel / #4611) - @uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/companion,@uppy/transloadit,@uppy/xhr-upload: use uppercase HTTP method names (Antoine du Hamel / #4612) - @uppy/aws-s3,@uppy/aws-s3-multipart: update types (bdirito / #4576) -- @uppy/aws-s3,@uppy/tus,@uppy/xhr-upload: Invoke headers function for remote uploads (Dominik Schmidt / #4596) -## 3.2.1 +## 3.5.2 + +Released: 2023-07-24 +Included in: Uppy v3.13.1 + +- @uppy/aws-s3-multipart: refresh file before calling user-defined functions (mjlumetta / #4557) + +## 3.5.1 + +Released: 2023-07-20 +Included in: Uppy v3.13.0 + +- @uppy/aws-s3-multipart: fix crash on pause/resume (Merlijn Vos / #4581) +- @uppy/aws-s3-multipart: do not access `globalThis.crypto` on the top-level (Bryan J Swift / #4584) + +## 3.5.0 + +Released: 2023-07-13 +Included in: Uppy v3.12.0 + +- @uppy/aws-s3-multipart: add support for signing on the client (Antoine du Hamel / #4519) +- @uppy/aws-s3-multipart: fix lint warning (Antoine du Hamel / #4569) +- @uppy/aws-s3-multipart: fix support for non-multipart PUT upload (Antoine du Hamel / #4568) + +## 3.4.1 Released: 2023-07-06 Included in: Uppy v3.11.0 -- @uppy/aws-s3: fix remote uploads (Antoine du Hamel / #4546) +- @uppy/aws-s3-multipart: increase priority of abort and complete (Stefan Schonert / #4542) +- @uppy/aws-s3-multipart: fix upload retry using an outdated ID (Antoine du Hamel / #4544) +- @uppy/aws-s3-multipart: fix Golden Retriever integration (Antoine du Hamel / #4526) +- @uppy/aws-s3-multipart: add types to internal fields (Antoine du Hamel / #4535) +- @uppy/aws-s3-multipart: fix pause/resume (Antoine du Hamel / #4523) +- @uppy/aws-s3-multipart: fix resume single-chunk multipart uploads (Antoine du Hamel / #4528) +- @uppy/aws-s3-multipart: disable pause/resume for remote uploads in the UI (Artur Paikin / #4500) -## 3.2.0 +## 3.4.0 Released: 2023-06-19 Included in: Uppy v3.10.0 -- @uppy/aws-s3: add `shouldUseMultipart` option (Antoine du Hamel / #4299) +- @uppy/aws-s3-multipart: fix the chunk size calculation (Antoine du Hamel / #4508) - @uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/tus,@uppy/utils,@uppy/xhr-upload: When file is removed (or all are canceled), controller.abort queued requests (Artur Paikin / #4504) +- @uppy/aws-s3-multipart,@uppy/tus,@uppy/xhr-upload: Don't close socket while upload is still in progress (Artur Paikin / #4479) +- @uppy/aws-s3-multipart: fix `getUploadParameters` option (Antoine du Hamel / #4465) -## 3.1.1 +## 3.3.0 Released: 2023-05-02 Included in: Uppy v3.9.0 -- @uppy/aws-s3: deprecate `timeout` option (Antoine du Hamel / #4298) +- @uppy/aws-s3-multipart: allowedMetaFields: null means “include all” (Artur Paikin / #4437) +- @uppy/aws-s3-multipart: add `shouldUseMultipart ` option (Antoine du Hamel / #4205) +- @uppy/aws-s3-multipart: make retries more robust (Antoine du Hamel / #4424) -## 3.0.6 +## 3.1.3 Released: 2023-04-04 Included in: Uppy v3.7.0 - @uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/tus,@uppy/xhr-upload: make sure that we reset serverToken when an upload fails (Mikael Finstad / #4376) -- @uppy/aws-s3: Update types (Minh Hieu / #4294) +- @uppy/aws-s3-multipart: do not auto-open sockets, clean them up on abort (Antoine du Hamel) -## 3.0.5 +## 3.1.2 Released: 2023-01-26 Included in: Uppy v3.4.0 -- @uppy/aws-s3: fix: add https:// to digital oceans link (Le Gia Hoang / #4165) +- @uppy/aws-s3-multipart: fix metadata shape (Antoine du Hamel / #4267) +- @uppy/aws-s3-multipart: add support for `allowedMetaFields` option (Antoine du Hamel / #4215) +- @uppy/aws-s3-multipart: fix singPart type (Stefan Schonert / #4224) -## 3.0.4 +## 3.1.1 -Released: 2022-10-24 -Included in: Uppy v3.2.2 +Released: 2022-11-16 +Included in: Uppy v3.3.1 -- @uppy/aws-s3,@uppy/tus,@uppy/xhr-upload: replace `this.getState().files` with `this.uppy.getState().files` (Artur Paikin / #4167) +- @uppy/aws-s3-multipart: handle slow connections better (Antoine du Hamel / #4213) +- @uppy/aws-s3-multipart: Fix typo in url check (Christian Franke / #4211) -## 3.0.3 +## 3.1.0 -Released: 2022-10-19 -Included in: Uppy v3.2.0 +Released: 2022-11-10 +Included in: Uppy v3.3.0 -- @uppy/aws-s3,@uppy/xhr-upload: fix `Cannot mark a queued request as done` in `MiniXHRUpload` (Antoine du Hamel / #4151) +- @uppy/aws-s3-multipart: empty the queue when pausing (Antoine du Hamel / #4203) +- @uppy/aws-s3-multipart: refactor rate limiting approach (Antoine du Hamel / #4187) +- @uppy/aws-s3-multipart: change limit to 6 (Antoine du Hamel / #4199) +- @uppy/aws-s3-multipart: remove unused `timeout` option (Antoine du Hamel / #4186) +- @uppy/aws-s3-multipart,@uppy/tus: fix `Timed out waiting for socket` (Antoine du Hamel / #4177) ## 3.0.2 @@ -101,71 +165,72 @@ Included in: Uppy v3.1.0 Released: 2022-08-22 Included in: Uppy v3.0.0 -- @uppy/aws-s3,@uppy/tus,@uppy/xhr-upload: @uppy/tus, @uppy/xhr-upload, @uppy/aws-s3: `metaFields` -> `allowedMetaFields` (Merlijn Vos / #4023) -- @uppy/aws-s3: aws-s3: fix incorrect comparison for `file-removed` (Merlijn Vos / #3962) - Switch to ESM -## 3.0.0-beta.3 +## 3.0.0-beta.4 Released: 2022-08-16 Included in: Uppy v3.0.0-beta.5 -- @uppy/aws-s3: Export AwsS3UploadParameters & AwsS3Options interfaces (Antonina Vertsinskaya / #3956) +- @uppy/aws-s3-multipart: Fix when using Companion (Merlijn Vos / #3969) +- @uppy/aws-s3-multipart: Fix race condition in `#uploadParts` (Morgan Zolob / #3955) +- @uppy/aws-s3-multipart: ignore exception inside `abortMultipartUpload` (Antoine du Hamel / #3950) + +## 3.0.0-beta.3 + +Released: 2022-08-03 +Included in: Uppy v3.0.0-beta.4 + +- @uppy/aws-s3-multipart: Correctly handle errors for `prepareUploadParts` (Merlijn Vos / #3912) ## 3.0.0-beta.2 Released: 2022-07-27 Included in: Uppy v3.0.0-beta.3 -- @uppy/aws-s3,@uppy/core,@uppy/dashboard,@uppy/store-redux,@uppy/xhr-upload: upgrade `nanoid` to v4 (Antoine du Hamel / #3904) +- @uppy/aws-s3-multipart: make `headers` part indexed too in `prepareUploadParts` (Merlijn Vos / #3895) -## 2.2.1 +## 2.4.1 Released: 2022-06-07 Included in: Uppy v2.12.0 - @uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/tus: queue socket token requests for remote files (Merlijn Vos / #3797) +- @uppy/aws-s3-multipart: allow `companionHeaders` to be modified with `setOptions` (Paulo Lemos Neto / #3770) -## 2.2.0 +## 2.4.0 Released: 2022-05-30 Included in: Uppy v2.11.0 - @uppy/angular,@uppy/audio,@uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/box,@uppy/core,@uppy/dashboard,@uppy/drag-drop,@uppy/dropbox,@uppy/facebook,@uppy/file-input,@uppy/form,@uppy/golden-retriever,@uppy/google-drive,@uppy/image-editor,@uppy/informer,@uppy/instagram,@uppy/onedrive,@uppy/progress-bar,@uppy/react,@uppy/redux-dev-tools,@uppy/robodog,@uppy/screen-capture,@uppy/status-bar,@uppy/store-default,@uppy/store-redux,@uppy/thumbnail-generator,@uppy/transloadit,@uppy/tus,@uppy/unsplash,@uppy/url,@uppy/vue,@uppy/webcam,@uppy/xhr-upload,@uppy/zoom: doc: update bundler recommendation (Antoine du Hamel / #3763) -- @uppy/aws-s3: fix JSDoc type error (Antoine du Hamel / #3785) -- @uppy/aws-s3: refactor to ESM (Antoine du Hamel / #3673) +- @uppy/aws-s3-multipart: refactor to ESM (Antoine du Hamel / #3672) -## 2.1.0 +## 2.3.0 Released: 2022-05-14 Included in: Uppy v2.10.0 - @uppy/aws-s3-multipart,@uppy/aws-s3,@uppy/core,@uppy/react,@uppy/transloadit,@uppy/tus,@uppy/xhr-upload: proposal: Cancel assemblies optional (Mikael Finstad / #3575) +- @uppy/aws-s3-multipart: export interface AwsS3MultipartOptions (Matteo Padovano / #3709) -## 2.0.9 - -Released: 2022-04-07 -Included in: Uppy v2.9.2 +## 2.2.2 -- @uppy/aws-s3,@uppy/companion-client,@uppy/transloadit,@uppy/utils: Propagate `isNetworkError` through error wrappers (Renée Kooi / #3620) +Released: 2022-04-27 +Included in: Uppy v2.9.4 -## 2.0.8 +- @uppy/aws-s3-multipart: Add `companionCookiesRule` type to @uppy/aws-s3-multipart (Mauricio Ribeiro / #3623) -Released: 2022-03-16 -Included in: Uppy v2.8.0 - -- @uppy/aws-s3: fix wrong events being sent to companion (Mikael Finstad / #3576) - -## 2.0.7 +## 2.2.1 -Released: 2021-12-09 -Included in: Uppy v2.3.1 +Released: 2022-03-02 +Included in: Uppy v2.7.0 -- @uppy/aws-s3,@uppy/core,@uppy/dashboard,@uppy/store-redux,@uppy/xhr-upload: deps: use `nanoid/non-secure` to workaround react-native limitation (Antoine du Hamel / #3350) +- @uppy/aws-s3-multipart: Add chunks back to prepareUploadParts, indexed by partNumber (Kevin West / #3520) -## 2.0.6 +## 2.2.0 Released: 2021-12-07 Included in: Uppy v2.3.0 -- @uppy/aws-s3,@uppy/box,@uppy/core,@uppy/dashboard,@uppy/drag-drop,@uppy/dropbox,@uppy/facebook,@uppy/file-input,@uppy/google-drive,@uppy/image-editor,@uppy/instagram,@uppy/locales,@uppy/onedrive,@uppy/screen-capture,@uppy/status-bar,@uppy/thumbnail-generator,@uppy/transloadit,@uppy/url,@uppy/webcam,@uppy/xhr-upload,@uppy/zoom: Refactor locale scripts & generate types and docs (Merlijn Vos / #3276) +- @uppy/aws-s3-multipart: Drop `lockedCandidatesForBatch` and mark chunks as 'busy' when preparing (Yegor Yarko / #3342) diff --git a/packages/@uppy/aws-s3/package.json b/packages/@uppy/aws-s3/package.json index 45a7a93eee..1e05f53c84 100644 --- a/packages/@uppy/aws-s3/package.json +++ b/packages/@uppy/aws-s3/package.json @@ -11,7 +11,8 @@ "amazon s3", "s3", "uppy", - "uppy-plugin" + "uppy-plugin", + "multipart" ], "homepage": "https://uppy.io", "bugs": { @@ -22,13 +23,13 @@ "url": "git+https://github.com/transloadit/uppy.git" }, "dependencies": { - "@uppy/aws-s3-multipart": "workspace:^", "@uppy/companion-client": "workspace:^", - "@uppy/utils": "workspace:^", - "@uppy/xhr-upload": "workspace:^", - "nanoid": "^4.0.0" + "@uppy/utils": "workspace:^" }, "devDependencies": { + "@aws-sdk/client-s3": "^3.362.0", + "@aws-sdk/s3-request-presigner": "^3.362.0", + "nock": "^13.1.0", "vitest": "^1.2.1", "whatwg-fetch": "3.6.2" }, diff --git a/packages/@uppy/aws-s3-multipart/src/HTTPCommunicationQueue.ts b/packages/@uppy/aws-s3/src/HTTPCommunicationQueue.ts similarity index 100% rename from packages/@uppy/aws-s3-multipart/src/HTTPCommunicationQueue.ts rename to packages/@uppy/aws-s3/src/HTTPCommunicationQueue.ts diff --git a/packages/@uppy/aws-s3/src/MiniXHRUpload.js b/packages/@uppy/aws-s3/src/MiniXHRUpload.js deleted file mode 100644 index e4fbc1e28c..0000000000 --- a/packages/@uppy/aws-s3/src/MiniXHRUpload.js +++ /dev/null @@ -1,235 +0,0 @@ -import { nanoid } from 'nanoid/non-secure' -import EventManager from '@uppy/utils/lib/EventManager' -import ProgressTimeout from '@uppy/utils/lib/ProgressTimeout' -import ErrorWithCause from '@uppy/utils/lib/ErrorWithCause' -import NetworkError from '@uppy/utils/lib/NetworkError' -import isNetworkError from '@uppy/utils/lib/isNetworkError' -import { internalRateLimitedQueue } from '@uppy/utils/lib/RateLimitedQueue' - -// See XHRUpload -function buildResponseError (xhr, error) { - if (isNetworkError(xhr)) return new NetworkError(error, xhr) - - const err = new ErrorWithCause('Upload error', { cause: error }) - err.request = xhr - return err -} - -// See XHRUpload -function setTypeInBlob (file) { - const dataWithUpdatedType = file.data.slice(0, file.data.size, file.meta.type) - return dataWithUpdatedType -} - -function addMetadata (formData, meta, opts) { - const allowedMetaFields = Array.isArray(opts.allowedMetaFields) - ? opts.allowedMetaFields - // Send along all fields by default. - : Object.keys(meta) - allowedMetaFields.forEach((item) => { - formData.append(item, meta[item]) - }) -} - -function createFormDataUpload (file, opts) { - const formPost = new FormData() - - addMetadata(formPost, file.meta, opts) - - const dataWithUpdatedType = setTypeInBlob(file) - - if (file.name) { - formPost.append(opts.fieldName, dataWithUpdatedType, file.meta.name) - } else { - formPost.append(opts.fieldName, dataWithUpdatedType) - } - - return formPost -} - -const createBareUpload = file => file.data - -export default class MiniXHRUpload { - constructor (uppy, opts) { - this.uppy = uppy - this.opts = { - validateStatus (status) { - return status >= 200 && status < 300 - }, - ...opts, - } - - this.requests = opts[internalRateLimitedQueue] - this.uploaderEvents = Object.create(null) - this.i18n = opts.i18n - } - - getOptions (file) { - const { uppy } = this - - const overrides = uppy.getState().xhrUpload - const opts = { - ...this.opts, - ...(overrides || {}), - ...(file.xhrUpload || {}), - headers: { - ...this.opts.headers, - ...overrides?.headers, - ...file.xhrUpload?.headers, - }, - } - - return opts - } - - #addEventHandlerForFile (eventName, fileID, eventHandler) { - this.uploaderEvents[fileID].on(eventName, (fileOrID) => { - // TODO (major): refactor Uppy events to consistently send file objects (or consistently IDs) - // We created a generic `addEventListenerForFile` but not all events - // use file IDs, some use files, so we need to do this weird check. - const id = fileOrID?.id ?? fileOrID - if (fileID === id) eventHandler() - }) - } - - #addEventHandlerIfFileStillExists (eventName, fileID, eventHandler) { - this.uploaderEvents[fileID].on(eventName, (...args) => { - if (this.uppy.getFile(fileID)) eventHandler(...args) - }) - } - - uploadLocalFile (file) { - const opts = this.getOptions(file) - - return new Promise((resolve, reject) => { - // This is done in index.js in the S3 plugin. - // this.uppy.emit('upload-started', file) - - const data = opts.formData - ? createFormDataUpload(file, opts) - : createBareUpload(file, opts) - - const xhr = new XMLHttpRequest() - this.uploaderEvents[file.id] = new EventManager(this.uppy) - - const timer = new ProgressTimeout(opts.timeout, () => { - xhr.abort() - // eslint-disable-next-line no-use-before-define - queuedRequest.done() - const error = new Error(this.i18n('timedOut', { seconds: Math.ceil(opts.timeout / 1000) })) - this.uppy.emit('upload-error', file, error) - reject(error) - }) - - const id = nanoid() - - xhr.upload.addEventListener('loadstart', () => { - this.uppy.log(`[AwsS3/XHRUpload] ${id} started`) - }) - - xhr.upload.addEventListener('progress', (ev) => { - this.uppy.log(`[AwsS3/XHRUpload] ${id} progress: ${ev.loaded} / ${ev.total}`) - // Begin checking for timeouts when progress starts, instead of loading, - // to avoid timing out requests on browser concurrency queue - timer.progress() - - if (ev.lengthComputable) { - this.uppy.emit('upload-progress', this.uppy.getFile(file.id), { - uploader: this, - bytesUploaded: ev.loaded, - bytesTotal: ev.total, - }) - } - }) - - xhr.addEventListener('load', (ev) => { - this.uppy.log(`[AwsS3/XHRUpload] ${id} finished`) - timer.done() - // eslint-disable-next-line no-use-before-define - queuedRequest.done() - if (this.uploaderEvents[file.id]) { - this.uploaderEvents[file.id].remove() - this.uploaderEvents[file.id] = null - } - - if (opts.validateStatus(ev.target.status, xhr.responseText, xhr)) { - const body = opts.getResponseData(xhr.responseText, xhr) - const uploadURL = body[opts.responseUrlFieldName] - - const uploadResp = { - status: ev.target.status, - body, - uploadURL, - } - - this.uppy.emit('upload-success', this.uppy.getFile(file.id), uploadResp) - - if (uploadURL) { - this.uppy.log(`Download ${file.name} from ${uploadURL}`) - } - - return resolve(file) - } - const body = opts.getResponseData(xhr.responseText, xhr) - const error = buildResponseError(xhr, opts.getResponseError(xhr.responseText, xhr)) - - const response = { - status: ev.target.status, - body, - } - - this.uppy.emit('upload-error', file, error, response) - return reject(error) - }) - - xhr.addEventListener('error', () => { - this.uppy.log(`[AwsS3/XHRUpload] ${id} errored`) - timer.done() - // eslint-disable-next-line no-use-before-define - queuedRequest.done() - if (this.uploaderEvents[file.id]) { - this.uploaderEvents[file.id].remove() - this.uploaderEvents[file.id] = null - } - - const error = buildResponseError(xhr, opts.getResponseError(xhr.responseText, xhr)) - this.uppy.emit('upload-error', file, error) - return reject(error) - }) - - xhr.open(opts.method.toUpperCase(), opts.endpoint, true) - // IE10 does not allow setting `withCredentials` and `responseType` - // before `open()` is called. It’s important to set withCredentials - // to a boolean, otherwise React Native crashes - xhr.withCredentials = Boolean(opts.withCredentials) - if (opts.responseType !== '') { - xhr.responseType = opts.responseType - } - - Object.keys(opts.headers).forEach((header) => { - xhr.setRequestHeader(header, opts.headers[header]) - }) - - const queuedRequest = this.requests.run(() => { - xhr.send(data) - return () => { - // eslint-disable-next-line no-use-before-define - timer.done() - xhr.abort() - } - }, { priority: 1 }) - - this.#addEventHandlerForFile('file-removed', file.id, () => { - queuedRequest.abort() - reject(new Error('File removed')) - }) - - this.#addEventHandlerIfFileStillExists('cancel-all', file.id, ({ reason } = {}) => { - if (reason === 'user') { - queuedRequest.abort() - } - reject(new Error('Upload cancelled')) - }) - }) - } -} diff --git a/packages/@uppy/aws-s3-multipart/src/MultipartUploader.ts b/packages/@uppy/aws-s3/src/MultipartUploader.ts similarity index 100% rename from packages/@uppy/aws-s3-multipart/src/MultipartUploader.ts rename to packages/@uppy/aws-s3/src/MultipartUploader.ts diff --git a/packages/@uppy/aws-s3-multipart/src/createSignedURL.test.ts b/packages/@uppy/aws-s3/src/createSignedURL.test.ts similarity index 100% rename from packages/@uppy/aws-s3-multipart/src/createSignedURL.test.ts rename to packages/@uppy/aws-s3/src/createSignedURL.test.ts diff --git a/packages/@uppy/aws-s3-multipart/src/createSignedURL.ts b/packages/@uppy/aws-s3/src/createSignedURL.ts similarity index 100% rename from packages/@uppy/aws-s3-multipart/src/createSignedURL.ts rename to packages/@uppy/aws-s3/src/createSignedURL.ts diff --git a/packages/@uppy/aws-s3/src/index.js b/packages/@uppy/aws-s3/src/index.js deleted file mode 100644 index 230ebb7864..0000000000 --- a/packages/@uppy/aws-s3/src/index.js +++ /dev/null @@ -1,367 +0,0 @@ -/** - * This plugin is currently a A Big Hack™! The core reason for that is how this plugin - * interacts with Uppy's current pipeline design. The pipeline can handle files in steps, - * including preprocessing, uploading, and postprocessing steps. This plugin initially - * was designed to do its work in a preprocessing step, and let XHRUpload deal with the - * actual file upload as an uploading step. However, Uppy runs steps on all files at once, - * sequentially: first, all files go through a preprocessing step, then, once they are all - * done, they go through the uploading step. - * - * For S3, this causes severely broken behaviour when users upload many files. The - * preprocessing step will request S3 upload URLs that are valid for a short time only, - * but it has to do this for _all_ files, which can take a long time if there are hundreds - * or even thousands of files. By the time the uploader step starts, the first URLs may - * already have expired. If not, the uploading might take such a long time that later URLs - * will expire before some files can be uploaded. - * - * The long-term solution to this problem is to change the upload pipeline so that files - * can be sent to the next step individually. That requires a breaking change, so it is - * planned for some future Uppy version. - * - * In the mean time, this plugin is stuck with a hackier approach: the necessary parts - * of the XHRUpload implementation were copied into this plugin, as the MiniXHRUpload - * class, and this plugin calls into it immediately once it receives an upload URL. - * This isn't as nicely modular as we'd like and requires us to maintain two copies of - * the XHRUpload code, but at least it's not horrifically broken :) - */ - -import BasePlugin from '@uppy/core/lib/BasePlugin.js' -import AwsS3Multipart from '@uppy/aws-s3-multipart' -import { RateLimitedQueue, internalRateLimitedQueue } from '@uppy/utils/lib/RateLimitedQueue' -import { RequestClient } from '@uppy/companion-client' -import { filterNonFailedFiles, filterFilesToEmitUploadStarted } from '@uppy/utils/lib/fileFilters' - -import packageJson from '../package.json' -import MiniXHRUpload from './MiniXHRUpload.js' -import isXml from './isXml.js' -import locale from './locale.js' - -function resolveUrl (origin, link) { - // DigitalOcean doesn’t return the protocol from Location - // without it, the `new URL` constructor will fail - if (!origin && !link.startsWith('https://') && !link.startsWith('http://')) { - link = `https://${link}` // eslint-disable-line no-param-reassign - } - return new URL(link, origin || undefined).toString() -} - -/** - * Get the contents of a named tag in an XML source string. - * - * @param {string} source - The XML source string. - * @param {string} tagName - The name of the tag. - * @returns {string} The contents of the tag, or the empty string if the tag does not exist. - */ -function getXmlValue (source, tagName) { - const start = source.indexOf(`<${tagName}>`) - const end = source.indexOf(``, start) - return start !== -1 && end !== -1 - ? source.slice(start + tagName.length + 2, end) - : '' -} - -function assertServerError (res) { - if (res && res.error) { - const error = new Error(res.message) - Object.assign(error, res.error) - throw error - } - return res -} - -function validateParameters (file, params) { - const valid = params != null - && typeof params.url === 'string' - && (typeof params.fields === 'object' || params.fields == null) - - if (!valid) { - const err = new TypeError(`AwsS3: got incorrect result from 'getUploadParameters()' for file '${file.name}', expected an object '{ url, method, fields, headers }' but got '${JSON.stringify(params)}' instead.\nSee https://uppy.io/docs/aws-s3/#getUploadParameters-file for more on the expected format.`) - throw err - } - - const methodIsValid = params.method == null || /^p(u|os)t$/i.test(params.method) - - if (!methodIsValid) { - const err = new TypeError(`AwsS3: got incorrect method from 'getUploadParameters()' for file '${file.name}', expected 'PUT' or 'POST' but got '${params.method}' instead.\nSee https://uppy.io/docs/aws-s3/#getUploadParameters-file for more on the expected format.`) - throw err - } -} - -// Get the error data from a failed XMLHttpRequest instance. -// `content` is the S3 response as a string. -// `xhr` is the XMLHttpRequest instance. -function defaultGetResponseError (content, xhr) { - // If no response, we don't have a specific error message, use the default. - if (!isXml(content, xhr)) { - return undefined - } - const error = getXmlValue(content, 'Message') - return new Error(error) -} - -// warning deduplication flag: see `getResponseData()` XHRUpload option definition -let warnedSuccessActionStatus = false - -// TODO deprecate this, will use s3-multipart instead -export default class AwsS3 extends BasePlugin { - static VERSION = packageJson.version - - #client - - #requests - - #uploader - - constructor (uppy, opts) { - // Opt-in to using the multipart plugin, which is going to be the only S3 plugin as of the next semver. - if (opts?.shouldUseMultipart != null) { - return new AwsS3Multipart(uppy, opts) - } - super(uppy, opts) - this.type = 'uploader' - this.id = this.opts.id || 'AwsS3' - this.title = 'AWS S3' - - this.defaultLocale = locale - - const defaultOptions = { - timeout: 30 * 1000, - limit: 0, - allowedMetaFields: [], // have to opt in - getUploadParameters: this.getUploadParameters.bind(this), - shouldUseMultipart: false, - companionHeaders: {}, - } - - this.opts = { ...defaultOptions, ...opts } - - if (opts?.allowedMetaFields === undefined && 'metaFields' in this.opts) { - throw new Error('The `metaFields` option has been renamed to `allowedMetaFields`.') - } - - // TODO: remove i18n once we can depend on XHRUpload instead of MiniXHRUpload - this.i18nInit() - - this.#client = new RequestClient(uppy, opts) - this.#requests = new RateLimitedQueue(this.opts.limit) - } - - [Symbol.for('uppy test: getClient')] () { return this.#client } - - // TODO: remove getter and setter for #client on the next major release - get client () { return this.#client } - - set client (client) { this.#client = client } - - getUploadParameters (file) { - if (!this.opts.companionUrl) { - throw new Error('Expected a `companionUrl` option containing a Companion address.') - } - - const filename = file.meta.name - const { type } = file.meta - const metadata = Object.fromEntries( - this.opts.allowedMetaFields - .filter(key => file.meta[key] != null) - .map(key => [`metadata[${key}]`, file.meta[key].toString()]), - ) - - const query = new URLSearchParams({ filename, type, ...metadata }) - return this.#client.get(`s3/params?${query}`) - .then(assertServerError) - } - - #handleUpload = async (fileIDs) => { - /** - * keep track of `getUploadParameters()` responses - * so we can cancel the calls individually using just a file ID - * - * @type {Record>} - */ - const paramsPromises = Object.create(null) - - function onremove (file) { - const { id } = file - paramsPromises[id]?.abort() - } - this.uppy.on('file-removed', onremove) - - const files = this.uppy.getFilesByIds(fileIDs) - - const filesFiltered = filterNonFailedFiles(files) - const filesToEmit = filterFilesToEmitUploadStarted(filesFiltered) - this.uppy.emit('upload-start', filesToEmit) - - const getUploadParameters = this.#requests.wrapPromiseFunction((file) => { - return this.opts.getUploadParameters(file) - }) - - const numberOfFiles = fileIDs.length - - return Promise.allSettled(fileIDs.map((id, index) => { - paramsPromises[id] = getUploadParameters(this.uppy.getFile(id)) - return paramsPromises[id].then((params) => { - delete paramsPromises[id] - - const file = this.uppy.getFile(id) - validateParameters(file, params) - - const { - method = 'POST', - url, - fields, - headers, - } = params - const xhrOpts = { - method, - formData: method.toUpperCase() === 'POST', - endpoint: url, - allowedMetaFields: fields ? Object.keys(fields) : [], - } - - if (headers) { - xhrOpts.headers = headers - } - - this.uppy.setFileState(file.id, { - meta: { ...file.meta, ...fields }, - xhrUpload: xhrOpts, - }) - - return this.uploadFile(file.id, index, numberOfFiles) - }).catch((error) => { - delete paramsPromises[id] - - const file = this.uppy.getFile(id) - this.uppy.emit('upload-error', file, error) - return Promise.reject(error) - }) - })).finally(() => { - // cleanup. - this.uppy.off('file-removed', onremove) - }) - } - - #setCompanionHeaders = () => { - this.#client.setCompanionHeaders(this.opts.companionHeaders) - return Promise.resolve() - } - - #getCompanionClientArgs = (file) => { - const opts = this.#uploader.getOptions(file) - const allowedMetaFields = Array.isArray(opts.allowedMetaFields) - ? opts.allowedMetaFields - // Send along all fields by default. - : Object.keys(file.meta) - return { - ...file.remote.body, - protocol: 'multipart', - endpoint: opts.endpoint, - size: file.data.size, - fieldname: opts.fieldName, - metadata: Object.fromEntries(allowedMetaFields.map(name => [name, file.meta[name]])), - httpMethod: opts.method, - useFormData: opts.formData, - headers: typeof opts.headers === 'function' ? opts.headers(file) : opts.headers, - } - } - - uploadFile (id, current, total) { - const file = this.uppy.getFile(id) - this.uppy.log(`uploading ${current} of ${total}`) - - if (file.error) throw new Error(file.error) - - if (file.isRemote) { - const getQueue = () => this.#requests - const controller = new AbortController() - - const removedHandler = (removedFile) => { - if (removedFile.id === file.id) controller.abort() - } - this.uppy.on('file-removed', removedHandler) - - const uploadPromise = this.uppy.getRequestClientForFile(file).uploadRemoteFile( - file, - this.#getCompanionClientArgs(file), - { signal: controller.signal, getQueue }, - ) - - this.#requests.wrapSyncFunction(() => { - this.uppy.off('file-removed', removedHandler) - }, { priority: -1 })() - - return uploadPromise - } - - return this.#uploader.uploadLocalFile(file, current, total) - } - - install () { - const { uppy } = this - uppy.addPreProcessor(this.#setCompanionHeaders) - uppy.addUploader(this.#handleUpload) - - // Get the response data from a successful XMLHttpRequest instance. - // `content` is the S3 response as a string. - // `xhr` is the XMLHttpRequest instance. - function defaultGetResponseData (content, xhr) { - const opts = this - - // If no response, we've hopefully done a PUT request to the file - // in the bucket on its full URL. - if (!isXml(content, xhr)) { - if (opts.method.toUpperCase() === 'POST') { - if (!warnedSuccessActionStatus) { - uppy.log('[AwsS3] No response data found, make sure to set the success_action_status AWS SDK option to 201. See https://uppy.io/docs/aws-s3/#POST-Uploads', 'warning') - warnedSuccessActionStatus = true - } - // The responseURL won't contain the object key. Give up. - return { location: null } - } - - // responseURL is not available in older browsers. - if (!xhr.responseURL) { - return { location: null } - } - - // Trim the query string because it's going to be a bunch of presign - // parameters for a PUT request—doing a GET request with those will - // always result in an error - return { location: xhr.responseURL.replace(/\?.*$/, '') } - } - - return { - // Some S3 alternatives do not reply with an absolute URL. - // Eg DigitalOcean Spaces uses /$bucketName/xyz - location: resolveUrl(xhr.responseURL, getXmlValue(content, 'Location')), - bucket: getXmlValue(content, 'Bucket'), - key: getXmlValue(content, 'Key'), - etag: getXmlValue(content, 'ETag'), - } - } - - const xhrOptions = { - fieldName: 'file', - responseUrlFieldName: 'location', - timeout: this.opts.timeout, - // Share the rate limiting queue with XHRUpload. - [internalRateLimitedQueue]: this.#requests, - responseType: 'text', - getResponseData: this.opts.getResponseData || defaultGetResponseData, - getResponseError: defaultGetResponseError, - } - - // TODO: remove i18n once we can depend on XHRUpload instead of MiniXHRUpload - xhrOptions.i18n = this.i18n - - // Revert to `uppy.use(XHRUpload)` once the big comment block at the top of - // this file is solved - this.#uploader = new MiniXHRUpload(uppy, xhrOptions) - } - - uninstall () { - this.uppy.removePreProcessor(this.#setCompanionHeaders) - this.uppy.removeUploader(this.#handleUpload) - } -} diff --git a/packages/@uppy/aws-s3/src/index.test.js b/packages/@uppy/aws-s3/src/index.test.js deleted file mode 100644 index e93a9cecba..0000000000 --- a/packages/@uppy/aws-s3/src/index.test.js +++ /dev/null @@ -1,69 +0,0 @@ -import { beforeEach, describe, expect, it } from 'vitest' -import 'whatwg-fetch' -import Core from '@uppy/core' -import AwsS3 from './index.js' - -describe('AwsS3', () => { - it('Registers AwsS3 upload plugin', () => { - const core = new Core() - core.use(AwsS3) - - const pluginNames = core[Symbol.for('uppy test: getPlugins')]('uploader').map((plugin) => plugin.constructor.name) - expect(pluginNames).toContain('AwsS3') - }) - - describe('getUploadParameters', () => { - it('Throws an error if configured without companionUrl', () => { - const core = new Core() - core.use(AwsS3) - const awsS3 = core.getPlugin('AwsS3') - - expect(awsS3.opts.getUploadParameters).toThrow() - }) - - it('Does not throw an error with companionUrl configured', () => { - const core = new Core() - core.use(AwsS3, { companionUrl: 'https://companion.uppy.io/' }) - const awsS3 = core.getPlugin('AwsS3') - const file = { - meta: { - name: 'foo.jpg', - type: 'image/jpg', - }, - } - - expect(() => awsS3.opts.getUploadParameters(file)).not.toThrow() - }) - }) - - describe('dynamic companionHeader', () => { - let core - let awsS3 - const oldToken = 'old token' - const newToken = 'new token' - - beforeEach(() => { - core = new Core() - core.use(AwsS3, { - companionHeaders: { - authorization: oldToken, - }, - }) - awsS3 = core.getPlugin('AwsS3') - }) - - it('companionHeader is updated before uploading file', async () => { - awsS3.setOptions({ - companionHeaders: { - authorization: newToken, - }, - }) - - await core.upload() - - const client = awsS3[Symbol.for('uppy test: getClient')]() - - expect(client[Symbol.for('uppy test: getCompanionHeaders')]().authorization).toEqual(newToken) - }) - }) -}) diff --git a/packages/@uppy/aws-s3-multipart/src/index.test.ts b/packages/@uppy/aws-s3/src/index.test.ts similarity index 100% rename from packages/@uppy/aws-s3-multipart/src/index.test.ts rename to packages/@uppy/aws-s3/src/index.test.ts diff --git a/packages/@uppy/aws-s3/src/index.ts b/packages/@uppy/aws-s3/src/index.ts new file mode 100644 index 0000000000..6e0aa32f70 --- /dev/null +++ b/packages/@uppy/aws-s3/src/index.ts @@ -0,0 +1,1010 @@ +import BasePlugin, { + type DefinePluginOpts, + type PluginOpts, +} from '@uppy/core/lib/BasePlugin.js' +import { RequestClient } from '@uppy/companion-client' +import type { RequestOptions } from '@uppy/utils/lib/CompanionClientProvider' +import type { Body as _Body, Meta, UppyFile } from '@uppy/utils/lib/UppyFile' +import type { Uppy } from '@uppy/core' +import EventManager from '@uppy/core/lib/EventManager.js' +import { RateLimitedQueue } from '@uppy/utils/lib/RateLimitedQueue' +import { + filterNonFailedFiles, + filterFilesToEmitUploadStarted, +} from '@uppy/utils/lib/fileFilters' +import { createAbortError } from '@uppy/utils/lib/AbortController' +import getAllowedMetaFields from '@uppy/utils/lib/getAllowedMetaFields' +import MultipartUploader from './MultipartUploader.ts' +import { throwIfAborted } from './utils.ts' +import type { + UploadResult, + UploadResultWithSignal, + MultipartUploadResultWithSignal, + UploadPartBytesResult, + Body, +} from './utils.ts' +import createSignedURL from './createSignedURL.ts' +import { HTTPCommunicationQueue } from './HTTPCommunicationQueue.ts' +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore We don't want TS to generate types for the package.json +import packageJson from '../package.json' + +interface MultipartFile extends UppyFile { + s3Multipart: UploadResult +} + +type PartUploadedCallback = ( + file: UppyFile, + part: { PartNumber: number; ETag: string }, +) => void + +declare module '@uppy/core' { + export interface UppyEventMap { + 's3-multipart:part-uploaded': PartUploadedCallback + } +} + +function assertServerError(res: T): T { + if ((res as any)?.error) { + const error = new Error((res as any).message) + Object.assign(error, (res as any).error) + throw error + } + return res +} + +export interface AwsS3STSResponse { + credentials: { + AccessKeyId: string + SecretAccessKey: string + SessionToken: string + Expiration?: string + } + bucket: string + region: string +} + +/** + * Computes the expiry time for a request signed with temporary credentials. If + * no expiration was provided, or an invalid value (e.g. in the past) is + * provided, undefined is returned. This function assumes the client clock is in + * sync with the remote server, which is a requirement for the signature to be + * validated for AWS anyway. + */ +function getExpiry( + credentials: AwsS3STSResponse['credentials'], +): number | undefined { + const expirationDate = credentials.Expiration + if (expirationDate) { + const timeUntilExpiry = Math.floor( + ((new Date(expirationDate) as any as number) - Date.now()) / 1000, + ) + if (timeUntilExpiry > 9) { + return timeUntilExpiry + } + } + return undefined +} + +function getAllowedMetadata>({ + meta, + allowedMetaFields, + querify = false, +}: { + meta: M + allowedMetaFields?: string[] | null + querify?: boolean +}) { + const metaFields = allowedMetaFields ?? Object.keys(meta) + + if (!meta) return {} + + return Object.fromEntries( + metaFields + .filter((key) => meta[key] != null) + .map((key) => { + const realKey = querify ? `metadata[${key}]` : key + const value = String(meta[key]) + return [realKey, value] + }), + ) +} + +type MaybePromise = T | Promise + +type SignPartOptions = { + uploadId: string + key: string + partNumber: number + body: Blob + signal?: AbortSignal +} + +export type AwsS3UploadParameters = + | { + method: 'POST' + url: string + fields: Record + expires?: number + headers?: Record + } + | { + method?: 'PUT' + url: string + fields?: Record + expires?: number + headers?: Record + } + +export interface AwsS3Part { + PartNumber?: number + Size?: number + ETag?: string +} + +type AWSS3WithCompanion = { + companionUrl: string + companionHeaders?: Record + companionCookiesRule?: string + getTemporarySecurityCredentials?: true +} +type AWSS3WithoutCompanion = { + getTemporarySecurityCredentials?: (options?: { + signal?: AbortSignal + }) => MaybePromise + uploadPartBytes?: (options: { + signature: AwsS3UploadParameters + body: FormData | Blob + size?: number + onProgress: any + onComplete: any + signal?: AbortSignal + }) => Promise +} + +type AWSS3NonMultipartWithCompanionMandatory = { + // No related options +} + +type AWSS3NonMultipartWithoutCompanionMandatory< + M extends Meta, + B extends Body, +> = { + getUploadParameters: ( + file: UppyFile, + options: RequestOptions, + ) => MaybePromise +} +type AWSS3NonMultipartWithCompanion = AWSS3WithCompanion & + AWSS3NonMultipartWithCompanionMandatory & { + shouldUseMultipart: false + } + +type AWSS3NonMultipartWithoutCompanion< + M extends Meta, + B extends Body, +> = AWSS3WithoutCompanion & + AWSS3NonMultipartWithoutCompanionMandatory & { + shouldUseMultipart: false + } + +type AWSS3MultipartWithoutCompanionMandatorySignPart< + M extends Meta, + B extends Body, +> = { + signPart: ( + file: UppyFile, + opts: SignPartOptions, + ) => MaybePromise +} +/** @deprecated Use signPart instead */ +type AWSS3MultipartWithoutCompanionMandatoryPrepareUploadParts< + M extends Meta, + B extends Body, +> = { + /** @deprecated Use signPart instead */ + prepareUploadParts: ( + file: UppyFile, + partData: { + uploadId: string + key: string + parts: [{ number: number; chunk: Blob }] + signal?: AbortSignal + }, + ) => MaybePromise<{ + presignedUrls: Record + headers?: Record> + }> +} +type AWSS3MultipartWithoutCompanionMandatory = { + getChunkSize?: (file: UppyFile) => number + createMultipartUpload: (file: UppyFile) => MaybePromise + listParts: ( + file: UppyFile, + opts: UploadResultWithSignal, + ) => MaybePromise + abortMultipartUpload: ( + file: UppyFile, + opts: UploadResultWithSignal, + ) => MaybePromise + completeMultipartUpload: ( + file: UppyFile, + opts: { + uploadId: string + key: string + parts: AwsS3Part[] + signal: AbortSignal + }, + ) => MaybePromise<{ location?: string }> +} & ( + | AWSS3MultipartWithoutCompanionMandatorySignPart + | AWSS3MultipartWithoutCompanionMandatoryPrepareUploadParts +) + +type AWSS3MultipartWithoutCompanion< + M extends Meta, + B extends Body, +> = AWSS3WithoutCompanion & + AWSS3MultipartWithoutCompanionMandatory & { + shouldUseMultipart?: true + } + +type AWSS3MultipartWithCompanion< + M extends Meta, + B extends Body, +> = AWSS3WithCompanion & + Partial> & { + shouldUseMultipart?: true + } + +type AWSS3MaybeMultipartWithCompanion< + M extends Meta, + B extends Body, +> = AWSS3WithCompanion & + Partial> & + AWSS3NonMultipartWithCompanionMandatory & { + shouldUseMultipart: (file: UppyFile) => boolean + } + +type AWSS3MaybeMultipartWithoutCompanion< + M extends Meta, + B extends Body, +> = AWSS3WithoutCompanion & + AWSS3MultipartWithoutCompanionMandatory & + AWSS3NonMultipartWithoutCompanionMandatory & { + shouldUseMultipart: (file: UppyFile) => boolean + } + +type RequestClientOptions = Partial< + ConstructorParameters>[1] +> + +interface _AwsS3MultipartOptions extends PluginOpts, RequestClientOptions { + allowedMetaFields?: string[] | boolean + limit?: number + retryDelays?: number[] | null +} + +export type AwsS3MultipartOptions< + M extends Meta, + B extends Body, +> = _AwsS3MultipartOptions & + ( + | AWSS3NonMultipartWithCompanion + | AWSS3NonMultipartWithoutCompanion + | AWSS3MultipartWithCompanion + | AWSS3MultipartWithoutCompanion + | AWSS3MaybeMultipartWithCompanion + | AWSS3MaybeMultipartWithoutCompanion + ) + +const defaultOptions = { + allowedMetaFields: true, + limit: 6, + getTemporarySecurityCredentials: false as any, + shouldUseMultipart: ((file: UppyFile) => + file.size !== 0) as any as true, // TODO: Switch default to: + // eslint-disable-next-line no-bitwise + // shouldUseMultipart: (file) => file.size >> 10 >> 10 > 100, + retryDelays: [0, 1000, 3000, 5000], + companionHeaders: {}, +} satisfies Partial> + +export default class AwsS3Multipart< + M extends Meta, + B extends Body, +> extends BasePlugin< + DefinePluginOpts, keyof typeof defaultOptions> & + // We also have a few dynamic options defined below: + Pick< + AWSS3MultipartWithoutCompanionMandatory, + | 'getChunkSize' + | 'createMultipartUpload' + | 'listParts' + | 'abortMultipartUpload' + | 'completeMultipartUpload' + > & + Required> & + AWSS3MultipartWithoutCompanionMandatorySignPart & + AWSS3NonMultipartWithoutCompanionMandatory, + M, + B +> { + static VERSION = packageJson.version + + #companionCommunicationQueue + + #client: RequestClient + + protected requests: any + + protected uploaderEvents: Record | null> + + protected uploaders: Record | null> + + protected uploaderSockets: Record + + constructor(uppy: Uppy, opts?: AwsS3MultipartOptions) { + super(uppy, { + ...defaultOptions, + uploadPartBytes: AwsS3Multipart.uploadPartBytes, + createMultipartUpload: null as any, + listParts: null as any, + abortMultipartUpload: null as any, + completeMultipartUpload: null as any, + signPart: null as any, + getUploadParameters: null as any, + ...opts, + }) + // We need the `as any` here because of the dynamic default options. + this.type = 'uploader' + this.id = this.opts.id || 'AwsS3Multipart' + // @ts-expect-error TODO: remove unused + this.title = 'AWS S3 Multipart' + // TODO: only initiate `RequestClient` is `companionUrl` is defined. + this.#client = new RequestClient(uppy, opts as any) + + const dynamicDefaultOptions = { + createMultipartUpload: this.createMultipartUpload, + listParts: this.listParts, + abortMultipartUpload: this.abortMultipartUpload, + completeMultipartUpload: this.completeMultipartUpload, + signPart: + opts?.getTemporarySecurityCredentials ? + this.createSignedURL + : this.signPart, + getUploadParameters: + opts?.getTemporarySecurityCredentials ? + (this.createSignedURL as any) + : this.getUploadParameters, + } satisfies Partial> + + for (const key of Object.keys(dynamicDefaultOptions)) { + if (this.opts[key as keyof typeof dynamicDefaultOptions] == null) { + this.opts[key as keyof typeof dynamicDefaultOptions] = + dynamicDefaultOptions[key as keyof typeof dynamicDefaultOptions].bind( + this, + ) + } + } + if ( + (opts as AWSS3MultipartWithoutCompanionMandatoryPrepareUploadParts) + ?.prepareUploadParts != null && + (opts as AWSS3MultipartWithoutCompanionMandatorySignPart) + .signPart == null + ) { + this.opts.signPart = async ( + file: UppyFile, + { uploadId, key, partNumber, body, signal }: SignPartOptions, + ) => { + const { presignedUrls, headers } = await ( + opts as AWSS3MultipartWithoutCompanionMandatoryPrepareUploadParts< + M, + B + > + ).prepareUploadParts(file, { + uploadId, + key, + parts: [{ number: partNumber, chunk: body }], + signal, + }) + return { + url: presignedUrls?.[partNumber], + headers: headers?.[partNumber], + } + } + } + + /** + * Simultaneous upload limiting is shared across all uploads with this plugin. + * + * @type {RateLimitedQueue} + */ + this.requests = + (this.opts as any).rateLimitedQueue ?? + new RateLimitedQueue(this.opts.limit) + this.#companionCommunicationQueue = new HTTPCommunicationQueue( + this.requests, + this.opts, + this.#setS3MultipartState, + this.#getFile, + ) + + this.uploaders = Object.create(null) + this.uploaderEvents = Object.create(null) + this.uploaderSockets = Object.create(null) + } + + private [Symbol.for('uppy test: getClient')]() { + return this.#client + } + + setOptions(newOptions: Partial>): void { + this.#companionCommunicationQueue.setOptions(newOptions) + super.setOptions(newOptions) + this.#setCompanionHeaders() + } + + /** + * Clean up all references for a file's upload: the MultipartUploader instance, + * any events related to the file, and the Companion WebSocket connection. + * + * Set `opts.abort` to tell S3 that the multipart upload is cancelled and must be removed. + * This should be done when the user cancels the upload, not when the upload is completed or errored. + */ + resetUploaderReferences(fileID: string, opts?: { abort: boolean }): void { + if (this.uploaders[fileID]) { + this.uploaders[fileID]!.abort({ really: opts?.abort || false }) + this.uploaders[fileID] = null + } + if (this.uploaderEvents[fileID]) { + this.uploaderEvents[fileID]!.remove() + this.uploaderEvents[fileID] = null + } + if (this.uploaderSockets[fileID]) { + // @ts-expect-error TODO: remove this block in the next major + this.uploaderSockets[fileID].close() + // @ts-expect-error TODO: remove this block in the next major + this.uploaderSockets[fileID] = null + } + } + + // TODO: make this a private method in the next major + assertHost(method: string): void { + if (!this.opts.companionUrl) { + throw new Error( + `Expected a \`companionUrl\` option containing a Companion address, or if you are not using Companion, a custom \`${method}\` implementation.`, + ) + } + } + + createMultipartUpload( + file: UppyFile, + signal?: AbortSignal, + ): Promise { + this.assertHost('createMultipartUpload') + throwIfAborted(signal) + + const allowedMetaFields = getAllowedMetaFields( + this.opts.allowedMetaFields, + file.meta, + ) + const metadata = getAllowedMetadata({ meta: file.meta, allowedMetaFields }) + + return this.#client + .post( + 's3/multipart', + { + filename: file.name, + type: file.type, + metadata, + }, + { signal }, + ) + .then(assertServerError) + } + + listParts( + file: UppyFile, + { key, uploadId, signal }: UploadResultWithSignal, + oldSignal?: AbortSignal, + ): Promise { + signal ??= oldSignal // eslint-disable-line no-param-reassign + this.assertHost('listParts') + throwIfAborted(signal) + + const filename = encodeURIComponent(key) + return this.#client + .get(`s3/multipart/${uploadId}?key=${filename}`, { signal }) + .then(assertServerError) + } + + completeMultipartUpload( + file: UppyFile, + { key, uploadId, parts, signal }: MultipartUploadResultWithSignal, + oldSignal?: AbortSignal, + ): Promise { + signal ??= oldSignal // eslint-disable-line no-param-reassign + this.assertHost('completeMultipartUpload') + throwIfAborted(signal) + + const filename = encodeURIComponent(key) + const uploadIdEnc = encodeURIComponent(uploadId) + return this.#client + .post( + `s3/multipart/${uploadIdEnc}/complete?key=${filename}`, + { parts }, + { signal }, + ) + .then(assertServerError) + } + + #cachedTemporaryCredentials: MaybePromise + + async #getTemporarySecurityCredentials(options?: RequestOptions) { + throwIfAborted(options?.signal) + + if (this.#cachedTemporaryCredentials == null) { + // We do not await it just yet, so concurrent calls do not try to override it: + if (this.opts.getTemporarySecurityCredentials === true) { + this.assertHost('getTemporarySecurityCredentials') + this.#cachedTemporaryCredentials = this.#client + .get('s3/sts', options) + .then(assertServerError) + } else { + this.#cachedTemporaryCredentials = + this.opts.getTemporarySecurityCredentials(options) + } + this.#cachedTemporaryCredentials = await this.#cachedTemporaryCredentials + setTimeout( + () => { + // At half the time left before expiration, we clear the cache. That's + // an arbitrary tradeoff to limit the number of requests made to the + // remote while limiting the risk of using an expired token in case the + // clocks are not exactly synced. + // The HTTP cache should be configured to ensure a client doesn't request + // more tokens than it needs, but this timeout provides a second layer of + // security in case the HTTP cache is disabled or misconfigured. + this.#cachedTemporaryCredentials = null as any + }, + (getExpiry(this.#cachedTemporaryCredentials.credentials) || 0) * 500, + ) + } + + return this.#cachedTemporaryCredentials + } + + async createSignedURL( + file: UppyFile, + options: SignPartOptions, + ): Promise { + const data = await this.#getTemporarySecurityCredentials(options) + const expires = getExpiry(data.credentials) || 604_800 // 604 800 is the max value accepted by AWS. + + const { uploadId, key, partNumber } = options + + // Return an object in the correct shape. + return { + method: 'PUT', + expires, + fields: {}, + url: `${await createSignedURL({ + accountKey: data.credentials.AccessKeyId, + accountSecret: data.credentials.SecretAccessKey, + sessionToken: data.credentials.SessionToken, + expires, + bucketName: data.bucket, + Region: data.region, + Key: key ?? `${crypto.randomUUID()}-${file.name}`, + uploadId, + partNumber, + })}`, + // Provide content type header required by S3 + headers: { + 'Content-Type': file.type as string, + }, + } + } + + signPart( + file: UppyFile, + { uploadId, key, partNumber, signal }: SignPartOptions, + ): Promise { + this.assertHost('signPart') + throwIfAborted(signal) + + if (uploadId == null || key == null || partNumber == null) { + throw new Error( + 'Cannot sign without a key, an uploadId, and a partNumber', + ) + } + + const filename = encodeURIComponent(key) + return this.#client + .get( + `s3/multipart/${uploadId}/${partNumber}?key=${filename}`, + { signal }, + ) + .then(assertServerError) + } + + abortMultipartUpload( + file: UppyFile, + { key, uploadId, signal }: UploadResultWithSignal, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + oldSignal?: AbortSignal, // TODO: remove in next major + ): Promise { + signal ??= oldSignal // eslint-disable-line no-param-reassign + this.assertHost('abortMultipartUpload') + + const filename = encodeURIComponent(key) + const uploadIdEnc = encodeURIComponent(uploadId) + return this.#client + .delete(`s3/multipart/${uploadIdEnc}?key=${filename}`, undefined, { + signal, + }) + .then(assertServerError) + } + + getUploadParameters( + file: UppyFile, + options: RequestOptions, + ): Promise { + const { meta } = file + const { type, name: filename } = meta + const allowedMetaFields = getAllowedMetaFields( + this.opts.allowedMetaFields, + file.meta, + ) + const metadata = getAllowedMetadata({ + meta, + allowedMetaFields, + querify: true, + }) + + const query = new URLSearchParams({ filename, type, ...metadata } as Record< + string, + string + >) + + return this.#client.get(`s3/params?${query}`, options) + } + + static async uploadPartBytes({ + signature: { url, expires, headers, method = 'PUT' }, + body, + size = (body as Blob).size, + onProgress, + onComplete, + signal, + }: { + signature: AwsS3UploadParameters + body: FormData | Blob + size?: number + onProgress: any + onComplete: any + signal?: AbortSignal + }): Promise { + throwIfAborted(signal) + + if (url == null) { + throw new Error('Cannot upload to an undefined URL') + } + + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest() + xhr.open(method, url, true) + if (headers) { + Object.keys(headers).forEach((key) => { + xhr.setRequestHeader(key, headers[key]) + }) + } + xhr.responseType = 'text' + if (typeof expires === 'number') { + xhr.timeout = expires * 1000 + } + + function onabort() { + xhr.abort() + } + function cleanup() { + signal?.removeEventListener('abort', onabort) + } + signal?.addEventListener('abort', onabort) + + xhr.upload.addEventListener('progress', (ev) => { + onProgress(ev) + }) + + xhr.addEventListener('abort', () => { + cleanup() + + reject(createAbortError()) + }) + + xhr.addEventListener('timeout', () => { + cleanup() + + const error = new Error('Request has expired') + ;(error as any).source = { status: 403 } + reject(error) + }) + xhr.addEventListener('load', (ev) => { + cleanup() + + if ( + xhr.status === 403 && + xhr.responseText.includes('Request has expired') + ) { + const error = new Error('Request has expired') + ;(error as any).source = xhr + reject(error) + return + } + if (xhr.status < 200 || xhr.status >= 300) { + const error = new Error('Non 2xx') + ;(error as any).source = xhr + reject(error) + return + } + + // todo make a proper onProgress API (breaking change) + onProgress?.({ loaded: size, lengthComputable: true }) + + // NOTE This must be allowed by CORS. + const etag = xhr.getResponseHeader('ETag') + const location = xhr.getResponseHeader('Location') + + if (method.toUpperCase() === 'POST' && location === null) { + // Not being able to read the Location header is not a fatal error. + // eslint-disable-next-line no-console + console.warn( + 'AwsS3/Multipart: Could not read the Location header. This likely means CORS is not configured correctly on the S3 Bucket. See https://uppy.io/docs/aws-s3-multipart#S3-Bucket-Configuration for instructions.', + ) + } + if (etag === null) { + reject( + new Error( + 'AwsS3/Multipart: Could not read the ETag header. This likely means CORS is not configured correctly on the S3 Bucket. See https://uppy.io/docs/aws-s3-multipart#S3-Bucket-Configuration for instructions.', + ), + ) + return + } + + onComplete?.(etag) + resolve({ + ETag: etag, + ...(location ? { location } : undefined), + }) + }) + + xhr.addEventListener('error', (ev) => { + cleanup() + + const error = new Error('Unknown error') + ;(error as any).source = ev.target + reject(error) + }) + + xhr.send(body) + }) + } + + #setS3MultipartState = ( + file: UppyFile, + { key, uploadId }: UploadResult, + ) => { + const cFile = this.uppy.getFile(file.id) + if (cFile == null) { + // file was removed from store + return + } + + this.uppy.setFileState(file.id, { + s3Multipart: { + ...(cFile as MultipartFile).s3Multipart, + key, + uploadId, + }, + } as Partial>) + } + + #getFile = (file: UppyFile) => { + return this.uppy.getFile(file.id) || file + } + + #uploadLocalFile(file: UppyFile) { + return new Promise((resolve, reject) => { + const onProgress = (bytesUploaded: number, bytesTotal: number) => { + this.uppy.emit('upload-progress', this.uppy.getFile(file.id), { + // @ts-expect-error TODO: figure out if we need this + uploader: this, + bytesUploaded, + bytesTotal, + }) + } + + const onError = (err: unknown) => { + this.uppy.log(err as Error) + this.uppy.emit('upload-error', file, err as Error) + + this.resetUploaderReferences(file.id) + reject(err) + } + + const onSuccess = (result: B) => { + const uploadResp = { + body: { + ...result, + }, + status: 200, + uploadURL: result.location, + } + + this.resetUploaderReferences(file.id) + + this.uppy.emit('upload-success', this.#getFile(file), uploadResp) + + if (result.location) { + this.uppy.log(`Download ${file.name} from ${result.location}`) + } + + resolve() + } + + const upload = new MultipartUploader(file.data, { + // .bind to pass the file object to each handler. + companionComm: this.#companionCommunicationQueue, + + log: (...args: Parameters['log']>) => this.uppy.log(...args), + getChunkSize: + this.opts.getChunkSize ? this.opts.getChunkSize.bind(this) : null, + + onProgress, + onError, + onSuccess, + onPartComplete: (part) => { + this.uppy.emit( + 's3-multipart:part-uploaded', + this.#getFile(file), + part, + ) + }, + + file, + shouldUseMultipart: this.opts.shouldUseMultipart, + + ...(file as MultipartFile).s3Multipart, + }) + + this.uploaders[file.id] = upload + const eventManager = new EventManager(this.uppy) + this.uploaderEvents[file.id] = eventManager + + eventManager.onFileRemove(file.id, (removed) => { + upload.abort() + this.resetUploaderReferences(file.id, { abort: true }) + resolve(`upload ${removed} was removed`) + }) + + eventManager.onCancelAll(file.id, (options) => { + if (options?.reason === 'user') { + upload.abort() + this.resetUploaderReferences(file.id, { abort: true }) + } + resolve(`upload ${file.id} was canceled`) + }) + + eventManager.onFilePause(file.id, (isPaused) => { + if (isPaused) { + upload.pause() + } else { + upload.start() + } + }) + + eventManager.onPauseAll(file.id, () => { + upload.pause() + }) + + eventManager.onResumeAll(file.id, () => { + upload.start() + }) + + upload.start() + }) + } + + // eslint-disable-next-line class-methods-use-this + #getCompanionClientArgs(file: UppyFile) { + return { + ...file.remote?.body, + protocol: 's3-multipart', + size: file.data.size, + metadata: file.meta, + } + } + + #upload = async (fileIDs: string[]) => { + if (fileIDs.length === 0) return undefined + + const files = this.uppy.getFilesByIds(fileIDs) + const filesFiltered = filterNonFailedFiles(files) + const filesToEmit = filterFilesToEmitUploadStarted(filesFiltered) + + this.uppy.emit('upload-start', filesToEmit) + + const promises = filesFiltered.map((file) => { + if (file.isRemote) { + const getQueue = () => this.requests + this.#setResumableUploadsCapability(false) + const controller = new AbortController() + + const removedHandler = (removedFile: UppyFile) => { + if (removedFile.id === file.id) controller.abort() + } + this.uppy.on('file-removed', removedHandler) + + const uploadPromise = this.uppy + .getRequestClientForFile>(file) + .uploadRemoteFile(file, this.#getCompanionClientArgs(file), { + signal: controller.signal, + getQueue, + }) + + this.requests.wrapSyncFunction( + () => { + this.uppy.off('file-removed', removedHandler) + }, + { priority: -1 }, + )() + + return uploadPromise + } + + return this.#uploadLocalFile(file) + }) + + const upload = await Promise.all(promises) + // After the upload is done, another upload may happen with only local files. + // We reset the capability so that the next upload can use resumable uploads. + this.#setResumableUploadsCapability(true) + return upload + } + + #setCompanionHeaders = () => { + this.#client.setCompanionHeaders(this.opts.companionHeaders) + } + + #setResumableUploadsCapability = (boolean: boolean) => { + const { capabilities } = this.uppy.getState() + this.uppy.setState({ + capabilities: { + ...capabilities, + resumableUploads: boolean, + }, + }) + } + + #resetResumableCapability = () => { + this.#setResumableUploadsCapability(true) + } + + install(): void { + this.#setResumableUploadsCapability(true) + this.uppy.addPreProcessor(this.#setCompanionHeaders) + this.uppy.addUploader(this.#upload) + this.uppy.on('cancel-all', this.#resetResumableCapability) + } + + uninstall(): void { + this.uppy.removePreProcessor(this.#setCompanionHeaders) + this.uppy.removeUploader(this.#upload) + this.uppy.off('cancel-all', this.#resetResumableCapability) + } +} + +export type uploadPartBytes = (typeof AwsS3Multipart< + any, + any +>)['uploadPartBytes'] diff --git a/packages/@uppy/aws-s3/src/isXml.js b/packages/@uppy/aws-s3/src/isXml.js deleted file mode 100644 index 8426e789e7..0000000000 --- a/packages/@uppy/aws-s3/src/isXml.js +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Remove parameters like `charset=utf-8` from the end of a mime type string. - * - * @param {string} mimeType - The mime type string that may have optional parameters. - * @returns {string} The "base" mime type, i.e. only 'category/type'. - */ -function removeMimeParameters (mimeType) { - return mimeType.replace(/;.*$/, '') -} - -/** - * Check if a response contains XML based on the response object and its text content. - * - * @param {string} content - The text body of the response. - * @param {object|XMLHttpRequest} xhr - The XHR object or response object from Companion. - * @returns {bool} Whether the content is (probably) XML. - */ -function isXml (content, xhr) { - const rawContentType = (xhr.headers ? xhr.headers['content-type'] : xhr.getResponseHeader('Content-Type')) - - if (typeof rawContentType === 'string') { - const contentType = removeMimeParameters(rawContentType).toLowerCase() - if (contentType === 'application/xml' || contentType === 'text/xml') { - return true - } - // GCS uses text/html for some reason - // https://github.com/transloadit/uppy/issues/896 - if (contentType === 'text/html' && /^<\?xml /.test(content)) { - return true - } - } - return false -} - -export default isXml diff --git a/packages/@uppy/aws-s3/src/isXml.test.js b/packages/@uppy/aws-s3/src/isXml.test.js deleted file mode 100644 index f22515930e..0000000000 --- a/packages/@uppy/aws-s3/src/isXml.test.js +++ /dev/null @@ -1,65 +0,0 @@ -import { describe, expect, it } from 'vitest' -import isXml from './isXml.js' - -describe('AwsS3', () => { - describe('isXml', () => { - it('returns true for XML documents', () => { - const content = 'image.jpg' - expect(isXml(content, { - getResponseHeader: () => 'application/xml', - })).toEqual(true) - expect(isXml(content, { - getResponseHeader: () => 'text/xml', - })).toEqual(true) - expect(isXml(content, { - getResponseHeader: () => 'text/xml; charset=utf-8', - })).toEqual(true) - expect(isXml(content, { - getResponseHeader: () => 'application/xml; charset=iso-8859-1', - })).toEqual(true) - }) - - it('returns true for GCS XML documents', () => { - const content = 'image.jpg' - expect(isXml(content, { - getResponseHeader: () => 'text/html', - })).toEqual(true) - expect(isXml(content, { - getResponseHeader: () => 'text/html; charset=utf8', - })).toEqual(true) - }) - - it('returns true for remote response objects', () => { - const content = 'image.jpg' - expect(isXml(content, { - headers: { 'content-type': 'application/xml' }, - })).toEqual(true) - expect(isXml(content, { - headers: { 'content-type': 'application/xml' }, - })).toEqual(true) - expect(isXml(content, { - headers: { 'content-type': 'text/html' }, - })).toEqual(true) - }) - - it('returns false when content-type is missing', () => { - const content = 'image.jpg' - expect(isXml(content, { - getResponseHeader: () => null, - })).toEqual(false) - expect(isXml(content, { - headers: { 'content-type': null }, - })).toEqual(false) - expect(isXml(content, { - headers: {}, - })).toEqual(false) - }) - - it('returns false for HTML documents', () => { - const content = '' - expect(isXml(content, { - getResponseHeader: () => 'text/html', - })).toEqual(false) - }) - }) -}) diff --git a/packages/@uppy/aws-s3/src/locale.js b/packages/@uppy/aws-s3/src/locale.js deleted file mode 100644 index 24316a7786..0000000000 --- a/packages/@uppy/aws-s3/src/locale.js +++ /dev/null @@ -1,5 +0,0 @@ -export default { - strings: { - timedOut: 'Upload stalled for %{seconds} seconds, aborting.', - }, -} diff --git a/packages/@uppy/aws-s3-multipart/src/utils.ts b/packages/@uppy/aws-s3/src/utils.ts similarity index 100% rename from packages/@uppy/aws-s3-multipart/src/utils.ts rename to packages/@uppy/aws-s3/src/utils.ts diff --git a/packages/@uppy/aws-s3-multipart/tsconfig.build.json b/packages/@uppy/aws-s3/tsconfig.build.json similarity index 100% rename from packages/@uppy/aws-s3-multipart/tsconfig.build.json rename to packages/@uppy/aws-s3/tsconfig.build.json diff --git a/packages/@uppy/aws-s3-multipart/tsconfig.json b/packages/@uppy/aws-s3/tsconfig.json similarity index 100% rename from packages/@uppy/aws-s3-multipart/tsconfig.json rename to packages/@uppy/aws-s3/tsconfig.json From 5528b2d308d4e2bc7523ededae767faeb33ffebb Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Wed, 10 Apr 2024 15:50:02 +0200 Subject: [PATCH 2/3] `yarn.lock` --- yarn.lock | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/yarn.lock b/yarn.lock index 503695f735..031c704190 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9347,8 +9347,8 @@ __metadata: dependencies: tslib: ^2.0.0 peerDependencies: - "@angular/common": ^16.2.0 - "@angular/core": ^16.2.0 + "@angular/common": ^17.3.0 + "@angular/core": ^17.3.0 "@uppy/core": "workspace:^" "@uppy/dashboard": "workspace:^" "@uppy/drag-drop": "workspace:^" @@ -9374,15 +9374,7 @@ __metadata: version: 0.0.0-use.local resolution: "@uppy/aws-s3-multipart@workspace:packages/@uppy/aws-s3-multipart" dependencies: - "@aws-sdk/client-s3": ^3.362.0 - "@aws-sdk/s3-request-presigner": ^3.362.0 - "@uppy/companion-client": "workspace:^" - "@uppy/utils": "workspace:^" - nock: ^13.1.0 - vitest: ^1.2.1 - whatwg-fetch: 3.6.2 - peerDependencies: - "@uppy/core": "workspace:^" + "@uppy/aws-s3": "workspace:^" languageName: unknown linkType: soft @@ -9390,11 +9382,11 @@ __metadata: version: 0.0.0-use.local resolution: "@uppy/aws-s3@workspace:packages/@uppy/aws-s3" dependencies: - "@uppy/aws-s3-multipart": "workspace:^" + "@aws-sdk/client-s3": ^3.362.0 + "@aws-sdk/s3-request-presigner": ^3.362.0 "@uppy/companion-client": "workspace:^" "@uppy/utils": "workspace:^" - "@uppy/xhr-upload": "workspace:^" - nanoid: ^4.0.0 + nock: ^13.1.0 vitest: ^1.2.1 whatwg-fetch: 3.6.2 peerDependencies: From cbac25015e0187fab24431dfd042ee0246485a19 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Wed, 10 Apr 2024 16:06:23 +0200 Subject: [PATCH 3/3] fix e2e --- e2e/clients/dashboard-aws-multipart/app.js | 1 + e2e/clients/dashboard-aws/app.js | 1 + 2 files changed, 2 insertions(+) diff --git a/e2e/clients/dashboard-aws-multipart/app.js b/e2e/clients/dashboard-aws-multipart/app.js index d43ee926a0..67a743ae87 100644 --- a/e2e/clients/dashboard-aws-multipart/app.js +++ b/e2e/clients/dashboard-aws-multipart/app.js @@ -10,6 +10,7 @@ const uppy = new Uppy() .use(AwsS3Multipart, { limit: 2, companionUrl: process.env.VITE_COMPANION_URL, + shouldUseMultipart: true, // This way we can test that the user provided API still works // as expected in the flow. We call the default internal function for this, // otherwise we would have to run another server to pre-sign requests diff --git a/e2e/clients/dashboard-aws/app.js b/e2e/clients/dashboard-aws/app.js index 9a4307126d..eeaa7effe6 100644 --- a/e2e/clients/dashboard-aws/app.js +++ b/e2e/clients/dashboard-aws/app.js @@ -10,6 +10,7 @@ const uppy = new Uppy() .use(AwsS3, { limit: 2, companionUrl: process.env.VITE_COMPANION_URL, + shouldUseMultipart: false, }) // Keep this here to access uppy in tests