feat: upload-client funcs have options.piece to opt in to piece link …

…calculation (storacha#1220)

Motivation:
* @Gozala suggested it to help debug
* allow callers to opt into piece generation, and it's disabled by
default (until we can ensure it'll work well for huge files)
* we may not keep this, but it could be helpful to allow `w3 up` to be
used to upload huge files without the piece link calculation (in case it
is erroring or taking too much memory)

packages/upload-client/src/index.js

@@ -130,12 +130,14 @@ async function uploadBlockStream(conf, blocks, options = {}) {
         const bytes = new Uint8Array(await car.arrayBuffer())
         const [cid, piece] = await Promise.all([
           Store.add(conf, bytes, options),
-          (async () => {
-            const multihashDigest = await PieceHasher.digest(bytes)
-            return /** @type {import('@web3-storage/capabilities/types').PieceLink} */ (
-              Link.create(raw.code, multihashDigest)
-            )
-          })(),
+          options.piece
+            ? (async () => {
+                const multihashDigest = await PieceHasher.digest(bytes)
+                return /** @type {import('@web3-storage/capabilities/types').PieceLink} */ (
+                  Link.create(raw.code, multihashDigest)
+                )
+              })()
+            : undefined,
         ])
         const { version, roots, size } = car
         return { version, roots, size, cid, piece }

packages/upload-client/src/types.ts

@@ -167,7 +167,7 @@ export interface CARMetadata extends CARHeaderInfo {
    *
    * @see https://github.com/filecoin-project/FIPs/pull/758/files
    */
-  piece: PieceLink
+  piece?: PieceLink
   /**
    * Size of the CAR file in bytes.
    */
@@ -253,6 +253,8 @@ export interface UploadOptions
     ShardStoringOptions,
     UploadProgressTrackable {
   onShardStored?: (meta: CARMetadata) => void
+  /** when true, uploading will calculate filecoin piece link */
+  piece?: true
 }
 
 export interface UploadDirectoryOptions

packages/upload-client/test/index.test.js

@@ -705,17 +705,36 @@ describe('uploadCAR', () => {
       car,
       {
         connection,
-        onShardStored: (meta) => pieceCIDs.push(meta.piece),
+        onShardStored: (meta) => meta.piece && pieceCIDs.push(meta.piece),
       }
     )
 
     assert(service.store.add.called)
     assert.equal(service.store.add.callCount, 1)
     assert(service.upload.add.called)
     assert.equal(service.upload.add.callCount, 1)
-    assert.equal(pieceCIDs.length, 1)
+    // pieceCID calculation is disabled by default
+    assert.equal(pieceCIDs.length, 0)
+
+    // can opt in to calculating piece link
+    /** @type {Array<import('@web3-storage/upload-client/types').CARMetadata>} */
+    const shards2 = []
+    await uploadCAR(
+      { issuer: agent, with: space.did(), proofs, audience: serviceSigner },
+      car,
+      {
+        connection,
+        onShardStored: (meta) => shards2.push(meta),
+        piece: true,
+      }
+    )
+    assert.equal(shards2.length, 1)
+    assert.ok(
+      shards2[0].piece,
+      'shard piece cid is truthy because options.piece=true'
+    )
     assert.equal(
-      pieceCIDs[0].toString(),
+      shards2[0].piece.toString(),
       'bafkzcibcoibrsisrq3nrfmsxvynduf4kkf7qy33ip65w7ttfk7guyqod5w5mmei'
     )
   })