From c38000ecff19f9d402d1f1912e67153a0fe7284c Mon Sep 17 00:00:00 2001
From: Benjamin Goering <171782+gobengo@users.noreply.github.com>
Date: Wed, 6 Dec 2023 13:01:15 -0800
Subject: [PATCH] feat: upload-client funcs have options.piece to opt in to
 piece link calculation (#1220)

Motivation:
* @Gozala suggested it to help debug
* allow callers to opt into piece generation, and it's disabled by
default (until we can ensure it'll work well for huge files)
* we may not keep this, but it could be helpful to allow `w3 up` to be
used to upload huge files without the piece link calculation (in case it
is erroring or taking too much memory)
---
 packages/upload-client/src/index.js       | 14 +++++++------
 packages/upload-client/src/types.ts       |  4 +++-
 packages/upload-client/test/index.test.js | 25 ++++++++++++++++++++---
 3 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js
index 6c6233d64..ef634fbe6 100644
--- a/packages/upload-client/src/index.js
+++ b/packages/upload-client/src/index.js
@@ -130,12 +130,14 @@ async function uploadBlockStream(conf, blocks, options = {}) {
         const bytes = new Uint8Array(await car.arrayBuffer())
         const [cid, piece] = await Promise.all([
           Store.add(conf, bytes, options),
-          (async () => {
-            const multihashDigest = await PieceHasher.digest(bytes)
-            return /** @type {import('@web3-storage/capabilities/types').PieceLink} */ (
-              Link.create(raw.code, multihashDigest)
-            )
-          })(),
+          options.piece
+            ? (async () => {
+                const multihashDigest = await PieceHasher.digest(bytes)
+                return /** @type {import('@web3-storage/capabilities/types').PieceLink} */ (
+                  Link.create(raw.code, multihashDigest)
+                )
+              })()
+            : undefined,
         ])
         const { version, roots, size } = car
         return { version, roots, size, cid, piece }
diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts
index 65886f9c6..582bb114d 100644
--- a/packages/upload-client/src/types.ts
+++ b/packages/upload-client/src/types.ts
@@ -167,7 +167,7 @@ export interface CARMetadata extends CARHeaderInfo {
    *
    * @see https://github.com/filecoin-project/FIPs/pull/758/files
    */
-  piece: PieceLink
+  piece?: PieceLink
   /**
    * Size of the CAR file in bytes.
    */
@@ -253,6 +253,8 @@ export interface UploadOptions
     ShardStoringOptions,
     UploadProgressTrackable {
   onShardStored?: (meta: CARMetadata) => void
+  /** when true, uploading will calculate filecoin piece link */
+  piece?: true
 }
 
 export interface UploadDirectoryOptions
diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js
index 11627871c..b48c54438 100644
--- a/packages/upload-client/test/index.test.js
+++ b/packages/upload-client/test/index.test.js
@@ -705,7 +705,7 @@ describe('uploadCAR', () => {
       car,
       {
         connection,
-        onShardStored: (meta) => pieceCIDs.push(meta.piece),
+        onShardStored: (meta) => meta.piece && pieceCIDs.push(meta.piece),
       }
     )
 
@@ -713,9 +713,28 @@ describe('uploadCAR', () => {
     assert.equal(service.store.add.callCount, 1)
     assert(service.upload.add.called)
     assert.equal(service.upload.add.callCount, 1)
-    assert.equal(pieceCIDs.length, 1)
+    // pieceCID calculation is disabled by default
+    assert.equal(pieceCIDs.length, 0)
+
+    // can opt in to calculating piece link
+    /** @type {Array<import('@web3-storage/upload-client/types').CARMetadata>} */
+    const shards2 = []
+    await uploadCAR(
+      { issuer: agent, with: space.did(), proofs, audience: serviceSigner },
+      car,
+      {
+        connection,
+        onShardStored: (meta) => shards2.push(meta),
+        piece: true,
+      }
+    )
+    assert.equal(shards2.length, 1)
+    assert.ok(
+      shards2[0].piece,
+      'shard piece cid is truthy because options.piece=true'
+    )
     assert.equal(
-      pieceCIDs[0].toString(),
+      shards2[0].piece.toString(),
       'bafkzcibcoibrsisrq3nrfmsxvynduf4kkf7qy33ip65w7ttfk7guyqod5w5mmei'
     )
   })