diff --git a/packages/upload-client/package.json b/packages/upload-client/package.json index 7673d8f8a..0982b13e3 100644 --- a/packages/upload-client/package.json +++ b/packages/upload-client/package.json @@ -76,6 +76,7 @@ "@ucanto/transport": "^9.1.0", "@web3-storage/capabilities": "workspace:^", "@web3-storage/data-segment": "^5.1.0", + "@web3-storage/filecoin-client": "workspace:^", "ipfs-utils": "^9.0.14", "multiformats": "^12.1.2", "p-retry": "^5.1.2", diff --git a/packages/upload-client/src/index.js b/packages/upload-client/src/index.js index bcb1b8c99..3ded4f59a 100644 --- a/packages/upload-client/src/index.js +++ b/packages/upload-client/src/index.js @@ -1,4 +1,5 @@ import * as PieceHasher from '@web3-storage/data-segment/multihash' +import { Storefront } from '@web3-storage/filecoin-client' import * as Link from 'multiformats/link' import * as raw from 'multiformats/codecs/raw' import * as Store from './store.js' @@ -129,7 +130,30 @@ async function uploadBlockStream(conf, blocks, options = {}) { const multihashDigest = await hasher.digest(bytes) /** @type {import('@web3-storage/capabilities/types').PieceLink} */ const piece = Link.create(raw.code, multihashDigest) + + // Invoke store/add and write bytes to write target const cid = await Store.add(conf, bytes, options) + // Invoke filecoin/offer for data + const result = await Storefront.filecoinOffer( + { + issuer: conf.issuer, + audience: conf.audience, + // Resource of invocation is the issuer did for being self issued + with: conf.issuer.did(), + proofs: conf.proofs, + }, + cid, + piece, + options + ) + + if (result.out.error) { + throw new Error( + 'failed to offer piece for aggregation into filecoin deal', + { cause: result.out.error } + ) + } + const { version, roots, size } = car controller.enqueue({ version, roots, size, cid, piece }) }, diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index 9954e0c34..adb74a05c 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -44,6 +44,7 @@ import { UsageReportSuccess, UsageReportFailure, } from '@web3-storage/capabilities/types' +import { StorefrontService } from '@web3-storage/filecoin-client/storefront' import { code as pieceHashCode } from '@web3-storage/data-segment/multihash' type Override = Omit & R @@ -93,7 +94,7 @@ export interface ProgressStatus extends XHRProgressStatus { export type ProgressFn = (status: ProgressStatus) => void -export interface Service { +export interface Service extends StorefrontService { store: { add: ServiceMethod get: ServiceMethod diff --git a/packages/upload-client/test/helpers/filecoin.js b/packages/upload-client/test/helpers/filecoin.js new file mode 100644 index 000000000..c89a9f321 --- /dev/null +++ b/packages/upload-client/test/helpers/filecoin.js @@ -0,0 +1,32 @@ +import * as StorefrontCapabilities from '@web3-storage/capabilities/filecoin/storefront' +import * as Server from '@ucanto/server' + +/** + * @param {Server.Signer<`did:${string}:${string}`, Server.API.SigAlg>} id + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {Pick<{ content: Server.API.Link; piece: import('@web3-storage/data-segment').PieceLink; }, 'content' | 'piece'>} args + */ +export async function getFilecoinOfferResponse(id, piece, args) { + // Create effect for receipt with self signed queued operation + const submitfx = await StorefrontCapabilities.filecoinSubmit + .invoke({ + issuer: id, + audience: id, + with: id.did(), + nb: args, + expiration: Infinity, + }) + .delegate() + + const acceptfx = await StorefrontCapabilities.filecoinAccept + .invoke({ + issuer: id, + audience: id, + with: id.did(), + nb: args, + expiration: Infinity, + }) + .delegate() + + return Server.ok({ piece }).fork(submitfx.link()).join(acceptfx.link()) +} diff --git a/packages/upload-client/test/helpers/mocks.js b/packages/upload-client/test/helpers/mocks.js index d9000d064..9df92b7fd 100644 --- a/packages/upload-client/test/helpers/mocks.js +++ b/packages/upload-client/test/helpers/mocks.js @@ -9,6 +9,7 @@ const notImplemented = () => { * store: Partial * upload: Partial * usage: Partial + * filecoin: Partial * }>} impl */ export function mockService(impl) { @@ -28,6 +29,12 @@ export function mockService(impl) { usage: { report: withCallCount(impl.usage?.report ?? notImplemented), }, + filecoin: { + offer: withCallCount(impl.filecoin?.offer ?? notImplemented), + submit: withCallCount(impl.filecoin?.submit ?? notImplemented), + accept: withCallCount(impl.filecoin?.accept ?? notImplemented), + info: withCallCount(impl.filecoin?.info ?? notImplemented), + }, } } diff --git a/packages/upload-client/test/index.test.js b/packages/upload-client/test/index.test.js index 97b3eee28..3e12ef521 100644 --- a/packages/upload-client/test/index.test.js +++ b/packages/upload-client/test/index.test.js @@ -6,6 +6,8 @@ import * as CAR from '@ucanto/transport/car' import * as Signer from '@ucanto/principal/ed25519' import * as StoreCapabilities from '@web3-storage/capabilities/store' import * as UploadCapabilities from '@web3-storage/capabilities/upload' +import * as StorefrontCapabilities from '@web3-storage/capabilities/filecoin/storefront' +import { Piece } from '@web3-storage/data-segment' import { uploadFile, uploadDirectory, @@ -24,6 +26,7 @@ import { headerEncodingLength, } from '../src/car.js' import { toBlock } from './helpers/block.js' +import { getFilecoinOfferResponse } from './helpers/filecoin.js' describe('uploadFile', () => { it('uploads a file to the service', async () => { @@ -32,6 +35,7 @@ describe('uploadFile', () => { const bytes = await randomBytes(128) const file = new Blob([bytes]) const expectedCar = await toCAR(bytes) + const piece = Piece.fromPayload(bytes).link /** @type {import('../src/types.js').CARLink|undefined} */ let carCID @@ -70,6 +74,18 @@ describe('uploadFile', () => { return { ok: { ...res, allocated: capability.nb.size } } }), }, + filecoin: { + offer: Server.provideAdvanced({ + capability: StorefrontCapabilities.filecoinOffer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb) { + throw new Error('no params received') + } + return getFilecoinOfferResponse(context.id, piece, invCap.nb) + }, + }), + }, upload: { add: provide(UploadCapabilities.add, ({ invocation }) => { assert.equal(invocation.issuer.did(), agent.did()) @@ -113,6 +129,8 @@ describe('uploadFile', () => { assert(service.store.add.called) assert.equal(service.store.add.callCount, 1) + assert(service.filecoin.offer.called) + assert.equal(service.filecoin.offer.callCount, 1) assert(service.upload.add.called) assert.equal(service.upload.add.callCount, 1) @@ -123,7 +141,9 @@ describe('uploadFile', () => { it('allows custom shard size to be set', async () => { const space = await Signer.generate() const agent = await Signer.generate() // The "user" that will ask the service to accept the upload - const file = new Blob([await randomBytes(1024 * 1024 * 5)]) + const bytes = await randomBytes(1024 * 1024 * 5) + const file = new Blob([bytes]) + const piece = Piece.fromPayload(bytes).link /** @type {import('../src/types.js').CARLink[]} */ const carCIDs = [] @@ -162,6 +182,18 @@ describe('uploadFile', () => { }, })), }, + filecoin: { + offer: Server.provideAdvanced({ + capability: StorefrontCapabilities.filecoinOffer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb) { + throw new Error('no params received') + } + return getFilecoinOfferResponse(context.id, piece, invCap.nb) + }, + }), + }, upload: { add: provide(UploadCapabilities.add, ({ capability }) => { if (!capability.nb) throw new Error('nb must be present') @@ -200,16 +232,97 @@ describe('uploadFile', () => { assert.equal(carCIDs.length, 5) }) + + it('fails to upload a file to the service if `filecoin/piece` invocation fails', async () => { + const space = await Signer.generate() + const agent = await Signer.generate() // The "user" that will ask the service to accept the upload + const bytes = await randomBytes(128) + const file = new Blob([bytes]) + const expectedCar = await toCAR(bytes) + + const proofs = await Promise.all([ + StoreCapabilities.add.delegate({ + issuer: space, + audience: agent, + with: space.did(), + expiration: Infinity, + }), + UploadCapabilities.add.delegate({ + issuer: space, + audience: agent, + with: space.did(), + expiration: Infinity, + }), + ]) + + /** @type {Omit} */ + const res = { + status: 'upload', + headers: { 'x-test': 'true' }, + url: 'http://localhost:9200', + link: expectedCar.cid, + with: space.did(), + } + + const service = mockService({ + store: { + add: provide(StoreCapabilities.add, ({ invocation, capability }) => { + assert.equal(invocation.issuer.did(), agent.did()) + assert.equal(invocation.capabilities.length, 1) + assert.equal(capability.can, StoreCapabilities.add.can) + assert.equal(capability.with, space.did()) + return { ok: { ...res, allocated: capability.nb.size } } + }), + }, + filecoin: { + offer: Server.provideAdvanced({ + capability: StorefrontCapabilities.filecoinOffer, + handler: async ({ invocation, context }) => { + return { + error: new Server.Failure('did not find piece'), + } + }, + }), + }, + }) + + const server = Server.create({ + id: serviceSigner, + service, + codec: CAR.inbound, + validateAuthorization, + }) + const connection = Client.connect({ + id: serviceSigner, + codec: CAR.outbound, + channel: server, + }) + await assert.rejects(async () => + uploadFile( + { issuer: agent, with: space.did(), proofs, audience: serviceSigner }, + file, + { + connection, + } + ) + ) + + assert(service.store.add.called) + assert.equal(service.store.add.callCount, 1) + assert(service.filecoin.offer.called) + assert.equal(service.filecoin.offer.callCount, 1) + }) }) describe('uploadDirectory', () => { it('uploads a directory to the service', async () => { const space = await Signer.generate() const agent = await Signer.generate() - const files = [ - new File([await randomBytes(128)], '1.txt'), - new File([await randomBytes(32)], '2.txt'), - ] + const bytesList = [await randomBytes(128), await randomBytes(32)] + const files = bytesList.map( + (bytes, index) => new File([bytes], `${index}.txt`) + ) + const pieces = bytesList.map((bytes) => Piece.fromPayload(bytes).link) /** @type {import('../src/types.js').CARLink?} */ let carCID = null @@ -256,6 +369,18 @@ describe('uploadDirectory', () => { } }), }, + filecoin: { + offer: Server.provideAdvanced({ + capability: StorefrontCapabilities.filecoinOffer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb) { + throw new Error('no params received') + } + return getFilecoinOfferResponse(context.id, pieces[0], invCap.nb) + }, + }), + }, upload: { add: provide(UploadCapabilities.add, ({ invocation }) => { assert.equal(invocation.issuer.did(), agent.did()) @@ -295,6 +420,8 @@ describe('uploadDirectory', () => { assert(service.store.add.called) assert.equal(service.store.add.callCount, 1) + assert(service.filecoin.offer.called) + assert.equal(service.filecoin.offer.callCount, 1) assert(service.upload.add.called) assert.equal(service.upload.add.callCount, 1) @@ -305,7 +432,11 @@ describe('uploadDirectory', () => { it('allows custom shard size to be set', async () => { const space = await Signer.generate() const agent = await Signer.generate() // The "user" that will ask the service to accept the upload - const files = [new File([await randomBytes(500_000)], '1.txt')] + const bytesList = [await randomBytes(500_000)] + const files = bytesList.map( + (bytes, index) => new File([bytes], `${index}.txt`) + ) + const pieces = bytesList.map((bytes) => Piece.fromPayload(bytes).link) /** @type {import('../src/types.js').CARLink[]} */ const carCIDs = [] @@ -344,6 +475,18 @@ describe('uploadDirectory', () => { }, })), }, + filecoin: { + offer: Server.provideAdvanced({ + capability: StorefrontCapabilities.filecoinOffer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb) { + throw new Error('no params received') + } + return getFilecoinOfferResponse(context.id, pieces[0], invCap.nb) + }, + }), + }, upload: { add: provide(UploadCapabilities.add, ({ capability }) => { if (!capability.nb) throw new Error('nb must be present') @@ -379,6 +522,9 @@ describe('uploadDirectory', () => { it('sorts files unless options.customOrder', async () => { const space = await Signer.generate() const agent = await Signer.generate() // The "user" that will ask the service to accept the upload + const someBytes = await randomBytes(32) + const piece = Piece.fromPayload(someBytes).link + const proofs = await Promise.all([ StoreCapabilities.add.delegate({ issuer: space, @@ -416,6 +562,18 @@ describe('uploadDirectory', () => { } }), }, + filecoin: { + offer: Server.provideAdvanced({ + capability: StorefrontCapabilities.filecoinOffer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb) { + throw new Error('no params received') + } + return getFilecoinOfferResponse(context.id, piece, invCap.nb) + }, + }), + }, upload: { add: provide(UploadCapabilities.add, (invocation) => { invocations.push(invocation) @@ -522,6 +680,8 @@ describe('uploadCAR', () => { await randomBlock(128), ] const car = await encode(blocks, blocks.at(-1)?.cid) + const someBytes = new Uint8Array(await car.arrayBuffer()) + const piece = Piece.fromPayload(someBytes).link // Wanted: 2 shards // 2 * CAR header (34) + 2 * blocks (256), 2 * block encoding prefix (78) const shardSize = @@ -575,6 +735,18 @@ describe('uploadCAR', () => { } }), }, + filecoin: { + offer: Server.provideAdvanced({ + capability: StorefrontCapabilities.filecoinOffer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb) { + throw new Error('no params received') + } + return getFilecoinOfferResponse(context.id, piece, invCap.nb) + }, + }), + }, upload: { add: provide(UploadCapabilities.add, ({ invocation }) => { assert.equal(invocation.issuer.did(), agent.did()) @@ -618,6 +790,8 @@ describe('uploadCAR', () => { assert(service.store.add.called) assert.equal(service.store.add.callCount, 2) + assert(service.filecoin.offer.called) + assert.equal(service.filecoin.offer.callCount, 2) assert(service.upload.add.called) assert.equal(service.upload.add.callCount, 1) assert.equal(carCIDs.length, 2) @@ -631,6 +805,8 @@ describe('uploadCAR', () => { await toBlock(new Uint8Array([1, 1, 3, 8])), ] const car = await encode(blocks, blocks.at(-1)?.cid) + const someBytes = new Uint8Array(await car.arrayBuffer()) + const piece = Piece.fromPayload(someBytes).link /** @type {import('../src/types.js').PieceLink[]} */ const pieceCIDs = [] @@ -676,6 +852,18 @@ describe('uploadCAR', () => { } }), }, + filecoin: { + offer: Server.provideAdvanced({ + capability: StorefrontCapabilities.filecoinOffer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb) { + throw new Error('no params received') + } + return getFilecoinOfferResponse(context.id, piece, invCap.nb) + }, + }), + }, upload: { add: provide(UploadCapabilities.add, ({ invocation }) => { assert.equal(invocation.issuer.did(), agent.did()) @@ -715,6 +903,8 @@ describe('uploadCAR', () => { assert(service.store.add.called) assert.equal(service.store.add.callCount, 1) + assert(service.filecoin.offer.called) + assert.equal(service.filecoin.offer.callCount, 1) assert(service.upload.add.called) assert.equal(service.upload.add.callCount, 1) assert.equal(pieceCIDs.length, 1) diff --git a/packages/upload-client/tsconfig.json b/packages/upload-client/tsconfig.json index 1a7abd345..2f4f83572 100644 --- a/packages/upload-client/tsconfig.json +++ b/packages/upload-client/tsconfig.json @@ -6,7 +6,11 @@ }, "include": ["src", "scripts", "test", "package.json"], "exclude": ["**/node_modules/**"], - "references": [{ "path": "../access-client" }, { "path": "../capabilities" }], + "references": [ + { "path": "../access-client" }, + { "path": "../filecoin-client" }, + { "path": "../capabilities" } + ], "typedocOptions": { "entryPoints": ["./src"] } diff --git a/packages/w3up-client/test/client.test.js b/packages/w3up-client/test/client.test.js index 9ac29abd5..be37a3fb6 100644 --- a/packages/w3up-client/test/client.test.js +++ b/packages/w3up-client/test/client.test.js @@ -4,6 +4,7 @@ import { create as createServer, parseLink, provide, + provideAdvanced, error, } from '@ucanto/server' import * as CAR from '@ucanto/transport/car' @@ -11,6 +12,8 @@ import * as Signer from '@ucanto/principal/ed25519' import * as StoreCapabilities from '@web3-storage/capabilities/store' import * as UploadCapabilities from '@web3-storage/capabilities/upload' import * as UCANCapabilities from '@web3-storage/capabilities/ucan' +import * as StorefrontCapabilities from '@web3-storage/capabilities/filecoin/storefront' +import { Piece } from '@web3-storage/data-segment' import { AgentData } from '@web3-storage/access/agent' import { StoreItemNotFound } from '../../upload-api/src/store/lib.js' import { randomBytes, randomCAR } from './helpers/random.js' @@ -19,6 +22,7 @@ import { mockService, mockServiceConf } from './helpers/mocks.js' import { File } from './helpers/shims.js' import { Client } from '../src/client.js' import { validateAuthorization } from './helpers/utils.js' +import { getFilecoinOfferResponse } from './helpers/filecoin.js' describe('Client', () => { describe('uploadFile', () => { @@ -26,7 +30,7 @@ describe('Client', () => { const bytes = await randomBytes(128) const file = new Blob([bytes]) const expectedCar = await toCAR(bytes) - + const piece = Piece.fromPayload(bytes).link /** @type {import('@web3-storage/upload-client/types').CARLink|undefined} */ let carCID @@ -52,6 +56,18 @@ describe('Client', () => { } }), }, + filecoin: { + offer: provideAdvanced({ + capability: StorefrontCapabilities.filecoinOffer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb) { + throw new Error('no params received') + } + return getFilecoinOfferResponse(context.id, piece, invCap.nb) + }, + }), + }, upload: { add: provide(UploadCapabilities.add, ({ invocation }) => { assert.equal(invocation.issuer.did(), alice.agent.did()) @@ -118,10 +134,11 @@ describe('Client', () => { describe('uploadDirectory', () => { it('should upload a directory to the service', async () => { - const files = [ - new File([await randomBytes(128)], '1.txt'), - new File([await randomBytes(32)], '2.txt'), - ] + const bytesList = [await randomBytes(128), await randomBytes(32)] + const files = bytesList.map( + (bytes, index) => new File([bytes], `${index}.txt`) + ) + const pieces = bytesList.map((bytes) => Piece.fromPayload(bytes).link) /** @type {import('@web3-storage/upload-client/types').CARLink|undefined} */ let carCID @@ -147,6 +164,18 @@ describe('Client', () => { } }), }, + filecoin: { + offer: provideAdvanced({ + capability: StorefrontCapabilities.filecoinOffer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb) { + throw new Error('no params received') + } + return getFilecoinOfferResponse(context.id, pieces[0], invCap.nb) + }, + }), + }, upload: { add: provide(UploadCapabilities.add, ({ invocation }) => { assert.equal(invocation.issuer.did(), alice.agent.did()) @@ -200,6 +229,8 @@ describe('Client', () => { describe('uploadCAR', () => { it('uploads a CAR file to the service', async () => { const car = await randomCAR(32) + const someBytes = new Uint8Array(await car.arrayBuffer()) + const piece = Piece.fromPayload(someBytes).link /** @type {import('../src/types.js').CARLink?} */ let carCID @@ -225,6 +256,18 @@ describe('Client', () => { } }), }, + filecoin: { + offer: provideAdvanced({ + capability: StorefrontCapabilities.filecoinOffer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + if (!invCap.nb) { + throw new Error('no params received') + } + return getFilecoinOfferResponse(context.id, piece, invCap.nb) + }, + }), + }, upload: { add: provide(UploadCapabilities.add, ({ invocation }) => { assert.equal(invocation.issuer.did(), alice.agent.did()) diff --git a/packages/w3up-client/test/helpers/filecoin.js b/packages/w3up-client/test/helpers/filecoin.js new file mode 100644 index 000000000..c89a9f321 --- /dev/null +++ b/packages/w3up-client/test/helpers/filecoin.js @@ -0,0 +1,32 @@ +import * as StorefrontCapabilities from '@web3-storage/capabilities/filecoin/storefront' +import * as Server from '@ucanto/server' + +/** + * @param {Server.Signer<`did:${string}:${string}`, Server.API.SigAlg>} id + * @param {import('@web3-storage/data-segment').PieceLink} piece + * @param {Pick<{ content: Server.API.Link; piece: import('@web3-storage/data-segment').PieceLink; }, 'content' | 'piece'>} args + */ +export async function getFilecoinOfferResponse(id, piece, args) { + // Create effect for receipt with self signed queued operation + const submitfx = await StorefrontCapabilities.filecoinSubmit + .invoke({ + issuer: id, + audience: id, + with: id.did(), + nb: args, + expiration: Infinity, + }) + .delegate() + + const acceptfx = await StorefrontCapabilities.filecoinAccept + .invoke({ + issuer: id, + audience: id, + with: id.did(), + nb: args, + expiration: Infinity, + }) + .delegate() + + return Server.ok({ piece }).fork(submitfx.link()).join(acceptfx.link()) +} diff --git a/packages/w3up-client/tsconfig.json b/packages/w3up-client/tsconfig.json index a4a804f40..f0b372b74 100644 --- a/packages/w3up-client/tsconfig.json +++ b/packages/w3up-client/tsconfig.json @@ -13,6 +13,7 @@ { "path": "../access-client" }, { "path": "../capabilities" }, { "path": "../upload-client" }, + { "path": "../filecoin-client" }, { "path": "../did-mailto" }, { "path": "../upload-api" } ] diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index de2c0c095..3ed85d28f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,4 @@ -lockfileVersion: '6.0' +lockfileVersion: '6.1' settings: autoInstallPeers: true @@ -470,6 +470,9 @@ importers: '@web3-storage/data-segment': specifier: ^5.1.0 version: 5.1.0 + '@web3-storage/filecoin-client': + specifier: workspace:^ + version: link:../filecoin-client ipfs-utils: specifier: ^9.0.14 version: 9.0.14