From bbc0433653befe27ede66245b2803ccd8f87df80 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 18 Apr 2024 12:40:12 +0100 Subject: [PATCH 1/5] feat!: add blob list and remove BREAKING CHANGE: allocations storage interface now requires remove to be implemented --- packages/capabilities/src/blob.js | 73 +++- packages/capabilities/src/index.js | 2 + packages/capabilities/src/types.ts | 18 + packages/upload-api/src/blob.js | 4 + packages/upload-api/src/blob/add.js | 4 +- packages/upload-api/src/blob/list.js | 15 + packages/upload-api/src/blob/remove.js | 26 ++ packages/upload-api/src/types.ts | 8 + packages/upload-api/src/types/blob.ts | 14 +- packages/upload-api/test/handlers/blob.js | 371 ++++++++++++++++++ .../upload-api/test/handlers/web3.storage.js | 191 ++++----- packages/upload-api/test/lib.js | 4 +- .../test/storage/allocations-storage-tests.js | 53 +++ .../test/storage/allocations-storage.js | 20 + 14 files changed, 703 insertions(+), 100 deletions(-) create mode 100644 packages/upload-api/src/blob/list.js create mode 100644 packages/upload-api/src/blob/remove.js diff --git a/packages/capabilities/src/blob.js b/packages/capabilities/src/blob.js index 7d8b89506..38b6adc15 100644 --- a/packages/capabilities/src/blob.js +++ b/packages/capabilities/src/blob.js @@ -11,7 +11,8 @@ * * @module */ -import { capability, Schema } from '@ucanto/validator' +import { equals } from 'uint8arrays/equals' +import { capability, Schema, fail, ok } from '@ucanto/validator' import { equalBlob, equalWith, SpaceDID } from './utils.js' /** @@ -70,6 +71,76 @@ export const add = capability({ derives: equalBlob, }) +/** + * Capability can be used to remove the stored Blob from the (memory) + * space identified by `with` field. + */ +export const remove = capability({ + can: 'blob/remove', + /** + * DID of the (memory) space where Blob is stored. + */ + with: SpaceDID, + nb: Schema.struct({ + /** + * A multihash digest of the blob payload bytes, uniquely identifying blob. + */ + content: Schema.bytes(), + }), + derives: (claimed, delegated) => { + if (claimed.with !== delegated.with) { + return fail( + `Expected 'with: "${delegated.with}"' instead got '${claimed.with}'` + ) + } else if ( + delegated.nb.content && + !equals(delegated.nb.content, claimed.nb.content) + ) { + return fail( + `Link ${ + claimed.nb.content ? `${claimed.nb.content}` : '' + } violates imposed ${delegated.nb.content} constraint.` + ) + } + return ok({}) + }, +}) + +/** + * Capability can be invoked to request a list of stored Blobs in the + * (memory) space identified by `with` field. + */ +export const list = capability({ + can: 'blob/list', + /** + * DID of the (memory) space where Blobs to be listed are stored. + */ + with: SpaceDID, + nb: Schema.struct({ + /** + * A pointer that can be moved back and forth on the list. + * It can be used to paginate a list for instance. + */ + cursor: Schema.string().optional(), + /** + * Maximum number of items per page. + */ + size: Schema.integer().optional(), + /** + * If true, return page of results preceding cursor. Defaults to false. + */ + pre: Schema.boolean().optional(), + }), + derives: (claimed, delegated) => { + if (claimed.with !== delegated.with) { + return fail( + `Expected 'with: "${delegated.with}"' instead got '${claimed.with}'` + ) + } + return ok({}) + }, +}) + // ⚠️ We export imports here so they are not omitted in generated typedefs // @see https://github.com/microsoft/TypeScript/issues/51548 export { Schema } diff --git a/packages/capabilities/src/index.js b/packages/capabilities/src/index.js index 0f666b3d9..5b2ba192c 100644 --- a/packages/capabilities/src/index.js +++ b/packages/capabilities/src/index.js @@ -95,6 +95,8 @@ export const abilitiesAsStrings = [ Usage.report.can, Blob.blob.can, Blob.add.can, + Blob.remove.can, + Blob.list.can, W3sBlob.blob.can, W3sBlob.allocate.can, W3sBlob.accept.can, diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index 279105585..3438425dc 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -452,6 +452,8 @@ export type HTTPPut = InferInvokedCapability // Blob export type Blob = InferInvokedCapability export type BlobAdd = InferInvokedCapability +export type BlobRemove = InferInvokedCapability +export type BlobList = InferInvokedCapability export type ServiceBlob = InferInvokedCapability export type BlobAllocate = InferInvokedCapability export type BlobAccept = InferInvokedCapability @@ -487,6 +489,20 @@ export interface BlobListItem { insertedAt: ISO8601Date } +// Blob remove +export interface BlobRemoveSuccess { + size: number +} + +// TODO: make types more specific +export type BlobRemoveFailure = Ucanto.Failure + +// Blob list +export interface BlobListSuccess extends ListResponse {} + +// TODO: make types more specific +export type BlobListFailure = Ucanto.Failure + // Blob allocate export interface BlobAllocateSuccess { size: number @@ -820,6 +836,8 @@ export type ServiceAbilityArray = [ UsageReport['can'], Blob['can'], BlobAdd['can'], + BlobRemove['can'], + BlobList['can'], ServiceBlob['can'], BlobAllocate['can'], BlobAccept['can'], diff --git a/packages/upload-api/src/blob.js b/packages/upload-api/src/blob.js index 84187cefd..78b4bb40b 100644 --- a/packages/upload-api/src/blob.js +++ b/packages/upload-api/src/blob.js @@ -1,4 +1,6 @@ import { blobAddProvider } from './blob/add.js' +import { blobListProvider } from './blob/list.js' +import { blobRemoveProvider } from './blob/remove.js' import * as API from './types.js' /** @@ -7,5 +9,7 @@ import * as API from './types.js' export function createService(context) { return { add: blobAddProvider(context), + list: blobListProvider(context), + remove: blobRemoveProvider(context), } } diff --git a/packages/upload-api/src/blob/add.js b/packages/upload-api/src/blob/add.js index 0d0fdc533..f287ca231 100644 --- a/packages/upload-api/src/blob/add.js +++ b/packages/upload-api/src/blob/add.js @@ -207,9 +207,7 @@ async function put({ context, blob, allocateTask }) { // of the `http/put` invocation. That way anyone with blob digest // could perform the invocation and issue receipt by deriving same // principal - const blobProvider = await ed25519.derive( - blob.digest.subarray(-32) - ) + const blobProvider = await ed25519.derive(blob.digest.subarray(-32)) const facts = [ { keys: blobProvider.toArchive(), diff --git a/packages/upload-api/src/blob/list.js b/packages/upload-api/src/blob/list.js new file mode 100644 index 000000000..4915fe16c --- /dev/null +++ b/packages/upload-api/src/blob/list.js @@ -0,0 +1,15 @@ +import * as Server from '@ucanto/server' +import * as Blob from '@web3-storage/capabilities/blob' +import * as API from '../types.js' + +/** + * @param {API.BlobServiceContext} context + * @returns {API.ServiceMethod} + */ +export function blobListProvider(context) { + return Server.provide(Blob.list, async ({ capability }) => { + const space = capability.with + const { cursor, size, pre } = capability.nb + return await context.allocationsStorage.list(space, { size, cursor, pre }) + }) +} diff --git a/packages/upload-api/src/blob/remove.js b/packages/upload-api/src/blob/remove.js new file mode 100644 index 000000000..954b0dbf7 --- /dev/null +++ b/packages/upload-api/src/blob/remove.js @@ -0,0 +1,26 @@ +import * as Server from '@ucanto/server' +import * as Blob from '@web3-storage/capabilities/blob' +import * as API from '../types.js' + +import { RecordNotFoundErrorName } from '../errors.js' + +/** + * @param {API.BlobServiceContext} context + * @returns {API.ServiceMethod} + */ +export function blobRemoveProvider(context) { + return Server.provide(Blob.remove, async ({ capability }) => { + const space = capability.with + const { content } = capability.nb + const res = await context.allocationsStorage.remove(space, content) + if (res.error && res.error.name === RecordNotFoundErrorName) { + return { + ok: { + size: 0, + }, + } + } + + return res + }) +} diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index ff91d076f..0b9ba35ae 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -57,6 +57,12 @@ import { BlobAdd, BlobAddSuccess, BlobAddFailure, + BlobList, + BlobListSuccess, + BlobListFailure, + BlobRemove, + BlobRemoveSuccess, + BlobRemoveFailure, BlobAllocate, BlobAllocateSuccess, BlobAllocateFailure, @@ -186,6 +192,8 @@ export type { AllocationsStorage, BlobsStorage, TasksStorage, BlobAddInput } export interface Service extends StorefrontService, W3sService { blob: { add: ServiceMethod + remove: ServiceMethod + list: ServiceMethod } store: { add: ServiceMethod diff --git a/packages/upload-api/src/types/blob.ts b/packages/upload-api/src/types/blob.ts index 755d9fed4..5f3e6433f 100644 --- a/packages/upload-api/src/types/blob.ts +++ b/packages/upload-api/src/types/blob.ts @@ -5,7 +5,11 @@ import type { Failure, DID, } from '@ucanto/interface' -import { BlobMultihash, BlobListItem } from '@web3-storage/capabilities/types' +import { + BlobMultihash, + BlobListItem, + BlobRemoveSuccess, +} from '@web3-storage/capabilities/types' import { RecordKeyConflict, ListOptions, ListResponse } from '../types.js' import { Storage } from './storage.js' @@ -29,6 +33,11 @@ export interface AllocationsStorage { space: DID, options?: ListOptions ) => Promise, Failure>> + /** Removes an item from the table but fails if the item does not exist. */ + remove: ( + space: DID, + digest: BlobMultihash + ) => Promise> } export interface BlobModel { @@ -42,8 +51,7 @@ export interface BlobAddInput { blob: BlobModel } -export interface BlobAddOutput - extends Omit {} +export interface BlobAddOutput extends Omit {} export interface BlobGetOutput { blob: { digest: Uint8Array; size: number } diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index ac9eb2366..48995e64c 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -334,4 +334,375 @@ export const test = { assert.ok(blobAdd.out.error, 'invocation should have failed') assert.equal(blobAdd.out.error.name, BlobSizeOutsideOfSupportedRangeName) }, + 'blob/remove returns receipt with blob size for content allocated in space': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + // Invoke `blob/add` to allocate content + const blobAdd = await blobAddInvocation.execute(connection) + if (!blobAdd.out.ok) { + throw new Error('invocation failed', { cause: blobAdd.out.error }) + } + + // invoke `blob/remove` + const blobRemoveInvocation = BlobCapabilities.remove.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + content: digest, + }, + proofs: [proof], + }) + const blobRemove = await blobRemoveInvocation.execute(connection) + if (!blobRemove.out.ok) { + throw new Error('invocation failed', { cause: blobRemove.out.error }) + } + + assert.ok(blobRemove.out.ok) + assert.equal(blobRemove.out.ok.size, size) + }, + 'blob/remove returns receipt with size 0 for non existent content in space': + async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // invoke `blob/remove` + const blobRemoveInvocation = BlobCapabilities.remove.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + content: digest, + }, + proofs: [proof], + }) + const blobRemove = await blobRemoveInvocation.execute(connection) + if (!blobRemove.out.ok) { + throw new Error('invocation failed', { cause: blobRemove.out.error }) + } + + assert.ok(blobRemove.out.ok) + assert.equal(blobRemove.out.ok.size, 0) + }, + 'blob/list does not fail for empty list': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + const blobList = await BlobCapabilities.list + .invoke({ + issuer: alice, + audience: connection.id, + with: spaceDid, + proofs: [proof], + nb: {}, + }) + .execute(connection) + + assert.deepEqual(blobList.out.ok, { results: [], size: 0 }) + }, + 'blob/list returns blobs previously stored by the user': async ( + assert, + context + ) => { + const { proof, spaceDid } = await registerSpace(alice, context) + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + const data = [ + new Uint8Array([11, 22, 34, 44, 55]), + new Uint8Array([22, 34, 44, 55, 66]), + ] + const receipts = [] + for (const datum of data) { + const multihash = await sha256.digest(datum) + const digest = multihash.bytes + const size = datum.byteLength + const blobAdd = await BlobCapabilities.add + .invoke({ + issuer: alice, + audience: connection.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + .execute(connection) + + if (blobAdd.out.error) { + throw new Error('invocation failed', { cause: blobAdd }) + } + + receipts.push(blobAdd) + } + + const blobList = await BlobCapabilities.list + .invoke({ + issuer: alice, + audience: connection.id, + with: spaceDid, + proofs: [proof], + nb: {}, + }) + .execute(connection) + + if (blobList.out.error) { + throw new Error('invocation failed', { cause: blobList }) + } + assert.equal(blobList.out.ok.size, receipts.length) + // list order last-in-first-out + const listReverse = await Promise.all( + data + .reverse() + .map(async (datum) => ({ digest: (await sha256.digest(datum)).bytes })) + ) + assert.deepEqual( + blobList.out.ok.results.map(({ blob }) => ({ digest: blob.digest })), + listReverse + ) + }, + 'blob/list can be paginated with custom size': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + const data = [ + new Uint8Array([11, 22, 34, 44, 55]), + new Uint8Array([22, 34, 44, 55, 66]), + ] + + for (const datum of data) { + const multihash = await sha256.digest(datum) + const digest = multihash.bytes + const size = datum.byteLength + const blobAdd = await BlobCapabilities.add + .invoke({ + issuer: alice, + audience: connection.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + .execute(connection) + + if (blobAdd.out.error) { + throw new Error('invocation failed', { cause: blobAdd }) + } + } + + // Get list with page size 1 (two pages) + const size = 1 + const listPages = [] + /** @type {string} */ + let cursor = '' + + do { + const blobList = await BlobCapabilities.list + .invoke({ + issuer: alice, + audience: connection.id, + with: spaceDid, + proofs: [proof], + nb: { + size, + ...(cursor ? { cursor } : {}), + }, + }) + .execute(connection) + + if (blobList.out.error) { + throw new Error('invocation failed', { cause: blobList }) + } + + // Add page if it has size + blobList.out.ok.size > 0 && listPages.push(blobList.out.ok.results) + + if (blobList.out.ok.after) { + cursor = blobList.out.ok.after + } else { + break + } + } while (cursor) + + assert.equal( + listPages.length, + data.length, + 'has number of pages of added CARs' + ) + + // Inspect content + const blobList = listPages.flat() + const listReverse = await Promise.all( + data + .reverse() + .map(async (datum) => ({ digest: (await sha256.digest(datum)).bytes })) + ) + assert.deepEqual( + blobList.map(({ blob }) => ({ digest: blob.digest })), + listReverse + ) + }, + 'blob/list can page backwards': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + const data = [ + new Uint8Array([11, 22, 33, 44, 55]), + new Uint8Array([22, 33, 44, 55, 66]), + new Uint8Array([33, 44, 55, 66, 77]), + new Uint8Array([44, 55, 66, 77, 88]), + new Uint8Array([55, 66, 77, 88, 99]), + new Uint8Array([66, 77, 88, 99, 11]), + ] + + for (const datum of data) { + const multihash = await sha256.digest(datum) + const digest = multihash.bytes + const size = datum.byteLength + const blobAdd = await BlobCapabilities.add + .invoke({ + issuer: alice, + audience: connection.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + .execute(connection) + + if (blobAdd.out.error) { + throw new Error('invocation failed', { cause: blobAdd }) + } + } + + const size = 3 + + const listResponse = await BlobCapabilities.list + .invoke({ + issuer: alice, + audience: connection.id, + with: spaceDid, + proofs: [proof], + nb: { + size, + }, + }) + .execute(connection) + if (listResponse.out.error) { + throw new Error('invocation failed', { cause: listResponse.out.error }) + } + + const secondListResponse = await BlobCapabilities.list + .invoke({ + issuer: alice, + audience: connection.id, + with: spaceDid, + proofs: [proof], + nb: { + size, + cursor: listResponse.out.ok.after, + }, + }) + .execute(connection) + if (secondListResponse.out.error) { + throw new Error('invocation failed', { + cause: secondListResponse.out.error, + }) + } + + const prevListResponse = await BlobCapabilities.list + .invoke({ + issuer: alice, + audience: connection.id, + with: spaceDid, + proofs: [proof], + nb: { + size, + cursor: secondListResponse.out.ok.before, + pre: true, + }, + }) + .execute(connection) + if (prevListResponse.out.error) { + throw new Error('invocation failed', { + cause: prevListResponse.out.error, + }) + } + + assert.equal(listResponse.out.ok.results.length, 3) + // listResponse is the first page. we used its after to get the second page, and then used the before of the second + // page with the `pre` caveat to list the first page again. the results and cursors should remain the same. + assert.deepEqual( + prevListResponse.out.ok.results[0], + listResponse.out.ok.results[0] + ) + assert.deepEqual( + prevListResponse.out.ok.results[1], + listResponse.out.ok.results[1] + ) + assert.deepEqual( + prevListResponse.out.ok.results[2], + listResponse.out.ok.results[2] + ) + assert.deepEqual(prevListResponse.out.ok.before, listResponse.out.ok.before) + assert.deepEqual(prevListResponse.out.ok.after, listResponse.out.ok.after) + }, } diff --git a/packages/upload-api/test/handlers/web3.storage.js b/packages/upload-api/test/handlers/web3.storage.js index fbff5b60f..a527e3707 100644 --- a/packages/upload-api/test/handlers/web3.storage.js +++ b/packages/upload-api/test/handlers/web3.storage.js @@ -520,98 +520,104 @@ export const test = { const retryBlobAllocate = await serviceBlobAllocate.execute(connection) assert.equal(retryBlobAllocate.out.error, undefined) }, - 'web3.storage/blob/accept returns site delegation': - async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - - // prepare data - const data = new Uint8Array([11, 22, 34, 44, 55]) - const multihash = await sha256.digest(data) - const digest = multihash.bytes - const size = data.byteLength - const content = createLink( - rawCode, - new Digest(sha256.code, 32, digest, digest) - ) - - // create service connection - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // create `blob/add` invocation - const blobAddInvocation = BlobCapabilities.add.invoke({ - issuer: alice, - audience: context.id, - with: spaceDid, - nb: { - blob: { - digest, - size, - }, + 'web3.storage/blob/accept returns site delegation': async ( + assert, + context + ) => { + const { proof, spaceDid } = await registerSpace(alice, context) + + // prepare data + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + const content = createLink( + rawCode, + new Digest(sha256.code, 32, digest, digest) + ) + + // create service connection + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // create `blob/add` invocation + const blobAddInvocation = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, }, - proofs: [proof], - }) - const blobAdd = await blobAddInvocation.execute(connection) - if (!blobAdd.out.ok) { - throw new Error('invocation failed', { cause: blobAdd }) - } - - // parse receipt next - const next = parseBlobAddReceiptNext(blobAdd) - - /** @type {import('@web3-storage/capabilities/types').BlobAddress} */ - // @ts-expect-error receipt type is unknown - const address = next.allocate.receipt.out.ok.address - - // Store the blob to the address - const goodPut = await fetch(address.url, { - method: 'PUT', - mode: 'cors', - body: data, - headers: address.headers, - }) - assert.equal(goodPut.status, 200, await goodPut.text()) - - // invoke `web3.storage/blob/accept` - const serviceBlobAccept = W3sBlobCapabilities.accept.invoke({ - issuer: context.id, - audience: context.id, - with: context.id.did(), - nb: { - blob: { - digest, - size, - }, - space: spaceDid, - _put: { 'ucan/await': ['.out.ok', next.put.task.link()] }, + }, + proofs: [proof], + }) + const blobAdd = await blobAddInvocation.execute(connection) + if (!blobAdd.out.ok) { + throw new Error('invocation failed', { cause: blobAdd }) + } + + // parse receipt next + const next = parseBlobAddReceiptNext(blobAdd) + + /** @type {import('@web3-storage/capabilities/types').BlobAddress} */ + // @ts-expect-error receipt type is unknown + const address = next.allocate.receipt.out.ok.address + + // Store the blob to the address + const goodPut = await fetch(address.url, { + method: 'PUT', + mode: 'cors', + body: data, + headers: address.headers, + }) + assert.equal(goodPut.status, 200, await goodPut.text()) + + // invoke `web3.storage/blob/accept` + const serviceBlobAccept = W3sBlobCapabilities.accept.invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + blob: { + digest, + size, }, - proofs: [proof], - }) - const blobAccept = await serviceBlobAccept.execute(connection) - if (!blobAccept.out.ok) { - throw new Error('invocation failed', { cause: blobAccept }) - } - // Validate out - assert.ok(blobAccept.out.ok) - assert.ok(blobAccept.out.ok.site) - - // Validate effect - assert.equal(blobAccept.fx.fork.length, 1) - /** @type {import('@ucanto/interface').Delegation} */ - // @ts-expect-error delegation not assignable to Effect per TS understanding - const delegation = blobAccept.fx.fork[0] - assert.equal(delegation.capabilities.length, 1) - assert.ok(delegation.capabilities[0].can, Assert.location.can) - // @ts-expect-error nb unknown - assert.ok(delegation.capabilities[0].nb.content.equals(content)) - // @ts-expect-error nb unknown - const locations = delegation.capabilities[0].nb.location - assert.equal(locations.length, 1) - assert.ok(locations[0].includes(`https://w3s.link/ipfs/${content.toString()}?origin`)) - }, - 'web3.storage/blob/accept fails to provide site delegation when blob was not stored': + space: spaceDid, + _put: { 'ucan/await': ['.out.ok', next.put.task.link()] }, + }, + proofs: [proof], + }) + const blobAccept = await serviceBlobAccept.execute(connection) + if (!blobAccept.out.ok) { + throw new Error('invocation failed', { cause: blobAccept }) + } + // Validate out + assert.ok(blobAccept.out.ok) + assert.ok(blobAccept.out.ok.site) + + // Validate effect + assert.equal(blobAccept.fx.fork.length, 1) + /** @type {import('@ucanto/interface').Delegation} */ + // @ts-expect-error delegation not assignable to Effect per TS understanding + const delegation = blobAccept.fx.fork[0] + assert.equal(delegation.capabilities.length, 1) + assert.ok(delegation.capabilities[0].can, Assert.location.can) + // @ts-expect-error nb unknown + assert.ok(delegation.capabilities[0].nb.content.equals(content)) + // @ts-expect-error nb unknown + const locations = delegation.capabilities[0].nb.location + assert.equal(locations.length, 1) + assert.ok( + locations[0].includes( + `https://w3s.link/ipfs/${content.toString()}?origin` + ) + ) + }, + 'web3.storage/blob/accept fails to provide site delegation when blob was not stored': async (assert, context) => { const { proof, spaceDid } = await registerSpace(alice, context) @@ -666,6 +672,9 @@ export const test = { const blobAccept = await serviceBlobAccept.execute(connection) // Validate out error assert.ok(blobAccept.out.error) - assert.equal(blobAccept.out.error?.name, AllocatedMemoryHadNotBeenWrittenToName) + assert.equal( + blobAccept.out.error?.name, + AllocatedMemoryHadNotBeenWrittenToName + ) }, } diff --git a/packages/upload-api/test/lib.js b/packages/upload-api/test/lib.js index 1ba33bae8..afccde264 100644 --- a/packages/upload-api/test/lib.js +++ b/packages/upload-api/test/lib.js @@ -33,7 +33,7 @@ export const test = { ...Blob.test, ...Upload.test, ...Web3Storage.test, - ...Ucan.test + ...Ucan.test, } export const storageTests = { @@ -45,7 +45,7 @@ export const storageTests = { ...allocationsStorageTests, ...blobsStorageTests, ...tasksStorageTests, - ...receiptsStorageTests + ...receiptsStorageTests, } export const handlerTests = { diff --git a/packages/upload-api/test/storage/allocations-storage-tests.js b/packages/upload-api/test/storage/allocations-storage-tests.js index 95ece5477..6ba345adb 100644 --- a/packages/upload-api/test/storage/allocations-storage-tests.js +++ b/packages/upload-api/test/storage/allocations-storage-tests.js @@ -306,4 +306,57 @@ export const test = { ) ) }, + 'should fail to remove non existent allocations on a space': async ( + assert, + context + ) => { + const { spaceDid } = await registerSpace(alice, context) + const allocationsStorage = context.allocationsStorage + + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + + const removeResult = await allocationsStorage.remove(spaceDid, digest) + + assert.ok(removeResult.error) + assert.equal(removeResult.error?.name, RecordNotFoundErrorName) + }, + 'should remove existent allocations on a space': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + const allocationsStorage = context.allocationsStorage + + const data = new Uint8Array([11, 22, 34, 44, 55]) + const multihash = await sha256.digest(data) + const digest = multihash.bytes + const size = data.byteLength + + // invoke `blob/add` + const blobAdd = BlobCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { + blob: { + digest, + size, + }, + }, + proofs: [proof], + }) + const cause = (await blobAdd.delegate()).link() + const allocationInsert0 = await allocationsStorage.insert({ + space: spaceDid, + blob: { + digest, + size, + }, + cause, + }) + assert.ok(allocationInsert0.ok) + + const removeResult = await allocationsStorage.remove(spaceDid, digest) + assert.ok(removeResult.ok) + assert.equal(removeResult.ok?.size, size) + }, } diff --git a/packages/upload-api/test/storage/allocations-storage.js b/packages/upload-api/test/storage/allocations-storage.js index 3b0ded4be..bf548ddf6 100644 --- a/packages/upload-api/test/storage/allocations-storage.js +++ b/packages/upload-api/test/storage/allocations-storage.js @@ -61,6 +61,26 @@ export class AllocationsStorage { return { ok: !!item } } + /** + * @param {Types.DID} space + * @param {Uint8Array} blobMultihash + * @returns {ReturnType} + */ + async remove(space, blobMultihash) { + const item = this.items.find( + (i) => i.space === space && equals(i.blob.digest, blobMultihash) + ) + if (!item) { + return { error: new RecordNotFound() } + } + this.items = this.items.filter((i) => i !== item) + return { + ok: { + size: item.blob.size, + }, + } + } + /** * @param {Types.DID} space * @param {Types.ListOptions} options From 9bbb24e3f803811a3445418ab414c19720a7df54 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 22 Apr 2024 10:39:48 +0100 Subject: [PATCH 2/5] fix: remove pre --- packages/upload-api/src/blob/list.js | 4 +- packages/upload-api/src/types/blob.ts | 7 +- packages/upload-api/test/handlers/blob.js | 112 ---------------------- 3 files changed, 8 insertions(+), 115 deletions(-) diff --git a/packages/upload-api/src/blob/list.js b/packages/upload-api/src/blob/list.js index 4915fe16c..694d3d3d0 100644 --- a/packages/upload-api/src/blob/list.js +++ b/packages/upload-api/src/blob/list.js @@ -9,7 +9,7 @@ import * as API from '../types.js' export function blobListProvider(context) { return Server.provide(Blob.list, async ({ capability }) => { const space = capability.with - const { cursor, size, pre } = capability.nb - return await context.allocationsStorage.list(space, { size, cursor, pre }) + const { cursor, size } = capability.nb + return await context.allocationsStorage.list(space, { size, cursor }) }) } diff --git a/packages/upload-api/src/types/blob.ts b/packages/upload-api/src/types/blob.ts index 5f3e6433f..2c9e338a9 100644 --- a/packages/upload-api/src/types/blob.ts +++ b/packages/upload-api/src/types/blob.ts @@ -11,7 +11,7 @@ import { BlobRemoveSuccess, } from '@web3-storage/capabilities/types' -import { RecordKeyConflict, ListOptions, ListResponse } from '../types.js' +import { RecordKeyConflict, ListResponse } from '../types.js' import { Storage } from './storage.js' export type TasksStorage = Storage @@ -40,6 +40,11 @@ export interface AllocationsStorage { ) => Promise> } +export interface ListOptions { + size?: number + cursor?: string +} + export interface BlobModel { digest: BlobMultihash size: number diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index 48995e64c..1eca22f4f 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -593,116 +593,4 @@ export const test = { listReverse ) }, - 'blob/list can page backwards': async (assert, context) => { - const { proof, spaceDid } = await registerSpace(alice, context) - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - const data = [ - new Uint8Array([11, 22, 33, 44, 55]), - new Uint8Array([22, 33, 44, 55, 66]), - new Uint8Array([33, 44, 55, 66, 77]), - new Uint8Array([44, 55, 66, 77, 88]), - new Uint8Array([55, 66, 77, 88, 99]), - new Uint8Array([66, 77, 88, 99, 11]), - ] - - for (const datum of data) { - const multihash = await sha256.digest(datum) - const digest = multihash.bytes - const size = datum.byteLength - const blobAdd = await BlobCapabilities.add - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - nb: { - blob: { - digest, - size, - }, - }, - proofs: [proof], - }) - .execute(connection) - - if (blobAdd.out.error) { - throw new Error('invocation failed', { cause: blobAdd }) - } - } - - const size = 3 - - const listResponse = await BlobCapabilities.list - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - proofs: [proof], - nb: { - size, - }, - }) - .execute(connection) - if (listResponse.out.error) { - throw new Error('invocation failed', { cause: listResponse.out.error }) - } - - const secondListResponse = await BlobCapabilities.list - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - proofs: [proof], - nb: { - size, - cursor: listResponse.out.ok.after, - }, - }) - .execute(connection) - if (secondListResponse.out.error) { - throw new Error('invocation failed', { - cause: secondListResponse.out.error, - }) - } - - const prevListResponse = await BlobCapabilities.list - .invoke({ - issuer: alice, - audience: connection.id, - with: spaceDid, - proofs: [proof], - nb: { - size, - cursor: secondListResponse.out.ok.before, - pre: true, - }, - }) - .execute(connection) - if (prevListResponse.out.error) { - throw new Error('invocation failed', { - cause: prevListResponse.out.error, - }) - } - - assert.equal(listResponse.out.ok.results.length, 3) - // listResponse is the first page. we used its after to get the second page, and then used the before of the second - // page with the `pre` caveat to list the first page again. the results and cursors should remain the same. - assert.deepEqual( - prevListResponse.out.ok.results[0], - listResponse.out.ok.results[0] - ) - assert.deepEqual( - prevListResponse.out.ok.results[1], - listResponse.out.ok.results[1] - ) - assert.deepEqual( - prevListResponse.out.ok.results[2], - listResponse.out.ok.results[2] - ) - assert.deepEqual(prevListResponse.out.ok.before, listResponse.out.ok.before) - assert.deepEqual(prevListResponse.out.ok.after, listResponse.out.ok.after) - }, } From 3d997a4b2041e844173963f1e37e3e0384f774b9 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 22 Apr 2024 16:08:20 +0100 Subject: [PATCH 3/5] fix: drop pre also in capability --- packages/capabilities/src/blob.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/capabilities/src/blob.js b/packages/capabilities/src/blob.js index 38b6adc15..142777ac4 100644 --- a/packages/capabilities/src/blob.js +++ b/packages/capabilities/src/blob.js @@ -126,10 +126,6 @@ export const list = capability({ * Maximum number of items per page. */ size: Schema.integer().optional(), - /** - * If true, return page of results preceding cursor. Defaults to false. - */ - pre: Schema.boolean().optional(), }), derives: (claimed, delegated) => { if (claimed.with !== delegated.with) { From 1331af04677523168dddf9f129f83b5378f8a719 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 22 Apr 2024 18:14:30 +0100 Subject: [PATCH 4/5] chore: correct comment on allocation remove --- packages/upload-api/src/types/blob.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/upload-api/src/types/blob.ts b/packages/upload-api/src/types/blob.ts index 2c9e338a9..21adf1094 100644 --- a/packages/upload-api/src/types/blob.ts +++ b/packages/upload-api/src/types/blob.ts @@ -33,7 +33,7 @@ export interface AllocationsStorage { space: DID, options?: ListOptions ) => Promise, Failure>> - /** Removes an item from the table but fails if the item does not exist. */ + /** Removes an item from the table, returning zero on size if non existent. */ remove: ( space: DID, digest: BlobMultihash From a1486fe66b16da0257c9dd185ac1938289e953cf Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 22 Apr 2024 18:16:30 +0100 Subject: [PATCH 5/5] fix: blob remove should have args digest instead of content --- packages/capabilities/src/blob.js | 10 +++++----- packages/upload-api/src/blob/remove.js | 4 ++-- packages/upload-api/test/handlers/blob.js | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/capabilities/src/blob.js b/packages/capabilities/src/blob.js index 142777ac4..eccadf173 100644 --- a/packages/capabilities/src/blob.js +++ b/packages/capabilities/src/blob.js @@ -85,7 +85,7 @@ export const remove = capability({ /** * A multihash digest of the blob payload bytes, uniquely identifying blob. */ - content: Schema.bytes(), + digest: Schema.bytes(), }), derives: (claimed, delegated) => { if (claimed.with !== delegated.with) { @@ -93,13 +93,13 @@ export const remove = capability({ `Expected 'with: "${delegated.with}"' instead got '${claimed.with}'` ) } else if ( - delegated.nb.content && - !equals(delegated.nb.content, claimed.nb.content) + delegated.nb.digest && + !equals(delegated.nb.digest, claimed.nb.digest) ) { return fail( `Link ${ - claimed.nb.content ? `${claimed.nb.content}` : '' - } violates imposed ${delegated.nb.content} constraint.` + claimed.nb.digest ? `${claimed.nb.digest}` : '' + } violates imposed ${delegated.nb.digest} constraint.` ) } return ok({}) diff --git a/packages/upload-api/src/blob/remove.js b/packages/upload-api/src/blob/remove.js index 954b0dbf7..e4e4d9400 100644 --- a/packages/upload-api/src/blob/remove.js +++ b/packages/upload-api/src/blob/remove.js @@ -11,8 +11,8 @@ import { RecordNotFoundErrorName } from '../errors.js' export function blobRemoveProvider(context) { return Server.provide(Blob.remove, async ({ capability }) => { const space = capability.with - const { content } = capability.nb - const res = await context.allocationsStorage.remove(space, content) + const { digest } = capability.nb + const res = await context.allocationsStorage.remove(space, digest) if (res.error && res.error.name === RecordNotFoundErrorName) { return { ok: { diff --git a/packages/upload-api/test/handlers/blob.js b/packages/upload-api/test/handlers/blob.js index 1eca22f4f..9d4eaafc2 100644 --- a/packages/upload-api/test/handlers/blob.js +++ b/packages/upload-api/test/handlers/blob.js @@ -375,7 +375,7 @@ export const test = { audience: context.id, with: spaceDid, nb: { - content: digest, + digest, }, proofs: [proof], }) @@ -408,7 +408,7 @@ export const test = { audience: context.id, with: spaceDid, nb: { - content: digest, + digest, }, proofs: [proof], })