From 0578432181099d526e4ffacbeafe178c8da1e532 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Mon, 12 Dec 2022 15:12:57 +0000 Subject: [PATCH 1/8] feat: adds `w3 rm ` cmd WIP. needs update to `upload-api` to return the removed item so we may use it to identify the shards that need removing. License: MIT Signed-off-by: Oli Evans --- bin.js | 5 +++++ index.js | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/bin.js b/bin.js index 92eb1fd..415801d 100755 --- a/bin.js +++ b/bin.js @@ -31,6 +31,11 @@ cli.command('ls') .option('--shards', 'Pretty print with shards in output') .action(list) +cli.command('rm ') + .example('rm bafy...') + .describe('Remove an upload from the uploads listing. Pass --shards to delete the actual data if you are sure no other uploads need them') + .option('--shards', 'Remove all shards referenced by the upload from the store. Use with caution and ensure other uploads do not reference the same shards.') + cli.command('whoami') .describe('Print information about the current agent.') .action(whoami) diff --git a/index.js b/index.js index a0f2942..1e035c6 100644 --- a/index.js +++ b/index.js @@ -1,7 +1,8 @@ -import fs from 'fs' -import ora from 'ora' +import fs, { readSync } from 'fs' +import ora, { oraPromise } from 'ora' import tree from 'pretty-tree' import { Readable } from 'stream' +import { CID } from 'multiformats/cid' import * as DID from '@ipld/dag-ucan/did' import { CarReader, CarWriter } from '@ipld/car' import { filesFromPath } from 'files-from-path' @@ -80,6 +81,34 @@ export async function list (opts) { console.log('⁂ Try out `w3 up ` to upload some') } } +/** + * @param {string} rootCid + * @param {object} opts + * @param {boolean} [opts.shards] + */ +export async function remove (rootCid, opts) { + let root + try { + root = CID.parse(rootCid.trim()) + } catch (err) { + console.error(`Error: ${rootCid} is not a CID`) + } + + const client = await getClient() + const res = await client.capability.upload.remove(root) + console.log(`Removed ${root}`) + if (!opts.shards) { + return + } + console.log('shards') + for (const shard of res.shards) { + oraPromise(client.capability.store.remove(shard, { + text: `Removing ${shard}`, + successText: `Removed ${shard}`, + failText: `Error removing ${shard}` + })) + } +} /** * @param {string} name From 0240ea94d42df7f68f31396b99d3af7d650eb561 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Thu, 15 Dec 2022 14:29:38 +0000 Subject: [PATCH 2/8] feat: w3 rm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit without `--shards`, `w3 rm` behaves like rm and silently removes the thing or errors. ```sh $ w3 rm bafybeidpouz7j6to2p7ps4j262ii32ov5wnp3rj4slavvsm4dfvonvrc6q ``` with `--shards`, as there might be many, we print each one with a spinner as we delete them ```sh $ ❯ w3 rm bafybeib5bn5onczh4n2qb25wqjvyemj6mwk6oshwiaibcef2testdboqra --shards ⁂ removing 4 shards ✔ bagbaieraxwxthv7ntlucs2lpghxv7wsrvoyca4gtna2ny4tg7mlkquqef7aq removed ✔ bagbaierawqhk2jn2qwzdm5ajdevwp2t5kari6d6qok7iu2jyfm6kndndemla removed ✔ bagbaierarihlji6ep47f42fv46atov4vh7kyhledn5shif5zwbpszsk3kltq removed ✔ bagbaieraz2bfnvj6rooqpjny2j3xjeqp2otndee5oylxl6jmdll7ydxa5lwq removed ``` License: MIT Signed-off-by: Oli Evans --- bin.js | 2 ++ index.js | 36 ++++++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/bin.js b/bin.js index d050e6d..0d81d6a 100755 --- a/bin.js +++ b/bin.js @@ -14,6 +14,7 @@ import { addProof, listProofs, upload, + remove, list, whoami } from './index.js' @@ -52,6 +53,7 @@ cli.command('rm ') .example('rm bafy...') .describe('Remove an upload from the uploads listing. Pass --shards to delete the actual data if you are sure no other uploads need them') .option('--shards', 'Remove all shards referenced by the upload from the store. Use with caution and ensure other uploads do not reference the same shards.') + .action(remove) cli.command('whoami') .describe('Print information about the current agent.') diff --git a/index.js b/index.js index d262e7d..15064c6 100644 --- a/index.js +++ b/index.js @@ -88,20 +88,36 @@ export async function remove (rootCid, opts) { } catch (err) { console.error(`Error: ${rootCid} is not a CID`) } - const client = await getClient() - const res = await client.capability.upload.remove(root) - console.log(`Removed ${root}`) + let upload + try { + upload = await client.capability.upload.remove(root) + } catch (err) { + console.error(`Error: remove failed: ${err.message ?? err}`) + process.exit(1) + } if (!opts.shards) { return } - console.log('shards') - for (const shard of res.shards) { - oraPromise(client.capability.store.remove(shard, { - text: `Removing ${shard}`, - successText: `Removed ${shard}`, - failText: `Error removing ${shard}` - })) + if (!upload) { + return console.log(`⁂ upload not found. could not determine shards to remove.`) + } + + const { shards } = upload + console.log(`⁂ removing ${shards.length} shard${ shards.length === 1 ? '' : 's'}`) + + function removeShard(shard) { + return oraPromise(client.capability.store.remove(shard), { + text: `${shard}`, + successText: `${shard} removed`, + failText: `${shard} failed` + }) + } + + const results = await Promise.allSettled(shards.map(removeShard)) + + if (results.some(res => res.status === 'rejected')) { + process.exit(1) } } From a1e82e57acf78db0bab6e5f5c7a356c518ac0aff Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Thu, 15 Dec 2022 15:02:13 +0000 Subject: [PATCH 3/8] chore: fixes License: MIT Signed-off-by: Oli Evans --- index.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 15064c6..0c2b1c8 100644 --- a/index.js +++ b/index.js @@ -1,4 +1,4 @@ -import fs, { readSync } from 'fs' +import fs from 'fs' import ora, { oraPromise } from 'ora' import tree from 'pretty-tree' import { Readable } from 'stream' @@ -102,6 +102,9 @@ export async function remove (rootCid, opts) { if (!upload) { return console.log(`⁂ upload not found. could not determine shards to remove.`) } + if (!upload.shards || !upload.shards.length) { + return console.log(`⁂ no shards to remove.`) + } const { shards } = upload console.log(`⁂ removing ${shards.length} shard${ shards.length === 1 ? '' : 's'}`) From 28ac24ca3ab0ab26766de46f49028fb255d5fec4 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Thu, 15 Dec 2022 15:03:03 +0000 Subject: [PATCH 4/8] chore: lint License: MIT Signed-off-by: Oli Evans --- index.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/index.js b/index.js index 0c2b1c8..f1ebc42 100644 --- a/index.js +++ b/index.js @@ -100,16 +100,16 @@ export async function remove (rootCid, opts) { return } if (!upload) { - return console.log(`⁂ upload not found. could not determine shards to remove.`) + return console.log('⁂ upload not found. could not determine shards to remove.') } if (!upload.shards || !upload.shards.length) { - return console.log(`⁂ no shards to remove.`) + return console.log('⁂ no shards to remove.') } - + const { shards } = upload - console.log(`⁂ removing ${shards.length} shard${ shards.length === 1 ? '' : 's'}`) + console.log(`⁂ removing ${shards.length} shard${shards.length === 1 ? '' : 's'}`) - function removeShard(shard) { + function removeShard (shard) { return oraPromise(client.capability.store.remove(shard), { text: `${shard}`, successText: `${shard} removed`, From b2f427fa802455cbec1e1d0b1235cd2737f954e8 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Thu, 15 Dec 2022 15:55:43 +0000 Subject: [PATCH 5/8] chore: add tests License: MIT Signed-off-by: Oli Evans --- index.js | 3 +- test/bin.spec.js | 94 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index f1ebc42..9c091f5 100644 --- a/index.js +++ b/index.js @@ -93,7 +93,8 @@ export async function remove (rootCid, opts) { try { upload = await client.capability.upload.remove(root) } catch (err) { - console.error(`Error: remove failed: ${err.message ?? err}`) + console.error(`Remove failed: ${err.message ?? err}`) + console.error(err) process.exit(1) } if (!opts.shards) { diff --git a/test/bin.spec.js b/test/bin.spec.js index e09de67..f3e4af3 100644 --- a/test/bin.spec.js +++ b/test/bin.spec.js @@ -18,6 +18,7 @@ import { mockService } from './helpers/mocks.js' import { createServer as createHTTPServer } from './helpers/http-server.js' import { createHTTPListener } from './helpers/ucanto.js' import { createEnv } from './helpers/env.js' +import { Store } from '@web3-storage/capabilities' /** * @typedef {{ @@ -167,6 +168,99 @@ test('w3 ls', async (t) => { t.notThrows(() => CID.parse(JSON.parse(list1.stdout).root)) }) +test('w3 remove', async t => { + const env = t.context.env.alice + + await execa('./bin.js', ['space', 'create'], { env }) + + const service = mockService({ + upload: { + remove: provide(UploadCapabilities.remove, ({ invocation }) => { + const { nb } = invocation.capabilities[0] + return { root: nb.root } + }) + } + }) + t.context.setService(service) + + t.throwsAsync(() => execa('./bin.js', ['rm', 'nope'], { env }), { message: /not a CID/ }) + + const rm = await execa('./bin.js', ['rm', 'bafybeih2k7ughhfwedltjviunmn3esueijz34snyay77zmsml5w24tqamm'], { env }) + t.is(rm.exitCode, 0) + t.is(service.upload.remove.callCount, 1) + t.is(service.store.remove.callCount, 0) + t.is(rm.stdout, '') +}) + +test('w3 remove - no such upload', async t => { + const env = t.context.env.alice + + await execa('./bin.js', ['space', 'create'], { env }) + + const service = mockService({ + upload: { + remove: provide(UploadCapabilities.remove, () => {}) + } + }) + t.context.setService(service) + + const rm = await execa('./bin.js', ['rm', 'bafybeih2k7ughhfwedltjviunmn3esueijz34snyay77zmsml5w24tqamm', '--shards'], { env }) + t.is(rm.exitCode, 0) + t.is(rm.stdout, '⁂ upload not found. could not determine shards to remove.') +}) + +test('w3 remove --shards', async t => { + const env = t.context.env.alice + + await execa('./bin.js', ['space', 'create'], { env }) + + const service = mockService({ + store: { + remove: provide(StoreCapabilities.remove, () => {}) + }, + upload: { + remove: provide(UploadCapabilities.remove, ({ invocation }) => { + const { nb } = invocation.capabilities[0] + return { root: nb.root, shards: [ + CID.parse('bagbaiera7ciaeifwrn7oo35gxdalocfj23vkvqus2eup27wt2qcxlvta2wya'), + CID.parse('bagbaiera7ciaeifwrn7oo35gxdalocfj23vkvqus2eup27wt2qcxlvta2wya') + ] } + }) + } + }) + t.context.setService(service) + + const rm = await execa('./bin.js', ['rm', 'bafybeih2k7ughhfwedltjviunmn3esueijz34snyay77zmsml5w24tqamm', '--shards'], { env }) + t.is(rm.exitCode, 0) + t.is(service.upload.remove.callCount, 1) + t.is(service.store.remove.callCount, 2) +}) + +test('w3 remove --shards - no shards to remove', async t => { + const env = t.context.env.alice + + await execa('./bin.js', ['space', 'create'], { env }) + + const service = mockService({ + store: { + remove: provide(StoreCapabilities.remove, () => {}) + }, + upload: { + remove: provide(UploadCapabilities.remove, ({ invocation }) => { + const { nb } = invocation.capabilities[0] + return { root: nb.root } + }) + } + }) + t.context.setService(service) + + const rm = await execa('./bin.js', ['rm', 'bafybeih2k7ughhfwedltjviunmn3esueijz34snyay77zmsml5w24tqamm', '--shards'], { env }) + t.is(rm.exitCode, 0) + t.is(service.upload.remove.callCount, 1) + t.is(service.store.remove.callCount, 0) + t.is(rm.stdout, '⁂ no shards to remove.') +}) + test('w3 delegation create', async t => { const env = t.context.env.alice From b972d5972ed8d867fb91a75d978f9fd8eb2b0d43 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Thu, 15 Dec 2022 15:56:46 +0000 Subject: [PATCH 6/8] chore: lint License: MIT Signed-off-by: Oli Evans --- test/bin.spec.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/bin.spec.js b/test/bin.spec.js index f3e4af3..146fe98 100644 --- a/test/bin.spec.js +++ b/test/bin.spec.js @@ -221,10 +221,13 @@ test('w3 remove --shards', async t => { upload: { remove: provide(UploadCapabilities.remove, ({ invocation }) => { const { nb } = invocation.capabilities[0] - return { root: nb.root, shards: [ - CID.parse('bagbaiera7ciaeifwrn7oo35gxdalocfj23vkvqus2eup27wt2qcxlvta2wya'), - CID.parse('bagbaiera7ciaeifwrn7oo35gxdalocfj23vkvqus2eup27wt2qcxlvta2wya') - ] } + return { + root: nb.root, + shards: [ + CID.parse('bagbaiera7ciaeifwrn7oo35gxdalocfj23vkvqus2eup27wt2qcxlvta2wya'), + CID.parse('bagbaiera7ciaeifwrn7oo35gxdalocfj23vkvqus2eup27wt2qcxlvta2wya') + ] + } }) } }) From f82c9d06ffa2f730eb6e6dfd72b035d143b6b6fc Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Thu, 15 Dec 2022 15:59:41 +0000 Subject: [PATCH 7/8] chore: lint lint License: MIT Signed-off-by: Oli Evans --- test/bin.spec.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/bin.spec.js b/test/bin.spec.js index 146fe98..e0c8d79 100644 --- a/test/bin.spec.js +++ b/test/bin.spec.js @@ -18,7 +18,6 @@ import { mockService } from './helpers/mocks.js' import { createServer as createHTTPServer } from './helpers/http-server.js' import { createHTTPListener } from './helpers/ucanto.js' import { createEnv } from './helpers/env.js' -import { Store } from '@web3-storage/capabilities' /** * @typedef {{ From 25cdddf372d0724b49c0be1bf36d69b177a192b0 Mon Sep 17 00:00:00 2001 From: Oli Evans Date: Thu, 15 Dec 2022 18:20:59 +0000 Subject: [PATCH 8/8] Update index.js Co-authored-by: Alan Shaw --- index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/index.js b/index.js index 9c091f5..5dcca5b 100644 --- a/index.js +++ b/index.js @@ -87,6 +87,7 @@ export async function remove (rootCid, opts) { root = CID.parse(rootCid.trim()) } catch (err) { console.error(`Error: ${rootCid} is not a CID`) + process.exit(1) } const client = await getClient() let upload