feat: Migrate store/* & upload/* APIs (#485)

This pull request moves `store/*` and `upload/*` capability providers
from w3infra repo to this one.

The goal is to consolidate implementations so that it's easier to update
ucanto and service implementations without having to sync between repos.
w3infra should be able to just pull `upload-api` package and expose it
through AWS lambdas.

This also migrates tests and makes them agnostic of the runtime. w3infra
could run same test suite but pass AWS specific context.

---------

Co-authored-by: Benjamin Goering <171782+gobengo@users.noreply.github.com>

.github/release-please-config.json

@@ -4,6 +4,7 @@
     "packages/access-client": {},
     "packages/access-api": {},
     "packages/capabilities": {},
+    "packages/upload-api": {},
     "packages/upload-client": {}
   }
 }

.github/release-please-manifest.json

@@ -2,5 +2,6 @@
   "packages/access-client": "9.4.0",
   "packages/access-api": "4.11.0",
   "packages/capabilities": "3.1.0",
+  "packages/upload-api": "0.0.0",
   "packages/upload-client": "6.0.0"
 }

.github/workflows/release.yml

@@ -34,7 +34,8 @@ jobs:
     if: |
       contains(fromJson(needs.release.outputs.paths_released), 'packages/access-client') ||
       contains(fromJson(needs.release.outputs.paths_released), 'packages/capabilities') ||
-      contains(fromJson(needs.release.outputs.paths_released), 'packages/upload-client')
+      contains(fromJson(needs.release.outputs.paths_released), 'packages/upload-client') ||
+      contains(fromJson(needs.release.outputs.paths_released), 'packages/upload-api')
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/upload-api.yml

@@ -0,0 +1,76 @@
+name: Upload API
+env:
+  CI: true
+  FORCE_COLOR: 1
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/upload-api/**'
+      - '.github/workflows/upload-api.yml'
+      - 'pnpm-lock.yaml'
+      - '.env.tpl'
+  pull_request:
+    paths:
+      - 'packages/upload-api/**'
+      - '.github/workflows/upload-api.yml'
+      - 'pnpm-lock.yaml'
+      - '.env.tpl'
+jobs:
+  check:
+    name: Typecheck
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install
+        uses: pnpm/action-setup@v2.2.3
+        with:
+          version: 7
+
+      - name: Setup
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          registry-url: https://registry.npmjs.org/
+          cache: 'pnpm'
+
+      - name: Prepare
+        run: pnpm install
+
+      - name: Check
+        uses: gozala/typescript-error-reporter-action@v1.0.8
+        with:
+          project: packages/upload-api/tsconfig.json
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install
+        uses: pnpm/action-setup@v2.2.3
+        with:
+          version: 7
+
+      - name: Setup
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          registry-url: https://registry.npmjs.org/
+          cache: 'pnpm'
+
+      - name: Build
+        run: |
+          pnpm install
+          pnpm run --if-present build
+
+      - name: Test
+        run: pnpm -r --filter @web3-storage/upload-api run test
+
+      - name: Dependency check
+        run: pnpm -r --filter @web3-storage/upload-api exec depcheck

packages/upload-api/package.json

@@ -0,0 +1,104 @@
+{
+  "name": "@web3-storage/upload-api",
+  "version": "3.0.0",
+  "type": "module",
+  "typesVersions": {
+    "*": {
+      "*": [
+        "dist/src/*"
+      ],
+      "dist/src/lib.d.ts": [
+        "dist/src/lib.d.ts"
+      ],
+      "test": [
+        "dist/test/*"
+      ]
+    }
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/src/lib.d.ts",
+      "import": "./src/lib.js"
+    },
+    "./types": {
+      "types": "./dist/src/types.d.ts",
+      "import": "./src/types.js"
+    },
+    "./store": {
+      "types": "./dist/src/store.d.ts",
+      "import": "./src/store.js"
+    },
+    "./upload": {
+      "types": "./dist/src/upload.d.ts",
+      "import": "./src/upload.js"
+    },
+    "./test": {
+      "types": "./dist/test/lib.d.ts",
+      "import": "./test/lib.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc --build",
+    "check": "tsc --build",
+    "lint": "tsc --build",
+    "test": "mocha --bail --timeout 10s -n no-warnings -n experimental-vm-modules test/**/*.spec.js",
+    "test-watch": "pnpm build && mocha --bail --timeout 10s --watch --parallel -n no-warnings -n experimental-vm-modules --watch-files src,test"
+  },
+  "dependencies": {
+    "@sentry/serverless": "^7.22.0",
+    "@ucanto/client": "^5.0.0",
+    "@ucanto/interface": "^5.0.0",
+    "@ucanto/principal": "^5.0.0",
+    "@ucanto/server": "^5.0.0",
+    "@ucanto/transport": "^5.0.0",
+    "@web3-storage/capabilities": "^3.0.0",
+    "multiformats": "^11.0.1",
+    "p-retry": "^5.1.2"
+  },
+  "devDependencies": {
+    "@ipld/car": "^5.0.1",
+    "@ucanto/core": "^5.0.0",
+    "@types/mocha": "^10.0.1",
+    "mocha": "^10.2.0",
+    "@web3-storage/sigv4": "^1.0.2"
+  },
+  "eslintConfig": {
+    "extends": [
+      "./node_modules/hd-scripts/eslint/index.js"
+    ],
+    "parserOptions": {
+      "project": "./tsconfig.json"
+    },
+    "rules": {
+      "unicorn/prefer-number-properties": "off",
+      "unicorn/prefer-export-from": "off",
+      "unicorn/no-array-reduce": "off",
+      "unicorn/no-null": "off",
+      "unicorn/no-zero-fractions": "off",
+      "unicorn/no-negated-condition": "off",
+      "@typescript-eslint/method-signature-style": "off",
+      "@typescript-eslint/no-empty-interface": "off",
+      "unicorn/no-useless-undefined": "off",
+      "no-nested-ternary": "off",
+      "yoda": "off",
+      "jsdoc/no-undefined-types": [
+        "error",
+        {
+          "definedTypes": [
+            "Iterable"
+          ]
+        }
+      ]
+    },
+    "env": {
+      "mocha": true
+    },
+    "ignorePatterns": [
+      "dist",
+      "coverage"
+    ]
+  },
+  "engines": {
+    "node": ">=16.15"
+  }
+}

packages/upload-api/src/index.js

@@ -0,0 +1,27 @@
+import * as Sentry from '@sentry/serverless'
+import { createService as createServiceRouter, createServer } from './lib.js'
+
+Sentry.AWSLambda.init({
+  environment: process.env.SST_STAGE,
+  dsn: process.env.SENTRY_DSN,
+  tracesSampleRate: 1.0,
+})
+
+export { createServiceRouter }
+
+/**
+ * @param {import('@ucanto/interface').Signer} servicePrincipal
+ * @param {import('./types').UcantoServerContext} context
+ */
+export const createUcantoServer = (servicePrincipal, context) =>
+  createServer({
+    ...context,
+    id: servicePrincipal,
+    errorReporter: {
+      catch: (/** @type {string | Error} */ err) => {
+        // eslint-disable-next-line no-console
+        console.warn(err)
+        Sentry.AWSLambda.captureException(err)
+      },
+    },
+  })

packages/upload-api/src/lib.js

@@ -0,0 +1,54 @@
+import * as Server from '@ucanto/server'
+import * as Client from '@ucanto/client'
+import * as Types from './types.js'
+import * as CAR from '@ucanto/transport/car'
+import * as CBOR from '@ucanto/transport/cbor'
+import { createService as createStoreService } from './store.js'
+import { createService as createUploadService } from './upload.js'
+
+/**
+ * @param {Types.UcantoServerContext} options
+ */
+export const createServer = ({
+  id,
+  decoder = CAR,
+  encoder = CBOR,
+  ...context
+}) =>
+  Server.create({
+    id,
+    encoder,
+    decoder,
+    service: createService(context),
+    catch: (error) => context.errorReporter.catch(error),
+  })
+
+/**
+ * @param {Types.ServiceContext} context
+ * @returns {Types.Service}
+ */
+export const createService = (context) => ({
+  store: createStoreService(context),
+  upload: createUploadService(context),
+})
+
+/**
+ * @param {object} options
+ * @param {Types.Principal} options.id
+ * @param {Types.Transport.Channel<Types.Service>} options.channel
+ * @param {Types.Transport.RequestEncoder} [options.encoder]
+ * @param {Types.Transport.ResponseDecoder} [options.decoder]
+ */
+export const connect = ({ id, channel, encoder = CAR, decoder = CBOR }) =>
+  Client.connect({
+    id,
+    channel,
+    encoder,
+    decoder,
+  })
+
+export {
+  createService as createUploadService,
+  createServer as createUploadServer,
+  connect as createUploadClient,
+}

packages/upload-api/src/store.js

@@ -0,0 +1,15 @@
+import { storeAddProvider } from './store/add.js'
+import { storeListProvider } from './store/list.js'
+import { storeRemoveProvider } from './store/remove.js'
+import * as API from './types.js'
+
+/**
+ * @param {API.StoreServiceContext} context
+ */
+export function createService(context) {
+  return {
+    add: storeAddProvider(context),
+    list: storeListProvider(context),
+    remove: storeRemoveProvider(context),
+  }
+}

packages/upload-api/src/store/add.js

@@ -0,0 +1,66 @@
+import * as Server from '@ucanto/server'
+import * as Store from '@web3-storage/capabilities/store'
+import * as API from '../types.js'
+
+/**
+ * @param {API.StoreServiceContext} context
+ * @returns {API.ServiceMethod<API.StoreAdd, API.StoreAddOk, API.Failure>}
+ */
+export function storeAddProvider({
+  access,
+  storeTable,
+  carStoreBucket,
+  maxUploadSize,
+}) {
+  return Server.provide(Store.add, async ({ capability, invocation }) => {
+    const { link, origin, size } = capability.nb
+    const space = Server.DID.parse(capability.with).did()
+    const issuer = invocation.issuer.did()
+    const [allocated, carIsLinkedToAccount, carExists] = await Promise.all([
+      access.allocateSpace(invocation),
+      storeTable.exists(space, link),
+      carStoreBucket.has(link),
+    ])
+
+    // If failed to allocate space, fail with allocation error
+    if (allocated.error) {
+      return allocated
+    }
+
+    if (!carIsLinkedToAccount) {
+      await storeTable.insert({
+        space,
+        link,
+        size,
+        origin,
+        issuer,
+        invocation: invocation.cid,
+      })
+    }
+
+    if (carExists) {
+      return {
+        status: 'done',
+        with: space,
+        link,
+      }
+    }
+
+    if (size > maxUploadSize) {
+      // checking this last, as larger CAR may already exist in bucket from pinning service fetch.
+      // we only want to prevent this here so we don't give the user a PUT url they can't use.
+      return new Server.Failure(
+        `Size must not exceed ${maxUploadSize}. Split CAR into smaller shards`
+      )
+    }
+
+    const { url, headers } = await carStoreBucket.createUploadUrl(link, size)
+    return {
+      status: 'upload',
+      with: space,
+      link,
+      url,
+      headers,
+    }
+  })
+}

packages/upload-api/src/store/list.js

@@ -0,0 +1,20 @@
+import * as Server from '@ucanto/server'
+import * as Store from '@web3-storage/capabilities/store'
+import * as API from '../types.js'
+
+/**
+ * @param {API.StoreServiceContext} context
+ * @returns {API.ServiceMethod<API.StoreList, API.StoreListOk, API.Failure>}
+ */
+export function storeListProvider(context) {
+  return Server.provide(Store.list, async ({ capability }) => {
+    const { cursor, size, pre } = capability.nb
+    const space = Server.DID.parse(capability.with).did()
+
+    return await context.storeTable.list(space, {
+      size,
+      cursor,
+      pre,
+    })
+  })
+}