diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 1a0819c23..8ce045a6f 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -3,10 +3,6 @@ ARG VARIANT=bullseye FROM --platform=linux/amd64 mcr.microsoft.com/devcontainers/base:0-${VARIANT} ARG DENO_VERSION=1.31.1 -ARG POLKADOT_VERSION=v0.9.37 -ARG POLKADOT_PARACHAIN_VERSION=v0.9.370 -ARG ZOMBIENET_VERSION=v1.3.37 -ARG SUBSTRATE_CONTRACTS_NODE_VERSION=v0.24.0 ENV DENO_INSTALL=/deno ENV DENO_INSTALL_ROOT=/usr/local @@ -21,14 +17,6 @@ ENV PATH=${DENO_INSTALL}/bin:${PATH} \ RUN export DEBIAN_FRONTEND=noninteractive \ && apt-get update \ && apt-get install -y unzip curl git procps \ - && curl -fsSL -o /usr/local/bin/polkadot https://github.com/paritytech/polkadot/releases/download/${POLKADOT_VERSION}/polkadot \ - && chmod +x /usr/local/bin/polkadot \ - && curl -fsSL -o /usr/local/bin/polkadot-parachain https://github.com/paritytech/cumulus/releases/download/${POLKADOT_PARACHAIN_VERSION}/polkadot-parachain \ - && chmod +x /usr/local/bin/polkadot-parachain \ - && curl -fsSL -o /usr/local/bin/zombienet-linux-x64 https://github.com/paritytech/zombienet/releases/download/${ZOMBIENET_VERSION}/zombienet-linux-x64 \ - && chmod +x /usr/local/bin/zombienet-linux-x64 \ - && curl -fsSL https://github.com/paritytech/substrate-contracts-node/releases/download/${SUBSTRATE_CONTRACTS_NODE_VERSION}/substrate-contracts-node-linux.tar.gz | tar -zx \ - && mv ./artifacts/substrate-contracts-node-linux/substrate-contracts-node /usr/local/bin/ \ && curl -fsSL https://dprint.dev/install.sh | DPRINT_INSTALL=/usr/local sh \ && curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \ && apt-get install -y nodejs \ diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index efeeaa928..c1a7996b3 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -5,6 +5,10 @@ on: push: branches: - main + +env: + CAPI_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + jobs: fmt: name: Formatting @@ -33,29 +37,54 @@ jobs: - uses: actions/checkout@v3 - uses: streetsidesoftware/cspell-action@6eb87310ae9cb344fb342ea01a2f74979fb31b86 # v2.7.0 + cache: + name: Cache + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v3 + - uses: denoland/setup-deno@9db7f66e8e16b5699a514448ce994936c63f0d54 # v1.1.0 + with: + deno-version: v1.x + - uses: actions/cache@v3 + with: + path: ~/.cache/deno + key: cache-${{ env.CAPI_SHA }} + - run: deno run -A _tasks/await_deployment.ts + timeout-minutes: 1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - run: deno run -A _tasks/use_remote.ts + - run: deno task run _tasks/star.ts + # Attempt caching thrice; #856 + - run: deno cache target/star.ts || deno cache target/star.ts || deno cache target/star.ts + + sync: + name: Sync + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v3 + - uses: denoland/setup-deno@9db7f66e8e16b5699a514448ce994936c63f0d54 # v1.1.0 + with: + deno-version: v1.x + - run: deno task sync --check + star: name: Star runs-on: ubuntu-latest timeout-minutes: 10 + needs: cache steps: - uses: actions/checkout@v3 - uses: denoland/setup-deno@9db7f66e8e16b5699a514448ce994936c63f0d54 # v1.1.0 with: deno-version: v1.x - - name: Cache Deno dependencies - uses: actions/cache@v3 - with: - path: | - ~/.deno - ~/.cache/deno - key: ${{ runner.os }}-deno-${{ hashFiles('deps/**/*.ts') }} - - name: Cache metadata - uses: actions/cache@v3 + - uses: actions/cache@v3 with: - path: | - target/capi/**/_metadata - target/capi/**/_chainName - key: ${{ runner.os }}-capi-metadata-${{ hashFiles('import_map.json') }} + path: ~/.cache/deno + key: cache-${{ env.CAPI_SHA }} + - run: deno run -A _tasks/use_remote.ts - run: deno task run _tasks/star.ts - run: deno task cache target/star.ts - run: deno task check target/star.ts @@ -64,47 +93,32 @@ jobs: name: Tests runs-on: ubuntu-latest timeout-minutes: 10 + needs: cache steps: - uses: actions/checkout@v3 - uses: denoland/setup-deno@9db7f66e8e16b5699a514448ce994936c63f0d54 # v1.1.0 with: deno-version: v1.x - - name: Cache Deno dependencies - uses: actions/cache@v3 + - uses: actions/cache@v3 with: - path: | - ~/.deno - ~/.cache/deno - key: ${{ runner.os }}-deno-${{ hashFiles('deps/**/*.ts') }} - - name: Cache metadata - uses: actions/cache@v3 - with: - path: | - target/capi/**/_metadata - target/capi/**/_chainName - key: ${{ runner.os }}-capi-metadata-${{ hashFiles('import_map.json') }} + path: ~/.cache/deno + key: cache-${{ env.CAPI_SHA }} + - run: deno run -A _tasks/use_remote.ts - run: deno task test + examples-deno: name: Examples (Deno) runs-on: ubuntu-latest timeout-minutes: 15 + needs: cache steps: - uses: actions/checkout@v3 - uses: denoland/setup-deno@9db7f66e8e16b5699a514448ce994936c63f0d54 # v1.1.0 with: deno-version: v1.x - - name: Cache Deno dependencies - uses: actions/cache@v3 - with: - path: | - ~/.deno - ~/.cache/deno - key: ${{ runner.os }}-deno-${{ hashFiles('deps/**/*.ts') }} - - name: Cache metadata - uses: actions/cache@v3 + - uses: actions/cache@v3 with: - path: | - target/capi/**/_metadata - target/capi/**/_chainName - key: ${{ runner.os }}-capi-metadata-${{ hashFiles('import_map.json') }} + path: ~/.cache/deno + key: cache-${{ env.CAPI_SHA }} + - run: deno run -A _tasks/use_remote.ts - run: deno task test:examples:deno diff --git a/.vscode/settings.json b/.vscode/settings.json index 737a584f1..cdd74e735 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,8 +24,7 @@ "editor.tabSize": 2, "files.watcherExclude": { "**/.git/objects/**": true, - "**/.git/subtree-cache/**": true, - "target/**": true + "**/.git/subtree-cache/**": true }, "prettier.printWidth": 100, "ltex.disabledRules": { diff --git a/_tasks/await_deployment.ts b/_tasks/await_deployment.ts new file mode 100644 index 000000000..a9409f25d --- /dev/null +++ b/_tasks/await_deployment.ts @@ -0,0 +1,10 @@ +import { delay } from "../deps/std/async.ts" +import { _getDeploymentUrl } from "../server/capi.dev/delegator.ts" + +const sha = Deno.env.get("CAPI_SHA")! + +const deploymentPollInterval = 5_000 + +while (!(await _getDeploymentUrl(sha))) { + await delay(deploymentPollInterval) +} diff --git a/_tasks/generate_artifacts.ts b/_tasks/generate_artifacts.ts index 6d92e855b..8de9c9c19 100755 --- a/_tasks/generate_artifacts.ts +++ b/_tasks/generate_artifacts.ts @@ -1,8 +1,8 @@ import { hex } from "../crypto/mod.ts" -import { testUser } from "../crypto/mod.ts" import * as $ from "../deps/scale.ts" import { emptyDir } from "../deps/std/fs.ts" import * as path from "../deps/std/path.ts" +import { testUser } from "../devnets/mod.ts" import dprintConfig from "../dprint.json" assert { type: "json" } import { compress } from "../util/compression.ts" diff --git a/_tasks/use_remote.ts b/_tasks/use_remote.ts new file mode 100644 index 000000000..7307017ba --- /dev/null +++ b/_tasks/use_remote.ts @@ -0,0 +1,37 @@ +// This task is used in CI to edit the import map to use capi.dev for codegen +// instead of the local server. + +import { config } from "../capi.config.ts" +import { checkCodegenUploaded, syncConfig } from "../mod.ts" + +const shaAbbrevLength = 8 +const sha = Deno.env.get("CAPI_SHA")!.slice(0, shaAbbrevLength) + +const oldServer = config.server +const capiServer = `https://capi.dev/@${sha}/` +config.server = capiServer + +const importMap = JSON.parse(await Deno.readTextFile("import_map.json")) +const oldCodegenUrl = importMap.imports["@capi/"] + +if (!oldCodegenUrl.startsWith(oldServer)) { + throw new Error("_tasks/use_remote.ts run twice") +} + +const codegenUrl = capiServer + oldCodegenUrl.slice(oldServer.length) + +console.log(codegenUrl) + +importMap.imports["@capi/"] = codegenUrl +importMap.imports[`${capiServer}capi/`] = "./" + +await Deno.writeTextFile("import_map.json", JSON.stringify(importMap, null, 2)) + +const codegenHash = oldCodegenUrl.slice(oldServer.length).split("/")[0] + +if (await checkCodegenUploaded(capiServer, codegenHash)) { + console.log("codegen already uploaded") +} else { + console.log("uploading codegen") + await syncConfig("target/capi", config) +} diff --git a/capi.config.ts b/capi.config.ts new file mode 100644 index 000000000..56a3678f5 --- /dev/null +++ b/capi.config.ts @@ -0,0 +1,51 @@ +import { binary, CapiConfig } from "./mod.ts" + +const polkadot = binary("polkadot", "v0.9.38") +const polkadotParachain = binary("polkadot-parachain", "v0.9.380") +const substrateContractsNode = binary("substrate-contracts-node", "v0.24.0") + +export const config: CapiConfig = { + server: "http://localhost:4646/", + chains: { + polkadot: { + url: "wss://rpc.polkadot.io/", + version: "v0.9.40", + }, + westend: { + url: "wss://westend-rpc.polkadot.io/", + version: "latest", + }, + statemint: { + url: "wss://statemint-rpc.polkadot.io/", + version: "latest", + }, + polkadotDev: { + binary: polkadot, + chain: "polkadot-dev", + }, + westendDev: { + binary: polkadot, + chain: "westend-dev", + }, + contractsDev: { + binary: substrateContractsNode, + chain: "dev", + }, + rococoDev: { + binary: polkadot, + chain: "rococo-local", + parachains: { + statemine: { + id: 1000, + binary: polkadotParachain, + chain: "statemine-local", + }, + contracts: { + id: 2000, + binary: polkadotParachain, + chain: "contracts-rococo-local", + }, + }, + }, + }, +} diff --git a/cli/bin.ts b/cli/bin.ts index 5dae91c5a..6e9e9db23 100644 --- a/cli/bin.ts +++ b/cli/bin.ts @@ -3,22 +3,12 @@ import { download } from "../deps/capi_binary_builds.ts" export default async function( binary: string, version: string, - dashDash?: string, ...args: string[] ) { if (!binary || !version) throw new Error("Must specify binary and version") const binaryPath = await download(binary, version) - if (!dashDash) { - console.log(binaryPath) - Deno.exit(0) - } - - if (dashDash !== "--") { - throw new Error("Arguments to bin must begin with --") - } - const child = new Deno.Command(binaryPath, { args, stdin: "inherit", diff --git a/cli/resolveConfig.ts b/cli/resolveConfig.ts new file mode 100644 index 000000000..43d668880 --- /dev/null +++ b/cli/resolveConfig.ts @@ -0,0 +1,19 @@ +import * as flags from "../deps/std/flags.ts" +import * as path from "../deps/std/path.ts" +import { CapiConfig } from "../devnets/mod.ts" + +export async function resolveConfig(...args: string[]) { + const { config: rawConfigPath } = flags.parse(args, { + string: ["config"], + default: { + config: "./capi.config.ts", + }, + }) + const configPath = path.resolve(rawConfigPath) + await Deno.stat(configPath) + const configModule = await import(path.toFileUrl(configPath).toString()) + const config = configModule.config + if (typeof config !== "object") throw new Error("config file must have a config export") + + return config as CapiConfig +} diff --git a/cli/serve.ts b/cli/serve.ts index 3558996f7..14f2fd588 100644 --- a/cli/serve.ts +++ b/cli/serve.ts @@ -1,18 +1,16 @@ import * as flags from "../deps/std/flags.ts" import { serve } from "../deps/std/http.ts" -import { - ContractsDevProvider, - PolkadotDevProvider, - ProjectProvider, - WssProvider, - ZombienetProvider, -} from "../providers/frame/mod.ts" -import { handler } from "../server/handler.ts" -import { Env } from "../server/mod.ts" +import { createTempDir } from "../devnets/createTempDir.ts" +import { createDevnetsHandler } from "../devnets/mod.ts" +import { createCorsHandler } from "../server/corsHandler.ts" +import { createErrorHandler } from "../server/errorHandler.ts" +import { createCodegenHandler } from "../server/mod.ts" +import { InMemoryCache } from "../util/cache/memory.ts" import { FsCache } from "../util/cache/mod.ts" +import { resolveConfig } from "./resolveConfig.ts" export default async function(...args: string[]) { - const { port: portStr, "--": cmd, out } = flags.parse(args, { + const { port, "--": cmd, out } = flags.parse(args, { string: ["port", "out"], default: { port: "4646", @@ -21,24 +19,17 @@ export default async function(...args: string[]) { "--": true, }) - const href = `http://localhost:${portStr}/` + const config = await resolveConfig(...args) + + const tempDir = await createTempDir() + + const href = `http://localhost:${port}/` const controller = new AbortController() const { signal } = controller - const cache = new FsCache(out, signal) - const modSpecifier = JSON.stringify(import.meta.resolve("./mod.ts")) - cache.getString("mod.ts", 0, async () => `export * from ${modSpecifier}`) - - const env = new Env(href, cache, signal, (env) => ({ - frame: { - wss: new WssProvider(env), - dev: new PolkadotDevProvider(env), - zombienet: new ZombienetProvider(env), - project: new ProjectProvider(env), - contracts_dev: new ContractsDevProvider(env), - }, - })) + const dataCache = new FsCache(out, signal) + const tempCache = new InMemoryCache(signal) const running = await fetch(`${href}capi_cwd`) .then((r) => r.text()) @@ -46,9 +37,21 @@ export default async function(...args: string[]) { .catch(() => false) if (!running) { - await serve(handler(env), { + const devnetsHandler = createDevnetsHandler(tempDir, config, signal) + const codegenHandler = createCodegenHandler(dataCache, tempCache) + const handler = createCorsHandler(createErrorHandler(async (request) => { + const { pathname } = new URL(request.url) + if (pathname === "/capi_cwd") { + return new Response(Deno.cwd()) + } + if (pathname.startsWith("/devnets/")) { + return await devnetsHandler(request) + } + return await codegenHandler(request) + })) + await serve(handler, { hostname: "[::]", - port: +portStr, + port: +port, signal, onError(error) { throw error @@ -66,16 +69,16 @@ export default async function(...args: string[]) { async function onReady() { const [bin, ...args] = cmd if (bin) { - const command = new Deno.Command(bin, { args, signal }) + const command = new Deno.Command(bin, { + args, + signal, + env: { + DEVNETS_SERVER: `http://localhost:${port}/devnets/`, + }, + }) const status = await command.spawn().status - console.log("inner command exited with status code", status.code) self.addEventListener("unload", () => Deno.exit(status.code)) controller.abort() - Deno.unrefTimer(setTimeout(() => { - // todo: fix - console.log("failed to exit gracefully") - Deno.exit(status.code) - }, 30_000)) } } } diff --git a/cli/sync.ts b/cli/sync.ts new file mode 100644 index 000000000..b49c53e6a --- /dev/null +++ b/cli/sync.ts @@ -0,0 +1,37 @@ +import * as flags from "../deps/std/flags.ts" +import { assertEquals } from "../deps/std/testing/asserts.ts" +import { createTempDir } from "../devnets/createTempDir.ts" +import { syncConfig } from "../devnets/mod.ts" +import { resolveConfig } from "./resolveConfig.ts" + +export default async function(...args: string[]) { + const { + "import-map": importMapFile, + "package-json": packageJsonFile, + check, + } = flags.parse(args, { + string: ["config", "import-map", "package-json"], + boolean: ["check"], + }) + + const config = await resolveConfig(...args) + + const tempDir = await createTempDir() + + const baseUrl = await syncConfig(tempDir, config) + console.log(baseUrl) + + if (importMapFile) { + const importMap = JSON.parse(await Deno.readTextFile(importMapFile)) + if (check) { + assertEquals(importMap.imports["@capi/"], baseUrl) + } else { + importMap.imports["@capi/"] = baseUrl + await Deno.writeTextFile(importMapFile, JSON.stringify(importMap, null, 2) + "\n") + } + } + + if (packageJsonFile) { + throw new Error("not yet supported") + } +} diff --git a/codegen/FrameCodegen.ts b/codegen/FrameCodegen.ts index 999dc390f..58781950e 100644 --- a/codegen/FrameCodegen.ts +++ b/codegen/FrameCodegen.ts @@ -24,13 +24,13 @@ export class FrameCodegen { "chain.js", ` import * as _codecs from "./codecs.js" -import { connectionCtor, discoveryValue } from "./connection.js" +import { connect } from "./connection.js" import * as C from "./capi.js" import * as t from "./types/mod.js" export const metadata = ${this.codecCodegen.print(this.metadata)} -export const chain = C.ChainRune.from(connectionCtor, discoveryValue, metadata) +export const chain = C.ChainRune.from(connect, metadata) `, ) files.set( diff --git a/crypto/hashers.ts b/crypto/hashers.ts index 57d678484..3dbda4bd6 100644 --- a/crypto/hashers.ts +++ b/crypto/hashers.ts @@ -56,7 +56,7 @@ export function $hash(hasher: Hasher, $inner: $.Codec): $.Codec { export class Blake2Hasher extends Hasher { digestLength - constructor(size: 128 | 256 | 512, public concat: boolean) { + constructor(size: 64 | 128 | 256 | 512, public concat: boolean) { super() this.digestLength = size / 8 } @@ -106,6 +106,7 @@ export interface Hashing { dispose?(): void } +export const blake2_64 = new Blake2Hasher(64, false) export const blake2_128 = new Blake2Hasher(128, false) export const blake2_128Concat = new Blake2Hasher(128, true) export const blake2_256 = new Blake2Hasher(256, false) diff --git a/crypto/test_pairs.ts b/crypto/test_pairs.ts index bfc0785f5..02e0a0437 100644 --- a/crypto/test_pairs.ts +++ b/crypto/test_pairs.ts @@ -1,5 +1,3 @@ -import { ArrayOfLength } from "../util/mod.ts" -import { blake2_256 } from "./hashers.ts" import { decode } from "./hex.ts" import { Sr25519 } from "./Sr25519.ts" @@ -32,27 +30,3 @@ export const bobStash = pair( function pair(secret: string) { return Sr25519.fromSecret(decode(secret)) } - -export function testUser(userId: number) { - return Sr25519.fromSeed(blake2_256.hash(new TextEncoder().encode(`capi-test-user-${userId}`))) -} -export function testUserFactory(url: string) { - return createUsers - function createUsers(): Promise> - function createUsers(count: N): Promise> - async function createUsers(count?: number): Promise | Sr25519[]> { - const response = await fetch(url, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ count: count ?? NAMES.length }), - }) - if (!response.ok) throw new Error(await response.text()) - const { index }: { index: number } = await response.json() - return typeof count === "number" - ? Array.from({ length: count }, (_, i) => testUser(index + i)) - : Object.fromEntries(NAMES.map((name, i) => [name, testUser(index + i)])) - } -} - -// dprint-ignore-next-line -const NAMES = ["alexa", "billy", "carol", "david", "ellie", "felix", "grace", "harry", "india", "jason", "kiera", "laura", "matty", "nadia", "oscar", "piper", "quinn", "ryann", "steff", "tisix", "usher", "vicky", "wendy", "xenia", "yetis", "zelda"] as const diff --git a/cspell.json b/cspell.json index 927bbd2ed..502f30ac5 100644 --- a/cspell.json +++ b/cspell.json @@ -8,15 +8,17 @@ "addWords": true } ], - "dictionaries": [ - "project-words" - ], + "enableGlobDot": true, + "dictionaries": ["project-words"], "ignorePaths": [ + ".git", "**/*.wasm", "target", "**/__snapshots__/*.snap", "**/*.contract", "examples/ink/erc20.json", - "**/*.scale" + "**/*.scale", + "**/*.svg", + "util/_artifacts" ] } diff --git a/deno.jsonc b/deno.jsonc index e87ed75e4..5bbc915b1 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -32,14 +32,15 @@ "debug": "deno run -A --inspect-brk", "udd": "deno run -A _tasks/udd.ts", "dnt": "deno task run _tasks/dnt.ts", - "test": "deno task capi -- deno test -A -L=info --ignore=target --parallel -r=http://localhost:4646/", - "test:examples:deno": "deno task capi -- deno run -A -r=http://localhost:4646/ https://deno.land/x/trun@v0.1.0-beta.2/main.ts --include \"**/*.eg.ts\" --import-map import_map.json --concurrency 4 --reload http://localhost:4646/", + "test": "deno task capi serve -- deno test -A -L=info --ignore=target --parallel -r=http://localhost:4646/", + "test:examples:deno": "deno task capi serve -- deno run -A -r=http://localhost:4646/ https://deno.land/x/trun@v0.1.0-beta.2/main.ts --include \"**/*.eg.ts\" --import-map import_map.json --concurrency 4 --reload http://localhost:4646/", "test:update": "deno task test -- --update", "moderate": "deno run -A https://deno.land/x/moderate@0.0.5/mod.ts --exclude '*.test.ts' && dprint fmt", "capi": "deno run -A main.ts", - "cache": "deno task capi -- deno cache -r=http://localhost:4646/", - "check": "deno task capi -- deno cache --check", + "cache": "deno task capi serve -- deno cache -r=http://localhost:4646/", + "check": "deno task capi serve -- deno cache --check", "star": "deno task run _tasks/star.ts && deno task check target/star.ts", - "run": "deno task capi -- deno run -A -r=http://localhost:4646/" + "sync": "mkdir -p target && deno task capi serve -- deno task capi sync --import-map import_map.json", + "run": "deno task capi serve -- deno run -A -r=http://localhost:4646/" } } diff --git a/deps/zombienet.ts b/deps/zombienet.ts deleted file mode 100644 index 890df083e..000000000 --- a/deps/zombienet.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "npm:@zombienet/orchestrator@0.0.34" -export * from "npm:@zombienet/utils@0.0.18" diff --git a/devnets/CapiConfig.ts b/devnets/CapiConfig.ts new file mode 100644 index 000000000..ddee8c353 --- /dev/null +++ b/devnets/CapiConfig.ts @@ -0,0 +1,25 @@ +import { Binary } from "./binary.ts" + +export interface WsChain { + url: string + binary?: never + version: string +} + +export interface NetworkConfig { + url?: never + binary: Binary + chain: string + nodes?: number + parachains?: Record +} + +export interface CapiConfig { + server: string + chains?: Record +} diff --git a/devnets/DevnetConnection.ts b/devnets/DevnetConnection.ts new file mode 100644 index 000000000..d3000420a --- /dev/null +++ b/devnets/DevnetConnection.ts @@ -0,0 +1,11 @@ +import { WsConnection } from "../rpc/mod.ts" + +export class DevnetConnection extends WsConnection { + constructor(path: string) { + const server = Deno.env.get("DEVNETS_SERVER") + if (!server) throw new Error("Must be run with a devnets server") + const url = new URL(path, server) + url.protocol = "ws" + super(url.toString()) + } +} diff --git a/devnets/binary.ts b/devnets/binary.ts new file mode 100644 index 000000000..5a201f65b --- /dev/null +++ b/devnets/binary.ts @@ -0,0 +1,15 @@ +import { download } from "../deps/capi_binary_builds.ts" + +export type Binary = string | { binary: string; version: string; resolved?: Promise } + +export function binary(binary: string, version: string): Binary { + return { binary, version } +} + +export async function resolveBinary(binary: Binary, _signal: AbortSignal) { + if (typeof binary === "string") { + return binary + } else { + return await (binary.resolved ??= download(binary.binary, binary.version)) + } +} diff --git a/devnets/capnHandler.ts b/devnets/capnHandler.ts new file mode 100644 index 000000000..8d3643c84 --- /dev/null +++ b/devnets/capnHandler.ts @@ -0,0 +1,72 @@ +import { deferred } from "../deps/std/async.ts" +import * as path from "../deps/std/path.ts" +import { $ } from "../mod.ts" +import * as f from "../server/factories.ts" +import { PermanentMemo } from "../util/memo.ts" +import { CapiConfig } from "./CapiConfig.ts" +import { Network, startNetwork } from "./startNetwork.ts" +import { testUserPublicKeys } from "./testUsers.ts" + +const rDevnetsApi = /^\/devnets\/([\w-]+)(?:\/([\w-]+))?$/ + +export function createDevnetsHandler(tempDir: string, config: CapiConfig, signal: AbortSignal) { + const networkMemo = new PermanentMemo() + return async (request: Request) => { + const { pathname, searchParams } = new URL(request.url) + const match = rDevnetsApi.exec(pathname) + if (!match) return f.notFound() + const name = match[1]! + const paraName = match[2] + const networkConfig = config.chains?.[name!] + if (networkConfig?.binary == null) return f.notFound() + const network = await networkMemo.run(name!, async () => { + return startNetwork(path.join(tempDir, name!), networkConfig, signal) + }) + const chain = paraName ? network.paras[paraName] : network.relay + if (!chain) return f.notFound() + if (request.headers.get("Upgrade") === "websocket") { + const port = chain.ports.shift()! + chain.ports.push(port) + return proxyWebSocket(request, `ws://localhost:${port}`) + } + if (request.method === "POST" && searchParams.has("users")) { + const count = +searchParams.get("users")! + if (!$.is($.u32, count)) return f.badRequest() + const index = chain.testUserIndex + const newCount = index + count + if (newCount < testUserPublicKeys.length) chain.testUserIndex = newCount + else throw new Error("Maximum test user count reached") + return new Response(`${index}`, { status: 200 }) + } + return new Response("Network launched") + } +} + +function proxyWebSocket(request: Request, url: string) { + const server = new WebSocket(url) + const { socket: client, response } = Deno.upgradeWebSocket(request) + setup(client, server) + setup(server, client) + return response + + function setup(a: WebSocket, b: WebSocket) { + const ready = deferred() + b.addEventListener("open", () => { + ready.resolve() + }) + a.addEventListener("close", async () => { + try { + b.close() + } catch {} + }) + a.addEventListener("message", async (event) => { + try { + await ready + b.send(event.data) + } catch { + a.close() + b.close() + } + }) + } +} diff --git a/devnets/chainSpec.ts b/devnets/chainSpec.ts new file mode 100644 index 000000000..8a922b2df --- /dev/null +++ b/devnets/chainSpec.ts @@ -0,0 +1,69 @@ +import { ensureDir } from "../deps/std/fs.ts" +import * as path from "../deps/std/path.ts" + +export async function createCustomChainSpec( + tempDir: string, + binary: string, + chain: string, + customize: (chainSpec: ChainSpec) => void, +) { + await ensureDir(tempDir) + + const specResult = await new Deno.Command(binary, { + args: ["build-spec", "--disable-default-bootnode", "--chain", chain], + }).output() + if (!specResult.success) { + // TODO: improve error message + throw new Error("build-spec failed") + } + const spec = JSON.parse(new TextDecoder().decode(specResult.stdout)) + customize(spec) + + const specPath = path.join(tempDir, `chainspec.json`) + await Deno.writeTextFile(specPath, JSON.stringify(spec, undefined, 2)) + + const rawResult = await new Deno.Command(binary, { + args: ["build-spec", "--disable-default-bootnode", "--chain", specPath, "--raw"], + }).output() + if (!rawResult.success) { + // TODO: improve error message + throw new Error("build-spec --raw failed") + } + + const rawPath = path.join(tempDir, `chainspec-raw.json`) + await Deno.writeFile(rawPath, rawResult.stdout) + + return rawPath +} + +export function getGenesisConfig(chainSpec: ChainSpec) { + return chainSpec.genesis.runtime.runtime_genesis_config ?? chainSpec.genesis.runtime +} + +export interface ChainSpec { + bootNodes: string[] + para_id?: number + genesis: { + runtime: + | { runtime_genesis_config: GenesisConfig } + | GenesisConfig + } +} + +interface GenesisConfig { + runtime_genesis_config?: never + paras: { + paras: [ + [ + parachainId: number, + genesis: [state: string, wasm: string, kind: boolean], + ], + ] + } + parachainInfo: { + parachainId: number + } + balances: { + balances: [account: string, initialBalance: number][] + } +} diff --git a/devnets/createTempDir.ts b/devnets/createTempDir.ts new file mode 100644 index 000000000..065515924 --- /dev/null +++ b/devnets/createTempDir.ts @@ -0,0 +1,10 @@ +import * as path from "../deps/std/path.ts" + +export async function createTempDir() { + const dir = path.resolve("target/devnets") + await Deno.mkdir(dir, { recursive: true }) + return await Deno.makeTempDir({ + dir, + prefix: `devnets-${new Date().toISOString()}-`, + }) +} diff --git a/devnets/mod.ts b/devnets/mod.ts new file mode 100644 index 000000000..2ee418534 --- /dev/null +++ b/devnets/mod.ts @@ -0,0 +1,11 @@ +// moderate + +export * from "./binary.ts" +export * from "./CapiConfig.ts" +export * from "./capnHandler.ts" +export * from "./chainSpec.ts" +export * from "./createTempDir.ts" +export * from "./DevnetConnection.ts" +export * from "./startNetwork.ts" +export * from "./syncConfig.ts" +export * from "./testUsers.ts" diff --git a/devnets/startNetwork.ts b/devnets/startNetwork.ts new file mode 100644 index 000000000..f4a8f42e5 --- /dev/null +++ b/devnets/startNetwork.ts @@ -0,0 +1,218 @@ +import { Narrow } from "../deps/scale.ts" +import * as path from "../deps/std/path.ts" +import { writableStreamFromWriter } from "../deps/std/streams.ts" +import { getFreePort, portReady } from "../util/port.ts" +import { resolveBinary } from "./binary.ts" +import { NetworkConfig } from "./CapiConfig.ts" +import { createCustomChainSpec, getGenesisConfig } from "./chainSpec.ts" +import { addTestUsers } from "./testUsers.ts" + +export interface Network { + relay: NetworkChain + paras: Record +} + +export interface NetworkChain { + testUserIndex: number + bootnodes: string + ports: number[] +} + +export async function startNetwork( + tempDir: string, + config: NetworkConfig, + signal: AbortSignal, +): Promise { + const paras = await Promise.all( + Object.entries(config.parachains ?? {}).map(async ([name, chain]) => { + const binary = await resolveBinary(chain.binary, signal) + + const spec = await createCustomChainSpec( + path.join(tempDir, name), + binary, + chain.chain, + (chainSpec) => { + chainSpec.para_id = chain.id + const genesisConfig = getGenesisConfig(chainSpec) + genesisConfig.parachainInfo.parachainId = chain.id + addTestUsers(genesisConfig.balances.balances) + }, + ) + + const genesis = await exportParachainGenesis(binary, spec, signal) + + return { + id: chain.id, + name, + binary, + spec, + genesis, + count: chain.nodes ?? 2, + } + }), + ) + + const relayBinary = await resolveBinary(config.binary, signal) + + const relaySpec = await createCustomChainSpec( + path.join(tempDir, "relay"), + relayBinary, + config.chain, + (chainSpec) => { + const genesisConfig = getGenesisConfig(chainSpec) + if (paras.length) { + genesisConfig.paras.paras.push( + ...paras.map(({ id, genesis }) => [id, [...genesis, true]] satisfies Narrow), + ) + } + addTestUsers(genesisConfig.balances.balances) + }, + ) + + const relay = await spawnChain( + path.join(tempDir, "relay"), + relayBinary, + relaySpec, + config.nodes ?? 2, + [], + signal, + ) + + return { + relay, + paras: Object.fromEntries( + await Promise.all( + paras.map(async ({ name, binary, spec, count }) => { + const chain = await spawnChain( + path.join(tempDir, name), + binary, + spec, + count, + [ + "--", + "--execution", + "wasm", + "--chain", + relaySpec, + "--bootnodes", + relay.bootnodes, + ], + signal, + ) + return [name, chain] satisfies Narrow + }), + ), + ), + } +} + +async function exportParachainGenesis( + binary: string, + chain: string, + signal: AbortSignal, +) { + return await Promise.all(["state", "wasm"].map(async (type) => { + const { success, stdout } = await new Deno.Command(binary, { + args: [`export-genesis-${type}`, "--chain", chain], + signal, + }).output() + if (!success) { + // TODO: improve error message + throw new Error(`export-genesis-${type} failed`) + } + return new TextDecoder().decode(stdout) + })) satisfies string[] as [state: string, wasm: string] +} + +function generateBootnodeString(port: number, peerId: string) { + return `/ip4/127.0.0.1/tcp/${port}/p2p/${peerId}` +} + +async function generateNodeKey(binary: string, signal?: AbortSignal) { + const { success, stdout, stderr } = await new Deno.Command(binary, { + args: ["key", "generate-node-key"], + signal, + }).output() + if (!success) { + throw new Error() + } + const decoder = new TextDecoder() + const nodeKey = decoder.decode(stdout).trim() + const peerId = decoder.decode(stderr).trim() + return { nodeKey, peerId } +} + +const keystoreAccounts = ["alice", "bob", "charlie", "dave", "eve", "ferdie", "one", "two"] +async function spawnChain( + tempDir: string, + binary: string, + chain: string, + count: number, + extraArgs: string[], + signal: AbortSignal, +): Promise { + let bootnodes: string | undefined + const ports = [] + + for (let i = 0; i < count; i++) { + const keystoreAccount = keystoreAccounts[i] + if (!keystoreAccount) throw new Error("ran out of keystore accounts") + const nodeDir = path.join(tempDir, keystoreAccount) + await Deno.mkdir(nodeDir, { recursive: true }) + const httpPort = getFreePort() + const wsPort = getFreePort() + ports.push(wsPort) + const args = [ + "--validator", + `--${keystoreAccount}`, + "--base-path", + nodeDir, + "--chain", + chain, + "--port", + `${httpPort}`, + "--ws-port", + `${wsPort}`, + ] + if (bootnodes) { + args.push("--bootnodes", bootnodes) + } else { + const { nodeKey, peerId } = await generateNodeKey(binary) + args.push("--node-key", nodeKey) + bootnodes = generateBootnodeString(httpPort, peerId) + } + args.push(...extraArgs) + spawnNode(nodeDir, binary, args, signal) + await portReady(wsPort) + } + + if (!bootnodes) throw new Error("count must be > 1") + return { testUserIndex: 0, bootnodes, ports } +} + +async function spawnNode(tempDir: string, binary: string, args: string[], signal: AbortSignal) { + const child = new Deno.Command(binary, { + args, + signal, + stdout: "piped", + stderr: "piped", + }).spawn() + + child.stdout.pipeTo( + writableStreamFromWriter( + await Deno.open(path.join(tempDir, "stdout"), { write: true, create: true }), + ), + ) + + child.stderr.pipeTo( + writableStreamFromWriter( + await Deno.open(path.join(tempDir, "stderr"), { write: true, create: true }), + ), + ) + + child.status.then((status) => { + if (!signal.aborted) { + throw new Error(`process exited with code ${status.code} (${tempDir})`) + } + }) +} diff --git a/devnets/syncConfig.ts b/devnets/syncConfig.ts new file mode 100644 index 000000000..da3dcbdfa --- /dev/null +++ b/devnets/syncConfig.ts @@ -0,0 +1,94 @@ +export * from "./binary.ts" + +import { blake2_512, blake2_64, Hasher } from "../crypto/hashers.ts" +import { hex } from "../crypto/mod.ts" +import * as path from "../deps/std/path.ts" +import { WsConnection } from "../mod.ts" +import { $codegenSpec, CodegenEntry, CodegenSpec } from "../server/codegenSpec.ts" +import { normalizePackageName, withSignal } from "../util/mod.ts" +import { normalizeTypeName } from "../util/normalize.ts" +import { CapiConfig } from "./CapiConfig.ts" +import { startNetwork } from "./startNetwork.ts" + +export async function syncConfig(tempDir: string, config: CapiConfig) { + return withSignal(async (signal) => { + const { server } = config + const entries = new Map() + await Promise.all( + Object.entries(config.chains ?? {}).map(async ([name, chain]) => { + if (chain.url != null) { + const metadata = await uploadMetadata(server, chain.url) + entries.set(normalizePackageName(name), { + type: "frame", + metadata, + chainName: normalizeTypeName(name), + connection: { type: "WsConnection", discovery: chain.url }, + }) + return + } + const network = await startNetwork(path.join(tempDir, name), chain, signal) + await Promise.all( + [ + [undefined, network.relay] as const, + ...Object.entries(network.paras), + ].map( + async ([paraName, chain]) => { + const metadata = await uploadMetadata( + server, + `ws://localhost:${chain.ports[0]}`, + ) + entries.set( + normalizePackageName(name) + (paraName ? `/${normalizePackageName(paraName)}` : ""), + { + type: "frame", + metadata: metadata, + chainName: normalizeTypeName(name), + connection: { + type: "DevnetConnection", + discovery: name + (paraName ? `/${paraName}` : ""), + }, + }, + ) + }, + ), + ) + }), + ) + const sortedEntries = new Map([...entries].sort((a, b) => a[0] < b[0] ? 1 : -1)) + const codegenHash = await uploadCodegenSpec(server, { + type: "v0", + codegen: sortedEntries, + }) + return new URL(codegenHash + "/", server).toString() + }) +} + +async function _upload(server: string, kind: string, data: Uint8Array, hasher: Hasher) { + const hash = hasher.hash(data) + const url = new URL(`upload/${kind}/${hex.encode(hash)}`, server) + const exists = await fetch(url, { method: "HEAD" }) + if (exists.ok) return hash + const response = await fetch(url, { method: "PUT", body: data }) + if (!response.ok) throw new Error(await response.text()) + return hash +} + +async function uploadMetadata(server: string, url: string) { + const metadata = await withSignal(async (signal) => { + const connection = WsConnection.connect(url, signal) + const response = await connection.call("state_getMetadata", []) + if (response.error) throw new Error("Error getting metadata") + return hex.decode(response.result as string) + }) + return await _upload(server, "metadata", metadata, blake2_512) +} + +async function uploadCodegenSpec(server: string, spec: CodegenSpec) { + return hex.encode(await _upload(server, "codegen", $codegenSpec.encode(spec), blake2_64)) +} + +export async function checkCodegenUploaded(server: string, hash: string) { + const url = new URL(`upload/codegen/${hash}`, server) + const exists = await fetch(url, { method: "HEAD" }) + return exists.ok +} diff --git a/devnets/testUsers.ts b/devnets/testUsers.ts new file mode 100644 index 000000000..53ebbcc8a --- /dev/null +++ b/devnets/testUsers.ts @@ -0,0 +1,68 @@ +import { blake2_256, Sr25519, ss58 } from "../crypto/mod.ts" +import * as $ from "../deps/scale.ts" +import { testUserPublicKeysData } from "../util/_artifacts/testUserPublicKeysData.ts" +import { ArrayOfLength } from "../util/ArrayOfLength.ts" + +const testUserInitialFunds = 1_000_000_000_000_000_000 + +export const testUserPublicKeys = $.array($.sizedUint8Array(32)).decode(testUserPublicKeysData) + +export function addTestUsers(balances: [string, number][]) { + const networkPrefix = ss58.decode(balances[0]![0])[0] + for (const publicKey of testUserPublicKeys) { + balances.push([ss58.encode(networkPrefix, publicKey), testUserInitialFunds]) + } +} + +export function testUser(userId: number) { + return Sr25519.fromSeed(blake2_256.hash(new TextEncoder().encode(`capi-test-user-${userId}`))) +} +export function testUserFactory(endpoint: string) { + return createUsers + function createUsers(): Promise> + function createUsers(count: N): Promise> + async function createUsers(count?: number): Promise | Sr25519[]> { + const server = Deno.env.get("DEVNETS_SERVER") + if (!server) throw new Error("Must be run with a devnets server") + const response = await fetch( + new URL(`${endpoint}?users=${count ?? testUserNames.length}`, server), + { + method: "POST", + }, + ) + if (!response.ok) throw new Error(await response.text()) + const index = +(await response.text()) + return typeof count === "number" + ? Array.from({ length: count }, (_, i) => testUser(index + i)) + : Object.fromEntries(testUserNames.map((name, i) => [name, testUser(index + i)])) + } +} + +export const testUserNames = [ + "alexa", + "billy", + "carol", + "david", + "ellie", + "felix", + "grace", + "harry", + "india", + "jason", + "kiera", + "laura", + "matty", + "nadia", + "oscar", + "piper", + "quinn", + "ryann", + "steff", + "tisix", + "usher", + "vicky", + "wendy", + "xenia", + "yetis", + "zelda", +] as const diff --git a/examples/blocks.eg.ts b/examples/blocks.eg.ts index bfcb5c583..c788d7fdf 100644 --- a/examples/blocks.eg.ts +++ b/examples/blocks.eg.ts @@ -7,9 +7,9 @@ * pieces of data pertaining to that block. */ +import { chain, metadata, types } from "@capi/polkadot/mod.js" import { $, $extrinsic, known, Rune } from "capi" import { babeBlockAuthor } from "capi/patterns/consensus/mod.ts" -import { chain, metadata, types } from "polkadot/mod.js" // Reference the latest block hash. const blockHash = chain.blockHash() diff --git a/examples/dev/metadata.eg.ts b/examples/dev/metadata.eg.ts index 01696808e..3e60764a0 100644 --- a/examples/dev/metadata.eg.ts +++ b/examples/dev/metadata.eg.ts @@ -10,7 +10,7 @@ * building an advanced Capi-based library, chances are that you don't need to work with the metadata directly. */ -import { chain } from "polkadot_dev/mod.js" +import { chain } from "@capi/polkadot-dev/mod.js" // Execute the metadata Rune. const metadata = await chain.metadata.run() diff --git a/examples/dev/storage_sizes.eg.ts b/examples/dev/storage_sizes.eg.ts index faaa6d2e0..c8668cac2 100644 --- a/examples/dev/storage_sizes.eg.ts +++ b/examples/dev/storage_sizes.eg.ts @@ -7,9 +7,9 @@ * That being said, this can be helpful in the context of chain development. */ +import { chain } from "@capi/polkadot-dev/mod.js" import { $ } from "capi" import { storageSizes } from "capi/patterns/storage_sizes.ts" -import { chain } from "polkadot_dev/mod.js" // Use the storageSizes factory to produce a Rune. Then execute it. const sizes = await storageSizes(chain).run() diff --git a/examples/dev/test_users.eg.ts b/examples/dev/test_users.eg.ts index 9c6c0bd9e..a93553f4a 100644 --- a/examples/dev/test_users.eg.ts +++ b/examples/dev/test_users.eg.ts @@ -7,8 +7,8 @@ * funds. This simplifies signing extrinsics for submission to the given test chain. */ +import { createUsers } from "@capi/polkadot-dev/mod.js" import { $, $sr25519 } from "capi" -import { createUsers } from "polkadot_dev/mod.js" // Test users can be initialized with no count. The resulting collection is // a record with 26 `Sr25519`s (one for every letter of the alphabet). diff --git a/examples/dynamic.eg.ts b/examples/dynamic.eg.ts index 22d72dc9f..75954c4df 100644 --- a/examples/dynamic.eg.ts +++ b/examples/dynamic.eg.ts @@ -16,7 +16,7 @@ import { $, ChainRune, WsConnection } from "capi" // We could also initialize a `ChainRune` with `WsConnection` and an RPC node WebSocket URL. -const wsChain = ChainRune.from(WsConnection, "wss://rpc.polkadot.io") +const wsChain = ChainRune.from(WsConnection.bind("wss://rpc.polkadot.io")) // Create a binding to the `System` pallet. const System = wsChain.pallet("System") diff --git a/examples/ink/deploy.eg.ts b/examples/ink/deploy.eg.ts index c2d80983b..bdecf0344 100644 --- a/examples/ink/deploy.eg.ts +++ b/examples/ink/deploy.eg.ts @@ -7,10 +7,10 @@ * @todo utilize `createUsers` instead of `alice` */ +import { chain, System } from "@capi/contracts-dev/mod.js" import { $, alice, ss58 } from "capi" import { InkMetadataRune } from "capi/patterns/ink/mod.ts" import { signature } from "capi/patterns/signature/polkadot.ts" -import { chain, System } from "contracts_dev/mod.js" // Initialize an `InkMetadataRune` with the raw Ink metadata text. const metadata = InkMetadataRune.fromMetadataText( diff --git a/examples/ink/interact.eg.ts b/examples/ink/interact.eg.ts index f426e3d96..2d69f653d 100644 --- a/examples/ink/interact.eg.ts +++ b/examples/ink/interact.eg.ts @@ -7,11 +7,11 @@ * @todo utilize `createUsers` instead of `alice` and `bob`. */ +import { chain } from "@capi/contracts-dev/mod.js" import { assert } from "asserts" import { $, alice, bob } from "capi" import { InkMetadataRune } from "capi/patterns/ink/mod.ts" import { signature } from "capi/patterns/signature/polkadot.ts" -import { chain } from "contracts_dev/mod.js" import { parse } from "../../deps/std/flags.ts" // Attempt to read contract address from command line argument (optional) diff --git a/examples/misc/identity.eg.ts b/examples/misc/identity.eg.ts index 68aab94e4..9400b4cc8 100644 --- a/examples/misc/identity.eg.ts +++ b/examples/misc/identity.eg.ts @@ -7,10 +7,10 @@ * Set a user's identity, potentially with metadata of arbitrary user-defined shape. */ +import { createUsers, Identity } from "@capi/polkadot-dev/mod.js" import { $ } from "capi" import { IdentityInfoTranscoders } from "capi/patterns/identity.ts" import { signature } from "capi/patterns/signature/polkadot.ts" -import { createUsers, Identity } from "polkadot_dev/mod.js" const { alexa } = await createUsers() diff --git a/examples/misc/indices.eg.ts b/examples/misc/indices.eg.ts index 81e1ee0f4..d00fd6154 100644 --- a/examples/misc/indices.eg.ts +++ b/examples/misc/indices.eg.ts @@ -6,9 +6,9 @@ * using the index. */ +import { createUsers, Indices } from "@capi/polkadot-dev/mod.js" import { assertEquals } from "asserts" import { signature } from "capi/patterns/signature/polkadot.ts" -import { createUsers, Indices } from "polkadot_dev/mod.js" const { alexa } = await createUsers() diff --git a/examples/multisig/basic.eg.ts b/examples/multisig/basic.eg.ts index 771a2a5ae..3ddf1595f 100644 --- a/examples/multisig/basic.eg.ts +++ b/examples/multisig/basic.eg.ts @@ -6,11 +6,11 @@ * that multisig. */ +import { Balances, chain, createUsers, System } from "@capi/polkadot-dev/mod.js" import { assert } from "asserts" import { $ } from "capi" import { MultisigRune } from "capi/patterns/multisig/mod.ts" import { signature } from "capi/patterns/signature/polkadot.ts" -import { Balances, chain, createUsers, System } from "polkadot_dev/mod.js" const { alexa, billy, carol, david } = await createUsers() diff --git a/examples/multisig/stash.eg.ts b/examples/multisig/stash.eg.ts index f2e49c231..13e583ae3 100644 --- a/examples/multisig/stash.eg.ts +++ b/examples/multisig/stash.eg.ts @@ -5,12 +5,12 @@ * Administrate a stash account (pure proxy) through a multisig with three signatories. */ +import { Balances, chain, createUsers, Proxy, System } from "@capi/polkadot-dev/mod.js" +import { MultiAddress } from "@capi/polkadot-dev/types/sp_runtime/multiaddress.js" import { assert } from "asserts" import { MultisigRune } from "capi/patterns/multisig/mod.ts" import { filterPureCreatedEvents } from "capi/patterns/proxy/mod.ts" import { signature } from "capi/patterns/signature/polkadot.ts" -import { Balances, chain, createUsers, Proxy, System } from "polkadot_dev/mod.js" -import { MultiAddress } from "polkadot_dev/types/sp_runtime/multiaddress.js" const { alexa, billy, carol } = await createUsers() diff --git a/examples/multisig/virtual.eg.ts b/examples/multisig/virtual.eg.ts index a51933863..d71d23fc9 100644 --- a/examples/multisig/virtual.eg.ts +++ b/examples/multisig/virtual.eg.ts @@ -18,12 +18,12 @@ * members ratify a call to give ownership of the stash account to the new multisig. */ +import { Balances, chain, createUsers, System, Utility } from "@capi/polkadot-dev/mod.js" +import { MultiAddress } from "@capi/polkadot-dev/types/sp_runtime/multiaddress.js" import { assert } from "asserts" import { $, Rune, Sr25519 } from "capi" import { VirtualMultisigRune } from "capi/patterns/multisig/mod.ts" import { signature } from "capi/patterns/signature/polkadot.ts" -import { Balances, chain, createUsers, System, Utility } from "polkadot_dev/mod.js" -import { MultiAddress } from "polkadot_dev/types/sp_runtime/multiaddress.js" import { parse } from "../../deps/std/flags.ts" const { alexa, billy, carol, david } = await createUsers() diff --git a/examples/paginate.eg.ts b/examples/paginate.eg.ts index 055b2e557..2bea04f71 100644 --- a/examples/paginate.eg.ts +++ b/examples/paginate.eg.ts @@ -5,8 +5,8 @@ * Read pages (either of keys or entries) from storage maps. */ +import { System, types } from "@capi/polkadot-dev/mod.js" import { $ } from "capi" -import { System, types } from "polkadot_dev/mod.js" // Reference the first 10 keys of a polkadot dev chain's system account map. const accountKeys = await System.Account.keyPage(10, null).run() diff --git a/examples/raw_rpc/call.eg.ts b/examples/raw_rpc/call.eg.ts index dfc05d2a2..0de09f34b 100644 --- a/examples/raw_rpc/call.eg.ts +++ b/examples/raw_rpc/call.eg.ts @@ -4,8 +4,8 @@ * @description Interact directly with the RPC node's call methods. */ +import { chain } from "@capi/polkadot-dev/mod.js" import { $ } from "capi" -import { chain } from "polkadot_dev/mod.js" // Make a call. const hash = await chain.connection diff --git a/examples/raw_rpc/subscription.eg.ts b/examples/raw_rpc/subscription.eg.ts index 18cb83c0f..94c6020c6 100644 --- a/examples/raw_rpc/subscription.eg.ts +++ b/examples/raw_rpc/subscription.eg.ts @@ -4,8 +4,8 @@ * @description Interact directly with the RPC node's subscription methods. */ +import { chain } from "@capi/polkadot-dev/mod.js" import { $, known } from "capi" -import { chain } from "polkadot_dev/mod.js" // Get an async iterator, which yields subscription events. const headerIter = chain.connection diff --git a/examples/read/account_info.eg.ts b/examples/read/account_info.eg.ts index 344cd17e5..7e02b5639 100644 --- a/examples/read/account_info.eg.ts +++ b/examples/read/account_info.eg.ts @@ -5,8 +5,8 @@ * Read the value (an `AccountInfo`) from the system account map. */ +import { createUsers, System, types } from "@capi/polkadot-dev/mod.js" import { $ } from "capi" -import { createUsers, System, types } from "polkadot_dev/mod.js" const { alexa } = await createUsers() diff --git a/examples/read/era_reward_points.eg.ts b/examples/read/era_reward_points.eg.ts index 5a6fc1b23..8d1f81afd 100644 --- a/examples/read/era_reward_points.eg.ts +++ b/examples/read/era_reward_points.eg.ts @@ -6,8 +6,8 @@ * is used as the key to another piece of storage (the corresponding reward points). */ +import { Staking, types } from "@capi/westend/mod.js" import { $ } from "capi" -import { Staking, types } from "westend/mod.js" const idx = Staking.ActiveEra .value() diff --git a/examples/read/now.eg.ts b/examples/read/now.eg.ts index da7bd06a6..75434f9e7 100644 --- a/examples/read/now.eg.ts +++ b/examples/read/now.eg.ts @@ -5,8 +5,8 @@ * Read the current timestamp as agreed upon by validators. */ +import { Timestamp } from "@capi/polkadot/mod.js" import { $ } from "capi" -import { Timestamp } from "polkadot/mod.js" const now = await Timestamp.Now.value().run() console.log("Now:", now) diff --git a/examples/read/para_heads.eg.ts b/examples/read/para_heads.eg.ts index d81c7b999..d3d641b66 100644 --- a/examples/read/para_heads.eg.ts +++ b/examples/read/para_heads.eg.ts @@ -7,8 +7,8 @@ * (the corresponding parachain heads). */ +import { Paras } from "@capi/polkadot/mod.js" import { $, ArrayRune, ValueRune } from "capi" -import { Paras } from "polkadot/mod.js" const heads = await Paras.Parachains .value() diff --git a/examples/sign/ed25519.eg.ts b/examples/sign/ed25519.eg.ts index 21d80bf53..21ba1f353 100644 --- a/examples/sign/ed25519.eg.ts +++ b/examples/sign/ed25519.eg.ts @@ -5,11 +5,11 @@ * Utilize an Ed25519 library for signing. */ +import { Balances, createUsers, System, types } from "@capi/westend-dev/mod.js" import { assert } from "asserts" import { Rune } from "capi" import { signature } from "capi/patterns/signature/polkadot.ts" import * as ed from "https://esm.sh/@noble/ed25519@1.7.3" -import { Balances, createUsers, System, types } from "westend_dev/mod.js" const { alexa, billy } = await createUsers() diff --git a/examples/sign/offline.ts b/examples/sign/offline.eg.ts similarity index 93% rename from examples/sign/offline.ts rename to examples/sign/offline.eg.ts index 44433082f..d64da1755 100644 --- a/examples/sign/offline.ts +++ b/examples/sign/offline.eg.ts @@ -5,9 +5,9 @@ * Finally, rehydrate the extrinsic and submit it. */ +import { Balances, chain, createUsers } from "@capi/westend-dev/mod.js" import { $, SignedExtrinsicRune } from "capi" import { signature } from "capi/patterns/signature/polkadot.ts" -import { Balances, chain, createUsers } from "westend_dev/mod.js" const { alexa, billy } = await createUsers() diff --git a/examples/sign/pjs.eg.ts b/examples/sign/pjs.eg.ts index bfccf2bba..1dbb0de2a 100644 --- a/examples/sign/pjs.eg.ts +++ b/examples/sign/pjs.eg.ts @@ -6,12 +6,12 @@ * sign a Capi extrinsic. */ +import { Balances, chain, createUsers, System } from "@capi/polkadot-dev/mod.js" import { ss58 } from "capi" import { pjsSender, PjsSigner } from "capi/patterns/compat/pjs_sender.ts" import { signature } from "capi/patterns/signature/polkadot.ts" import { createPair } from "https://deno.land/x/polkadot@0.2.34/keyring/mod.ts" import { TypeRegistry } from "https://deno.land/x/polkadot@0.2.34/types/mod.ts" -import { Balances, chain, createUsers, System } from "polkadot_dev/mod.js" const { alexa, billy } = await createUsers() diff --git a/examples/smoldot.eg.ts b/examples/smoldot.eg.ts index fe516dd13..64261d289 100644 --- a/examples/smoldot.eg.ts +++ b/examples/smoldot.eg.ts @@ -15,7 +15,7 @@ const relayChainSpec = await (await fetch( )).text() // Initialize a `ChainRune` with `SmoldotConnection` and the chainspec. -const smoldotChain = ChainRune.from(SmoldotConnection, { relayChainSpec }) +const smoldotChain = ChainRune.from(SmoldotConnection.bind({ relayChainSpec })) const [metadata, extrinsics] = await Rune .tuple([ diff --git a/examples/tx/balances_transfer.eg.ts b/examples/tx/balances_transfer.eg.ts index ecba05b44..fa2190a96 100644 --- a/examples/tx/balances_transfer.eg.ts +++ b/examples/tx/balances_transfer.eg.ts @@ -5,9 +5,9 @@ * Transfer some funds from one user to another. */ +import { Balances, createUsers, System } from "@capi/westend-dev/mod.js" import { assert } from "asserts" import { signature } from "capi/patterns/signature/polkadot.ts" -import { Balances, createUsers, System } from "westend_dev/mod.js" // Create two test users. Alexa will send the funds to Billy. const { alexa, billy } = await createUsers() diff --git a/examples/tx/handle_errors.eg.ts b/examples/tx/handle_errors.eg.ts index 94dcfaa7e..b213b562a 100644 --- a/examples/tx/handle_errors.eg.ts +++ b/examples/tx/handle_errors.eg.ts @@ -6,10 +6,10 @@ * (and handling) of dispatch errors. */ +import { Balances } from "@capi/contracts-dev/mod.js" import { assertInstanceOf } from "asserts" import { alice, bob, ExtrinsicError } from "capi" import { signature } from "capi/patterns/signature/polkadot.ts" -import { Balances } from "contracts_dev/mod.js" // The following should reject with an `ExtrinsicError`. const extrinsicError = await Balances diff --git a/examples/tx/utility_batch.eg.ts b/examples/tx/utility_batch.eg.ts index 9190b09b5..67950daa9 100644 --- a/examples/tx/utility_batch.eg.ts +++ b/examples/tx/utility_batch.eg.ts @@ -5,10 +5,10 @@ * Sign and submit multiple calls within a single extrinsic. */ +import { Balances, createUsers, System, Utility } from "@capi/westend-dev/mod.js" import { assert } from "asserts" import { Rune } from "capi" import { signature } from "capi/patterns/signature/polkadot.ts" -import { Balances, createUsers, System, Utility } from "westend_dev/mod.js" // Create four test users, one of whom will be the batch sender. The other // three will be recipients of balance transfers described in the batch. diff --git a/examples/watch.eg.ts b/examples/watch.eg.ts index 0d9e6f1a2..ef100c309 100644 --- a/examples/watch.eg.ts +++ b/examples/watch.eg.ts @@ -7,8 +7,8 @@ * subsequent states. */ +import { chain, Timestamp } from "@capi/polkadot/mod.js" import { $ } from "capi" -import { chain, Timestamp } from "polkadot/mod.js" // Specifying `chain.latestBlockHash` indicates that (A) this Rune tree // can be treated as reactive and (B) is a dependent of a "timeline" associated diff --git a/examples/xcm/asset_teleportation.eg.ts b/examples/xcm/asset_teleportation.eg.ts index dacf73e73..234c4a4e6 100644 --- a/examples/xcm/asset_teleportation.eg.ts +++ b/examples/xcm/asset_teleportation.eg.ts @@ -7,13 +7,13 @@ * balance of the user to whom the asset was transferred. */ +import { types, XcmPallet } from "@capi/rococo-dev/mod.js" +import { chain as parachain, System } from "@capi/rococo-dev/statemine/mod.js" +import { Event } from "@capi/rococo-dev/statemine/types/cumulus_pallet_parachain_system/pallet.js" +import { RuntimeEvent } from "@capi/rococo-dev/statemine/types/statemine_runtime.js" import { assert } from "asserts" import { alice, Rune } from "capi" import { signature } from "capi/patterns/signature/polkadot.ts" -import { types, XcmPallet } from "zombienet/statemine.toml/alice/@latest/mod.js" -import { chain as parachain, System } from "zombienet/statemine.toml/collator/@latest/mod.js" -import { Event } from "zombienet/statemine.toml/collator/@latest/types/cumulus_pallet_parachain_system/pallet.js" -import { RuntimeEvent } from "zombienet/statemine.toml/collator/@latest/types/statemine_runtime.js" // Destructure the various type factories for convenient access. const { diff --git a/fluent/ChainRune.ts b/fluent/ChainRune.ts index 51f4a9c42..92f370785 100644 --- a/fluent/ChainRune.ts +++ b/fluent/ChainRune.ts @@ -1,7 +1,7 @@ import { hex } from "../crypto/mod.ts" import * as $ from "../deps/scale.ts" import { decodeMetadata, FrameMetadata } from "../frame_metadata/mod.ts" -import { Connection, ConnectionCtorLike } from "../rpc/mod.ts" +import { Connection } from "../rpc/mod.ts" import { Rune, RunicArgs, ValueRune } from "../rune/mod.ts" import { BlockHashRune } from "./BlockHashRune.ts" import { ConnectionRune } from "./ConnectionRune.ts" @@ -51,14 +51,11 @@ export namespace Chain { // TODO: do we want to represent the discovery value and conn type within the type system? export class ChainRune extends Rune { - static from( - connectionCtor: ConnectionCtorLike, - discovery: D, + static from( + connect: (signal: AbortSignal) => Connection, staticMetadata?: M, ) { - const connection = ConnectionRune.from(async (signal) => - connectionCtor.connect(discovery, signal) - ) + const connection = ConnectionRune.from(connect) const metadata = staticMetadata ?? Rune .fn(hex.decode) .call(connection.call("state_getMetadata")) diff --git a/fluent/ConnectionRune.ts b/fluent/ConnectionRune.ts index 53046a9d1..c2a8fd44a 100644 --- a/fluent/ConnectionRune.ts +++ b/fluent/ConnectionRune.ts @@ -3,7 +3,10 @@ import { Connection, ConnectionError, RpcSubscriptionMessage, ServerError } from import { Batch, MetaRune, Run, Rune, RunicArgs, RunStream } from "../rune/mod.ts" class RunConnection extends Run { - constructor(ctx: Batch, readonly initConnection: (signal: AbortSignal) => Promise) { + constructor( + ctx: Batch, + readonly initConnection: (signal: AbortSignal) => Connection | Promise, + ) { super(ctx) } @@ -14,7 +17,7 @@ class RunConnection extends Run { } export class ConnectionRune extends Rune { - static from(init: (signal: AbortSignal) => Promise) { + static from(init: (signal: AbortSignal) => Connection | Promise) { return Rune.new(RunConnection, init).into(ConnectionRune) } diff --git a/frame_metadata/raw/v14.ts b/frame_metadata/raw/v14.ts index 2b76ba0e8..224bf5475 100644 --- a/frame_metadata/raw/v14.ts +++ b/frame_metadata/raw/v14.ts @@ -2,7 +2,6 @@ import { blake2_128, blake2_128Concat, blake2_256, - Hasher, identity, twox128, twox256, @@ -21,7 +20,7 @@ import { $storageKey, } from "../key_codecs.ts" -const $hasher = $.literalUnion([ +const hashers = { blake2_128, blake2_256, blake2_128Concat, @@ -29,6 +28,16 @@ const $hasher = $.literalUnion([ twox256, twox64Concat, identity, +} + +const $hasher = $.literalUnion([ + "blake2_128", + "blake2_256", + "blake2_128Concat", + "twox128", + "twox256", + "twox64Concat", + "identity", ]) const $storageEntry = $.object( @@ -92,6 +101,8 @@ export const $metadata = $.object( $.field("tys", $.array($ty)), $.field("pallets", $.array($pallet)), $.field("extrinsic", $extrinsicDef), + // TODO: is this useful? + $.field("runtime", $tyId), ) export function transformMetadata(metadata: $.Native): FrameMetadata { @@ -109,11 +120,11 @@ export function transformMetadata(metadata: $.Native): FrameMe key = $emptyKey partialKey = $partialEmptyKey } else if (storage.hashers.length === 1) { - key = storage.hashers[0]!.$hash(types[storage.key]!) + key = hashers[storage.hashers[0]!].$hash(types[storage.key]!) partialKey = $partialSingleKey(key) } else { const codecs = extractTupleMembersVisitor.visit(types[storage.key]!).map((codec, i) => - storage.hashers[i]!.$hash(codec) + hashers[storage.hashers[i]!].$hash(codec) ) key = $.tuple(...codecs) partialKey = $partialMultiKey(...codecs) diff --git a/import_map.json b/import_map.json index 9d28ae25f..0b05ccc47 100644 --- a/import_map.json +++ b/import_map.json @@ -1,32 +1,15 @@ { "imports": { - "polkadot/": "http://localhost:4646/frame/wss/rpc.polkadot.io/@latest/", - "polkadot_dev/": "http://localhost:4646/frame/dev/polkadot/@v0.9.39-1/", - "kusama/": "http://localhost:4646/frame/wss/kusama-rpc.polkadot.io/@latest/", - "kusama_dev/": "http://localhost:4646/frame/dev/kusama/@v0.9.39-1/", - "westend/": "http://localhost:4646/frame/wss/westend-rpc.polkadot.io/@latest/", - "westend_dev/": "http://localhost:4646/frame/dev/westend/@v0.9.39-1/", - "rococo/": "http://localhost:4646/frame/wss/rococo-contracts-rpc.polkadot.io/@latest/", - "rococo_contracts/": "http://localhost:4646/frame/wss/rococo-contracts-rpc.polkadot.io/@latest/", - "rococo_dev/": "http://localhost:4646/frame/dev/rococo/@v0.9.39-1/", - "acala/": "http://localhost:4646/frame/wss/acala-polkadot.api.onfinality.io/public-ws/@latest/", - "moonbeam/": "http://localhost:4646/frame/wss/wss.api.moonbeam.network/@latest/", - "statemint/": "http://localhost:4646/frame/wss/statemint-rpc.polkadot.io/@latest/", - "phala/": "http://localhost:4646/frame/wss/api.phala.network/ws#/rpc/@latest/", - "subsocial/": "http://localhost:4646/frame/para.subsocial.network/@latest/" + "@capi/": "http://localhost:4646/5a2452ab3d4473be/" }, "scopes": { "examples/": { - "capi": "http://localhost:4646/mod.ts", - "capi/patterns/": "http://localhost:4646/patterns/", - "zombienet/": "http://localhost:4646/frame/zombienet/zombienets/", - "project/": "http://localhost:4646/frame/project/", - "contracts_dev/": "http://localhost:4646/frame/contracts_dev/@v0.24.0/", + "capi": "./mod.ts", + "capi/": "./", "asserts": "./deps/std/testing/asserts.ts" }, "http://localhost:4646/": { - "http://localhost:4646/": "./", - "http://localhost:4646/frame/": "http://localhost:4646/frame/" + "http://localhost:4646/capi/": "./" } } } diff --git a/main.ts b/main.ts index 4b7b10893..abd489375 100644 --- a/main.ts +++ b/main.ts @@ -1,10 +1,11 @@ import bin from "./cli/bin.ts" import serve from "./cli/serve.ts" +import sync from "./cli/sync.ts" -const commands: Record void> = { bin, serve } +const commands: Record void> = { bin, serve, sync } if (Deno.args[0]! in commands) { commands[Deno.args[0]!]!(...Deno.args.slice(1)) } else { - serve(...Deno.args) + throw new Error("Unrecognized command") } diff --git a/mod.ts b/mod.ts index 651b4025a..ce108ea3a 100644 --- a/mod.ts +++ b/mod.ts @@ -1,9 +1,10 @@ export * as $ from "./deps/scale.ts" export { BitSequence } from "./deps/scale.ts" -// moderate --exclude main.ts providers server util +// moderate --exclude main.ts server util capi.config.ts export * from "./crypto/mod.ts" +export * from "./devnets/mod.ts" export * from "./fluent/mod.ts" export * from "./frame_metadata/mod.ts" export * from "./rpc/mod.ts" diff --git a/patterns/consensus/babeBlockAuthor.ts b/patterns/consensus/babeBlockAuthor.ts index f471b9604..24c21898f 100644 --- a/patterns/consensus/babeBlockAuthor.ts +++ b/patterns/consensus/babeBlockAuthor.ts @@ -1,4 +1,4 @@ -import { $preDigest } from "polkadot/types/sp_consensus_babe/digests.js" +import { $preDigest } from "@capi/polkadot/types/sp_consensus_babe/digests.js" import { AddressPrefixChain, ChainRune } from "../../fluent/mod.ts" import { AccountIdRune } from "../../fluent/mod.ts" import { Rune, RunicArgs, ValueRune } from "../../rune/mod.ts" diff --git a/patterns/consensus/preRuntimeDigest.ts b/patterns/consensus/preRuntimeDigest.ts index 48be97547..12c8111fc 100644 --- a/patterns/consensus/preRuntimeDigest.ts +++ b/patterns/consensus/preRuntimeDigest.ts @@ -1,4 +1,4 @@ -import { $digestItem, DigestItem } from "polkadot/types/sp_runtime/generic/digest.js" +import { $digestItem, DigestItem } from "@capi/polkadot/types/sp_runtime/generic/digest.js" import { hex } from "../../crypto/mod.ts" import { Chain, ChainRune } from "../../fluent/mod.ts" import { RunicArgs, ValueRune } from "../../rune/mod.ts" diff --git a/patterns/identity.ts b/patterns/identity.ts index ef9a9c303..c486eae69 100644 --- a/patterns/identity.ts +++ b/patterns/identity.ts @@ -1,4 +1,4 @@ -import { Data, type IdentityInfo } from "polkadot_dev/types/pallet_identity/types.js" +import { Data, type IdentityInfo } from "@capi/polkadot-dev/types/pallet_identity/types.js" import * as $ from "../deps/scale.ts" import { Rune, RunicArgs } from "../rune/mod.ts" diff --git a/patterns/multisig/MultisigRune.ts b/patterns/multisig/MultisigRune.ts index 00800f6a2..56c001c7e 100644 --- a/patterns/multisig/MultisigRune.ts +++ b/patterns/multisig/MultisigRune.ts @@ -1,4 +1,4 @@ -import { MultiAddress } from "polkadot/types/sp_runtime/multiaddress.js" +import { MultiAddress } from "@capi/polkadot/types/sp_runtime/multiaddress.js" import * as bytes from "../../deps/std/bytes.ts" import { $, Chain, ChainRune, PatternRune, Rune, RunicArgs, ValueRune } from "../../mod.ts" import { multisigAccountId } from "./multisigAccountId.ts" diff --git a/patterns/multisig/VirtualMultisigRune.ts b/patterns/multisig/VirtualMultisigRune.ts index 15fc65639..12d876e66 100644 --- a/patterns/multisig/VirtualMultisigRune.ts +++ b/patterns/multisig/VirtualMultisigRune.ts @@ -1,4 +1,4 @@ -import { MultiAddress } from "polkadot/types/sp_runtime/multiaddress.js" +import { MultiAddress } from "@capi/polkadot/types/sp_runtime/multiaddress.js" import { equals } from "../../deps/std/bytes.ts" import { $, diff --git a/patterns/proxy/events.ts b/patterns/proxy/events.ts index 719694116..e76a3edca 100644 --- a/patterns/proxy/events.ts +++ b/patterns/proxy/events.ts @@ -1,5 +1,5 @@ -import { type Event as ProxyEvent } from "polkadot/types/pallet_proxy/pallet.js" -import { type RuntimeEvent } from "polkadot/types/polkadot_runtime.js" +import { type Event as ProxyEvent } from "@capi/polkadot/types/pallet_proxy/pallet.js" +import { type RuntimeEvent } from "@capi/polkadot/types/polkadot_runtime.js" import { Rune, RunicArgs } from "../../mod.ts" export function filterPureCreatedEvents(...[events]: RunicArgs) { diff --git a/patterns/proxy/replaceDelegateCalls.ts b/patterns/proxy/replaceDelegateCalls.ts index 236f45d1b..a29bcae9c 100644 --- a/patterns/proxy/replaceDelegateCalls.ts +++ b/patterns/proxy/replaceDelegateCalls.ts @@ -1,4 +1,4 @@ -import { MultiAddress } from "polkadot/types/sp_runtime/multiaddress.js" +import { MultiAddress } from "@capi/polkadot/types/sp_runtime/multiaddress.js" import { Chain, ChainRune, Rune, RunicArgs } from "../../mod.ts" // TODO: constrain diff --git a/patterns/signature/polkadot.ts b/patterns/signature/polkadot.ts index a9a8629b7..b8993738d 100644 --- a/patterns/signature/polkadot.ts +++ b/patterns/signature/polkadot.ts @@ -1,4 +1,4 @@ -import { Polkadot } from "polkadot/mod.js" +import { Polkadot } from "@capi/polkadot/mod.js" import { AddressPrefixChain, Chain, ChainRune } from "../../fluent/ChainRune.ts" import { ExtrinsicSender, SignatureData } from "../../fluent/ExtrinsicRune.ts" import { $, hex, ss58, ValueRune } from "../../mod.ts" diff --git a/providers/frame/FrameBinProvider.ts b/providers/frame/FrameBinProvider.ts deleted file mode 100644 index 11f67fb91..000000000 --- a/providers/frame/FrameBinProvider.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { download } from "../../deps/capi_binary_builds.ts" -import { deadline } from "../../deps/std/async.ts" -import { Env, PathInfo } from "../../server/mod.ts" -import { PermanentMemo } from "../../util/mod.ts" -import { ready } from "../../util/port.ts" -import { FrameProxyProvider } from "./FrameProxyProvider.ts" - -const readyTimeout = 3 * 60 * 1000 - -export abstract class FrameBinProvider extends FrameProxyProvider { - constructor(env: Env, readonly bin: string) { - super(env) - } - - abstract launch(pathInfo: PathInfo): Promise - - dynamicUrlMemo = new PermanentMemo() - async dynamicUrl(pathInfo: PathInfo) { - return this.dynamicUrlMemo.run(pathInfo.target ?? "", () => - deadline( - (async () => { - const port = await this.launch(pathInfo) - await ready(port) - return `ws://localhost:${port}` - })(), - readyTimeout, - )) - } - - async getBinPath(pathInfo: PathInfo) { - if (pathInfo.vRuntime === "local") return this.bin - return download(this.bin, pathInfo.vRuntime!) - } - - async runBin(pathInfo: PathInfo, args: string[]): Promise { - const command = new Deno.Command(await this.getBinPath(pathInfo), { - args, - stdout: "piped", - stderr: "piped", - signal: this.env.signal, - }) - const child = command.spawn() - // TODO: get rid of this without breaking CI - this.env.signal.addEventListener("abort", () => child.kill("SIGKILL")) - return child - } -} diff --git a/providers/frame/FrameProvider.ts b/providers/frame/FrameProvider.ts deleted file mode 100644 index 7a11d0b25..000000000 --- a/providers/frame/FrameProvider.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { FrameCodegen } from "../../codegen/FrameCodegen.ts" -import { hex } from "../../crypto/mod.ts" -import { posix as path } from "../../deps/std/path.ts" -import { decodeMetadata, FrameMetadata } from "../../frame_metadata/mod.ts" -import { Connection, ServerError } from "../../rpc/mod.ts" -import { f, PathInfo, Provider } from "../../server/mod.ts" -import { fromPathInfo } from "../../server/PathInfo.ts" -import { WeakMemo } from "../../util/mod.ts" -import { normalizeIdent } from "../../util/normalize.ts" -import { tsFormatter } from "../../util/tsFormatter.ts" -import { withSignal } from "../../util/withSignal.ts" -import { generateTar } from "./common.ts" - -export abstract class FrameProvider extends Provider { - codegenCtxsPending: Record> = {} - - abstract connect(pathInfo: PathInfo, signal: AbortSignal): Promise - - abstract connectionCode(pathInfo: PathInfo, isTypes: boolean): Promise - - async handle(request: Request, pathInfo: PathInfo): Promise { - const { vRuntime, filePath } = pathInfo - if (!vRuntime) return f.serverError("Must specify vRuntime") - if (!filePath) { - return f.redirect(fromPathInfo({ - ...pathInfo, - filePath: "mod.ts", - })) - } - if (filePath === "capi.js" || filePath === "capi.d.ts") { - const capiPath = path.relative(path.dirname(new URL(request.url).pathname), "/mod.ts") - return f.code( - this.env.cache, - request, - async () => `export * from ${JSON.stringify(capiPath)}`, - ) - } - if (filePath === "pkg.tar") { - return new Response( - await this.env.cache.getRaw(`${this.cacheKey(pathInfo)}/pkg.tar`, async () => { - const files = await this.codegen(pathInfo) - const chainName = await this.chainName(pathInfo) - return await generateTar(files, chainName, vRuntime) - }), - ) - } - return await f.code(this.env.cache, request, async () => { - const codegen = await this.codegen(pathInfo) - const code = codegen.get(filePath) - if (!code) throw f.notFound() - return tsFormatter.formatText(filePath, code) - }) - } - - cacheKey(pathInfo: PathInfo) { - return fromPathInfo({ ...pathInfo, filePath: "" }) - } - - codegenMemo = new WeakMemo>() - codegen(pathInfo: PathInfo) { - return this.codegenMemo.run(this.cacheKey(pathInfo), async () => { - const [metadata, connectionCode, connectionTypes, chainName] = await Promise.all([ - this.getMetadata(pathInfo), - this.connectionCode(pathInfo, false), - this.connectionCode(pathInfo, true), - this.chainName(pathInfo), - ]) - const files = new Map() - files.set("connection.js", connectionCode) - files.set("connection.d.ts", connectionTypes) - new FrameCodegen(metadata, chainName).write(files) - return files - }) - } - - metadataMemo = new WeakMemo() - async getMetadata(pathInfo: PathInfo) { - const cacheKey = this.cacheKey(pathInfo) - return this.metadataMemo.run(cacheKey, async () => { - const raw = await this.env.cache.getRaw( - `${cacheKey}/_metadata`, - async () => { - return hex.decode(await this.call(pathInfo, "state_getMetadata", [])) - }, - ) - return decodeMetadata(raw) - }) - } - - async call( - pathInfo: PathInfo, - method: string, - params: unknown[] = [], - ): Promise { - return withSignal(async (signal) => { - const connection = await this.connect(pathInfo, signal) - const result = await connection.call(method, params) - if (result.error) throw new ServerError(result) - return result.result as T - }) - } - - async chainName(pathInfo: PathInfo) { - return this.env.cache.getString( - `${this.cacheKey(pathInfo)}/_chainName`, - chainNameTtl, - async () => normalizeIdent(await this.call(pathInfo, "system_chain")), - ) - } -} - -const chainNameTtl = 60_000 diff --git a/providers/frame/FrameProxyProvider.ts b/providers/frame/FrameProxyProvider.ts deleted file mode 100644 index d73ebb107..000000000 --- a/providers/frame/FrameProxyProvider.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { deferred } from "../../deps/std/async.ts" -import { WsConnection } from "../../rpc/mod.ts" -import { PathInfo } from "../../server/mod.ts" -import { fromPathInfo } from "../../server/PathInfo.ts" -import { FrameProvider } from "./FrameProvider.ts" - -export abstract class FrameProxyProvider extends FrameProvider { - override async handle(request: Request, pathInfo: PathInfo): Promise { - if ( - !pathInfo.filePath && request.headers.get("upgrade") === "websocket" - ) { - return this.proxyWs(request, pathInfo) - } - return super.handle(request, pathInfo) - } - - async proxyWs(request: Request, pathInfo: PathInfo) { - const url = await this.dynamicUrl(pathInfo) - const server = new WebSocket(url) - const { socket: client, response } = Deno.upgradeWebSocket(request) - setup(client, server) - setup(server, client) - return response - - function setup(a: WebSocket, b: WebSocket) { - const ready = deferred() - b.addEventListener("open", () => { - ready.resolve() - }) - a.addEventListener("close", async () => { - try { - b.close() - } catch {} - }) - a.addEventListener("message", async (event) => { - try { - await ready - b.send(event.data) - } catch { - a.close() - b.close() - } - }) - } - } - - abstract dynamicUrl(pathInfo: PathInfo): Promise - - staticUrl(pathInfo: PathInfo) { - return new URL( - fromPathInfo({ - ...pathInfo, - filePath: "", - }), - this.env.upgradedHref, - ).toString() - } - - async connect(pathInfo: PathInfo, signal: AbortSignal) { - return WsConnection.connect(await this.dynamicUrl(pathInfo), signal) - } - - async connectionCode(pathInfo: PathInfo, isTypes: boolean) { - const url = this.staticUrl(pathInfo) - return ` -import * as C from "./capi.js" - -export const connectionCtor ${isTypes ? `: typeof C.WsConnection` : `= C.WsConnection`} -export const discoveryValue ${isTypes ? ":" : "="} "${url}" - ` - } -} diff --git a/providers/frame/common.ts b/providers/frame/common.ts deleted file mode 100644 index 1fd38815c..000000000 --- a/providers/frame/common.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { ss58 } from "../../crypto/mod.ts" -import * as $ from "../../deps/scale.ts" -import { Tar } from "../../deps/std/archive.ts" -import { Buffer } from "../../deps/std/io.ts" -import { readableStreamFromReader, writableStreamFromWriter } from "../../deps/std/streams.ts" -import { tsFormatter } from "../../util/tsFormatter.ts" - -export const DEFAULT_TEST_USER_COUNT = 100_000 -const DEFAULT_TEST_USER_INITIAL_FUNDS = 1_000_000_000_000_000_000 -export const publicKeysUrl = import.meta.resolve("./test_users_public_keys.scale") - -export async function createCustomChainSpec( - bin: string, - chain: string, - networkPrefix: number, - signal: AbortSignal, -): Promise { - const buildSpecCmd = new Deno.Command(bin, { - args: ["build-spec", "--disable-default-bootnode", "--chain", chain], - signal, - }) - const chainSpec = JSON.parse(new TextDecoder().decode((await buildSpecCmd.output()).stdout)) - const balances: [string, number][] = chainSpec.genesis.runtime.balances.balances - const publicKeys = $.array($.sizedUint8Array(32)).decode( - new Uint8Array(await (await fetch(publicKeysUrl)).arrayBuffer()), - ) - for (let i = 0; i < DEFAULT_TEST_USER_COUNT; i++) { - balances.push([ss58.encode(networkPrefix, publicKeys[i]!), DEFAULT_TEST_USER_INITIAL_FUNDS]) - } - const customChainSpecPath = await Deno.makeTempFile({ - prefix: `custom-${chain}-chain-spec`, - suffix: ".json", - }) - await Deno.writeTextFile(customChainSpecPath, JSON.stringify(chainSpec, undefined, 2)) - const buildSpecRawCmd = new Deno.Command(bin, { - args: ["build-spec", "--disable-default-bootnode", "--chain", customChainSpecPath, "--raw"], - signal, - }) - const chainSpecRaw = JSON.parse(new TextDecoder().decode((await buildSpecRawCmd.output()).stdout)) - const customChainSpecRawPath = await Deno.makeTempFile({ - prefix: `custom-${chain}-chain-spec-raw`, - suffix: ".json", - }) - await Deno.writeTextFile(customChainSpecRawPath, JSON.stringify(chainSpecRaw, undefined, 2)) - return customChainSpecRawPath -} - -export function connectionCodeWithUsers(code: string, isTypes: boolean, url: string): string { - return ` -${code} - -export const createUsers ${ - isTypes - ? `: ReturnType` - : `= C.testUserFactory(${JSON.stringify(url)})` - } - ` -} - -export async function handleCount(request: Request, cache: { count: number }): Promise { - const body = await request.json() - $.assert($.field("count", $.u32), body) - const { count } = body - const index = cache.count - const newCount = index + count - if (newCount < DEFAULT_TEST_USER_COUNT) cache.count = newCount - else throw new Error("Maximum test user count reached") - return new Response(JSON.stringify({ index }), { - status: 200, - headers: { "Content-Type": "application/json" }, - }) -} - -export async function generateTar(_files: Map, chainName: string, version: string) { - const files = new Map(_files) - - files.set("capi.js", `export * from "capi"`) - files.set("capi.d.ts", `export * from "capi"`) - files.set( - "package.json", - JSON.stringify( - { - name: packageName(chainName), - version, - type: "module", - main: "./mod.js", - peerDependencies: { - capi: "*", - }, - }, - null, - 2, - ), - ) - - const tar = new Tar() - for (const [name, content] of files) { - const formatted = /\.(js|ts)$/.test(name) ? tsFormatter.formatText(name, content) : content - const data = new TextEncoder().encode(formatted) - tar.append(`package/${name}`, { - contentSize: data.length, - reader: new Buffer(data), - }) - } - - const buffer = new Buffer() - - await readableStreamFromReader(tar.getReader()) - .pipeTo(writableStreamFromWriter(buffer)) - - return buffer.bytes() -} - -function packageName(chainName: string) { - return `@capi/` + chainName.replace(/([a-z])(?=[A-Z])/g, (x) => `${x}-`).toLowerCase() -} diff --git a/providers/frame/contracts_dev.ts b/providers/frame/contracts_dev.ts deleted file mode 100644 index d10bce3fd..000000000 --- a/providers/frame/contracts_dev.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { Env, PathInfo } from "../../server/mod.ts" -import { fromPathInfo } from "../../server/PathInfo.ts" -import { getAvailable } from "../../util/port.ts" -import { connectionCodeWithUsers, createCustomChainSpec, handleCount } from "./common.ts" -import { FrameBinProvider } from "./FrameBinProvider.ts" - -export class ContractsDevProvider extends FrameBinProvider { - userCount = { count: 0 } - - constructor(env: Env) { - super(env, "substrate-contracts-node") - } - - override async connectionCode(pathInfo: PathInfo, isTypes: boolean): Promise { - const url = new URL(fromPathInfo({ ...pathInfo, filePath: "user_i" }), this.env.href).toString() - return connectionCodeWithUsers(await super.connectionCode(pathInfo, isTypes), isTypes, url) - } - - override async handle(request: Request, pathInfo: PathInfo): Promise { - if (pathInfo.filePath === "user_i") return handleCount(request, this.userCount) - return super.handle(request, pathInfo) - } - - async launch(pathInfo: PathInfo) { - const port = getAvailable() - const chainSpec = await createCustomChainSpec( - await this.getBinPath(pathInfo), - "dev", - 42, - this.env.signal, - ) - await this.runBin(pathInfo, [ - "--tmp", - "--alice", - "--ws-port", - port.toString(), - "--chain", - chainSpec, - ]) - return port - } -} diff --git a/providers/frame/mod.ts b/providers/frame/mod.ts deleted file mode 100644 index 6bc687612..000000000 --- a/providers/frame/mod.ts +++ /dev/null @@ -1,11 +0,0 @@ -// moderate --exclude zombienet_worker.ts - -export * from "./common.ts" -export * from "./contracts_dev.ts" -export * from "./FrameBinProvider.ts" -export * from "./FrameProvider.ts" -export * from "./FrameProxyProvider.ts" -export * from "./polkadot_dev.ts" -export * from "./project.ts" -export * from "./wss.ts" -export * from "./zombienet.ts" diff --git a/providers/frame/polkadot_dev.ts b/providers/frame/polkadot_dev.ts deleted file mode 100644 index be9399d11..000000000 --- a/providers/frame/polkadot_dev.ts +++ /dev/null @@ -1,57 +0,0 @@ -import * as $ from "../../deps/scale.ts" -import { Env, PathInfo } from "../../server/mod.ts" -import { fromPathInfo } from "../../server/PathInfo.ts" -import { getAvailable } from "../../util/port.ts" -import { getOrInit } from "../../util/state.ts" -import { connectionCodeWithUsers, createCustomChainSpec, handleCount } from "./common.ts" -import { FrameBinProvider } from "./FrameBinProvider.ts" - -export class PolkadotDevProvider extends FrameBinProvider { - userCount = new Map() - - constructor(env: Env) { - super(env, "polkadot") - } - - override async connectionCode(pathInfo: PathInfo, isTypes: boolean): Promise { - const url = new URL(fromPathInfo({ ...pathInfo, filePath: "user_i" }), this.env.href).toString() - return connectionCodeWithUsers(await super.connectionCode(pathInfo, isTypes), isTypes, url) - } - - override async handle(request: Request, pathInfo: PathInfo): Promise { - if (pathInfo.filePath === "user_i") { - $.assert($devRuntimeName, pathInfo.target) - return handleCount(request, getOrInit(this.userCount, pathInfo.target, () => ({ count: 0 }))) - } - return super.handle(request, pathInfo) - } - - async launch(pathInfo: PathInfo) { - const runtimeName = pathInfo.target - $.assert($devRuntimeName, runtimeName) - const port = getAvailable() - const chainSpec = await createCustomChainSpec( - await this.getBinPath(pathInfo), - `${runtimeName}-dev`, - DEV_RUNTIME_PREFIXES[runtimeName], - this.env.signal, - ) - const args: string[] = ["--tmp", "--alice", "--ws-port", port.toString(), "--chain", chainSpec] - await this.runBin(pathInfo, args) - return port - } - - override async chainName(pathInfo: PathInfo): Promise { - return pathInfo.target!.replace(/^./, (x) => x.toUpperCase()) + "Dev" - } -} - -type DevRuntimeName = $.Native -const $devRuntimeName = $.literalUnion(["polkadot", "kusama", "westend", "rococo"]) - -const DEV_RUNTIME_PREFIXES: Record = { - polkadot: 0, - kusama: 2, - westend: 42, - rococo: 42, -} diff --git a/providers/frame/project.ts b/providers/frame/project.ts deleted file mode 100644 index 49e18f798..000000000 --- a/providers/frame/project.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Env } from "../../server/mod.ts" -import { PathInfo } from "../../server/PathInfo.ts" -import { getAvailable } from "../../util/port.ts" -import { FrameBinProvider } from "./FrameBinProvider.ts" - -export class ProjectProvider extends FrameBinProvider { - constructor(env: Env) { - super(env, "cargo") - } - - override async getBinPath(): Promise { - return "cargo" - } - - async launch(pathInfo: PathInfo) { - const port = getAvailable() - await this.runBin(pathInfo, ["run", "--release", "--", "--dev", "--ws-port", port.toString()]) - return port - } -} diff --git a/providers/frame/test_users_public_keys.scale b/providers/frame/test_users_public_keys.scale deleted file mode 100644 index 0dd2a655e..000000000 Binary files a/providers/frame/test_users_public_keys.scale and /dev/null differ diff --git a/providers/frame/wss.ts b/providers/frame/wss.ts deleted file mode 100644 index 38ed28865..000000000 --- a/providers/frame/wss.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { PathInfo } from "../../server/mod.ts" -import { FrameProxyProvider } from "./FrameProxyProvider.ts" - -export class WssProvider extends FrameProxyProvider { - async dynamicUrl(pathInfo: PathInfo) { - return this.staticUrl(pathInfo) - } - - override staticUrl(pathInfo: PathInfo) { - return `wss://${pathInfo.target}` - } -} diff --git a/providers/frame/zombienet.ts b/providers/frame/zombienet.ts deleted file mode 100644 index fba5fe429..000000000 --- a/providers/frame/zombienet.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { Network, readNetworkConfig, start } from "../../deps/zombienet.ts" -import { PathInfo } from "../../server/mod.ts" -import { PermanentMemo } from "../../util/mod.ts" -import { FrameProxyProvider } from "./FrameProxyProvider.ts" - -export interface ZombienetProviderProps { - zombienetPath?: string -} - -export class ZombienetProvider extends FrameProxyProvider { - networkMemo = new PermanentMemo() - async dynamicUrl(pathInfo: PathInfo) { - const target = pathInfo.target! - const i = target.lastIndexOf("/") - const configPath = target.slice(0, i) - const network = await this.networkMemo.run(configPath, async () => { - const zombiecache = await Deno.realPath(await Deno.makeTempDir({ prefix: `capi_zombienet_` })) - const config = readNetworkConfig(configPath) - ;(config.settings ??= { provider: "native", timeout: 1200 }).provider = "native" - const options = { - monitor: false, - spawnConcurrency: 1, - dir: zombiecache, - force: true, - inCI: false, - } - const network = await start("", config, options) - this.env.signal.addEventListener("abort", () => { - network.stop() - }) - return network - }) - const nodeName = target.slice(i + 1) - const url = network.nodesByName[nodeName]?.wsUri - if (!url) throw new Error() - return url - } -} diff --git a/providers/mod.ts b/providers/mod.ts deleted file mode 100644 index a8df1580e..000000000 --- a/providers/mod.ts +++ /dev/null @@ -1,3 +0,0 @@ -// moderate - -export * from "./frame/mod.ts" diff --git a/rpc/Connection.ts b/rpc/Connection.ts index d10e94f6d..f8eb2faf4 100644 --- a/rpc/Connection.ts +++ b/rpc/Connection.ts @@ -4,27 +4,24 @@ import { RpcCallMessage, RpcIngressMessage, RpcSubscriptionHandler } from "./rpc const connectionMemos = new Map Connection, Map>() -export interface ConnectionCtorLike { - new(discovery: D): Connection - connect: (discovery: D, signal: AbortSignal) => Connection -} - export abstract class Connection { nextId = 0 references = 0 - signal - #controller + #controller = new AbortController() + signal = this.#controller.signal - constructor() { - this.#controller = new AbortController() - this.signal = this.#controller.signal + static bind( + this: new(discovery: D) => Connection, + discovery: D, + ): (signal: AbortSignal) => Connection { + return (signal) => (Connection.connect).call(this, discovery, signal) } static connect( this: new(discovery: D) => Connection, discovery: D, signal: AbortSignal, - ): Connection { + ) { const memo = getOrInit(connectionMemos, this, () => new Map()) return getOrInit(memo, discovery, () => { const connection = new this(discovery) diff --git a/rpc/ws.ts b/rpc/ws.ts index 9671f544d..1be823d06 100644 --- a/rpc/ws.ts +++ b/rpc/ws.ts @@ -7,11 +7,13 @@ export class WsConnection extends Connection { constructor(readonly url: string) { super() this.ws = new WebSocket(url) - this.ws.addEventListener("message", (e) => this.handle(JSON.parse(e.data))) + this.ws.addEventListener("message", (e) => this.handle(JSON.parse(e.data)), { + signal: this.signal, + }) this.ws.addEventListener("error", (e) => { console.log(e) throw new Error("TODO: more graceful error messaging / recovery") - }) + }, { signal: this.signal }) } async ready() { diff --git a/server/Env.ts b/server/Env.ts deleted file mode 100644 index fc8216650..000000000 --- a/server/Env.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { CacheBase } from "../util/cache/base.ts" -import { Provider } from "./Provider.ts" - -export class Env { - upgradedHref - providers - - constructor( - readonly href: string, - readonly cache: CacheBase, - readonly signal: AbortSignal, - providersFactory: (env: Env) => Record>, - ) { - this.upgradedHref = href.replace(/^http/, "ws") - this.providers = providersFactory(this) - } -} diff --git a/server/PathInfo.test.ts b/server/PathInfo.test.ts deleted file mode 100644 index aadd1bb02..000000000 --- a/server/PathInfo.test.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { assertEquals } from "../deps/std/testing/asserts.ts" -import { fromPathInfo, parsePathInfo, PathInfo } from "./PathInfo.ts" - -const vCapi = "v0.1.0" -const generatorId = "frame" -const providerId = "dev" -const target = "polkadot" -const vRuntime = "v9.36.0" -const filePath = "types/mod.ts" - -testPathInfo( - `/${generatorId}/${providerId}`, - { generatorId, providerId }, -) - -testPathInfo( - `/${generatorId}/${providerId}/${target}`, - { generatorId, providerId, target }, -) - -testPathInfo( - `/${generatorId}/${providerId}/@${vRuntime}`, - { generatorId, providerId, vRuntime }, -) - -testPathInfo( - `/${generatorId}/${providerId}/${target}/@${vRuntime}`, - { generatorId, providerId, target, vRuntime }, -) - -testPathInfo( - `/${generatorId}/${providerId}/@${vRuntime}/${filePath}`, - { generatorId, providerId, vRuntime, filePath }, -) - -testPathInfo( - `/${generatorId}/${providerId}/${target}/@${vRuntime}/${filePath}`, - { generatorId, providerId, target, vRuntime, filePath }, -) - -function testPathInfo(path: string, pathInfo: PathInfo) { - pathInfo = { - vCapi: undefined, - target: undefined, - vRuntime: undefined, - filePath: undefined, - ...pathInfo, - } - Deno.test(path, () => { - const parsed = parsePathInfo(path) - assertEquals(parsed, pathInfo) - assertEquals(fromPathInfo(pathInfo), path) - }) - const capiPath = `/@${vCapi}${path}` - const capiPathInfo = { ...pathInfo, vCapi } - Deno.test(capiPath, () => { - const parsed = parsePathInfo(capiPath) - assertEquals(parsed, capiPathInfo) - assertEquals(fromPathInfo(capiPathInfo), capiPath) - }) -} diff --git a/server/PathInfo.ts b/server/PathInfo.ts deleted file mode 100644 index ff10aa6fa..000000000 --- a/server/PathInfo.ts +++ /dev/null @@ -1,26 +0,0 @@ -export interface PathInfo { - vCapi?: string - generatorId: string - providerId: string - target?: string - vRuntime?: string - filePath?: string -} - -const rPathInfo = - /^\/(@(?.+?)\/)?(?.+?)\/(?.+?)(\/(?.+?))??(\/@(?.+?)(\/(?.*))?)?$/ - -export function parsePathInfo(src: string): PathInfo | undefined { - return rPathInfo.exec(src)?.groups as PathInfo | undefined -} - -export function fromPathInfo( - { vCapi, generatorId, providerId, target, vRuntime, filePath }: PathInfo, -): string { - let src = vCapi ? `/@${vCapi}/` : "/" - src += [generatorId, providerId].join("/") - if (target) src += `/${target}` - if (vRuntime) src += `/@${vRuntime}` - if (filePath) src += `/${filePath}` - return src -} diff --git a/server/Provider.ts b/server/Provider.ts deleted file mode 100644 index 25d012070..000000000 --- a/server/Provider.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { Env } from "./Env.ts" -import { PathInfo } from "./PathInfo.ts" - -export abstract class Provider { - constructor(readonly env: Env) {} - - abstract handle(request: Request, pathInfo: PathInfo): Promise -} diff --git a/server/capi.dev/delegatee.ts b/server/capi.dev/delegatee.ts index 048501be8..acb7a1fe7 100644 --- a/server/capi.dev/delegatee.ts +++ b/server/capi.dev/delegatee.ts @@ -1,26 +1,23 @@ import { serve } from "../../deps/std/http.ts" -import { WssProvider } from "../../providers/frame/wss.ts" import { S3Cache } from "../../util/cache/mod.ts" -import { handler } from "../handler.ts" -import { Env } from "../mod.ts" - -// this is only used by the dev providers, which should target the local server -const href = "http://localhost:4646/" +import { createCodegenHandler } from "../codegenHandler.ts" +import { createErrorHandler } from "../errorHandler.ts" const controller = new AbortController() const { signal } = controller -const cache = new S3Cache(Deno.env.get("DENO_DEPLOYMENT_ID")!, { +const dataCache = new S3Cache("", { accessKeyID: Deno.env.get("S3_ACCESS_KEY")!, secretKey: Deno.env.get("S3_SECRET_KEY")!, region: Deno.env.get("S3_REGION")!, - bucket: Deno.env.get("S3_BUCKET")!, + bucket: Deno.env.get("S3_BUCKET_DATA")!, }, signal) -const env = new Env(href, cache, signal, (env) => ({ - frame: { - wss: new WssProvider(env), - }, -})) +const generatedCache = new S3Cache(Deno.env.get("DENO_DEPLOYMENT_ID")! + "/", { + accessKeyID: Deno.env.get("S3_ACCESS_KEY")!, + secretKey: Deno.env.get("S3_SECRET_KEY")!, + region: Deno.env.get("S3_REGION")!, + bucket: Deno.env.get("S3_BUCKET_TEMP")!, +}, signal) -serve(handler(env)) +serve(createErrorHandler(createCodegenHandler(dataCache, generatedCache))) diff --git a/server/capi.dev/delegator.ts b/server/capi.dev/delegator.ts index 64b2ed64b..f4d953c6c 100644 --- a/server/capi.dev/delegator.ts +++ b/server/capi.dev/delegator.ts @@ -1,8 +1,13 @@ import { serve } from "../../deps/std/http.ts" import { TimedMemo } from "../../util/memo.ts" -import { f, handleCors, handleErrors } from "../mod.ts" +import { createCorsHandler, createErrorHandler, f } from "../mod.ts" -serve(handleCors(handleErrors(handler))) +if (import.meta.main) { + serve(createCorsHandler(createErrorHandler(handler))) +} + +const githubToken = Deno.env.get("GITHUB_TOKEN") +if (!githubToken) throw new Error("GITHUB_TOKEN not set") const ttl = 60_000 const shaAbbrevLength = 8 @@ -63,18 +68,18 @@ async function getSha(version: string): Promise { const deploymentMemo = new TimedMemo(ttl, signal) async function getDeployment(sha: string) { return await deploymentMemo.run(sha, async () => { - const deployments = await github(`deployments?sha=${sha}`) - const deployment = deployments.find((x) => x.payload.project_id === delegateeProjectId) - if (!deployment) throw f.notFound() - const statuses = await github(deployment.statuses_url) - const url = statuses.map((x) => x.environment_url).find((x) => x) + const url = await _getDeploymentUrl(sha) if (!url) throw f.notFound() return url }) } async function github(url: string): Promise { - const response = await fetch(new URL(url, githubApiBase)) + const response = await fetch(new URL(url, githubApiBase), { + headers: { + Authorization: `token ${githubToken}`, + }, + }) if (!response.ok) throw new Error(`${url}: invalid response`) return await response.json() } @@ -95,3 +100,12 @@ interface GithubRelease { interface GithubCommit { sha: string } + +export async function _getDeploymentUrl(sha: string) { + const deployments = await github(`deployments?sha=${sha}`) + const deployment = deployments.find((x) => x.payload.project_id === delegateeProjectId) + if (!deployment) return + const statuses = await github(deployment.statuses_url) + const url = statuses.map((x) => x.environment_url).find((x) => x) + return url +} diff --git a/server/codegenHandler.ts b/server/codegenHandler.ts new file mode 100644 index 000000000..621ebe0ae --- /dev/null +++ b/server/codegenHandler.ts @@ -0,0 +1,166 @@ +import { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts" +import { FrameCodegen } from "../codegen/FrameCodegen.ts" +import { blake2_512, blake2_64 } from "../crypto/hashers.ts" +import { hex } from "../crypto/mod.ts" +import { posix as path } from "../deps/std/path.ts" +import { decodeMetadata } from "../frame_metadata/decodeMetadata.ts" +import { $metadata } from "../frame_metadata/raw/v14.ts" +import { CacheBase } from "../util/cache/base.ts" +import { WeakMemo } from "../util/memo.ts" +import { tsFormatter } from "../util/tsFormatter.ts" +import { $codegenSpec, CodegenEntry } from "./codegenSpec.ts" +import * as f from "./factories.ts" + +const { relative } = path + +const rCodegenUrl = /^\/([\da-f]{16})(\/.+)?$/ +const rUploadUrl = /^\/upload\/(codegen\/[\da-f]{16}|metadata\/[\da-f]{128})$/ + +const codeTtl = 60_000 + +export function createCodegenHandler(dataCache: CacheBase, generatedCache: CacheBase) { + const filesMemo = new WeakMemo>() + return handle + + async function handle(request: Request) { + const url = new URL(request.url) + const { pathname } = url + if (pathname === "/") return await fetch(import.meta.resolve("./static/index.html")) + let match + if ((match = rUploadUrl.exec(pathname))) { + const key = match[1]! + return handleUpload(request, key) + } else if ((match = rCodegenUrl.exec(pathname))) { + const hash = match[1]! + const path = match[2] ?? "/" + return handleCodegen(request, hash, path) + } + if (pathname.startsWith("/capi/")) { + return f.code(generatedCache, request, async () => { + const url = new URL(pathname.slice("/capi/".length), import.meta.resolve("../")) + const response = await fetch(url) + if (!response.ok) throw f.notFound() + return response.text() + }) + } + const response = await fetch(new URL(pathname.slice(1), import.meta.resolve("./static/"))) + if (!response.ok) return f.notFound() + return new Response(response.body, { + headers: { + "Content-Type": mime.getType(pathname) ?? "text/plain", + }, + }) + } + + async function handleUpload(request: Request, key: string) { + const [kind, untrustedHash] = key.split("/") as ["codegen" | "metadata", string] + if (request.method === "HEAD") { + const exists = await dataCache.has(key) + return new Response(null, { status: exists ? 204 : 404 }) + } else if (request.method === "PUT") { + if (await dataCache.has(key)) { + return new Response(null, { status: 204 }) + } + const untrustedData = new Uint8Array(await request.arrayBuffer()) + const hasher = kind === "codegen" ? blake2_64 : blake2_512 + const codec = kind === "codegen" ? $codegenSpec : $metadata + let data: Uint8Array + try { + const value = codec.decode(untrustedData) + codec.assert(value) + data = codec.encode(value as any) + } catch { + return new Response("invalid request body data", { status: 400 }) + } + const hash = hex.encode(hasher.hash(data)) + if (hash !== untrustedHash) { + return new Response("request body does not match provided hash", { status: 400 }) + } + dataCache.getRaw(key, async () => data) + return new Response(null, { status: 204 }) + } else { + return new Response(null, { status: 405 }) + } + } + + async function handleCodegen(request: Request, hash: string, path: string) { + return f.code( + generatedCache, + request, + () => + generatedCache.getString(hash + path, codeTtl, async () => { + const codegenSpec = await dataCache.get(`codegen/${hash}`, $codegenSpec, () => { + throw new Response(`${hash} not found`, { status: 404 }) + }) + + let match: [string, CodegenEntry] | undefined = undefined + for (const [key, value] of codegenSpec.codegen) { + if ( + path.startsWith(`/${key}/`) + && key.length >= (match?.[0].length ?? 0) + ) { + match = [key, value] + } + } + + if (!match) throw f.notFound() + + const [key, entry] = match + + const files = await filesMemo.run(`${hash}/${key}`, async () => { + const metadataHash = hex.encode(entry.metadata) + const metadata = decodeMetadata( + await dataCache.getRaw(`metadata/${metadataHash}`, async () => { + throw new Response(`${hash} not found`, { status: 404 }) + }), + ) + + const codegen = new FrameCodegen(metadata, entry.chainName) + const files = new Map() + codegen.write(files) + const capiCode = `export * from "${relative(`${hash}/${key}/`, "capi/mod.ts")}"` + files.set("capi.js", capiCode) + files.set("capi.d.ts", capiCode) + files.set( + "connection.js", + ` +import * as C from "./capi.js" + +export const connect = C.${entry.connection.type}.bind(${ + JSON.stringify(entry.connection.discovery) + }) + +${ + entry.connection.type === "DevnetConnection" + ? `export const createUsers = C.testUserFactory(${ + JSON.stringify(entry.connection.discovery) + })` + : "" + } +`, + ) + files.set( + "connection.d.ts", + ` +import * as C from "./capi.js" + +export const connect: (signal: AbortSignal) => C.Connection + +${ + entry.connection.type === "DevnetConnection" + ? `export const createUsers: ReturnType` + : "" + } +`, + ) + return files + }) + + const subpath = path.slice(`/${key}/`.length) + + if (!files.has(subpath)) throw f.notFound() + return tsFormatter.formatText(path, files.get(subpath)!) + }), + ) + } +} diff --git a/server/codegenSpec.ts b/server/codegenSpec.ts new file mode 100644 index 000000000..f68080c56 --- /dev/null +++ b/server/codegenSpec.ts @@ -0,0 +1,25 @@ +import * as $ from "../deps/scale.ts" + +export type CodegenEntry = $.Native +const $codegenEntry = $.taggedUnion("type", [ + $.variant( + "frame", + $.field("metadata", $.sizedUint8Array(64)), + $.field("chainName", $.str), + $.field( + "connection", + $.taggedUnion("type", [ + $.variant("WsConnection", $.field("discovery", $.str)), + $.variant("DevnetConnection", $.field("discovery", $.str)), + ]), + ), + ), +]) + +export type CodegenSpec = $.Native +export const $codegenSpec = $.taggedUnion("type", [ + $.variant( + "v0", + $.field("codegen", $.map($.str, $codegenEntry)), + ), +]) diff --git a/server/corsHandler.ts b/server/corsHandler.ts new file mode 100644 index 000000000..e23d12236 --- /dev/null +++ b/server/corsHandler.ts @@ -0,0 +1,35 @@ +import { Handler, Status } from "../deps/std/http.ts" + +export function createCorsHandler(handler: Handler): Handler { + return async (request, connInfo) => { + const newHeaders = new Headers() + newHeaders.set("Access-Control-Allow-Origin", "*") + newHeaders.set("Access-Control-Allow-Headers", "*") + newHeaders.set("Access-Control-Allow-Methods", "*") + newHeaders.set("Access-Control-Allow-Credentials", "true") + + if (request.method === "OPTIONS") { + return new Response(null, { + headers: newHeaders, + status: Status.NoContent, + }) + } + + const res = await handler(request, connInfo) + + // Deno.upgradeWebSocket response objects cannot be modified + if (res.headers.get("upgrade") !== "websocket") { + for (const [k, v] of res.headers) { + newHeaders.append(k, v) + } + + return new Response(res.body, { + headers: newHeaders, + status: res.status, + statusText: res.statusText, + }) + } + + return res + } +} diff --git a/server/errorHandler.ts b/server/errorHandler.ts new file mode 100644 index 000000000..f7c2226f4 --- /dev/null +++ b/server/errorHandler.ts @@ -0,0 +1,13 @@ +import * as f from "./factories.ts" + +export function createErrorHandler(handler: (request: Request) => Promise) { + return async (request: Request) => { + try { + return await handler(request) + } catch (e) { + if (e instanceof Response) return e.clone() + console.error(e) + return f.serverError(Deno.inspect(e)) + } + } +} diff --git a/server/factories.ts b/server/factories.ts index a8f8721a5..855f9cf20 100644 --- a/server/factories.ts +++ b/server/factories.ts @@ -20,7 +20,7 @@ export async function code(cache: CacheBase, request: Request, genCode: () => Pr headers: path.endsWith(".js") ? { "Content-Type": "application/javascript", - "X-TypeScript-Types": request.url.slice(0, -3) + ".d.ts", + "X-TypeScript-Types": `./${path.split("/").at(-1)!.slice(0, -3)}.d.ts`, } : { "Content-Type": "application/typescript" }, }) @@ -38,6 +38,10 @@ export function notFound() { return new Response("404", { status: Status.NotFound }) } +export function badRequest() { + return new Response("400", { status: Status.BadRequest }) +} + export function serverError(message?: string) { return new Response(message || "500", { status: Status.InternalServerError }) } diff --git a/server/handler.ts b/server/handler.ts deleted file mode 100644 index 43ff05e03..000000000 --- a/server/handler.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { Handler, Status } from "../deps/std/http.ts" -import { Env } from "./Env.ts" -import * as f from "./factories.ts" -import { parsePathInfo } from "./PathInfo.ts" - -export function handler(env: Env): Handler { - return handleCors(handleErrors(async (request) => { - const url = new URL(request.url) - const { pathname } = url - if (pathname === "/") return new Response("capi dev server active") - if (pathname === "/capi_cwd") return new Response(Deno.cwd()) - const pathInfo = parsePathInfo(pathname) - if (pathInfo) { - const { vCapi, providerId, generatorId } = pathInfo - if (vCapi) { - return f.serverError( - "The local Capi sever assumes the same version as itself; another cannot be specified.", - ) - } - const provider = env.providers[generatorId]?.[providerId] - if (provider) { - return await provider.handle(request, pathInfo) - } - } - for (const dir of staticDirs) { - try { - const url = new URL(pathname.slice(1), dir) - const res = await fetch(url) - if (!res.ok) continue - if (f.acceptsHtml(request)) { - return f.html(await f.renderCode(await res.text())) - } - return res - } catch (_e) {} - } - return f.notFound() - })) -} - -const staticDirs = ["../", "./static/"].map((p) => import.meta.resolve(p)) - -export function handleErrors(handler: Handler): Handler { - return async (request, connInfo) => { - try { - return await handler(request, connInfo) - } catch (e) { - if (e instanceof Response) return e.clone() - console.error(e) - return f.serverError(Deno.inspect(e)) - } - } -} - -export function handleCors(handler: Handler): Handler { - return async (request, connInfo) => { - const newHeaders = new Headers() - newHeaders.set("Access-Control-Allow-Origin", "*") - newHeaders.set("Access-Control-Allow-Headers", "*") - newHeaders.set("Access-Control-Allow-Methods", "*") - newHeaders.set("Access-Control-Allow-Credentials", "true") - - if (request.method === "OPTIONS") { - return new Response(null, { - headers: newHeaders, - status: Status.NoContent, - }) - } - - const res = await handler(request, connInfo) - - // Deno.upgradeWebSocket response objects cannot be modified - if (res.headers.get("upgrade") !== "websocket") { - for (const [k, v] of res.headers) { - newHeaders.append(k, v) - } - - return new Response(res.body, { - headers: newHeaders, - status: res.status, - statusText: res.statusText, - }) - } - - return res - } -} diff --git a/server/mod.ts b/server/mod.ts index 44fec260d..225b5301b 100644 --- a/server/mod.ts +++ b/server/mod.ts @@ -1,5 +1,8 @@ -export * from "./Env.ts" export * as f from "./factories.ts" -export * from "./handler.ts" -export * from "./PathInfo.ts" -export * from "./Provider.ts" + +// moderate --exclude factories.ts + +export * from "./codegenHandler.ts" +export * from "./codegenSpec.ts" +export * from "./corsHandler.ts" +export * from "./errorHandler.ts" diff --git a/server/static/index.html b/server/static/index.html new file mode 100644 index 000000000..8df8abc4b --- /dev/null +++ b/server/static/index.html @@ -0,0 +1,27 @@ + + + + + + capi.dev + + + + + + + diff --git a/server/static/logo.svg b/server/static/logo.svg new file mode 100644 index 000000000..caaaecba3 --- /dev/null +++ b/server/static/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/util/cache/base.ts b/util/cache/base.ts index e9ba949f9..26e17d6bc 100644 --- a/util/cache/base.ts +++ b/util/cache/base.ts @@ -1,4 +1,5 @@ import * as $ from "../../deps/scale.ts" +import { AsyncMemo } from "../memo.ts" import { getOrInit, TimedMemo, WeakMemo } from "../mod.ts" export abstract class CacheBase { @@ -8,6 +9,13 @@ export abstract class CacheBase { abstract _getRaw(key: string, init: () => Promise): Promise + abstract _has(key: string): Promise + + hasMemo = new AsyncMemo() + has(key: string) { + return this.hasMemo.run(key, () => this._has(key)) + } + rawMemo = new WeakMemo() getRaw(key: string, init: () => Promise): Promise { return this.rawMemo.run(key, () => this._getRaw(key, init)) diff --git a/util/cache/fs.ts b/util/cache/fs.ts index c57eabf93..2d0fd7b2d 100644 --- a/util/cache/fs.ts +++ b/util/cache/fs.ts @@ -7,6 +7,16 @@ export class FsCache extends CacheBase { super(signal) } + async _has(key: string) { + const file = path.join(this.location, key) + try { + await Deno.lstat(file) + return true + } catch { + return false + } + } + async _getRaw(key: string, init: () => Promise) { const file = path.join(this.location, key) try { diff --git a/util/cache/memory.ts b/util/cache/memory.ts index e35b8b0c1..8cacf0da2 100644 --- a/util/cache/memory.ts +++ b/util/cache/memory.ts @@ -6,4 +6,8 @@ export class InMemoryCache extends CacheBase { _getRaw(key: string, init: () => Promise): Promise { return Promise.resolve(this.memo.run(key, init)) } + + async _has(key: string): Promise { + return this.memo.done.has(key) + } } diff --git a/util/cache/s3.ts b/util/cache/s3.ts index e4f6d8cf4..f2df29d3d 100644 --- a/util/cache/s3.ts +++ b/util/cache/s3.ts @@ -19,4 +19,8 @@ export class S3Cache extends CacheBase { await this.bucket.putObject(key, value) return value } + + async _has(key: string): Promise { + return !!(await this.bucket.headObject(key)) + } } diff --git a/util/mod.ts b/util/mod.ts index 107674bbf..fedc98233 100644 --- a/util/mod.ts +++ b/util/mod.ts @@ -2,6 +2,7 @@ export * from "./ArrayOfLength.ts" export * from "./clock.ts" +export * from "./compression.ts" export * from "./error.ts" export * from "./key.ts" export * from "./memo.ts" diff --git a/util/normalize.ts b/util/normalize.ts index 764015dcd..a9b4f2829 100644 --- a/util/normalize.ts +++ b/util/normalize.ts @@ -6,3 +6,11 @@ export function normalizeIdent(ident: string) { export function normalizeDocs(docs: string[] | undefined): string { return docs?.join("\n") ?? "" } + +export function normalizePackageName(name: string) { + return name.replace(/[A-Z]/g, (x) => `-` + x.toLowerCase()) +} + +export function normalizeTypeName(name: string) { + return normalizeIdent(name).replace(/^./, (x) => x.toUpperCase()) +} diff --git a/util/port.ts b/util/port.ts index 487dcf720..e5a4016b9 100644 --- a/util/port.ts +++ b/util/port.ts @@ -1,11 +1,11 @@ -export function getAvailable(): number { +export function getFreePort(): number { const tmp = Deno.listen({ port: 0 }) const { port } = tmp.addr as Deno.NetAddr tmp.close() return port } -export async function ready(port: number): Promise { +export async function portReady(port: number): Promise { while (true) { try { const connection = await Deno.connect({ port }) diff --git a/util/withSignal.ts b/util/withSignal.ts index 0b54dfd70..93d21cb07 100644 --- a/util/withSignal.ts +++ b/util/withSignal.ts @@ -1,5 +1,13 @@ -export async function withSignal(cb: (signal: AbortSignal) => Promise) { +export async function withSignal( + cb: (signal: AbortSignal) => Promise, + outerSignal?: AbortSignal, +) { const controller = new AbortController() + if (outerSignal) { + outerSignal.addEventListener("abort", () => { + controller.abort() + }, { signal: controller.signal }) + } try { return await cb(controller.signal) } finally { diff --git a/words.txt b/words.txt index 5d4d4ce7c..a7970337b 100644 --- a/words.txt +++ b/words.txt @@ -1,10 +1,15 @@ -acala alexa +amannn +autoremove aventus +azuretools backronym bootnode +bootnodes +bungcip callables capi +capn chainspec chelios chev @@ -12,25 +17,31 @@ childstate codegen concat deno +denoland +devcontainers +devnet +devnets dispatchable dprint egdoc +esbenp extrinsics ferdie ffect finalised framesystem +harrysolovay hasher hashers inherents instanceof instantiator kiera -kusama -lparachain +ltex lxkf matty merkle +morfologik multiaddr multiaddress multiasset @@ -38,18 +49,18 @@ multichain multilocation multisigs multistep +noninteractive offchain parachain parachains paritytech pendings -phala polkadot precommits prevotes +procps relaychain runtimes -rustc ryann serde shiki @@ -64,6 +75,7 @@ sufficients suri timepoint tisix +tjjfvi tpeyg transcoders twox @@ -71,6 +83,5 @@ unfollow unguard unhandle unioned +vadimcn westend -zombiecache -zombienet diff --git a/zombienets/rococo_contracts.toml b/zombienets/rococo_contracts.toml deleted file mode 100644 index aa88412da..000000000 --- a/zombienets/rococo_contracts.toml +++ /dev/null @@ -1,30 +0,0 @@ -[relaychain] -default_command = "`deno task capi bin polkadot v0.9.39-1`" -default_args = ["-lparachain=debug"] -chain = "rococo-local" - -[[relaychain.nodes]] -name = "alice" -validator = true - -[[relaychain.nodes]] -name = "bob" -validator = true - -[[relaychain.nodes]] -name = "charlie" -validator = true - -[[relaychain.nodes]] -name = "dave" -validator = true - -[[parachains]] -id = 1000 -cumulus_based = true -chain = "contracts-rococo-local" - -[parachains.collator] -name = "collator" -command = "`deno task capi bin polkadot-parachain v0.9.380`" -args = ["-lparachain=debug"] diff --git a/zombienets/statemine.toml b/zombienets/statemine.toml deleted file mode 100644 index 0629b687d..000000000 --- a/zombienets/statemine.toml +++ /dev/null @@ -1,30 +0,0 @@ -[relaychain] -default_command = "`deno task capi bin polkadot v0.9.39-1`" -default_args = ["-lparachain=debug"] -chain = "rococo-local" - -[[relaychain.nodes]] -name = "alice" -validator = true - -[[relaychain.nodes]] -name = "bob" -validator = true - -[[relaychain.nodes]] -name = "charlie" -validator = true - -[[relaychain.nodes]] -name = "dave" -validator = true - -[[parachains]] -id = 1000 -cumulus_based = true -chain = "statemine-local" - -[parachains.collator] -name = "collator" -command = "`deno task capi bin polkadot-parachain v0.9.380`" -args = ["-lparachain=debug"]