From d6595169bba974428899a8fd18f596718aaa5e41 Mon Sep 17 00:00:00 2001 From: Benjamin Goering <171782+gobengo@users.noreply.github.com> Date: Fri, 14 Apr 2023 11:06:19 -0700 Subject: [PATCH] feat: change access-api wrangler.toml to be no_bundle=false. use wrangler bundling (#739) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation: * #623 * verify our assumption that we have to use `no_bundle=true` along with d1. Experiment * I wanted to try some new ways of building and whether they would work, but without risking staging, so I used the 'dev' environment from wrangler.toml to try some deploys from local. * I made access-api use of wrangler more like toucan-js@3 [wrangler-basic](https://github.com/robertcepa/toucan-js/blob/master/examples/wrangler-basic/wrangler.toml) * I did a sample deploy from my laptop to 'dev' workers environment, with the sentry/toucan `release` called `bengo-dev-0` [sentry errors here](https://protocol-labs-it.sentry.io/issues/?query=+release%3Abengo-dev-0&referrer=issue-list&statsPeriod=14d) * trigger error via `GET /.debug/error` * expectation: this to lead to an in-route error and show up in sentry with helpful stack traces - because this is what [example](https://github.com/robertcepa/toucan-js/blob/master/examples/wrangler-basic/wrangler.toml) implies should work * trigger error via route that uses d1 binding (e.g. ucanto routes [triggered via w3up+ucanto observable pointing to dev env](https://observablehq.com/d/a76545064f82a998)) * expectation: d1 error - because supposedly `no_bundle=false` (or omitting `no_bundle`) is incompatible with wrangler d1 bindings Findings: * omg it works! * [error triggered by GET /.debug/error (no d1)](https://protocol-labs-it.sentry.io/issues/4078173859/?query=+release%3Abengo-dev-0&referrer=issue-stream&statsPeriod=14d&stream_index=3) * [error triggered by using d1 via access protocol via observable](https://protocol-labs-it.sentry.io/issues/4078175792/?query=is%3Aunresolved+release%3Abengo-dev-0&referrer=issue-stream&statsPeriod=14d&stream_index=2) * The error did not happen in the `access/authorize` handler. That worked fine, and I got an email from dev env. The error happened when I clicked the email (invoking `access/confirm`), when the `access/confirm` invoacation handler tried to write to d1 * It got a D1_ERROR, but I could see from the sentry report > Error: ERROR 9009: SQL prepare error: no such table: delegations_v3 * which makes sense, because `dev` doesn't have latest migrations applied that created that table. So I provisioned a new d1 database for this dev env, and ran `npx wrangler --env=dev d1 migrations apply __D1_BETA__`. This would error every couple migrations, but if I re-ran it it would make progress and error again (`Internal error [code: 7501]`), but eventually it would succeed at all migrations. * then I re-triggered things via the observable, get email, click email. I saw 'email validated'! 🎉 * I continued using the observable to invoke `access/claim` (reads from D1), and got no error all the way through invoking from second device
Screenshot 2023-04-10 at 5 48 41 PM
--- .github/workflows/reusable-deploy-api.yml | 36 +++++++++++-- packages/access-api/.gitignore | 1 + packages/access-api/scripts/release.js | 52 +++++++++++++++++++ packages/access-api/src/config.js | 21 +++----- packages/access-api/src/utils/context.js | 5 +- .../access-api/src/utils/release.build.js | 9 ++++ packages/access-api/src/utils/release.node.js | 39 ++++++++++++++ packages/access-api/wrangler.toml | 23 ++++---- 8 files changed, 155 insertions(+), 31 deletions(-) create mode 100644 packages/access-api/.gitignore create mode 100644 packages/access-api/scripts/release.js create mode 100644 packages/access-api/src/utils/release.build.js create mode 100644 packages/access-api/src/utils/release.node.js diff --git a/.github/workflows/reusable-deploy-api.yml b/.github/workflows/reusable-deploy-api.yml index bfb52779f..dce2116ac 100644 --- a/.github/workflows/reusable-deploy-api.yml +++ b/.github/workflows/reusable-deploy-api.yml @@ -42,8 +42,23 @@ jobs: with: node-version: 18 cache: 'pnpm' - - run: pnpm install - # Migration database + - run: pnpm --filter '@web3-storage/access-api...' install + # get release name that will be used for sentry + - id: set-release-name + env: + ENV: ${{ inputs.environment }} + run: | + echo "release_name=$(node packages/access-api/scripts/release.js)" >> "$GITHUB_OUTPUT" + - run: echo "release name is ${{ steps.set-release-name.outputs.release_name }}" + # write release info to ./src so it can be imported at runtime (e.g. used by toucan-js for `opts.release`) + # be sure to keep this before wrangler-action bundles+deploys + - name: write release info to src + working-directory: packages/access-api/ + env: + ENV: ${{ vars.ENV }} + run: | + node scripts/release.js esm > src/utils/release.build.js + # Apply D1 Migrations - run: pnpm -r --filter @web3-storage/access-api exec wrangler d1 migrations apply __D1_BETA__ --env ${{ inputs.environment }} env: CLOUDFLARE_API_TOKEN: ${{ secrets.CF_TOKEN }} @@ -53,6 +68,7 @@ jobs: with: # preCommands: git config --global --add safe.directory "*" apiToken: ${{ secrets.CF_TOKEN }} + command: publish --env "${{ vars.ENV }}" --outdir=dist workingDirectory: 'packages/access-api' environment: ${{ inputs.environment }} secrets: | @@ -65,7 +81,19 @@ jobs: POSTMARK_TOKEN: ${{ secrets.POSTMARK_TOKEN }} PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} - SENTRY_UPLOAD: ${{ secrets.SENTRY_UPLOAD }} - SENTRY_TOKEN: ${{ secrets.SENTRY_TOKEN }} LOGTAIL_TOKEN: ${{ secrets.LOGTAIL_TOKEN }} UCAN_LOG_BASIC_AUTH: ${{ secrets.UCAN_LOG_BASIC_AUTH }} + - name: create sentry release + working-directory: packages/access-api + run: | + ls -alh ./dist + # create sentry release + pnpm exec sentry-cli releases new "$RELEASE_NAME" --finalize + # associate wrangler-built src+map to sentry release + pnpm exec sentry-cli releases files "$RELEASE_NAME" upload-sourcemaps ./dist + env: + ENV: ${{ vars.ENV }} + RELEASE_NAME: ${{ steps.set-release-name.outputs.release_name }} + SENTRY_ORG: ${{ vars.SENTRY_ORG }} + SENTRY_PROJECT: ${{ vars.SENTRY_PROJECT }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_TOKEN }} diff --git a/packages/access-api/.gitignore b/packages/access-api/.gitignore new file mode 100644 index 000000000..babca1bb1 --- /dev/null +++ b/packages/access-api/.gitignore @@ -0,0 +1 @@ +.dev.vars diff --git a/packages/access-api/scripts/release.js b/packages/access-api/scripts/release.js new file mode 100644 index 000000000..5880f0cdf --- /dev/null +++ b/packages/access-api/scripts/release.js @@ -0,0 +1,52 @@ +/* eslint-disable unicorn/prefer-top-level-await */ +/* eslint-disable no-console */ +/* eslint-disable unicorn/prefer-module */ +/** + * @file + * use this at build time (node.js) to create files that can be + * imported at runtime (maybe workerd). + */ +import sade from 'sade' +import * as url from 'node:url' +import { getReleaseName } from '../src/utils/release.node.js' +import path from 'path' +import { fileURLToPath } from 'url' +// @ts-ignore +import git from 'git-rev-sync' + +function __dirname() { + return path.dirname(fileURLToPath(import.meta.url)) +} + +if (import.meta.url.startsWith('file:')) { + const modulePath = url.fileURLToPath(import.meta.url) + if (process.argv[1] === modulePath) { + main().catch((error) => { + throw error + }) + } +} + +async function main(argv = process.argv) { + const cli = sade('access-api-release') + cli + .command('name', '', { default: true }) + .option('--env', 'Environment', process.env.ENV) + .action((opts) => { + const releaseName = getReleaseName(opts.env) + console.log(releaseName) + }) + cli + .command('esm', 'print release info as ES Module string') + .option('--env', 'Environment', process.env.ENV) + .action((opts) => { + const lines = [ + `/** @type {string|undefined} */`, + `export const gitRevShort = '${git.short(__dirname())}'`, + `/** @type {string|undefined} */`, + `export const name = '${getReleaseName(opts.env)}'`, + ] + console.log(lines.join('\n')) + }) + cli.parse(argv) +} diff --git a/packages/access-api/src/config.js b/packages/access-api/src/config.js index a623f712d..bef538025 100644 --- a/packages/access-api/src/config.js +++ b/packages/access-api/src/config.js @@ -2,6 +2,7 @@ // eslint-disable-next-line no-unused-vars import * as UCAN from '@ucanto/interface' import { DID } from '@ucanto/core' +import * as release from './utils/release.build.js' /** * Loads configuration variables from the global environment and returns a JS object @@ -14,15 +15,7 @@ export function loadConfig(env) { const vars = {} /** @type {Array} */ - const required = [ - 'DID', - 'ENV', - 'DEBUG', - 'PRIVATE_KEY', - 'SENTRY_DSN', - 'POSTMARK_TOKEN', - 'LOGTAIL_TOKEN', - ] + const required = ['DID', 'ENV', 'DEBUG', 'PRIVATE_KEY'] for (const name of required) { const val = env[name] @@ -47,21 +40,21 @@ export function loadConfig(env) { POSTMARK_TOKEN: vars.POSTMARK_TOKEN, POSTMARK_SENDER: env.POSTMARK_SENDER, - SENTRY_DSN: vars.SENTRY_DSN, - LOGTAIL_TOKEN: vars.LOGTAIL_TOKEN, + SENTRY_DSN: env.SENTRY_DSN, + LOGTAIL_TOKEN: env.LOGTAIL_TOKEN, UCAN_LOG_BASIC_AUTH: env.UCAN_LOG_BASIC_AUTH, UCAN_LOG_URL: env.UCAN_LOG_URL, // These are injected in esbuild // @ts-ignore // eslint-disable-next-line no-undef - BRANCH: ACCOUNT_BRANCH, + BRANCH: env.ACCOUNT_BRANCH ?? '', // @ts-ignore // eslint-disable-next-line no-undef - VERSION: ACCOUNT_VERSION, + VERSION: env.ACCOUNT_VERSION ?? release.name ?? '', // @ts-ignore // eslint-disable-next-line no-undef - COMMITHASH: ACCOUNT_COMMITHASH, + COMMITHASH: env.ACCOUNT_COMMITHASH ?? release.gitRevShort ?? '', PRIVATE_KEY: vars.PRIVATE_KEY, DID: /** @type {UCAN.DID<"web">} */ (DID.parse(vars.DID).did()), diff --git a/packages/access-api/src/utils/context.js b/packages/access-api/src/utils/context.js index 0e38926ea..33562a5f4 100644 --- a/packages/access-api/src/utils/context.js +++ b/packages/access-api/src/utils/context.js @@ -15,6 +15,7 @@ import { } from '../models/delegations.js' import { createD1Database } from './d1.js' import { DbProvisions } from '../models/provisions.js' +import * as release from './release.build.js' /** * Obtains a route context object. @@ -56,14 +57,14 @@ export function getContext(request, env, ctx) { dsn: config.SENTRY_DSN, debug: false, environment: config.ENV, - release: config.VERSION, + release: release.name, }) // Logging const log = new Logging(request, ctx, { token: config.LOGTAIL_TOKEN, debug: config.DEBUG, - sentry: ['test', 'dev'].includes(config.ENV) ? undefined : sentry, + sentry: ['test'].includes(config.ENV) ? undefined : sentry, branch: config.BRANCH, version: config.VERSION, commit: config.COMMITHASH, diff --git a/packages/access-api/src/utils/release.build.js b/packages/access-api/src/utils/release.build.js new file mode 100644 index 000000000..ad64e7814 --- /dev/null +++ b/packages/access-api/src/utils/release.build.js @@ -0,0 +1,9 @@ +/** + * @file + * This file MAY be rewritten during build + * to make some buildtime variables available at runtime. + */ +/** @type {string|undefined} */ +export const gitRevShort = undefined +/** @type {string|undefined} */ +export const name = undefined diff --git a/packages/access-api/src/utils/release.node.js b/packages/access-api/src/utils/release.node.js new file mode 100644 index 000000000..07b875534 --- /dev/null +++ b/packages/access-api/src/utils/release.node.js @@ -0,0 +1,39 @@ +/** + * @file + * utils related to sentry that rely on node. + * These might be used at build time, + * but won't work at run-time in cloudflare workers + */ +/* eslint-disable no-console */ +/* eslint-disable unicorn/prefer-module */ + +// @ts-ignore +import git from 'git-rev-sync' +import path from 'path' +import { fileURLToPath } from 'url' +import { createRequire } from 'node:module' + +const packageJson = createRequire(import.meta.url)('../../package.json') +const __dirname = () => path.dirname(fileURLToPath(import.meta.url)) + +/** + * Create a string to be used for the sentry release value + * + * @param {string} [env] - environment name e.g. 'dev' + * @param {object} pkg - package.json info + * @param {string} pkg.name + * @param {string} pkg.version + * @param {string} gitShort - git-rev-parse short value + * @returns {string} release name e.g. `@web3-storage__access-api@6.0.0-staging+92a89d3` + */ +export function getReleaseName( + env, + pkg = packageJson, + gitShort = git.short(__dirname()) +) { + const version = `${pkg.name}@${pkg.version}-${env}+${gitShort}`.replace( + '/', + '__' + ) + return version +} diff --git a/packages/access-api/wrangler.toml b/packages/access-api/wrangler.toml index 1187a67f8..f443edfda 100644 --- a/packages/access-api/wrangler.toml +++ b/packages/access-api/wrangler.toml @@ -1,15 +1,12 @@ # Development name = "w3access-local" account_id = "fffa4b4363a7e5250af8357087263b3a" -main = "./dist/worker.js" +main = "./src/index.js" # Compatibility flags https://github.com/cloudflare/wrangler/pull/2009 compatibility_date = "2022-09-28" compatibility_flags = ["url_standard"] -# We need to let wrangler bundle while D1 is in Beta -no_bundle = false - [[kv_namespaces]] binding = "SPACES" id = "e9fad7e04b254bf49206e08e50074387" @@ -28,15 +25,16 @@ database_id = "7c676e0c-b9e7-4711-97c8-7b1c8eb229ae" [[r2_buckets]] binding = "DELEGATIONS_BUCKET" bucket_name = "w3up-delegations-dev-0" +preview_bucket_name = "w3up-delegations-dev-0" [vars] ENV = "dev" DEBUG = "true" DID = "did:web:local.web3.storage" +PRIVATE_KEY="MgCYWjE6vp0cn3amPan2xPO+f6EZ3I+KwuN1w2vx57vpJ9O0Bn4ci4jn8itwc121ujm7lDHkCW24LuKfZwIdmsifVysY=" UPLOAD_API_URL = "https://up.web3.storage" [build] -command = "scripts/cli.js build" watch_dir = "src" [miniflare] @@ -47,24 +45,28 @@ d1_persist = ".wrangler/miniflare" name = "w3access-dev" workers_dev = true vars = { ENV = "dev", DEBUG = "false", DID = "did:web:dev.web3.storage", UPLOAD_API_URL = "https://staging.up.web3.storage" } -build = { command = "scripts/cli.js build --env dev", watch_dir = "src" } kv_namespaces = [ { binding = "SPACES", id = "5697e95e1aaa436788e6d697fd3350be" }, { binding = "VALIDATIONS", id = "ea17f472b37a43d29c1faf7af9512e03" }, ] -d1_databases = [ - { binding = "__D1_BETA__", database_name = "access-dev", database_id = "4145a261-e54c-411d-a001-050fc30e4678" }, -] unsafe = { bindings = [ { type = "analytics_engine", dataset = "W3ACCESS_METRICS", name = "W3ACCESS_METRICS" }, ] } +[[env.dev.d1_databases]] +binding = "__D1_BETA__" +database_name = "access-dev" +database_id = "7f5c4ec7-610b-4885-b9f7-0886ce0639f6" + +[[env.dev.r2_buckets]] +binding = "DELEGATIONS_BUCKET" +bucket_name = "w3up-delegations-dev-0" +preview_bucket_name = "w3up-delegations-dev-0" # Staging [env.staging] name = "w3access-staging" workers_dev = true -build = { command = "scripts/cli.js build --env staging", watch_dir = "src" } kv_namespaces = [ { binding = "SPACES", id = "b0e5ca990dda4e3784a1741dfa28a52e" }, { binding = "VALIDATIONS", id = "b13f07c88fe848db9ccf651a0fea3fb6" }, @@ -90,7 +92,6 @@ bucket_name = "w3up-delegations-staging-0" [env.production] name = "w3access" routes = [{ pattern = "access.web3.storage", custom_domain = true }] -build = { command = "scripts/cli.js build --env production", watch_dir = "src" } kv_namespaces = [ { binding = "SPACES", id = "5437954e8cfd4f7d98557132b0a2e93f" }, { binding = "VALIDATIONS", id = "fb7cf10c725f45948321e88b8cb168ad" },