chore(backend): update OpenAPI specs (#799)

* chore(backend): update OpenAPI specs

* chore(localenv): add root level fetch-schemas command

Restructure local development / localenv instructions.

* chore(open-payments): update OpenAPI specs commit

* chore(backend): improve OpenAPI loading error message

* chore(auth): rename resource-server.yaml as token-introspection.yaml

* chore(open-payments): update OpenAPI specs commit

.github/workflows/lint_test_build.yml

@@ -26,6 +26,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: ./.github/workflows/rafiki/env-setup
       - run: pnpm --filter backend build:deps
+      - run: pnpm --filter backend fetch-schemas
       - run: pnpm --filter backend test
 
   frontend:
@@ -53,7 +54,7 @@ jobs:
           echo "{\"extends\":[\"spectral:oas\",\"spectral:asyncapi\"]}" >> .spectral.json
       - name: Validate Open API specs
         run: |
-          npx @stoplight/spectral-cli lint ./packages/auth/openapi/id-provider.yaml ./packages/auth/openapi/resource-server.yaml
+          npx @stoplight/spectral-cli lint ./packages/auth/openapi/*.yaml
 
   openapi:
     runs-on: ubuntu-latest

README.md

@@ -68,6 +68,13 @@ pnpm clean
 # install dependencies
 pnpm i
 
+# pull in latest openapi specs:
+pnpm fetch-schemas
+```
+
+### Local Development
+
+```sh
 # build all the packages in the repo:
 pnpm -r build
 # build specific package (backend):
@@ -81,9 +88,6 @@ pnpm --filter open-api test
 # run all tests
 pnpm -r --workspace-concurrency=1 test
 
-# pull in latest openapi specs for auth server:
-pnpm --filter auth fetch-schemas
-
 # format and lint code:
 pnpm format
 
@@ -97,29 +101,5 @@ pnpm check:format
 pnpm check:lint
 ```
 
-### Local Development
-
 The [infrastructure/local](infrastructure/local) directory contains resources for setting up Rafiki in
 common configurations.
-
-```sh
-# set up two instances of Rafiki
-pnpm localenv up -d
-
-# seed the postgres databases with the auth data creating an admin token
-pnpm localenv:seed:auth
-
-# tear down
-pnpm localenv down
-
-# delete database volumes (containers must be removed first with e.g. pnpm localenv down)
-pnpm localenv:dbvolumes:remove
-```
-
-The local environment consists of a primary Rafiki instance and a secondary Rafiki instance, each with
-its own docker compose files ([primary](infrastructure/local/docker-compose.yml), [secondary](infrastructure/local/peer-docker-compose.yml)).
-The primary `docker-compose.yml` includes the main Rafiki services `backend`, `auth`, and `rates`, as well
-as the required data stores tigerbeetle, redis, and postgres, so it can be run on its own.
-The `peer-docker-compose.yml` includes only the Rafiki services, not the data stores. It uses the
-data stores created by the primary Rafiki instance so it can't be run by itself.
-The `pnpm run localenv` command starts both the primary instance and the secondary.

infrastructure/local/README.md

@@ -5,6 +5,7 @@ admin GraphQL.
 
 Prerequisites:
 
+- [Rafiki local environment setup](../../README.md#environment-setup)
 - [docker](https://docs.docker.com/get-docker/)
 - [compose plugin](https://docs.docker.com/compose/install/compose-plugin/)
 - [postman](https://www.postman.com/downloads/)
@@ -21,8 +22,22 @@ pnpm localenv up -d --build
 
 // Seed auth tokens
 pnpm localenv:seed:auth
+
+// tear down
+pnpm localenv down
+
+// delete database volumes (containers must be removed first with e.g. pnpm localenv down)
+pnpm localenv:dbvolumes:remove
 ```
 
+The local environment consists of a primary Rafiki instance and a secondary Rafiki instance, each with
+its own docker compose files ([primary](./docker-compose.yml), [secondary](./peer-docker-compose.yml)).
+The primary `docker-compose.yml` includes the main Rafiki services `backend`, `auth`, and `rates`, as well
+as the required data stores tigerbeetle, redis, and postgres, so it can be run on its own.
+The `peer-docker-compose.yml` includes only the Rafiki services, not the data stores. It uses the
+data stores created by the primary Rafiki instance so it can't be run by itself.
+The `pnpm run localenv` command starts both the primary instance and the secondary.
+
 ## P2P payment
 
 This will demonstrate a P2P payment from Grace Franklin (Fynbos account) to Philip Fry (local bank account) using

package.json

@@ -17,6 +17,7 @@
     "check:prettier": "prettier --check .",
     "clean": "find . -name 'node_modules' -type d -prune -exec rm -rf '{}' +",
     "build": "tsc --build",
+    "fetch-schemas": "pnpm --filter auth --filter backend fetch-schemas",
     "localenv": "docker compose -f ./infrastructure/local/docker-compose.yml -f ./infrastructure/local/peer-docker-compose.yml",
     "localenv:build": "docker compose -f ./infrastructure/local/docker-compose.yml -f ./infrastructure/local/peer-docker-compose.yml -f ./infrastructure/local/build-override.yml build",
     "localenv:seed:auth": "pnpm -C ./packages/auth knex seed:run --env=development && pnpm -C ./packages/auth knex seed:run --env=peerdevelopment",

packages/auth/scripts/get-op-schema.sh

@@ -9,5 +9,5 @@ while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
 done
 OUTDIR=$( cd -P "$( dirname "$SOURCE" )/../src/openapi" >/dev/null 2>&1 && pwd )
 
-curl -o "$OUTDIR/schemas.yaml" https://raw.githubusercontent.com/interledger/open-payments/b363d33038fe789e5388f04f80ddd06a4fa97093/openapi/schemas.yaml
-curl -o "$OUTDIR/auth-server.yaml" https://raw.githubusercontent.com/interledger/open-payments/b363d33038fe789e5388f04f80ddd06a4fa97093/openapi/auth-server.yaml
+curl -o "$OUTDIR/schemas.yaml" https://raw.githubusercontent.com/interledger/open-payments/ae0e924a56fe9f12834f65da109a1042760c1605/openapi/schemas.yaml
+curl -o "$OUTDIR/auth-server.yaml" https://raw.githubusercontent.com/interledger/open-payments/ae0e924a56fe9f12834f65da109a1042760c1605/openapi/auth-server.yaml

packages/auth/src/accessToken/routes.test.ts

@@ -110,7 +110,7 @@ describe('Access Token Routes', (): void => {
       })
 
       const openApi = await deps.use('openApi')
-      jestOpenAPI(openApi.resourceServerSpec)
+      jestOpenAPI(openApi.tokenIntrospectionSpec)
     })
     test('Cannot introspect fake token', async (): Promise<void> => {
       const ctx = createContext(

packages/auth/src/app.ts

@@ -248,7 +248,7 @@ export class App {
     // Token Introspection
     this.publicRouter.post(
       '/introspect',
-      createValidatorMiddleware(openApi.resourceServerSpec, {
+      createValidatorMiddleware(openApi.tokenIntrospectionSpec, {
         path: '/introspect',
         method: HttpMethod.POST
       }),

packages/auth/src/index.ts

@@ -111,20 +111,28 @@ export function initIocContainer(
     })
   })
 
-  container.singleton('openApi', async () => {
-    const authServerSpec = await createOpenAPI(
-      path.resolve(__dirname, './openapi/auth-server.yaml')
-    )
-    const resourceServerSpec = await createOpenAPI(
-      path.resolve(__dirname, './openapi/resource-server.yaml')
-    )
-    const idpSpec = await createOpenAPI(
-      path.resolve(__dirname, './openapi/id-provider.yaml')
-    )
-    return {
-      authServerSpec,
-      resourceServerSpec,
-      idpSpec
+  container.singleton('openApi', async (deps: IocContract<AppServices>) => {
+    try {
+      const authServerSpec = await createOpenAPI(
+        path.resolve(__dirname, './openapi/auth-server.yaml')
+      )
+      const idpSpec = await createOpenAPI(
+        path.resolve(__dirname, './openapi/id-provider.yaml')
+      )
+      const tokenIntrospectionSpec = await createOpenAPI(
+        path.resolve(__dirname, './openapi/token-introspection.yaml')
+      )
+      return {
+        authServerSpec,
+        idpSpec,
+        tokenIntrospectionSpec
+      }
+    } catch (err) {
+      const logger = await deps.use('logger')
+      logger.error({ err }, 'error while loading OpenAPI files')
+      throw new Error(
+        'Could not load OpenAPI files. Did you run `pnpm fetch-schemas`?'
+      )
     }
   })
 

packages/backend/.gitignore

@@ -4,3 +4,5 @@ build
 .env
 src/coverage
 tmp
+/src/openapi/resource-server.yaml
+/src/openapi/schemas.yaml

packages/backend/package.json

@@ -8,6 +8,7 @@
     "build": "pnpm build:deps && tsc --build tsconfig.json && pnpm copy-files",
     "clean": "rm -fr dist/",
     "copy-files": "cp src/graphql/schema.graphql dist/graphql/",
+    "fetch-schemas": "./scripts/get-op-schema.sh",
     "prepack": "pnpm build"
   },
   "devDependencies": {

packages/backend/scripts/get-op-schema.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+## https://stackoverflow.com/questions/59895/how-do-i-get-the-directory-where-a-bash-script-is-located-from-within-the-script
+SOURCE=${BASH_SOURCE[0]}
+while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR=$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )
+  SOURCE=$(readlink "$SOURCE")
+  [[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+OUTDIR=$( cd -P "$( dirname "$SOURCE" )/../src/openapi" >/dev/null 2>&1 && pwd )
+
+curl -o "$OUTDIR/resource-server.yaml" https://raw.githubusercontent.com/interledger/open-payments/ae0e924a56fe9f12834f65da109a1042760c1605/openapi/resource-server.yaml
+curl -o "$OUTDIR/schemas.yaml" https://raw.githubusercontent.com/interledger/open-payments/ae0e924a56fe9f12834f65da109a1042760c1605/openapi/schemas.yaml

packages/backend/src/app.ts

@@ -256,7 +256,7 @@ export class App {
     )
     const quoteRoutes = await this.container.use('quoteRoutes')
     const connectionRoutes = await this.container.use('connectionRoutes')
-    const openApi = await this.container.use('openApi')
+    const { resourceServerSpec } = await this.container.use('openApi')
     const toRouterPath = (path: string): string =>
       path.replace(/{/g, ':').replace(/}/g, '')
 
@@ -290,8 +290,8 @@ export class App {
       [AccessAction.ListAll]: 'list'
     }
 
-    for (const path in openApi.paths) {
-      for (const method in openApi.paths[path]) {
+    for (const path in resourceServerSpec.paths) {
+      for (const method in resourceServerSpec.paths[path]) {
         if (isHttpMethod(method)) {
           const action = toAction({ path, method })
           if (!action) {
@@ -314,10 +314,13 @@ export class App {
               route = connectionRoutes.get
               router[method](
                 toRouterPath(path),
-                createValidatorMiddleware<ContextType<typeof route>>(openApi, {
-                  path,
-                  method
-                }),
+                createValidatorMiddleware<ContextType<typeof route>>(
+                  resourceServerSpec,
+                  {
+                    path,
+                    method
+                  }
+                ),
                 route
               )
             } else if (path !== '/' || method !== HttpMethod.GET) {
@@ -329,10 +332,13 @@ export class App {
           router[method](
             PAYMENT_POINTER_PATH + toRouterPath(path),
             createPaymentPointerMiddleware(),
-            createValidatorMiddleware<ContextType<typeof route>>(openApi, {
-              path,
-              method
-            }),
+            createValidatorMiddleware<ContextType<typeof route>>(
+              resourceServerSpec,
+              {
+                path,
+                method
+              }
+            ),
             createAuthMiddleware({
               type,
               action
@@ -348,7 +354,7 @@ export class App {
     router.get(
       PAYMENT_POINTER_PATH + '/jwks.json',
       createPaymentPointerMiddleware(),
-      createValidatorMiddleware<PaymentPointerContext>(openApi, {
+      createValidatorMiddleware<PaymentPointerContext>(resourceServerSpec, {
         path: '/jwks.json',
         method: HttpMethod.GET
       }),
@@ -363,7 +369,7 @@ export class App {
     router.get(
       PAYMENT_POINTER_PATH,
       createPaymentPointerMiddleware(),
-      createValidatorMiddleware<PaymentPointerContext>(openApi, {
+      createValidatorMiddleware<PaymentPointerContext>(resourceServerSpec, {
         path: '/',
         method: HttpMethod.GET
       }),

packages/backend/src/config/app.ts

@@ -109,14 +109,6 @@ export const Config = {
   signatureVersion: envInt('SIGNATURE_VERSION', 1),
   bypassSignatureValidation: envBool('BYPASS_SIGNATURE_VALIDATION', false),
 
-  openPaymentsSpec: envString(
-    'OPEN_PAYMENTS_SPEC',
-    'https://raw.githubusercontent.com/interledger/open-payments/f365dbec4b9dec98b9f622bc49a92aea9ee01568/openapi/RS/openapi.yaml'
-  ),
-  authServerSpec: envString(
-    'AUTH_SERVER_SPEC',
-    'https://raw.githubusercontent.com/interledger/open-payments/77462cd0872be8d0fa487a4b233defe2897a7ee4/auth-server-open-api-spec.yaml'
-  ),
   keyId: envString('KEY_ID', 'rafiki'),
   privateKey: parseOrProvisionKey(envString('PRIVATE_KEY_FILE', undefined)),
 

packages/backend/src/index.ts

@@ -1,5 +1,6 @@
 import { EventEmitter } from 'events'
 import { Server } from 'http'
+import path from 'path'
 import createLogger from 'pino'
 import { knex } from 'knex'
 import { Model } from 'objection'
@@ -109,13 +110,25 @@ export function initIocContainer(
       replica_addresses: config.tigerbeetleReplicaAddresses
     })
   })
-  container.singleton('openApi', async (deps) => {
-    const config = await deps.use('config')
-    return await createOpenAPI(config.openPaymentsSpec)
-  })
-  container.singleton('authOpenApi', async (deps) => {
-    const config = await deps.use('config')
-    return await createOpenAPI(config.authServerSpec)
+  container.singleton('openApi', async (deps: IocContract<AppServices>) => {
+    try {
+      const tokenIntrospectionSpec = await createOpenAPI(
+        path.resolve(__dirname, './openapi/token-introspection.yaml')
+      )
+      const resourceServerSpec = await createOpenAPI(
+        path.resolve(__dirname, './openapi/resource-server.yaml')
+      )
+      return {
+        resourceServerSpec,
+        tokenIntrospectionSpec
+      }
+    } catch (err) {
+      const logger = await deps.use('logger')
+      logger.error({ err }, 'error while loading OpenAPI files')
+      throw new Error(
+        'Could not load OpenAPI files. Did you run `pnpm fetch-schemas`?'
+      )
+    }
   })
   container.singleton('openPaymentsClient', async (deps) => {
     const config = await deps.use('config')
@@ -170,10 +183,11 @@ export function initIocContainer(
   })
   container.singleton('authService', async (deps) => {
     const config = await deps.use('config')
+    const { tokenIntrospectionSpec } = await deps.use('openApi')
     return await createAuthService({
       logger: await deps.use('logger'),
       authServerIntrospectionUrl: config.authServerIntrospectionUrl,
-      authOpenApi: await deps.use('authOpenApi')
+      tokenIntrospectionSpec
     })
   })
   container.singleton('authServerService', async (deps) => {

packages/backend/src/open_payments/auth/middleware.test.ts

@@ -123,9 +123,9 @@ describe('Auth Middleware', (): void => {
       type: AccessType.IncomingPayment,
       action: AccessAction.Read
     })
-    const authOpenApi = await deps.use('authOpenApi')
+    const { tokenIntrospectionSpec } = await deps.use('openApi')
     requestPath = '/introspect'
-    validateRequest = authOpenApi.createRequestValidator({
+    validateRequest = tokenIntrospectionSpec.createRequestValidator({
       path: requestPath,
       method: HttpMethod.POST
     })

packages/backend/src/open_payments/auth/service.ts

@@ -37,7 +37,7 @@ export interface AuthService {
 
 interface ServiceDependencies {
   authServerIntrospectionUrl: string
-  authOpenApi: OpenAPI
+  tokenIntrospectionSpec: OpenAPI
   logger: Logger
   validateResponse: ResponseValidator<TokenInfoJSON>
 }
@@ -49,7 +49,7 @@ export async function createAuthService(
     service: 'AuthService'
   })
   const validateResponse =
-    deps_.authOpenApi.createResponseValidator<TokenInfoJSON>({
+    deps_.tokenIntrospectionSpec.createResponseValidator<TokenInfoJSON>({
       path: '/introspect',
       method: HttpMethod.POST
     })

packages/backend/src/open_payments/connection/routes.test.ts

@@ -32,7 +32,8 @@ describe('Connection Routes', (): void => {
     deps = await initIocContainer(config)
     appContainer = await createTestApp(deps)
     knex = await deps.use('knex')
-    jestOpenAPI(await deps.use('openApi'))
+    const { resourceServerSpec } = await deps.use('openApi')
+    jestOpenAPI(resourceServerSpec)
   })
 
   const asset = {

packages/backend/src/open_payments/payment/incoming/routes.test.ts

@@ -34,7 +34,8 @@ describe('Incoming Payment Routes', (): void => {
     deps = await initIocContainer(config)
     appContainer = await createTestApp(deps)
     knex = await deps.use('knex')
-    jestOpenAPI(await deps.use('openApi'))
+    const { resourceServerSpec } = await deps.use('openApi')
+    jestOpenAPI(resourceServerSpec)
   })
 
   const asset = {