SchemaSafe library for OAS all version validation

package.json

@@ -25,19 +25,23 @@
     "typescript": "^4.7.3"
   },
   "dependencies": {
+    "@exodus/schemasafe": "^1.3.0",
+    "ajv": "^8.17.1",
+    "ajv-formats": "^3.0.1",
     "amqplib": "^0.10.0",
     "axios": "^0.26.1",
     "axios-retry": "^4.0.0",
     "config": "^3.3.7",
     "cors": "^2.8.5",
     "dotenv": "^16.4.1",
     "express": "^4.18.1",
-    "express-openapi-validator": "^5.1.6",
+    "express-openapi-validator": "^6.0.0-alpha.3",
     "express-status-monitor": "^1.3.4",
     "ioredis": "^5.0.6",
     "libsodium-wrappers": "^0.7.9",
     "mongodb": "^4.7.0",
     "node-mocks-http": "^1.15.0",
+    "openapi-types": "^12.1.3",
     "request-ip": "^3.3.0",
     "uuid": "^8.3.2",
     "winston": "^3.7.2",

schemas/context_2.0.1.json

@@ -0,0 +1,9 @@
+{
+  "domain": "context.domain",
+  "version": "context.version",
+  "bap_id": "context.bap_id ? context.bap_id : getConfig().app.subscriberId",
+  "bap_uri": "context.bap_uri ? context.bap_uri : getConfig().app.subscriberUri",
+  "location": "context.location",
+  "bpp_id": "context.bpp_id",
+  "bpp_uri": "context.bpp_uri"
+}

src/app.ts

@@ -13,6 +13,7 @@ import { getConfig } from "./utils/config.utils";
 import { GatewayUtils } from "./utils/gateway.utils";
 import logger from "./utils/logger.utils";
 import { OpenApiValidatorMiddleware } from "./middlewares/schemaValidator.middleware";
+import { Validator } from "./middlewares/schemaValidatorAjv.middleware";
 
 const app = Express();
 
@@ -134,7 +135,7 @@ const main = async () => {
       getConfig().app.gateway.mode.toLocaleUpperCase().substring(0, 1) +
       getConfig().app.gateway.mode.toLocaleUpperCase().substring(1)
     );
-    await OpenApiValidatorMiddleware.getInstance().initOpenApiMiddleware();
+    await Validator.getInstance().initialize();
     logger.info('Initialized openapi validator middleware');
   } catch (err) {
     if (err instanceof Exception) {

src/middlewares/schemaValidator.middleware.ts

@@ -8,16 +8,16 @@ import { v4 as uuid_v4 } from "uuid";
 import { Exception, ExceptionType } from "../models/exception.model";
 import { Locals } from "../interfaces/locals.interface";
 import { getConfig } from "../utils/config.utils";
-import { OpenAPIV3 } from "express-openapi-validator/dist/framework/types";
+import { OpenAPIV3 } from 'openapi-types';
 import logger from "../utils/logger.utils";
 import { AppMode } from "../schemas/configs/app.config.schema";
 import { GatewayMode } from "../schemas/configs/gateway.app.config.schema";
 import {
   RequestActions,
   ResponseActions
 } from "../schemas/configs/actions.app.config.schema";
+import { Validator } from './schemaValidatorAjv.middleware';
 
-const protocolServerLevel = `${getConfig().app.mode.toUpperCase()}-${getConfig().app.gateway.mode.toUpperCase()}`;
 const specFolder = 'schemas';
 
 export class OpenApiValidatorMiddleware {
@@ -72,6 +72,7 @@ export class OpenApiValidatorMiddleware {
           logger.info(`Intially cache Not found loadApiSpec file. Loading.... ${file}`);
           const apiSpec = this.getApiSpec(file);
           const requestHandler = OpenApiValidator.middleware({
+            // @ts-ignore
             apiSpec,
             validateRequests: true,
             validateResponses: false,
@@ -116,6 +117,7 @@ export class OpenApiValidatorMiddleware {
           apiSpec,
           count: 1,
           requestHandler: OpenApiValidator.middleware({
+            // @ts-ignore
             apiSpec,
             validateRequests: true,
             validateResponses: false,
@@ -200,6 +202,19 @@ export const schemaErrorHandler = (
   next: NextFunction
 ) => {
   logger.error('OpenApiValidator Error', err);
+  if (
+    getConfig().app.mode === AppMode.bap &&
+    getConfig().app.gateway.mode === GatewayMode.client
+  ){
+    const errorData = new Exception(
+      ExceptionType.OpenApiSchema_ParsingError,
+      `OpenApiValidator Error`,
+      500,
+      err
+    );
+    next(errorData);
+    return
+  }
   if (err instanceof Exception) {
     next(err);
   } else {
@@ -248,7 +263,7 @@ export const openApiValidatorMiddleware = async (
     ? req?.body?.context?.core_version
     : req?.body?.context?.version;
   let specFile = `${specFolder}/core_${version}.yaml`;
-
+  let specFileName = `core_${version}.yaml`;
   if (getConfig().app.useLayer2Config) {
     let doesLayer2ConfigExist = false;
     let layer2ConfigFilename = `${req?.body?.context?.domain}_${version}.yaml`;
@@ -263,7 +278,10 @@ export const openApiValidatorMiddleware = async (
     } catch (error) {
       doesLayer2ConfigExist = false;
     }
-    if (doesLayer2ConfigExist) specFile = `${specFolder}/${layer2ConfigFilename}`;
+    if (doesLayer2ConfigExist) {
+      specFile = `${specFolder}/${layer2ConfigFilename}`;
+      specFileName = layer2ConfigFilename;
+    }
     else {
       if (getConfig().app.mandateLayer2Config) {
         const message = `Layer 2 config file ${layer2ConfigFilename} is not installed and it is marked as required in configuration`
@@ -278,6 +296,13 @@ export const openApiValidatorMiddleware = async (
       }
     }
   }
-  const openApiValidator = OpenApiValidatorMiddleware.getInstance().getOpenApiMiddleware(specFile);
-  walkSubstack([...openApiValidator], req, res, next);
+  const apiSpecYAML = fs.readFileSync(specFile, "utf8");
+  const apiSpec = YAML.parse(apiSpecYAML);
+  const ajvValidatorInstance = Validator.getInstance();
+  const openApiValidator = await ajvValidatorInstance.getValidationMiddleware(specFile, specFileName);
+  (await openApiValidator)(req, res, () => {
+    logger.info('Validation Success');
+    next()
+  });
+
 };

src/middlewares/schemaValidatorAjv.middleware.ts

@@ -0,0 +1,215 @@
+import fs from 'fs';
+import YAML from 'yaml';
+import Ajv, { ValidateFunction } from 'ajv';
+import addFormats from 'ajv-formats';
+import { OpenAPIV3 } from 'openapi-types';
+import $RefParser from '@apidevtools/json-schema-ref-parser';
+import path from "path";
+import logger from '../utils/logger.utils';
+import { NextFunction, Request, Response } from 'express';
+import { Locals } from "../interfaces/locals.interface";
+import { getConfig } from '../utils/config.utils';
+import { parser, validator, ValidatorOptions } from '@exodus/schemasafe';
+const specFolder = 'schemas';
+type SchemaSafeValidator = (data: unknown) => boolean;
+export class Validator {
+    private static instance: Validator;
+    private ajv: Ajv;
+    private static schemaCache: {
+        [keyName: string]: {
+          count: number,
+          requestHandler: Function
+        }
+      } = {};
+    private initialized: boolean = false;
+    private constructor() {
+        this.ajv = new Ajv({ allErrors: true, coerceTypes: true, useDefaults: true, strict: false });
+        addFormats(this.ajv);
+        //this.schemaCache = new Map<string, Function>();
+    }
+
+    public static getInstance(): Validator {
+        if (!Validator.instance) {
+            Validator.instance = new Validator();
+        }
+        return Validator.instance;
+    }
+
+    async initialize() {
+        if (this.initialized) return;
+        console.time('SchemaValidation');
+        await this.compileEachSpecFiles();
+        console.timeEnd('SchemaValidation');
+        this.initialized = true;
+    }
+
+    private getApiSpec(specFile: string): OpenAPIV3.Document {
+        const apiSpecYAML = fs.readFileSync(specFile, "utf8");
+        const apiSpec = YAML.parse(apiSpecYAML);
+        return apiSpec;
+    };
+
+    async compileEachSpecFiles() {
+        const cachedFileLimit: number = getConfig().app?.openAPIValidator?.cachedFileLimit || 7;
+        logger.info(`OpenAPIValidator Cache count ${cachedFileLimit}`);
+        const files = fs.readdirSync(specFolder);
+        const fileNames = files.filter(file => fs.lstatSync(path.join(specFolder, file)).isFile() && (file.endsWith('.yaml') || file.endsWith('.yml')));
+        logger.info(`OpenAPIValidator loaded spec files ${fileNames}`);
+        let count = 0;
+        let i = 0;
+        logger.info(`Total file: ${fileNames.length}`);
+
+        while (true) {
+            if (count == cachedFileLimit || i >= fileNames.length) {
+                break;
+            }
+            const file = `${specFolder}/${fileNames[i]}`;
+
+            const options = {
+                continueOnError: true, // Continue dereferencing despite errors
+            };
+            let dereferencedSpec: any;
+            const spec = this.getApiSpec(file);
+
+            try {
+                dereferencedSpec = await $RefParser.dereference(spec, options) as OpenAPIV3.Document;
+            } catch (error) {
+                console.error('Dereferencing error:', error);
+            }
+
+            try {
+                await this.compileSchemas(dereferencedSpec, fileNames[i]);
+            } catch (error) {
+                logger.error(`Error derefencing doc: ${error}`);
+            }
+            count++;
+
+            i++;
+
+        }
+        logger.info(`Schema cache size: ${Object.keys(Validator.schemaCache).length}`);
+        const cacheStats = Object.entries(Validator.schemaCache).map((cache) => {
+            return {
+              count: cache[1].count,
+              specFile: cache[0]
+            }
+          });
+          console.table(cacheStats);
+
+    }
+
+    async compileSchemas(spec: OpenAPIV3.Document, file: string, schemaPath?: string | null | undefined, schemaMethod?: string | null | undefined) {
+        const regex = /\.(yml|yaml)$/;
+        const fileName = file.split(regex)[0];
+        logger.info(`OpenAPIValidator compile schema fileName:  ${fileName}`);
+
+        for (const path of Object.keys(spec.paths)) {
+            const methods: any = spec.paths[path];
+            if (!schemaPath || schemaPath === path) {
+                for (const method of Object.keys(methods)) {
+                    if (!schemaMethod || schemaMethod === method) {
+                        const operation = methods[method];
+                        const key = `${fileName}-${path}-${method}`;
+
+                        const options: ValidatorOptions = {
+
+                            includeErrors: true, // Include errors in the output
+                            allErrors: true, // Report all validation errors
+                            contentValidation: true, // Validate content based on formats,
+                            //requireSchema: true,
+                            $schemaDefault: 'http://json-schema.org/draft/2020-12/schema', // Specify the schema version
+                        };
+                        if (!Validator.schemaCache[key]) {
+                            try {
+                                const parse = validator(operation.requestBody?.content['application/json']?.schema, options)
+                                Validator.schemaCache[key] = {
+                                    count: 0,
+                                    requestHandler: parse
+                                }
+                                logger.info(`Schema compiled and cached for ${key}`);
+                            } catch (error: any) {
+                                console.error(`Error compiling schema for ${key}: ${error.message}`);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+    }
+
+    deleteEmptyKeys(obj: any) {
+        // Recursively iterate through the object
+        for (const key in obj) {
+          if (obj.hasOwnProperty(key)) {
+            const value = obj[key];
+            if (typeof value === 'object' && value !== null) {
+                this.deleteEmptyKeys(value); 
+
+            }
+            // If the value is undefined or null, delete the key
+            else if (value === undefined || value === null) {
+              delete obj[key];
+            }
+          }
+        }
+        return obj;
+      }
+
+    async getValidationMiddleware(specFile: string, specFileName: string) {
+        return async (req: any, res: any, next: any) => {
+            logger.info(`Spec file:  ${specFile}`);
+            const regex = /\.(yml|yaml)$/;
+            const fileName = specFileName.split(regex)[0];
+            logger.info(`File name: ${specFile}`);
+            const action = `/${req.body.context.action}`;
+            const method = req.method.toLowerCase();
+            const requestKey = `${fileName}-${action}-${method}`;
+            this.deleteEmptyKeys(req.body);
+            const validateKey = Validator.schemaCache[requestKey];
+            if (validateKey) {
+                logger.info(`Schemasafe Validation Cache HIT for ${specFileName}`);
+                const validate: any = validateKey.requestHandler;
+                try {
+                    const validationResult = validate(req.body);
+                    if (!validationResult) {
+                        return res.status(400).json({ error: validate.errors });
+                    }
+                    validateKey.count = validateKey.count ? validateKey.count + 1 : 1;
+                    console.table([{key: requestKey, count: Validator.schemaCache[requestKey].count}]);
+                } catch (error) {
+                    return res.status(400).json({ error: 'Schema Validation Failed' });
+                }
+            } else {
+                const cashedSpec = Object.entries(Validator.schemaCache);
+                let cachedFileLimit: number = getConfig().app?.openAPIValidator?.cachedFileLimit || 7;
+                cachedFileLimit = cachedFileLimit * 20;
+                if (cashedSpec.length >= cachedFileLimit) {
+                    const specWithLeastCount = cashedSpec.reduce((minEntry, currentEntry) => {
+                        return currentEntry[1].count < minEntry[1].count ? currentEntry : minEntry;
+                    }) || cashedSpec[0];
+                    logger.info(`Cache count reached limit. Deleting from cache.... ${specWithLeastCount[0]}`);
+                    delete Validator.schemaCache[specWithLeastCount[0]];
+                }
+                logger.info(`Schemasafe Validation Cache miss for ${specFileName}`);
+                const apiSpecYAML = this.getApiSpec(specFile);
+                const dereferencedSpec = await $RefParser.dereference(apiSpecYAML) as OpenAPIV3.Document;
+
+                try {
+                    await this.compileSchemas(dereferencedSpec, specFileName, action, method);
+                    const validateKey = Validator.schemaCache[requestKey];
+                    const validate: any = validateKey.requestHandler;
+                    const validationResult = validate(req.body);
+                    if (!validationResult) {
+                        return res.status(400).json({ error: validate.errors });
+                    }
+                    validateKey.count = validateKey.count ? validateKey.count + 1 : 1;
+                } catch (error) {
+                    console.error(`Error compiling doc: ${error}`);
+                }
+            }  
+            next();
+        };
+    }
+}
+