feat(evals): add new API for data validation (#1647)

genkit-tools/common/package.json

@@ -26,6 +26,8 @@
     "json-2-csv": "^5.5.1",
     "json-schema": "^0.4.0",
     "terminate": "^2.6.1",
+    "ajv": "^8.12.0",
+    "ajv-formats": "^3.0.1",
     "tsx": "^4.19.2",
     "uuid": "^9.0.1",
     "winston": "^3.11.0",

genkit-tools/common/src/eval/index.ts

@@ -21,6 +21,7 @@ export { InferenceDataset, InferenceDatasetSchema } from '../types/eval';
 export * from './evaluate';
 export * from './exporter';
 export * from './parser';
+export * from './validate';
 
 export function getEvalStore(): EvalStore {
   // TODO: This should provide EvalStore, based on tools config.

genkit-tools/common/src/eval/validate.ts

@@ -0,0 +1,111 @@
+/**
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Ajv, { ErrorObject, JSONSchemaType } from 'ajv';
+import addFormats from 'ajv-formats';
+import { getDatasetStore } from '.';
+import { RuntimeManager } from '../manager';
+import {
+  Action,
+  ErrorDetail,
+  InferenceDatasetSchema,
+  ValidateDataRequest,
+  ValidateDataResponse,
+} from '../types';
+
+// Setup for AJV
+type JSONSchema = JSONSchemaType<any> | any;
+const ajv = new Ajv();
+addFormats(ajv);
+
+/**
+ * Validate given data against a target action. Intended to be used via the
+ * reflection API.
+ */
+export async function validateSchema(
+  manager: RuntimeManager,
+  request: ValidateDataRequest
+): Promise<ValidateDataResponse> {
+  const { dataSource, actionRef } = request;
+  const { datasetId, data } = dataSource;
+  if (!datasetId && !data) {
+    throw new Error(`Either 'data' or 'datasetId' must be provided`);
+  }
+  const targetAction = await getAction(manager, actionRef);
+  const targetSchema = targetAction?.inputSchema;
+  if (!targetAction) {
+    throw new Error(`Could not find matching action for ${actionRef}`);
+  }
+  if (!targetSchema) {
+    return { valid: true };
+  }
+
+  const errorsMap: Record<string, ErrorDetail[]> = {};
+
+  if (datasetId) {
+    const datasetStore = await getDatasetStore();
+    const dataset = await datasetStore.getDataset(datasetId);
+    if (dataset.length === 0) {
+      return { valid: true };
+    }
+    dataset.forEach((sample, index) => {
+      const response = validate(targetSchema, sample.input);
+      if (!response.valid) {
+        errorsMap[sample.testCaseId] = response.errors ?? [];
+      }
+    });
+
+    return Object.keys(errorsMap).length === 0
+      ? { valid: true }
+      : { valid: false, errors: errorsMap };
+  } else {
+    const dataset = InferenceDatasetSchema.parse(data);
+    dataset.forEach((sample, index) => {
+      const response = validate(targetSchema, sample.input);
+      if (!response.valid) {
+        errorsMap[index.toString()] = response.errors ?? [];
+      }
+    });
+    return Object.keys(errorsMap).length === 0
+      ? { valid: true }
+      : { valid: false, errors: errorsMap };
+  }
+}
+
+function validate(
+  jsonSchema: JSONSchema,
+  data: unknown
+): { valid: boolean; errors?: ErrorDetail[] } {
+  const validator = ajv.compile(jsonSchema);
+  const valid = validator(data) as boolean;
+  const errors = validator.errors?.map((e) => e);
+  return { valid, errors: errors?.map(toErrorDetail) };
+}
+
+function toErrorDetail(error: ErrorObject): ErrorDetail {
+  return {
+    path: error.instancePath.substring(1).replace(/\//g, '.') || '(root)',
+    message: error.message!,
+  };
+}
+
+async function getAction(
+  manager: RuntimeManager,
+  actionRef: string
+): Promise<Action | undefined> {
+  const actions = await manager.listActions();
+  return actions[actionRef];
+}

genkit-tools/common/src/server/router.ts

@@ -15,7 +15,12 @@
  */
 import { initTRPC, TRPCError } from '@trpc/server';
 import { z } from 'zod';
-import { getDatasetStore, getEvalStore, runNewEvaluation } from '../eval';
+import {
+  getDatasetStore,
+  getEvalStore,
+  runNewEvaluation,
+  validateSchema,
+} from '../eval';
 import { RuntimeManager } from '../manager/manager';
 import { GenkitToolsError, RuntimeInfo } from '../manager/types';
 import { Action } from '../types/action';
@@ -239,6 +244,15 @@ export const TOOLS_SERVER_ROUTER = (manager: RuntimeManager) =>
         return response;
       }),
 
+    /** Validate given data against a target action schema */
+    validateDatasetSchema: loggedProcedure
+      .input(apis.ValidateDataRequestSchema)
+      .output(apis.ValidateDataResponseSchema)
+      .mutation(async ({ input }) => {
+        const response = await validateSchema(manager, input);
+        return response;
+      }),
+
     /** Send a screen view analytics event */
     sendPageView: t.procedure
       .input(apis.PageViewSchema)

genkit-tools/common/src/types/apis.ts

@@ -150,3 +150,29 @@ export const RunNewEvaluationRequestSchema = z.object({
 export type RunNewEvaluationRequest = z.infer<
   typeof RunNewEvaluationRequestSchema
 >;
+
+export const ValidateDataRequestSchema = z.object({
+  dataSource: z.object({
+    datasetId: z.string().optional(),
+    data: InferenceDatasetSchema.optional(),
+  }),
+  actionRef: z.string(),
+});
+export type ValidateDataRequest = z.infer<typeof ValidateDataRequestSchema>;
+
+export const ErrorDetailSchema = z.object({
+  path: z.string(),
+  message: z.string(),
+});
+export type ErrorDetail = z.infer<typeof ErrorDetailSchema>;
+
+export const ValidateDataResponseSchema = z.object({
+  valid: z.boolean(),
+  errors: z
+    .record(z.string(), z.array(ErrorDetailSchema))
+    .describe(
+      'Errors mapping, if any. The key is testCaseId if source is a dataset, otherewise it is the index number (stringified)'
+    )
+    .optional(),
+});
+export type ValidateDataResponse = z.infer<typeof ValidateDataResponseSchema>;