[FormRecognizer] Fix an issue accessing undefined object (#8748)

* [FormRecognizer] Fix an issue accessing undefined object

when input document to receipt recognition contains blank pages

* The receipt items can be missing in receipt.

also added back confidence for receipt type.

sdk/formrecognizer/ai-form-recognizer/.vscode/launch.json

@@ -12,6 +12,7 @@
         "<node_internals>/**"
       ],
       "program": "${file}",
+      "cwd": "${fileDirname}",
       "outFiles": [
         "${workspaceFolder}/dist/**/*.js"
       ]

sdk/formrecognizer/ai-form-recognizer/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## 1.0.0-preview.3 (Unreleased)
 
+- Blank pages in receipt recognition are now handled properly.
+
 ## 1.0.0-preview.2 (2020-05-06)
 
 - `FormTrainingClient.listModels()` now works correctly on NodeJs v8.

sdk/formrecognizer/ai-form-recognizer/review/ai-form-recognizer.api.md

@@ -580,7 +580,10 @@ export interface USReceiptItem {
 }
 
 // @public (undocumented)
-export type USReceiptType = "unrecognized" | "itemized" | "creditCard" | "gas" | "parking";
+export type USReceiptType = {
+    type: "Unrecognized" | "Itemized" | "CreditCard" | "Gas" | "Parking";
+    confidence?: number;
+};
 
 // @public (undocumented)
 export type ValueTypes = "string" | "date" | "time" | "phoneNumber" | "number" | "integer" | "array" | "object";

sdk/formrecognizer/ai-form-recognizer/samples/javascript/recognizeReceipt.js

@@ -44,7 +44,7 @@ async function main() {
 
   const usReceipt = response.receipts[0];
   console.log("First receipt:");
-  console.log(`Receipt type: ${usReceipt.receiptType}`);
+  console.log(`Receipt type: ${usReceipt.receiptType.type} with confidence ${usReceipt.receiptType.confidence}`);
   console.log(
     `Merchant Name: ${usReceipt.merchantName.value} (confidence: ${usReceipt.merchantName.confidence})`
   );

sdk/formrecognizer/ai-form-recognizer/samples/javascript/recognizeReceiptFromUrl.js

@@ -43,7 +43,7 @@ async function main() {
 
   const usReceipt = response.receipts[0];
   console.log("First receipt:");
-  console.log(`Receipt type: ${usReceipt.receiptType}`);
+  console.log(`Receipt type: ${usReceipt.receiptType.type} with confidence ${usReceipt.receiptType.confidence}`);
   console.log(
     `Merchant Name: ${usReceipt.merchantName.value} (confidence: ${usReceipt.merchantName.confidence})`
   );

sdk/formrecognizer/ai-form-recognizer/samples/typescript/src/recognizeReceipt.ts

@@ -45,7 +45,7 @@ export async function main() {
 
   const usReceipt = response.receipts[0];
   console.log("First receipt:");
-  console.log(`Receipt type: ${usReceipt.receiptType}`);
+  console.log(`Receipt type: ${usReceipt.receiptType.type} with confidence ${usReceipt.receiptType.confidence}`);
   console.log(
     `Merchant Name: ${usReceipt.merchantName.value} (confidence: ${usReceipt.merchantName.confidence})`
   );

sdk/formrecognizer/ai-form-recognizer/samples/typescript/src/recognizeReceiptFromUrl.ts

@@ -39,7 +39,7 @@ export async function main() {
 
   const usReceipt = response.receipts[0];
   console.log("First receipt:");
-  console.log(`Receipt type: ${usReceipt.receiptType}`);
+  console.log(`Receipt type: ${usReceipt.receiptType.type} with confidence ${usReceipt.receiptType.confidence}`);
   console.log(
     `Merchant Name: ${usReceipt.merchantName.value} (confidence: ${usReceipt.merchantName.confidence})`
   );

sdk/formrecognizer/ai-form-recognizer/src/models.ts

@@ -504,7 +504,13 @@ export interface USReceiptItem {
   totalPrice?: FormField;
 }
 
-export type USReceiptType = "unrecognized" | "itemized" | "creditCard" | "gas" | "parking";
+export type USReceiptType = {
+  type: "Unrecognized" | "Itemized" | "CreditCard" | "Gas" | "Parking";
+  /**
+   * Confidence value.
+   */
+  confidence?: number;
+}
 
 /**
  * United States receipt

sdk/formrecognizer/ai-form-recognizer/src/transforms.ts

@@ -397,16 +397,25 @@ function toRecognizedReceipt(result: DocumentResultModel, pages: FormPage[]): Re
   const form = toRecognizedForm(result, pages);
   return {
     recognizedForm: form,
-    locale: undefined
+    locale: undefined // in the future service would return locale info
   };
 }
 
-function toReceiptType(type: FormField): USReceiptType {
-  if (type.valueType === "string" && type.value === "Itemized") {
-    return "itemized";
-  } else {
-    return "unrecognized";
+function toReceiptType(field: FormField): USReceiptType {
+  if (field.valueType === "string") {
+    const stringValue = field.value as string;
+    switch (stringValue) {
+      case "Itemized":
+      case "CreditCard":
+      case "Gas":
+      case "Parking":
+        return  { confidence: field.confidence, type: stringValue };
+      default:
+        return  { confidence: field.confidence, type: "Unrecognized" };
+    }
   }
+
+  throw new Error(`Expect receipt type field to have 'string' type but got ${field.valueType}`);
 }
 
 function toUSReceiptItems(items: ReceiptItemArrayField): USReceiptItem[] {
@@ -470,7 +479,7 @@ function toUSReceipt(receipt: RecognizedReceipt): ReceiptWithLocale {
   return {
     locale: "US",
     recognizedForm: receipt.recognizedForm,
-    items: toUSReceiptItems((form.fields["Items"] as unknown) as ReceiptItemArrayField),
+    items: form.fields["Items"] ? toUSReceiptItems((form.fields["Items"] as unknown) as ReceiptItemArrayField) : [],
     merchantAddress: form.fields["MerchantAddress"],
     merchantName: form.fields["MerchantName"],
     merchantPhoneNumber: form.fields["MerchantPhoneNumber"],
@@ -506,11 +515,17 @@ export function toReceiptResultResponse(
     return common;
   }
 
+  if (!result.analyzeResult) {
+    throw new Error("Expecting valid analyzeResult from the service response")
+  }
+
   const pages = result.analyzeResult!.readResults.map(toFormPage);
   return {
     ...common,
     version: result.analyzeResult!.version,
-    receipts: result.analyzeResult!.documentResults!.map((d) => {
+    receipts: result.analyzeResult!.documentResults!.filter(d => {
+      return !!d.fields
+    }).map((d) => {
       const receipt = toRecognizedReceipt(d, pages);
       return toReceiptWithLocale({ ...receipt, locale: "US" }); // default to US until service returns locale info.
     })

sdk/formrecognizer/ai-form-recognizer/test/node/formrecognizerclient.spec.ts

@@ -140,7 +140,7 @@ describe("FormRecognizerClient NodeJS only", () => {
     const usReceipt = response!.receipts![0];
     assert.equal(usReceipt.recognizedForm.formType, "prebuilt:receipt");
     assert.equal(usReceipt.locale, "US"); // default to "US" for now
-    assert.equal(usReceipt.receiptType, "itemized");
+    assert.equal(usReceipt.receiptType.type, "Itemized");
     assert.equal(usReceipt.locale, "US");
     assert.ok(usReceipt.tax, "Expecting valid 'tax' field");
     assert.equal(usReceipt.tax!.name, "Tax");
@@ -185,4 +185,27 @@ describe("FormRecognizerClient NodeJS only", () => {
     const usReceipt = response!.receipts![0];
     assert.equal(usReceipt.recognizedForm.formType, "prebuilt:receipt");
   });
+
+  it("recognizes multi-page receipt with blank page", async () => {
+    const filePath = path.join(ASSET_PATH, "receipt", "multipage_invoice1.pdf");
+    const stream = fs.createReadStream(filePath);
+
+    const poller = await client.beginRecognizeReceipts(stream, "application/pdf", {
+      includeTextDetails: true
+    });
+    await poller.pollUntilDone();
+    const response = poller.getResult();
+
+    assert.ok(response, "Expect valid response object");
+    assert.equal(response!.status, "succeeded");
+    assert.ok(
+      response!.receipts && response!.receipts.length > 0,
+      `Expect no-empty pages but got ${response!.receipts}`
+    );
+    const usReceipt = response!.receipts![0];
+    assert.equal(usReceipt.recognizedForm.formType, "prebuilt:receipt");
+    assert.equal(usReceipt.locale, "US"); // default to "US" for now
+    assert.equal(usReceipt.receiptType.type, "Itemized");
+    assert.equal(usReceipt.locale, "US");
+  });
 }).timeout(60000);

sdk/formrecognizer/ai-form-recognizer/test/transforms.spec.ts

@@ -14,15 +14,17 @@ import {
   toFormTable,
   toRecognizeFormResultResponse,
   toReceiptResultResponse,
-  toFormModelResponse
+  toFormModelResponse,
+  toRecognizedForm
 } from "../src/transforms";
 import {
   GeneratedClientGetAnalyzeFormResultResponse as GetAnalyzeFormResultResponse,
   GeneratedClientGetAnalyzeReceiptResultResponse as GetAnalyzeReceiptResultResponse,
   GeneratedClientGetCustomModelResponse as GetCustomModelResponse,
   ReadResult as ReadResultModel,
   FieldValue as FieldValueModel,
-  DataTable as DataTableModel
+  DataTable as DataTableModel,
+  DocumentResult as DocumentResultModel
 } from "../src/generated/models";
 import {
   StringFieldValue,
@@ -456,6 +458,19 @@ describe("Transforms", () => {
     assert.equal(transformed.rows[2].cells[0].columnSpan, 2);
   });
 
+  it("toRecognizedForm() should handle empty page", () => {
+    const original: DocumentResultModel = {
+      docType: "prebuilt:receipt",
+      pageRange: [1, 1],
+      fields: {}
+    }
+
+    const transformed = toRecognizedForm(original, formPages);
+
+    assert.ok(transformed, "Expected valid recognized form");
+    assert.deepStrictEqual(transformed.fields, {}, "expected empty fields in recognzied form");
+  });
+
   it("toRecognizeFormResultResponse() converts unsupervised response into recognized forms", () => {
     const original: GetAnalyzeFormResultResponse = JSON.parse(unsupervisedResponseString);
     const transformed = toRecognizeFormResultResponse(original);
@@ -554,7 +569,8 @@ describe("Transforms", () => {
     const receiptResult = toReceiptResultResponse(original);
     const usReceipt = receiptResult.receipts![0];
 
-    assert.equal(usReceipt.receiptType, "itemized");
+    assert.equal(usReceipt.receiptType.confidence, 0.692);
+    assert.equal(usReceipt.receiptType.type, "Itemized");
     assert.equal(usReceipt.locale, "US");
     assert.ok(usReceipt.tax, "Expecting valid 'tax' field");
     assert.equal(usReceipt.tax!.name, "Tax");