notebooks/gcv_ocr.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# GCV OCR\n",
    "\n",
    "Send images to Google Cloud Vision for OCR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GCV Documentation for:\n",
    "* [client libraries](https://cloud.google.com/vision/docs/quickstart-client-libraries#client-libraries-install-python)\n",
    "* [OCR](https://cloud.google.com/vision/docs/ocr)\n",
    "\n",
    "TODO: take a look at whether any of these could be helpful:\n",
    "\n",
    "https://cloud.google.com/vision/docs/reference/rpc/google.cloud.vision.v1#google.cloud.vision.v1.TextDetectionParams\n",
    "https://cloud.google.com/vision/docs/reference/rpc/google.cloud.vision.v1#google.cloud.vision.v1.ImageContext"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### bash commands I've been using:\n",
    "- `ls -1 data/images/20210513 | grep -E '\\.jpg$' | wc -l && ls -lah data/images/20210513/ | grep -E \"_automl\\.json\" | wc -l && ls -lah data/images/20210513/ | grep -E \"_ocr\\.json\" | wc -l`\n",
    "- `rsync -av --exclude=\".*\" ./ /home/ariutta/Dropbox\\ \\(Gladstone\\)/archive/pathway-figure-ocr/`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import hashlib\n",
    "import io\n",
    "import json\n",
    "import os\n",
    "import re\n",
    "import subprocess\n",
    "import sys\n",
    "import warnings\n",
    "from itertools import zip_longest\n",
    "from pathlib import Path, PurePath\n",
    "\n",
    "from wand.image import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "google_application_credentials_path = Path(\n",
    "    Path.home(),\n",
    "    \".credentials/api-project-453052878726-f42cadc718aa.json\",\n",
    ")\n",
    "\n",
    "os.environ[\"GOOGLE_APPLICATION_CREDENTIALS\"] = str(\n",
    "    google_application_credentials_path\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Imports the Google Cloud client library\n",
    "\n",
    "from google.cloud import vision\n",
    "from google.protobuf.json_format import MessageToDict\n",
    "\n",
    "\n",
    "def get_text(input_path, language_hints=[]):\n",
    "\n",
    "    # Instantiates a client\n",
    "    client = vision.ImageAnnotatorClient()\n",
    "\n",
    "    with input_path.open(\"rb\") as f:\n",
    "        content = f.read()\n",
    "\n",
    "    image = vision.Image(content=content)\n",
    "\n",
    "    response = client.text_detection(\n",
    "        image=image,\n",
    "        image_context={\"language_hints\": language_hints},\n",
    "    )\n",
    "\n",
    "    if response.error.message:\n",
    "        raise Exception(\n",
    "            \"{}\\nFor more info on error messages, check: \"\n",
    "            \"https://cloud.google.com/apis/design/errors\".format(\n",
    "                response.error.message\n",
    "            )\n",
    "        )\n",
    "\n",
    "    text_annotations = [MessageToDict(x._pb) for x in response.text_annotations]\n",
    "\n",
    "    return text_annotations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OCR all images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_figure_path_count: 124447\n",
      "Figures OCRed in last run: 0\n"
     ]
    }
   ],
   "source": [
    "target_date = \"20210513\"\n",
    "images_dir = Path(f\"../data/images/{target_date}\")\n",
    "\n",
    "ocred_images_count_path = Path(\n",
    "    f\"../data/images/{target_date}/ocred_images_count.log\"\n",
    ")\n",
    "\n",
    "figure_paths = list()\n",
    "for ext in (\"*.jpg\", \"*.jpeg\", \"*.png\"):\n",
    "    for f in images_dir.rglob(ext):\n",
    "        figure_paths.append(f)\n",
    "total_figure_path_count = len(figure_paths)\n",
    "print(f\"total_figure_path_count: {total_figure_path_count}\")\n",
    "\n",
    "language_hints = []\n",
    "\n",
    "i = 0\n",
    "for figure_path in figure_paths:\n",
    "    ocr_output_path = figure_path.with_name(f\"{figure_path.stem}_ocr.json\")\n",
    "\n",
    "    # don't do the same figure more than once\n",
    "    if ocr_output_path.exists():\n",
    "        continue\n",
    "\n",
    "    automl_output_path = figure_path.with_name(\n",
    "        f\"{figure_path.stem}_automl.json\"\n",
    "    )\n",
    "\n",
    "    # GC AutoML rejected a small number of the JPGs as invalid\n",
    "    if not automl_output_path.exists():\n",
    "        continue\n",
    "\n",
    "    with automl_output_path.open(\"r\") as f:\n",
    "        automl_output = json.load(f)\n",
    "        if automl_output[\"displayName\"] != \"pathway\":\n",
    "            continue\n",
    "\n",
    "    try:\n",
    "        text_annotations = get_text(figure_path, language_hints)\n",
    "\n",
    "        # Specifying utf8 and 'ensure_ascii=False' ensures the output file uses\n",
    "        # Greek characters, not escape encoding like \"\\u...\"\n",
    "        with ocr_output_path.open(\"w\", encoding=\"utf8\") as f:\n",
    "            json.dump(text_annotations, f, ensure_ascii=False)\n",
    "\n",
    "        i += 1\n",
    "        with open(ocred_images_count_path, \"w\") as f:\n",
    "            f.write(f\"{i} of {total_figure_path_count}\\n\")\n",
    "    except:\n",
    "        e = sys.exc_info()[0]\n",
    "        print(f\"failed for {str(figure_path)}\")\n",
    "        print(\"<p>Error: %s</p>\" % e)\n",
    "\n",
    "print(f\"Figures OCRed in last run: {i}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OCR the sample images"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "target_batch = \"20210506sample\"\n",
    "images_dir = Path(f\"../data/images/{target_batch}\")\n",
    "\n",
    "figure_paths = list()\n",
    "for ext in (\"*.jpg\", \"*.jpeg\", \"*.png\"):\n",
    "    for f in images_dir.rglob(ext):\n",
    "        figure_paths.append(f)\n",
    "\n",
    "results = dict()\n",
    "\n",
    "# \"el\" is modern Greek\n",
    "# \"grc\" is ancient greek\n",
    "# \"got\" is Gothic, which maps to \"Latin script model\".\n",
    "# .     I think that means it just tries to recognize Latin script characters.\n",
    "# \"fy\" is western Frisian, mapping to generic Latin script model\n",
    "language_hints_sets = [\n",
    "    # I am calling the following \"unspecified\":\n",
    "    [],\n",
    "    #\n",
    "    [\"en\"],\n",
    "    #\n",
    "    # ==============================================\n",
    "    # the following lines produce identical results:\n",
    "    [\"el\"],\n",
    "    [\"en\", \"el\"],\n",
    "    [\"en\", \"grc\"],\n",
    "    # --------------------------\n",
    "    #\n",
    "    # ==============================================\n",
    "    # the following lines produce identical results:\n",
    "    [\"got\"],\n",
    "    # [\"got\", \"Grek\"],\n",
    "    # --------------------------\n",
    "    #\n",
    "    [\"fy\"],\n",
    "    #\n",
    "    # ==============================================\n",
    "    # the following lines produce identical results:\n",
    "    [\"grc\"],\n",
    "    [\"got\", \"grc\"],\n",
    "    [\"got\", \"el\"],\n",
    "    [\"got\", \"el\", \"grc\"],\n",
    "    # [\"el\", \"grc\"],\n",
    "    # --------------------------\n",
    "    #\n",
    "    [\"en\", \"got\", \"grc\"],\n",
    "    [\"en\", \"got\", \"el\"],\n",
    "]\n",
    "\n",
    "for figure_path in figure_paths:\n",
    "    # figure_path_key = str(figure_path)\n",
    "    figure_path_key = figure_path.name\n",
    "    results[figure_path_key] = dict()\n",
    "    for language_hints in language_hints_sets:\n",
    "        text_annotations = get_text(figure_path, language_hints)\n",
    "\n",
    "        readable_language_hints = language_hints.copy()\n",
    "        if len(readable_language_hints) == 0:\n",
    "            readable_language_hints.append(\"unspecified\")\n",
    "\n",
    "        results[figure_path_key][\n",
    "            tuple(readable_language_hints)\n",
    "        ] = text_annotations\n",
    "\n",
    "        # Specifying utf8 and 'ensure_ascii=False' ensures the output file uses\n",
    "        # Greek characters, not escape encoding like \"\\u...\"\n",
    "        with figure_path.with_name(\n",
    "            \"_\".join([figure_path.stem] + readable_language_hints)\n",
    "        ).with_suffix(\".json\").open(\"w\", encoding=\"utf8\") as f:\n",
    "            json.dump(text_annotations, f, ensure_ascii=False)\n",
    "\n",
    "        description = text_annotations[0][\"description\"]\n",
    "        with figure_path.with_name(\n",
    "            \"_\".join([figure_path.stem] + readable_language_hints)\n",
    "        ).with_suffix(\".txt\").open(\"w\", encoding=\"utf8\") as f:\n",
    "            f.write(repr(description))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3 - mypkgs",
   "language": "python",
   "name": "ipython_mypkgs"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}