spacy data structures - spacy dot io CHAPTER 2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "398aaf16",
   "metadata": {},
   "source": [
    "## spans, docs, vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d4970d58",
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy.matcher import PhraseMatcher\n",
    "from spacy.tokens import Doc, Span\n",
    "from spacy import displacy\n",
    "nlp = spacy.blank('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dcda3bb4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "coffee_hash = 3197928453018144401, nlp.vocab.strings[coffee_hash] = 'coffee'\n"
     ]
    }
   ],
   "source": [
    "# the vocabulary of a spaCy model is stored in a lookup table\n",
    "# with a two-way mapping between hashes and strings\n",
    "doc = nlp('I love coffee!!!')\n",
    "coffee_hash = nlp.vocab.strings['coffee']\n",
    "print(f'{coffee_hash = }, {nlp.vocab.strings[coffee_hash] = }')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9e9ea65c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "coffee 3197928453018144401 True\n"
     ]
    }
   ],
   "source": [
    "# a lexeme is an entry in the vocab\n",
    "lexeme = nlp.vocab['coffee']\n",
    "# the orth attribute is the hash\n",
    "print(lexeme.text, lexeme.orth, lexeme.is_alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "027e27b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Good morning USA!\n"
     ]
    }
   ],
   "source": [
    "# The words and spaces to create the doc from\n",
    "words = [\"Good\", \"morning\", \"USA\", \"!\"]\n",
    "spaces = [True, True, False, False]\n",
    "\n",
    "# Create a doc manually\n",
    "doc = Doc(nlp.vocab, words=words, spaces=spaces)\n",
    "print(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "013ebe37",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Good morning\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GREETING</span>\n",
       "</mark>\n",
       " \n",
       "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    USA\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
       "</mark>\n",
       "!</div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "greet_span = Span(doc, 0, 2, label='GREETING')\n",
    "usa_span = Span(doc, 2, 3, label='GPE')\n",
    "doc.ents = [greet_span, usa_span]\n",
    "displacy.render(doc, style='ent')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8e3febd",
   "metadata": {},
   "source": [
    "## word vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d8d53858",
   "metadata": {},
   "outputs": [],
   "source": [
    "# spacy.blank and en_core_web_sm don't have built-in word vectorization\n",
    "nlp = spacy.load('en_core_web_md')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "61604881",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\"I enjoy caffeinated beverages.\" <-> \"I LOVE COFFEE\": 0.8751647980586529\n",
      "\"I enjoy caffeinated beverages.\" <-> \"I hate Coca-Cola ☹\": 0.6316076821778656\n",
      "\"I enjoy caffeinated beverages.\" <-> \"Coca-Cola\": 0.4358014247194589\n",
      "\"I enjoy caffeinated beverages.\" <-> \"☹\": -0.032447879410338464\n",
      "\"I LOVE COFFEE\" <-> \"I hate Coca-Cola ☹\": 0.7047962756090318\n",
      "\"I LOVE COFFEE\" <-> \"Coca-Cola\": 0.3882343083308216\n",
      "\"I LOVE COFFEE\" <-> \"☹\": -0.005725138796304899\n",
      "\"I hate Coca-Cola ☹\" <-> \"Coca-Cola\": 0.7712480345362694\n",
      "\"I hate Coca-Cola ☹\" <-> \"☹\": 0.31047273144090287\n",
      "\"Coca-Cola\" <-> \"☹\": -0.0868011936545372\n",
      "\"angry face\" <-> \"☹\": -0.050709066160061725\n"
     ]
    }
   ],
   "source": [
    "doc1 = nlp('I enjoy caffeinated beverages.')\n",
    "doc2 = nlp('I LOVE COFFEE')\n",
    "doc3 = nlp('I hate Coca-Cola \\u2639')\n",
    "coke_span = Span(doc3, 2, 5, label = 'CONSUMABLE')\n",
    "emoji_span = Span(doc3, 5, 6, label = 'EMOJI')\n",
    "doc3.ents = [coke_span, emoji_span]\n",
    "docs = [doc1, doc2, doc3, coke_span, emoji_span]\n",
    "# spans and individual tokens can be compared to documents\n",
    "# default similarity metric is cosine similarity, but can be adjusted\n",
    "for ii, d1 in enumerate(docs):\n",
    "    for d2 in docs[ii+1:len(docs)]:\n",
    "        print(f'\"{d1.text}\" <-> \"{d2.text}\": {d1.similarity(d2)}')\n",
    "print(('\"angry face\" <-> \"\\u2639\": '\n",
    "      f'{nlp(\"angry face\").similarity(emoji_span)}'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "404a0b69",
   "metadata": {},
   "source": [
    "### PhraseMatcher: match sub-documents in a document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "55a306a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# The phrase matcher can match entire phrases as documents\n",
    "grimdark = [nlp(f'{adj} {noun}') \n",
    "            for adj in ['grim', 'menacing', 'depressing', 'forboding']\n",
    "            for noun in ['darkness', 'shadows', 'obscurity', 'void']\n",
    "]\n",
    "\n",
    "phrase_matcher = PhraseMatcher(nlp.vocab)\n",
    "phrase_matcher.add('GRIM_DARK', grimdark)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d61e9cc5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "grim darkness\n",
      "menacing shadows\n",
      "depressing void\n",
      "forboding obscurity\n"
     ]
    }
   ],
   "source": [
    "doc = nlp('''\n",
    "Chapter 1:\n",
    "In the grim darkness of the far future there is only war.\n",
    "Bojor crept through the menacing shadows, contemplating the \n",
    "depressing void of outer space.\n",
    "Out of the forboding obscurity skulked a Tyranid warrior! \n",
    "''')\n",
    "\n",
    "matches = phrase_matcher(doc)\n",
    "\n",
    "for match_id, start, end in matches:\n",
    "    matched_span = doc[start:end]\n",
    "    print(matched_span.text)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}