tokenizing example

rasbt · May 6, 2024 · cddcbc8 · cddcbc8
1 parent 258dcad
commit cddcbc8
Showing 1 changed file with 8 additions and 10 deletions.
diff --git a/ch06/01_main-chapter-code/ch06.ipynb b/ch06/01_main-chapter-code/ch06.ipynb
@@ -172,7 +172,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "File downloaded and saved as sms_spam_collection/SMSSpamCollection.tsv\n"
+      "sms_spam_collection/SMSSpamCollection.tsv already exists. Skipping download and extraction.\n"
      ]
     }
    ],
@@ -557,18 +557,16 @@
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "text/plain": [
-       "[42, 13, 314, 481, 1908, 340, 757]"
-      ]
-     },
-     "execution_count": 9,
-     "metadata": {},
-     "output_type": "execute_result"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[1212, 318, 262, 717, 2420, 3275]\n"
+     ]
     }
    ],
    "source": [
-    "tokenizer.encode(\"K. I will sent it again\")"
+    "token_ids = tokenizer.encode(\"This is the first text message\")\n",
+    "print(token_ids)"
    ]
   },
   {