cleaning_ARABIC_2.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "8f173bca",
      "metadata": {
        "id": "8f173bca"
      },
      "source": [
        "# Here is cleaning the Arabic texts for all books!!! ( please follow steps)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "id": "5e020991",
      "metadata": {
        "id": "5e020991"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd \n",
        "#Language\n",
        "LANGUAGE = \"arb\"\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "id": "1af197fa",
      "metadata": {
        "id": "1af197fa"
      },
      "outputs": [],
      "source": [
        "df = pd.read_csv('auther10fix.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "id": "aa07b583",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "aa07b583",
        "outputId": "445588a8-7f1f-4204-eed2-9f1a8fbe199c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Txt</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>شمس الدين ابن مفلح</td>\n",
              "      <td>\\nالكتاب : الآداب الشرعية\\n\\n[ الكتاب مشكول وم...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>الماوردي</td>\n",
              "      <td>\\nالكتاب : الأحكام السلطانية\\nالمؤلف : الماورد...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>عبدالملك بن هشام</td>\n",
              "      <td>مَبْعَثُ النّبِيّ صَلّى اللّهُ عَلَيْهِ وَعَلَ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ابن القيم</td>\n",
              "      <td>بِسْمِ اللَّهِ الرَّحْمَنِ الرَّحِيمِ الْحَمْد...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>محمد مهدي قشلان</td>\n",
              "      <td>الصحيفة السجَّادية\\nللإمام علي بن الحسين السجّ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 Name                                                Txt\n",
              "0  شمس الدين ابن مفلح  \\nالكتاب : الآداب الشرعية\\n\\n[ الكتاب مشكول وم...\n",
              "1            الماوردي  \\nالكتاب : الأحكام السلطانية\\nالمؤلف : الماورد...\n",
              "2    عبدالملك بن هشام  مَبْعَثُ النّبِيّ صَلّى اللّهُ عَلَيْهِ وَعَلَ...\n",
              "3           ابن القيم  بِسْمِ اللَّهِ الرَّحْمَنِ الرَّحِيمِ الْحَمْد...\n",
              "4     محمد مهدي قشلان  الصحيفة السجَّادية\\nللإمام علي بن الحسين السجّ..."
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df.head() # 10 books in here total"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "id": "13966f6a",
      "metadata": {
        "id": "13966f6a"
      },
      "outputs": [],
      "source": [
        "df1=df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "id": "e58ee0bd",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e58ee0bd",
        "outputId": "3aeb9c21-5ce6-46eb-bdf6-6a932a6914e8"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0    \\nالكتاب : الآداب الشرعية\\n\\n[ الكتاب مشكول وم...\n",
              "Name: Txt, dtype: object"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df1['Txt'].head(n=1) # check if loaded "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "id": "bf5ee03d",
      "metadata": {
        "id": "bf5ee03d"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "import string\n",
        "import sys\n",
        "import argparse\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from textblob import TextBlob\n",
        "import os\n",
        "import warnings\n",
        "warnings.simplefilter(\"ignore\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "id": "75128b88",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "75128b88",
        "outputId": "f0fc4388-5c20-4d63-d3c3-541920f86abc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "nltk.download('stopwords')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "74af1076",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "74af1076",
        "outputId": "f768311b-6180-42d8-dd65-dfffc5719b58"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "nltk.download('punkt')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "id": "51eb2b33",
      "metadata": {
        "id": "51eb2b33"
      },
      "outputs": [],
      "source": [
        "def clean_text(text):\n",
        "    search = [\"أ\",\"إ\",\"آ\",\"ة\",\"_\",\"-\",\"/\",\".\",\"،\",\" و \",\" يا \",'\"',\"ـ\",\"'\",\"ى\",\"\\\\\",'\\n', '\\t','&quot;','?','؟','!']\n",
        "    replace = [\"ا\",\"ا\",\"ا\",\"ه\",\" \",\" \",\"\",\"\",\"\",\" و\",\" يا\",\"\",\"\",\"\",\"ي\",\"\",' ', ' ',' ',' ? ',' ؟ ',' ! ','[',']',':','[ ']  \n",
        "    p_tashkeel = re.compile(r'[\\u0617-\\u061A\\u064B-\\u0652]')\n",
        "    text = re.sub(p_tashkeel,\"\", text)\n",
        "    p_longation = re.compile(r'(.)\\1+')\n",
        "    text_nonum = re.sub(r'\\d+', '', text)\n",
        "    subst = r\"\\1\\1\"\n",
        "    text = re.sub(p_longation, subst, text)\n",
        "    text = text.replace('وو', 'و')\n",
        "    text = text.replace('يي', 'ي')\n",
        "    text = text.replace('اا', 'ا')\n",
        "    \n",
        "    for i in range(0, len(search)):\n",
        "        text = text.replace(search[i], replace[i])\n",
        "        \n",
        "    text = text.strip()\n",
        "    \n",
        "    return text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "id": "677033a2",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "677033a2",
        "outputId": "5dc8d32b-df87-4281-dd91-564b5d59e3ef"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 10 entries, 0 to 9\n",
            "Data columns (total 2 columns):\n",
            " #   Column  Non-Null Count  Dtype \n",
            "---  ------  --------------  ----- \n",
            " 0   Name    10 non-null     object\n",
            " 1   Txt     10 non-null     object\n",
            "dtypes: object(2)\n",
            "memory usage: 288.0+ bytes\n"
          ]
        }
      ],
      "source": [
        "#df1.iloc[0,1]  # [0,1]second book and so on\n",
        "df1.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "id": "7033e2aa",
      "metadata": {
        "id": "7033e2aa"
      },
      "outputs": [],
      "source": [
        "\n",
        "b2= clean_text(df1.iloc[0,1])  # each books [0,1]-[9,1] "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "id": "5c9cb5fa",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "5c9cb5fa",
        "outputId": "88c4f2d2-853c-470c-c246-bf3552caa5d3"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'الكتاب : الاداب الشرعيه  [ الكتاب مشكول ومرقم اليا غير موافق للمطبوع ]بسم الله الرحمن الرح'"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "b2[0:90] #range of text all book might carsh so I took few words"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "id": "3147be65",
      "metadata": {
        "id": "3147be65"
      },
      "outputs": [],
      "source": [
        "#pip install mahad   # it has cleaning functions could be helpful "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "id": "6e2f579e",
      "metadata": {
        "id": "6e2f579e"
      },
      "outputs": [],
      "source": [
        "from maha.cleaners.functions import keep, remove, contains, replace\n",
        "from maha.processors import TextProcessor"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "id": "0337824d",
      "metadata": {
        "id": "0337824d"
      },
      "outputs": [],
      "source": [
        "processor = TextProcessor.from_text(b2, sep=\"\\n\")   # processor ur df (d2)\n",
        "cleaned_text = (\n",
        "processor.normalize(alef=True).keep(arabic_letters=True).drop_empty_lines().text\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "id": "794b0bdf",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "794b0bdf",
        "outputId": "0bb78121-89e7-4177-c633-f634203afec5"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'arabic_letters': True, 'english_letters': True}"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "contains(b2, arabic_letters=True, english_letters=True) # check Eng numbers too "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "id": "2ee162fb",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2ee162fb",
        "outputId": "e68eab9a-a1f2-4dca-e694-3ff76d5ede29"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1789324\n"
          ]
        }
      ],
      "source": [
        "print(len(b2))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "id": "14b27ed7",
      "metadata": {
        "id": "14b27ed7"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "import itertools\n",
        "from nltk.tokenize import TweetTokenizer\n",
        "from nltk.stem.isri import ISRIStemmer\n",
        "pd.options.display.max_colwidth = 100000000"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "id": "6cbfab07",
      "metadata": {
        "id": "6cbfab07"
      },
      "outputs": [],
      "source": [
        "TATWEEL = u\"\\u0640\"\n",
        "stemmer = ISRIStemmer()\n",
        "tknzr = TweetTokenizer()\n",
        "#text = tknzr.tokenize(b2) # if needed\n",
        "\n",
        "result = []     # cleaned strings are stored here\n",
        "\n",
        "for word in b2: # here dirctly use df. if needed more u can use result list\n",
        "    if word.startswith(('@', '#')):    # filter out hashtags\n",
        "        continue\n",
        "    word = word.lower()\n",
        "    word = ''.join([i for i in word if not i.isdigit()]) # Remove digits\n",
        "    #word = re.sub(r\"http\\S+\", \"\",word) # Remove links\n",
        "    #word = stemmer.norm(word, num=1) # #emove diacritics\n",
        "    word = re.sub(r'[^\\w\\s]','', word)  # Remove punctuation\n",
        "    #word = word.replace(TATWEEL, '')\n",
        "    #word = ''.join(i for i, _ in itertools.groupby(word)) # Remove consecutive duplicate\n",
        "    if word:\n",
        "        result.append(word)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "id": "d86c919b",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d86c919b",
        "outputId": "27ff2257-0557-44ba-84f2-44e962aed086"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1789324\n"
          ]
        }
      ],
      "source": [
        "print(len(b2))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "id": "781e7536",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "781e7536",
        "outputId": "aece63f7-5c17-4d6f-8a24-9968e2ef3118"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1761798\n"
          ]
        }
      ],
      "source": [
        "print(len(result))  # data cleand. check number of words !!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "id": "2ywb2Rqwpd7B",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2ywb2Rqwpd7B",
        "outputId": "488f7b6a-3bb0-4256-858a-6981ad87439e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['ا', 'ل', 'ك', 'ت', 'ا', 'ب', ' ', ' ', 'ا', 'ل']"
            ]
          },
          "execution_count": 40,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "result[0:10] # list and split each words in lists then join them agin for better cleaniing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "id": "ac281e7c",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "ac281e7c",
        "outputId": "91580b19-a303-4481-eac7-0bbc2434c83f"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'ا ل ك ت ا ب     ا ل ا د ا ب   ا ل ش ر ع ي ه       ا ل ك ت ا ب   م ش ك و ل   و م ر ق م   ا ل ي ا   غ '"
            ]
          },
          "execution_count": 41,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "' '.join(result)[0:100]  # print letters it will be splits No worries next steps will do thw work"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "id": "4F0nmB74oEwp",
      "metadata": {
        "id": "4F0nmB74oEwp"
      },
      "outputs": [],
      "source": [
        "# lets manipliate thses and keep cleaning and string them agin OK!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "id": "9d8f7905",
      "metadata": {
        "id": "9d8f7905"
      },
      "outputs": [],
      "source": [
        "def listToString(s): # here convert them back \n",
        "    \n",
        "    str1 = \"\" \n",
        "    \n",
        "    for ele in s: \n",
        "        str1 += ele  \n",
        "    return str1 "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "id": "3d5709a5",
      "metadata": {
        "id": "3d5709a5"
      },
      "outputs": [],
      "source": [
        "b12 =listToString(result)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "id": "e79bc540",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 244
        },
        "id": "e79bc540",
        "outputId": "079a12c5-2b93-44eb-8149-3307b2b23e61"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'الكتاب  الاداب الشرعيه   الكتاب مشكول ومرقم اليا غير موافق للمطبوع بسم الله الرحمن الرحيم رب يسر واعن ياكريم قال الشيخ الامام العلامه اقضي القضاه  شمس الدين ابو عبد الله محمد بن مفلح المقدسي الحنبلي رحمه الله تعالي ورضي عنه واثابه الجنه  الحمد لله رب العالمين  وصلي الله علي سيدنا محمد خاتم النبين  وعلي اله وصحبه وسلم  اما بعد   فهذا كتاب يشتمل علي جمله كثيره من الاداب الشرعيه  والمنح المرعيه  يحتاج الي معرفته او معرفه كثير منه كل عالم او عابد وكل مسلم  وقد صنف في هذا المعني كثير من اصحابنا كابي داود السجستاني صاحب السنن  وابي بكر الخلال  وابي بكر عبد العزيز  وابي حفص  وابي علي بن ابي موسي  والقاضي ابي يعلي  وابن عقيل وغيرهم  وصنف في بعض ما يتعلق به   كالامر بالمعروف والنهي عن المنكر والدعاء والطب واللباس وغير ذلك   الطبراني وابو بكر الاجري وابو محمد الخلال والقاضي ابو يعلي وابنه ابو الحسين وابن الجوزي وغيرهم   ثناء المؤلف علي الكتاب  وقد اشتمل هذا الكتاب بحمد الله وعونه وحسن توفيقه علي ما تضمنته هذه المصنفات من المسائل او علي اكثرها  وتضمن مع ذلك اشياء كثيره نافعه حسنه غريبه من اماكن متفرقه  فمن علمه علم قدره  وعلم انه قد علم من الفوائد المحتاج اليها ما لم يعلم اكثر الفقهاء او كثير منهم لاشتغالهم بغيره  وعزه الكتب الجامعه لهذا الفن  والله اسال حسن القصد والنيه  وان ينفع به من حفظه وقراه وكتبه  وان يجعله عام النفع والبركه بفضله ورحمته انه علي كل شيء قدير  فصل  في الخوف والصبر والرضا  يسن لكل مسلم مكلف خوف السابقه  والخاتمه والمكربه  والخديعه  والفضيحه  والصبر علي الطاعه والنعم والبلاء والنقم في بدنه وعرضه واهله وماله  وعن كل ماثم  واستدراك ما فات من الهفوات  وقصد القرب والطاعه بنيته وفعله  كقوله  وسائر حركاته وسكناته  والزهد في الدنيا  والرغبه في الاخره  والنظر في حاله وماله  وحشره ونشره وسؤاله  ويسن رجاء قبول الطاعه  والتوبه من المعصيه والقناعه  والاكتفاء بالكفايه المعتاده بلا اسراف ولا تقتير ذكر ذلك في الرعايه الكبري وغيرها  وقال في نهايه المبتدئين  هل يجب الرضا بالمرض والسقم والفقر  والعاهه وعدم العقل    قال القاضي  لا يلزم  وقيل  بلي قال ابن عقيل الرضا بقضاء الله تعالي واجب فيما كان من فعله تعالي كالامراض ونحوها قال  فاما ما نهي عنه من افعال العباد كالكفر والضلال فلا يجوز اجماعا اذ الرضا بالكفر والمعاصي كفر وعصيان  وذكر الشيخ تقي الدين ان الرضا بالقضاء ليس بواجب في اصح قولي العلماء انما الواجب الصبر وذكر في كتاب الايمان   انما المؤمنون الذين امنوا بالله ورسوله ثم لم يرتابوا   فلم يجعل لهم ريبا عند المحن التي تقلقل الايمان في القلوب  والريب يكون في علم القلب وعمله بخلاف الشك فانه لا يكون الا في العلم فلهذا لا يوصف باليقين الا من اطمان قلبه علما وعملا  والا فاذا كان عالما بالحق ولكن المصيبه او الخوف اورثه جزعا عظيما لم يكن صاحب يقين  وذكر الشيخ وجيه الدين من اصحابنا في شرح الهدايه انه يجوز البكاء علي الميت اذا تجرد عن فعل محرم من ندب ونياحه وتسخط بقضاء الله وقدره المحتوم  والجزع الذي يناقض الانقياد والاستسلام له  وقال ابن الجوزي في اخر كلامه في قوله تعالي  يااسفي علي يوسف   قال وروي عن الحسن ان اخاه مات فجزع الحسن جزعا شديدا فعوتب في ذلك فقال  ما سمعت الله عاب علي يعقوب عليه السلام الحزن حيث قال  يااسفي علي يوسف  وذكر الشيخ تقي الدين في التحفه العراقيه ان البكاء علي الميت علي وجه الرحمه مستحب وذلك لا ينافي الرضا بقضاء الله بخلاف البكاء عليه لفوا'"
            ]
          },
          "execution_count": 35,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "b12[0:3000] # check few chp of the book if u print all might slow: so I limit"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "22YyWgotqMqL",
      "metadata": {
        "id": "22YyWgotqMqL"
      },
      "source": [
        "\n",
        "\n",
        "\n",
        "## now u can save list of words or string as you want  this only way worked well with our arabic books\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9ae8dc93",
      "metadata": {
        "id": "9ae8dc93"
      },
      "outputs": [],
      "source": [
        "list = [] # save them in list\n",
        "for name in b12:\n",
        "    list.append(name.strip())\n",
        "with open(\"book1.csv\", \"w\") as file: # watch out it will took it as token list by list\n",
        "    for element in list:      \n",
        "        file.write(element + \"\\n\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "id": "28f8069c",
      "metadata": {
        "id": "28f8069c"
      },
      "outputs": [],
      "source": [
        "#save as txt tooo\n",
        "text_file = open(\"book1.txt\", \"w\") # it will take whole texts splited and cleaned\n",
        "n = text_file.write(b12)\n",
        " \n",
        "#close file\n",
        "text_file.close()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "cbc71873",
      "metadata": {
        "id": "cbc71873"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6b71051f",
      "metadata": {
        "id": "6b71051f"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "id": "626404f4",
      "metadata": {
        "id": "626404f4"
      },
      "outputs": [],
      "source": [
        "## if you need to remove Eng numbers or letters \n",
        "\n",
        "# b2 = re.sub(r\"$\\d+\\W+|\\b\\d+\\b|\\W+\\d+$\", \"\", b2)\n",
        "# b2 = re.sub(r\"(^|\\W)\\d+\", \"\", b2)\n",
        "# b2= re.sub(\"(\\s\\d+)\",\"\",foo) \n",
        "# b2= re.sub(r\"(\\b|\\s+\\-?|^\\-?)(\\d+|\\d*\\.\\d+)\\b\",\"\", b2)\n",
        "# b2= re.sub(r\"\\d*$\", \"\", b2)\n",
        "# b2= re.sub(r'[0-9]', '', b2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "id": "5JIDrjxzsYgA",
      "metadata": {
        "id": "5JIDrjxzsYgA"
      },
      "outputs": [],
      "source": [
        "# might help you if u encounter in problems for some books"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "id": "-3O1uC5esH-R",
      "metadata": {
        "id": "-3O1uC5esH-R"
      },
      "outputs": [],
      "source": [
        "# from maha.cleaners.functions import keep, remove, contains, replace\n",
        "# sample_text = \"\"\"\n",
        "# ... 1. بِسْمِ اللَّـهِ الرَّحْمَـٰـــنِ الرَّحِيمِ\n",
        "# ... 2. In the name of God, the most gracious, the most merciful\n",
        "# ... \"\"\"\n",
        "# keep(sample_text, arabic=True)\n",
        "# 'بِسْمِ اللَّهِ الرَّحْمَٰنِ الرَّحِيمِ'\n",
        "# keep(sample_text, arabic_letters=True)\n",
        "# 'بسم الله الرحمن الرحيم'\n",
        "# keep(sample_text, arabic_letters=True, harakat=True)\n",
        "# 'بِسْمِ اللَّهِ الرَّحْمَنِ الرَّحِيمِ'\n",
        "# remove(sample_text, numbers=True, punctuations=True)\n",
        "# 'بِسْمِ اللَّـهِ الرَّحْمَـٰـــنِ الرَّحِيمِ\\n In the name of God the most gracious the most merciful'\n",
        "# contains(sample_text, numbers=True)\n",
        "# True\n",
        "# contains(sample_text, hashtags=True, arabic=True, emails=True)\n",
        "# {'arabic': True, 'emails': False, 'hashtags': False}\n",
        "# replace(keep(sample_text, english_letters=True), \"God\", \"Allah\")\n",
        "# 'In the name of Allah the most gracious the most merciful'\n",
        "\n",
        "\n",
        "#https://githubmemory.com/repo/saedx1/Maha"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "Q7vYosj-sd_V",
      "metadata": {
        "id": "Q7vYosj-sd_V"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "cleaning_ARABIC-2.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    },
    "toc": {
      "base_numbering": 1,
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": false,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    },
    "varInspector": {
      "cols": {
        "lenName": 16,
        "lenType": 16,
        "lenVar": 40
      },
      "kernels_config": {
        "python": {
          "delete_cmd_postfix": "",
          "delete_cmd_prefix": "del ",
          "library": "var_list.py",
          "varRefreshCmd": "print(var_dic_list())"
        },
        "r": {
          "delete_cmd_postfix": ") ",
          "delete_cmd_prefix": "rm(",
          "library": "var_list.r",
          "varRefreshCmd": "cat(var_dic_list()) "
        }
      },
      "types_to_exclude": [
        "module",
        "function",
        "builtin_function_or_method",
        "instance",
        "_Feature"
      ],
      "window_display": false
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}