Letterboxd_Web_Scrape.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Letterboxd_Web_Scrape.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "CWxSrJkPsDyU"
      },
      "source": [
        "!pip install bs4"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8CpJh7ksr5dD"
      },
      "source": [
        "# import os\n",
        "import sys\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import json\n",
        "import itertools\n",
        "import threading\n",
        "import time\n",
        "from bs4 import BeautifulSoup # this module helps in web scrapping.\n",
        "import requests  # this module helps us to download a web page"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Kbc0PDQrzpa"
      },
      "source": [
        "username = ''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eRIGpLc2rc66"
      },
      "source": [
        "done = False\n",
        "#here is the animation\n",
        "def animate():\n",
        "    for c in itertools.cycle(['|', '/', '-', '\\\\']):\n",
        "        if done:\n",
        "            break\n",
        "        sys.stdout.write('\\rloading ' + c)\n",
        "        sys.stdout.flush()\n",
        "        time.sleep(0.1)\n",
        "    sys.stdout.write('\\rDone!     ')\n",
        "\n",
        "t = threading.Thread(target=animate)\n",
        "t.start()\n",
        "\n",
        "#long process here\n",
        "def my_films(username):\n",
        "    \n",
        "    url = requests.get(f'https://letterboxd.com/{username}/films/')\n",
        "    html = url.text\n",
        "    html = BeautifulSoup(html, 'html5lib')\n",
        "\n",
        "    def number_of_pages(html):\n",
        "        number_of_pages = html.find_all('li', class_=\"paginate-page\")\n",
        "        x = number_of_pages[-1]\n",
        "        x = str(x)\n",
        "        x = x.split('page/')[1]\n",
        "        x = x.split('/\">')[0]\n",
        "        x = int(x)\n",
        "        return x\n",
        "\n",
        "    number_of_pages = number_of_pages(html)\n",
        "\n",
        "    list_of_films = []\n",
        "    my_ratings = []\n",
        "    minutes_list = []\n",
        "    year_list = []\n",
        "    director_list = []\n",
        "    ratingCount_list = []\n",
        "    reviewCount_list = []\n",
        "    average_rating_list = []\n",
        "    film_description_list = []\n",
        "    genre_list = []\n",
        "    actors_list = []\n",
        "    countryOfOrigin_list = []\n",
        "\n",
        "    def get_film_data(html):\n",
        "        \n",
        "        title_names = html.find_all('li', class_=\"poster-container\")\n",
        "        num_of_title_names_at_page = len(title_names)\n",
        "        ratings = html.find_all('p', class_=\"poster-viewingdata -rated-and-liked\")\n",
        "\n",
        "        for i in range(0, num_of_title_names_at_page):\n",
        "            title = str(title_names[i])\n",
        "            title_ = title.split('<img alt=\"')[1]\n",
        "            title_name = title_.split('\" class=')[0]\n",
        "            list_of_films.append(title_name)\n",
        "\n",
        "            rating = ratings[i]\n",
        "            rating = str(rating)\n",
        "            if rating.find('★') != -1:\n",
        "                idx = rating.find('★')\n",
        "                rating = rating[idx:]\n",
        "                rating = rating.split('</span>')[0]\n",
        "                no_of_stars = rating.count('★')\n",
        "                half_star = rating.count('½') * 0.5\n",
        "                rating = no_of_stars + half_star\n",
        "            else:\n",
        "                rating = np.nan\n",
        "            my_ratings.append(rating)\n",
        "\n",
        "            film_path = title\n",
        "            film_path = film_path.split('data-target-link=\"')[1]\n",
        "            film_path = film_path.split('\" data-target-link-target=')[0]\n",
        "            url2 = requests.get(f'https://letterboxd.com/{film_path}/')\n",
        "            html2 = url2.text\n",
        "            html2 = BeautifulSoup(html2, 'html5lib') \n",
        "\n",
        "            film_info = html2.find_all('script')\n",
        "            film_info = str(film_info)\n",
        "            film_info = film_info.split('/* <![CDATA[ */\\n')[1]\n",
        "            film_info = film_info.split('\\n/* ]]> */')[0]\n",
        "            film_info = json.loads(film_info)\n",
        "\n",
        "            minutes = html2.find_all('p', class_=\"text-link text-footer\")\n",
        "            minutes = str(minutes)\n",
        "            minutes = minutes.replace('\\n','')\n",
        "            minutes = minutes.replace('\\t','')\n",
        "            minutes = minutes.split('[<p class=\"text-link text-footer\">')[1]\n",
        "            minutes = minutes.split('\\xa0mins')[0]\n",
        "            if minutes.find(',') != -1:\n",
        "                minutes = minutes.replace(',','')\n",
        "            minutes = int(minutes)\n",
        "            minutes_list.append(minutes)\n",
        "\n",
        "            year = film_info['releasedEvent'][0]['startDate']\n",
        "            year_list.append(year)\n",
        "            \n",
        "            if 'director' in film_info.keys():\n",
        "                if len(film_info['director']) == 1:\n",
        "                    director = film_info['director'][0]['name']\n",
        "                else:\n",
        "                    directors_in_this_film = []\n",
        "                    for j in range(0, len(film_info['director'])):\n",
        "                        directors_in_this_film.append(film_info['director'][j]['name'])\n",
        "                        director = directors_in_this_film\n",
        "            else:\n",
        "                director = np.nan\n",
        "            director_list.append(director)\n",
        "            \n",
        "            if 'aggregateRating' in film_info.keys():\n",
        "                if 'ratingCount' in film_info['aggregateRating']:\n",
        "                    ratingCount = film_info['aggregateRating']['ratingCount']\n",
        "                else:\n",
        "                    ratingCount = np.nan\n",
        "                if 'reviewCount' in film_info['aggregateRating']:\n",
        "                    reviewCount = film_info['aggregateRating']['reviewCount']\n",
        "                else:\n",
        "                    reviewCount = np.nan\n",
        "                if 'ratingValue' in film_info['aggregateRating']:\n",
        "                    average_rating = film_info['aggregateRating']['ratingValue']\n",
        "                else:\n",
        "                    average_rating = np.nan\n",
        "            else:\n",
        "                ratingCount = np.nan\n",
        "                reviewCount = np.nan\n",
        "                average_rating = np.nan\n",
        "            ratingCount_list.append(ratingCount)\n",
        "            reviewCount_list.append(reviewCount)\n",
        "            average_rating_list.append(average_rating)\n",
        "            \n",
        "            if 'genre' in film_info.keys():\n",
        "                genre = film_info['genre']\n",
        "            else:\n",
        "                genre = np.nan\n",
        "            genre_list.append(genre)\n",
        "\n",
        "            if 'actors' in film_info.keys():\n",
        "                if len(film_info['actors']) == 1:\n",
        "                    actors = film_info['actors'][0]['name']\n",
        "                else:\n",
        "                    actors_in_this_film = []\n",
        "                    for j in range(0, len(film_info['actors'])):\n",
        "                        actors_in_this_film.append(film_info['actors'][j]['name'])\n",
        "                        actors = actors_in_this_film\n",
        "            else:\n",
        "                actors = np.nan\n",
        "            actors_list.append(actors)\n",
        "\n",
        "            if 'countryOfOrigin' in film_info.keys():\n",
        "                if len(film_info['countryOfOrigin']) == 1:\n",
        "                    country = film_info['countryOfOrigin'][0]['name']\n",
        "                else:\n",
        "                    countries_this_film = []\n",
        "                    for j in range(0, len(film_info['countryOfOrigin'])):\n",
        "                        countries_this_film.append(film_info['countryOfOrigin'][j]['name'])\n",
        "                        country = countries_this_film\n",
        "            else:\n",
        "                country = np.nan\n",
        "            countryOfOrigin_list.append(country)\n",
        "\n",
        "            film_description = html2.find_all('div', class_=\"truncate\")\n",
        "            film_description = str(film_description)\n",
        "            if film_description.find('<p>') != -1:\n",
        "                film_description = film_description.split('<p>')\n",
        "                if len(film_description) > 0:\n",
        "                    film_description = film_description[1]\n",
        "                    film_description = film_description.split('</p>')[0]\n",
        "                else:\n",
        "                    film_description = np.nan\n",
        "            else:\n",
        "                    film_description = np.nan\n",
        "            film_description_list.append(film_description)\n",
        "\n",
        "        return     list_of_films, my_ratings, year_list, director_list, ratingCount_list, reviewCount_list, average_rating_list, film_description_list, genre_list, actors_list, countryOfOrigin_list, minutes_list\n",
        "\n",
        "    list_of_films, my_ratings, year_list, director_list, ratingCount_list, reviewCount_list, average_rating_list, film_description_list, genre_list, actors_list, countryOfOrigin_list, minutes_list = get_film_data(html)\n",
        "\n",
        "    for i in range(2, number_of_pages +1):\n",
        "        url3 = requests.get(f'https://letterboxd.com/{username}/films/page/{i}/')\n",
        "        html3 = url3.text\n",
        "        html3 = BeautifulSoup(html3, 'html5lib')\n",
        "    \n",
        "        list_of_films, my_ratings, year_list, director_list, ratingCount_list, reviewCount_list, average_rating_list, film_description_list, genre_list, actors_list, countryOfOrigin_list, minutes_list = get_film_data(html3)\n",
        "\n",
        "    data = {'Films':list_of_films,\n",
        "            'Minutes': minutes_list,\n",
        "            'Year':year_list,\n",
        "            'Director':director_list,\n",
        "            'My Ratings':my_ratings,\n",
        "            'Average Ratings':average_rating_list,\n",
        "            'Genres':genre_list,\n",
        "            'Actors':actors_list,\n",
        "            'Description':film_description_list,\n",
        "            'Countries':countryOfOrigin_list,\n",
        "            'RatingsCount':ratingCount_list,\n",
        "            'ReviewCount':reviewCount_list}\n",
        "    \n",
        "    df = pd.DataFrame(data)\n",
        "    df.reset_index(drop=True, inplace=True)\n",
        "\n",
        "    def list_to_str(x):\n",
        "        if x == x:\n",
        "            if type(x) != list:\n",
        "                x = str(x)\n",
        "            if type(x) == list:\n",
        "                outputs =[]\n",
        "                for j in range(0, len(x)):\n",
        "                    outputs.append(x[j])\n",
        "                x = ', '.join(outputs)\n",
        "        return x\n",
        "\n",
        "    df['Countries'] = df['Countries'].apply(lambda x: list_to_str(x))\n",
        "    df['Director'] = df['Director'].apply(lambda x: list_to_str(x))\n",
        "    df['Genres'] = df['Genres'].apply(lambda x: list_to_str(x))\n",
        "    df['Actors'] = df['Actors'].apply(lambda x: list_to_str(x))\n",
        "\n",
        "    return df\n",
        "\n",
        "df = my_films(f'{username}')\n",
        "\n",
        "time.sleep(10)\n",
        "done = True\n",
        "\n",
        "df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8D-gITX0rr8V"
      },
      "source": [
        "df.to_csv('', encoding='utf-8', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}