03-Spark/001.02 (optional) Google_Colab_setup_Spark_download_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Google Colab Setup\n",
    "## Get Spark and download the MovieLens dataset   \n",
    "\n",
    "The notebook here will explain how to download and install Apache Spark in a Google Colab environment.\n",
    "It will also demonstrate how to download the MovieLens dataset needed for the workshop. \n",
    "You can directly open this notebook in Colab using the following button:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/shauryashaurya/learn-data-munging/blob/main/03-Spark/001.01%20-%20Setup%20and%20suchlike.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rQ2J1c0zRg7D",
    "outputId": "46f7c939-4a29-45e2-da07-1cc353346e08"
   },
   "outputs": [],
   "source": [
    "# find out OS details\n",
    "! cat /etc/os-release"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_Pt_uDtJRko8",
    "outputId": "1afaa5ae-fa82-4e9a-bc28-fb25a12a1989"
   },
   "outputs": [],
   "source": [
    "# see if java is available\n",
    "! java -version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jycPMjnJWUlG",
    "outputId": "a4c44a9e-4348-437e-bf5a-de96c1701d56"
   },
   "outputs": [],
   "source": [
    "# spark needs JAVA_HOME and SPARK_HOME variables set.\n",
    "# to do that\n",
    "# we've got to locate java\n",
    "! whereis java"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2BkgXWcdWxje",
    "outputId": "602450d0-23f5-4495-e73c-5c13a513b261"
   },
   "outputs": [],
   "source": [
    "# let's check more details so we can supply exact details to JAVA_HOME\n",
    "! find / -iname \"*openjdk-*\"\n",
    "\n",
    "# typically - /usr/lib/jvm/java-11-openjdk-amd64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Akd__KdqSWRF"
   },
   "outputs": [],
   "source": [
    "# grab spark\n",
    "# as of 2023-06-23, the latest version is 3.4.1, get the link from Apache Spark's website\n",
    "! wget -q https://dlcdn.apache.org/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz\n",
    "# unzip spark\n",
    "!tar xf spark-3.4.1-bin-hadoop3.tgz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# install pyspark (needed?)\n",
    "!pip install pyspark plotly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "53U4TuMITsAi"
   },
   "outputs": [],
   "source": [
    "# install findspark package\n",
    "!pip install -q findspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jlhN6wvBVfrY",
    "outputId": "531952de-1908-40f2-9c9f-b081a30ec699"
   },
   "outputs": [],
   "source": [
    "# now the folder we are working in is \"content\"\n",
    "! ls ../content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "W6B0UuvPVtBM"
   },
   "outputs": [],
   "source": [
    "# got to provide JAVA_HOME and SPARK_HOME vairables\n",
    "import os\n",
    "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-11-openjdk-amd64\"\n",
    "# IMPORTANT - UPDATE THE SPARK_HOME PATH BASED ON THE PACKAGE YOU DOWNLOAD\n",
    "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.4.1-bin-hadoop3.tgz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xITc7EjHdHEu"
   },
   "outputs": [],
   "source": [
    "# Now we initialize spark like before"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2jEVYi6mUYZ-"
   },
   "outputs": [],
   "source": [
    "# Step 1: initialize findspark\n",
    "import findspark\n",
    "findspark.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "dbblGcx7csTM",
    "outputId": "8adce291-057c-469b-a678-9fd814cda511"
   },
   "outputs": [],
   "source": [
    "# Step 2: import pyspark\n",
    "import pyspark\n",
    "from pyspark.sql import SparkSession\n",
    "pyspark.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "yWqPQh1DcwTb"
   },
   "outputs": [],
   "source": [
    "# Step 3: Create a spark session\n",
    "\n",
    "# 'local[1]' indicates spark on 1 core on the local machine (the Ubuntu VM on colab in this case), \n",
    "# specify the number of cores needed - we'll use local[*] in this case to engage as many cores as available\n",
    "# use .config(\"spark.some.config.option\", \"some-value\") for additional configuration\n",
    "\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .master('local[*]') \\\n",
    "    .appName(\"Setup-Spark-in-Google-Colab\") \\\n",
    "    .getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 219
    },
    "id": "XjSJWTY8c4xj",
    "outputId": "a8f84a28-9b3e-45ea-e00e-82b5302327cc"
   },
   "outputs": [],
   "source": [
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PARHZ5SzdcNA"
   },
   "outputs": [],
   "source": [
    "# Let's download and unzip the MovieLens 25M Dataset as well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-rrkiGzrlAzH"
   },
   "outputs": [],
   "source": [
    "! wget -q https://files.grouplens.org/datasets/movielens/ml-25m.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "B1BqQDILlEZV",
    "outputId": "d5cb6a45-8c90-49de-82c7-254523da6b60"
   },
   "outputs": [],
   "source": [
    "! mkdir ./data\n",
    "! ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QLGyv3m0lKAN",
    "outputId": "434209f3-3688-4716-eea9-d92af3a3858a"
   },
   "outputs": [],
   "source": [
    "! unzip ./ml-25m.zip -d ./data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ho_rxhVMlLtD",
    "outputId": "58798053-ed2c-46ff-b6ea-9df54cf007a1"
   },
   "outputs": [],
   "source": [
    "! ls ./data/ml-25m/"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyNp7aqZtczcFGhiJ6YOlcwF",
   "include_colab_link": true,
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}