pyspark_initial_practice.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMPzWV1LFaKJeDFI4e9jUAf",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/armahdavi/BigData_pyspark_practice/blob/main/pySpark_initial_practice.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "CVKpiURhyO2Q"
      },
      "outputs": [],
      "source": [
        "import pyspark\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql.functions import when, count, broadcast"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a Spark Session\n",
        "spark = (SparkSession\n",
        "         .builder\n",
        "         .appName(\"SparkSQLExampleApp\")\n",
        "         .getOrCreate())\n",
        "\n",
        "# Path the to dataframe\n",
        "csv_file = \"/content/sample_data/onlinefoods.csv\"\n",
        "\n",
        "df = spark.read \\\n",
        "    .format(\"csv\") \\\n",
        "    .option(\"header\", \"true\") \\\n",
        "    .option(\"inferSchema\", \"true\") \\\n",
        "    .load(csv_file)\n",
        "\n",
        "# Create a temporary view to run SQL queries\n",
        "df.createOrReplaceGlobalTempView('u')"
      ],
      "metadata": {
        "id": "qOLtyWP8yk4c"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rmu8N1pHzjrL",
        "outputId": "c69ef40c-d447-4dd0-fe25-e4b3237a9ed1"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+------+--------------+--------------+---------------+--------------------------+-----------+--------+---------+--------+------+---------+----+\n",
            "|Age|Gender|Marital Status|    Occupation| Monthly Income|Educational Qualifications|Family size|latitude|longitude|Pin code|Output| Feedback|_c12|\n",
            "+---+------+--------------+--------------+---------------+--------------------------+-----------+--------+---------+--------+------+---------+----+\n",
            "| 20|Female|        Single|       Student|      No Income|             Post Graduate|          4| 12.9766|  77.5993|  560001|   Yes| Positive| Yes|\n",
            "| 24|Female|        Single|       Student| Below Rs.10000|                  Graduate|          3|  12.977|  77.5773|  560009|   Yes| Positive| Yes|\n",
            "| 22|  Male|        Single|       Student| Below Rs.10000|             Post Graduate|          3| 12.9551|  77.6593|  560017|   Yes|Negative | Yes|\n",
            "| 22|Female|        Single|       Student|      No Income|                  Graduate|          6| 12.9473|  77.5616|  560019|   Yes| Positive| Yes|\n",
            "| 22|  Male|        Single|       Student| Below Rs.10000|             Post Graduate|          4|  12.985|  77.5533|  560010|   Yes| Positive| Yes|\n",
            "| 27|Female|       Married|      Employee|More than 50000|             Post Graduate|          2| 12.9299|  77.6848|  560103|   Yes| Positive| Yes|\n",
            "| 22|  Male|        Single|       Student|      No Income|                  Graduate|          3|  12.977|  77.5773|  560009|   Yes| Positive| Yes|\n",
            "| 24|Female|        Single|       Student|      No Income|             Post Graduate|          3| 12.9828|  77.6131|  560042|   Yes| Positive| Yes|\n",
            "| 23|Female|        Single|       Student|      No Income|             Post Graduate|          2| 12.9766|  77.5993|  560001|   Yes| Positive| Yes|\n",
            "| 23|Female|        Single|       Student|      No Income|             Post Graduate|          4| 12.9854|  77.7081|  560048|   Yes| Positive| Yes|\n",
            "| 22|Female|        Single|       Student|      No Income|             Post Graduate|          5|  12.985|  77.5533|  560010|   Yes| Positive| Yes|\n",
            "| 23|  Male|        Single|       Student| Below Rs.10000|             Post Graduate|          2|  12.977|  77.5773|  560009|   Yes|Negative | Yes|\n",
            "| 23|  Male|        Single|       Student|      No Income|             Post Graduate|          5| 12.8988|  77.5764|  560078|   Yes| Positive| Yes|\n",
            "| 21|  Male|        Single|       Student|      No Income|                  Graduate|          4|  12.977|  77.5773|  560009|   Yes| Positive| Yes|\n",
            "| 23|Female|        Single|Self Employeed| 10001 to 25000|             Post Graduate|          5| 12.9438|  77.5738|  560004|   Yes| Positive| Yes|\n",
            "| 24|Female|        Single|       Student|      No Income|             Post Graduate|          6| 12.8893|  77.6399|  560068|   Yes| Positive| Yes|\n",
            "| 28|Female|        Single|      Employee| 25001 to 50000|             Post Graduate|          2| 12.9783|  77.6408|  560038|   Yes| Positive| Yes|\n",
            "| 23|Female|        Single|       Student|      No Income|                  Graduate|          3|  12.982|  77.6256|  560008|   Yes|Negative | Yes|\n",
            "| 25|  Male|        Single|       Student|      No Income|                  Graduate|          4| 12.8988|  77.5764|  560078|   Yes|Negative | Yes|\n",
            "| 21|Female|        Single|       Student| Below Rs.10000|             Post Graduate|          1| 12.9783|  77.6408|  560038|   Yes| Positive| Yes|\n",
            "+---+------+--------------+--------------+---------------+--------------------------+-----------+--------+---------+--------+------+---------+----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# comparison with Pandas. Display format is different\n",
        "import pandas as pd\n",
        "\n",
        "df2 = pd.read_csv(csv_file)\n",
        "df2.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "7f2ESXv7zuUb",
        "outputId": "c344524a-e696-4296-cc31-4a6fd8f6d392"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Age  Gender Marital Status Occupation  Monthly Income  \\\n",
              "0   20  Female         Single    Student       No Income   \n",
              "1   24  Female         Single    Student  Below Rs.10000   \n",
              "2   22    Male         Single    Student  Below Rs.10000   \n",
              "3   22  Female         Single    Student       No Income   \n",
              "4   22    Male         Single    Student  Below Rs.10000   \n",
              "\n",
              "  Educational Qualifications  Family size  latitude  longitude  Pin code  \\\n",
              "0              Post Graduate            4   12.9766    77.5993    560001   \n",
              "1                   Graduate            3   12.9770    77.5773    560009   \n",
              "2              Post Graduate            3   12.9551    77.6593    560017   \n",
              "3                   Graduate            6   12.9473    77.5616    560019   \n",
              "4              Post Graduate            4   12.9850    77.5533    560010   \n",
              "\n",
              "  Output   Feedback Unnamed: 12  \n",
              "0    Yes   Positive         Yes  \n",
              "1    Yes   Positive         Yes  \n",
              "2    Yes  Negative          Yes  \n",
              "3    Yes   Positive         Yes  \n",
              "4    Yes   Positive         Yes  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3e06c1b9-e111-4558-bfdc-bc2c811dd7a8\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Age</th>\n",
              "      <th>Gender</th>\n",
              "      <th>Marital Status</th>\n",
              "      <th>Occupation</th>\n",
              "      <th>Monthly Income</th>\n",
              "      <th>Educational Qualifications</th>\n",
              "      <th>Family size</th>\n",
              "      <th>latitude</th>\n",
              "      <th>longitude</th>\n",
              "      <th>Pin code</th>\n",
              "      <th>Output</th>\n",
              "      <th>Feedback</th>\n",
              "      <th>Unnamed: 12</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>20</td>\n",
              "      <td>Female</td>\n",
              "      <td>Single</td>\n",
              "      <td>Student</td>\n",
              "      <td>No Income</td>\n",
              "      <td>Post Graduate</td>\n",
              "      <td>4</td>\n",
              "      <td>12.9766</td>\n",
              "      <td>77.5993</td>\n",
              "      <td>560001</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Positive</td>\n",
              "      <td>Yes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>24</td>\n",
              "      <td>Female</td>\n",
              "      <td>Single</td>\n",
              "      <td>Student</td>\n",
              "      <td>Below Rs.10000</td>\n",
              "      <td>Graduate</td>\n",
              "      <td>3</td>\n",
              "      <td>12.9770</td>\n",
              "      <td>77.5773</td>\n",
              "      <td>560009</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Positive</td>\n",
              "      <td>Yes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>22</td>\n",
              "      <td>Male</td>\n",
              "      <td>Single</td>\n",
              "      <td>Student</td>\n",
              "      <td>Below Rs.10000</td>\n",
              "      <td>Post Graduate</td>\n",
              "      <td>3</td>\n",
              "      <td>12.9551</td>\n",
              "      <td>77.6593</td>\n",
              "      <td>560017</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Negative</td>\n",
              "      <td>Yes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>22</td>\n",
              "      <td>Female</td>\n",
              "      <td>Single</td>\n",
              "      <td>Student</td>\n",
              "      <td>No Income</td>\n",
              "      <td>Graduate</td>\n",
              "      <td>6</td>\n",
              "      <td>12.9473</td>\n",
              "      <td>77.5616</td>\n",
              "      <td>560019</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Positive</td>\n",
              "      <td>Yes</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>22</td>\n",
              "      <td>Male</td>\n",
              "      <td>Single</td>\n",
              "      <td>Student</td>\n",
              "      <td>Below Rs.10000</td>\n",
              "      <td>Post Graduate</td>\n",
              "      <td>4</td>\n",
              "      <td>12.9850</td>\n",
              "      <td>77.5533</td>\n",
              "      <td>560010</td>\n",
              "      <td>Yes</td>\n",
              "      <td>Positive</td>\n",
              "      <td>Yes</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3e06c1b9-e111-4558-bfdc-bc2c811dd7a8')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-3e06c1b9-e111-4558-bfdc-bc2c811dd7a8 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-3e06c1b9-e111-4558-bfdc-bc2c811dd7a8');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-27d2b3bd-9fd6-4cee-a184-656c6103ae04\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-27d2b3bd-9fd6-4cee-a184-656c6103ae04')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-27d2b3bd-9fd6-4cee-a184-656c6103ae04 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df2",
              "summary": "{\n  \"name\": \"df2\",\n  \"rows\": 388,\n  \"fields\": [\n    {\n      \"column\": \"Age\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2,\n        \"min\": 18,\n        \"max\": 33,\n        \"num_unique_values\": 16,\n        \"samples\": [\n          20,\n          24,\n          21\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Gender\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"Male\",\n          \"Female\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Marital Status\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"Single\",\n          \"Married\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Occupation\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          \"Employee\",\n          \"House wife\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Monthly Income\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"Below Rs.10000\",\n          \"25001 to 50000\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Educational Qualifications\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"Graduate\",\n          \"School\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Family size\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1,\n        \"min\": 1,\n        \"max\": 6,\n        \"num_unique_values\": 6,\n        \"samples\": [\n          4,\n          3\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"latitude\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.044489248628106076,\n        \"min\": 12.8652,\n        \"max\": 13.102,\n        \"num_unique_values\": 77,\n        \"samples\": [\n          12.985,\n          12.9335\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"longitude\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.05135391701271844,\n        \"min\": 77.4842,\n        \"max\": 77.7582,\n        \"num_unique_values\": 76,\n        \"samples\": [\n          77.5533,\n          77.5691\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Pin code\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 31,\n        \"min\": 560001,\n        \"max\": 560109,\n        \"num_unique_values\": 77,\n        \"samples\": [\n          560010,\n          560028\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Output\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"No\",\n          \"Yes\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Feedback\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"Negative \",\n          \"Positive\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Unnamed: 12\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"No\",\n          \"Yes\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(type(df), type(df2))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yx9q_AA5z7nw",
        "outputId": "7e99e979-3a6e-47d6-df4e-67bae5669685"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pyspark.sql.dataframe.DataFrame'> <class 'pandas.core.frame.DataFrame'>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(df.columns)\n",
        "print(df.count())\n",
        "\n",
        "# df.except"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i_qIp1R10sxP",
        "outputId": "d10fbde6-4a89-4a7a-d6b7-505101e7e324"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Age', 'Gender', 'Marital Status', 'Occupation', 'Monthly Income', 'Educational Qualifications', 'Family size', 'latitude', 'longitude', 'Pin code', 'Output', 'Feedback', '_c12']\n",
            "388\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.head(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XZOoned466Is",
        "outputId": "3e69582e-67a3-48b3-b18b-fb7b98a63419"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Row(Age=20, Gender='Female', Marital Status='Single', Occupation='Student', Monthly Income='No Income', Educational Qualifications='Post Graduate', Family size=4, latitude=12.9766, longitude=77.5993, Pin code=560001, Output='Yes', Feedback='Positive', _c12='Yes'),\n",
              " Row(Age=24, Gender='Female', Marital Status='Single', Occupation='Student', Monthly Income='Below Rs.10000', Educational Qualifications='Graduate', Family size=3, latitude=12.977, longitude=77.5773, Pin code=560009, Output='Yes', Feedback='Positive', _c12='Yes'),\n",
              " Row(Age=22, Gender='Male', Marital Status='Single', Occupation='Student', Monthly Income='Below Rs.10000', Educational Qualifications='Post Graduate', Family size=3, latitude=12.9551, longitude=77.6593, Pin code=560017, Output='Yes', Feedback='Negative ', _c12='Yes'),\n",
              " Row(Age=22, Gender='Female', Marital Status='Single', Occupation='Student', Monthly Income='No Income', Educational Qualifications='Graduate', Family size=6, latitude=12.9473, longitude=77.5616, Pin code=560019, Output='Yes', Feedback='Positive', _c12='Yes'),\n",
              " Row(Age=22, Gender='Male', Marital Status='Single', Occupation='Student', Monthly Income='Below Rs.10000', Educational Qualifications='Post Graduate', Family size=4, latitude=12.985, longitude=77.5533, Pin code=560010, Output='Yes', Feedback='Positive', _c12='Yes'),\n",
              " Row(Age=27, Gender='Female', Marital Status='Married', Occupation='Employee', Monthly Income='More than 50000', Educational Qualifications='Post Graduate', Family size=2, latitude=12.9299, longitude=77.6848, Pin code=560103, Output='Yes', Feedback='Positive', _c12='Yes'),\n",
              " Row(Age=22, Gender='Male', Marital Status='Single', Occupation='Student', Monthly Income='No Income', Educational Qualifications='Graduate', Family size=3, latitude=12.977, longitude=77.5773, Pin code=560009, Output='Yes', Feedback='Positive', _c12='Yes'),\n",
              " Row(Age=24, Gender='Female', Marital Status='Single', Occupation='Student', Monthly Income='No Income', Educational Qualifications='Post Graduate', Family size=3, latitude=12.9828, longitude=77.6131, Pin code=560042, Output='Yes', Feedback='Positive', _c12='Yes'),\n",
              " Row(Age=23, Gender='Female', Marital Status='Single', Occupation='Student', Monthly Income='No Income', Educational Qualifications='Post Graduate', Family size=2, latitude=12.9766, longitude=77.5993, Pin code=560001, Output='Yes', Feedback='Positive', _c12='Yes'),\n",
              " Row(Age=23, Gender='Female', Marital Status='Single', Occupation='Student', Monthly Income='No Income', Educational Qualifications='Post Graduate', Family size=4, latitude=12.9854, longitude=77.7081, Pin code=560048, Output='Yes', Feedback='Positive', _c12='Yes')]"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.describe().show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A8UnOsJE0FQ0",
        "outputId": "d146b038-24af-42b8-d642-c00a7c7ed631"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+------------------+------+--------------+----------+--------------+--------------------------+------------------+--------------------+------------------+------------------+------+---------+----+\n",
            "|summary|               Age|Gender|Marital Status|Occupation|Monthly Income|Educational Qualifications|       Family size|            latitude|         longitude|          Pin code|Output| Feedback|_c12|\n",
            "+-------+------------------+------+--------------+----------+--------------+--------------------------+------------------+--------------------+------------------+------------------+------+---------+----+\n",
            "|  count|               388|   388|           388|       388|           388|                       388|               388|                 388|               388|               388|   388|      388| 388|\n",
            "|   mean|24.628865979381445|  NULL|          NULL|      NULL|          NULL|                      NULL|3.2809278350515463|  12.972057989690706| 77.60015953608251| 560040.1134020619|  NULL|     NULL|NULL|\n",
            "| stddev| 2.975592660672904|  NULL|          NULL|      NULL|          NULL|                      NULL|1.3510249396453127|0.044489248628105924|0.0513539170127217|31.399608710261372|  NULL|     NULL|NULL|\n",
            "|    min|                18|Female|       Married|  Employee|10001 to 25000|                  Graduate|                 1|             12.8652|           77.4842|            560001|    No|Negative |  No|\n",
            "|    max|                33|  Male|        Single|   Student|     No Income|                Uneducated|                 6|              13.102|           77.7582|            560109|   Yes| Positive| Yes|\n",
            "+-------+------------------+------+--------------+----------+--------------+--------------------------+------------------+--------------------+------------------+------------------+------+---------+----+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.isEmpty()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D_bBT59Z0Uib",
        "outputId": "0233c3f0-6252-4a4c-d771-06a8de3fc22c"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wy-r6NNW3Ta7",
        "outputId": "2b178853-d4eb-4811-e1b1-3aed2ace6342"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- Age: integer (nullable = true)\n",
            " |-- Gender: string (nullable = true)\n",
            " |-- Marital Status: string (nullable = true)\n",
            " |-- Occupation: string (nullable = true)\n",
            " |-- Monthly Income: string (nullable = true)\n",
            " |-- Educational Qualifications: string (nullable = true)\n",
            " |-- Family size: integer (nullable = true)\n",
            " |-- latitude: double (nullable = true)\n",
            " |-- longitude: double (nullable = true)\n",
            " |-- Pin code: integer (nullable = true)\n",
            " |-- Output: string (nullable = true)\n",
            " |-- Feedback: string (nullable = true)\n",
            " |-- _c12: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.exceptAll"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "id": "zrHfIqs87bSe",
        "outputId": "ee31f6db-be36-4988-ffc2-ca2d33f0c0ba"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<bound method DataFrame.exceptAll of DataFrame[Age: int, Gender: string, Marital Status: string, Occupation: string, Monthly Income: string, Educational Qualifications: string, Family size: int, latitude: double, longitude: double, Pin code: int, Output: string, Feedback: string, _c12: string]>"
            ],
            "text/html": [
              "<div style=\"max-width:800px; border: 1px solid var(--colab-border-color);\"><style>\n",
              "      pre.function-repr-contents {\n",
              "        overflow-x: auto;\n",
              "        padding: 8px 12px;\n",
              "        max-height: 500px;\n",
              "      }\n",
              "\n",
              "      pre.function-repr-contents.function-repr-contents-collapsed {\n",
              "        cursor: pointer;\n",
              "        max-height: 100px;\n",
              "      }\n",
              "    </style>\n",
              "    <pre style=\"white-space: initial; background:\n",
              "         var(--colab-secondary-surface-color); padding: 8px 12px;\n",
              "         border-bottom: 1px solid var(--colab-border-color);\"><b>pyspark.sql.dataframe.DataFrame.exceptAll</b><br/>def exceptAll(other: &#x27;DataFrame&#x27;) -&gt; &#x27;DataFrame&#x27;</pre><pre class=\"function-repr-contents function-repr-contents-collapsed\" style=\"\"><a class=\"filepath\" style=\"display:none\" href=\"#\">/usr/local/lib/python3.10/dist-packages/pyspark/sql/dataframe.py</a>Return a new :class:`DataFrame` containing rows in this :class:`DataFrame` but\n",
              "not in another :class:`DataFrame` while preserving duplicates.\n",
              "\n",
              "This is equivalent to `EXCEPT ALL` in SQL.\n",
              "As standard in SQL, this function resolves columns by position (not by name).\n",
              "\n",
              ".. versionadded:: 2.4.0\n",
              "\n",
              ".. versionchanged:: 3.4.0\n",
              "    Supports Spark Connect.\n",
              "\n",
              "Parameters\n",
              "----------\n",
              "other : :class:`DataFrame`\n",
              "    The other :class:`DataFrame` to compare to.\n",
              "\n",
              "Returns\n",
              "-------\n",
              ":class:`DataFrame`\n",
              "\n",
              "Examples\n",
              "--------\n",
              "&gt;&gt;&gt; df1 = spark.createDataFrame(\n",
              "...         [(&quot;a&quot;, 1), (&quot;a&quot;, 1), (&quot;a&quot;, 1), (&quot;a&quot;, 2), (&quot;b&quot;,  3), (&quot;c&quot;, 4)], [&quot;C1&quot;, &quot;C2&quot;])\n",
              "&gt;&gt;&gt; df2 = spark.createDataFrame([(&quot;a&quot;, 1), (&quot;b&quot;, 3)], [&quot;C1&quot;, &quot;C2&quot;])\n",
              "&gt;&gt;&gt; df1.exceptAll(df2).show()\n",
              "+---+---+\n",
              "| C1| C2|\n",
              "+---+---+\n",
              "|  a|  1|\n",
              "|  a|  1|\n",
              "|  a|  2|\n",
              "|  c|  4|\n",
              "+---+---+</pre>\n",
              "      <script>\n",
              "      if (google.colab.kernel.accessAllowed && google.colab.files && google.colab.files.view) {\n",
              "        for (const element of document.querySelectorAll('.filepath')) {\n",
              "          element.style.display = 'block'\n",
              "          element.onclick = (event) => {\n",
              "            event.preventDefault();\n",
              "            event.stopPropagation();\n",
              "            google.colab.files.view(element.textContent, 745);\n",
              "          };\n",
              "        }\n",
              "      }\n",
              "      for (const element of document.querySelectorAll('.function-repr-contents')) {\n",
              "        element.onclick = (event) => {\n",
              "          event.preventDefault();\n",
              "          event.stopPropagation();\n",
              "          element.classList.toggle('function-repr-contents-collapsed');\n",
              "        };\n",
              "      }\n",
              "      </script>\n",
              "      </div>"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "new_df = df.select('Age', 'Monthly Income')\n",
        "new_df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rHhGd9-781J8",
        "outputId": "a7b43da8-0b8f-4bb2-a520-1d8b612f7b84"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+---------------+\n",
            "|Age| Monthly Income|\n",
            "+---+---------------+\n",
            "| 20|      No Income|\n",
            "| 24| Below Rs.10000|\n",
            "| 22| Below Rs.10000|\n",
            "| 22|      No Income|\n",
            "| 22| Below Rs.10000|\n",
            "| 27|More than 50000|\n",
            "| 22|      No Income|\n",
            "| 24|      No Income|\n",
            "| 23|      No Income|\n",
            "| 23|      No Income|\n",
            "| 22|      No Income|\n",
            "| 23| Below Rs.10000|\n",
            "| 23|      No Income|\n",
            "| 21|      No Income|\n",
            "| 23| 10001 to 25000|\n",
            "| 24|      No Income|\n",
            "| 28| 25001 to 50000|\n",
            "| 23|      No Income|\n",
            "| 25|      No Income|\n",
            "| 21| Below Rs.10000|\n",
            "+---+---------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "first_two_rows = df.take(2)\n",
        "\n",
        "for row in first_two_rows:\n",
        "  print(row)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_jggAGrm9AJv",
        "outputId": "5229ab7a-5942-4636-ee19-01b7549daf61"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Row(Age=20, Gender='Female', Marital Status='Single', Occupation='Student', Monthly Income='No Income', Educational Qualifications='Post Graduate', Family size=4, latitude=12.9766, longitude=77.5993, Pin code=560001, Output='Yes', Feedback='Positive', _c12='Yes')\n",
            "Row(Age=24, Gender='Female', Marital Status='Single', Occupation='Student', Monthly Income='Below Rs.10000', Educational Qualifications='Graduate', Family size=3, latitude=12.977, longitude=77.5773, Pin code=560009, Output='Yes', Feedback='Positive', _c12='Yes')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "first_row = df.first()\n",
        "\n",
        "# Convert the row to a dictionary\n",
        "row_dict = first_row.asDict()\n",
        "print(row_dict)\n",
        "type(row_dict)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qJa8KdwC9Vbq",
        "outputId": "e5e5205a-e4cb-4167-cadc-a40cee689784"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'Age': 20, 'Gender': 'Female', 'Marital Status': 'Single', 'Occupation': 'Student', 'Monthly Income': 'No Income', 'Educational Qualifications': 'Post Graduate', 'Family size': 4, 'latitude': 12.9766, 'longitude': 77.5993, 'Pin code': 560001, 'Output': 'Yes', 'Feedback': 'Positive', '_c12': 'Yes'}\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.withColumn('Post_Graduate_customer',\n",
        "                   when(df['Educational Qualifications'] == 'Post Graduate',\n",
        "                        'Yes')\n",
        "                   .otherwise('No'))\n",
        "df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ic0JC8DF9t0z",
        "outputId": "ea54c095-71eb-497f-b228-0292a48ae154"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+------+--------------+--------------+---------------+--------------------------+-----------+--------+---------+--------+------+---------+----+----------------------+\n",
            "|Age|Gender|Marital Status|    Occupation| Monthly Income|Educational Qualifications|Family size|latitude|longitude|Pin code|Output| Feedback|_c12|Post_Graduate_customer|\n",
            "+---+------+--------------+--------------+---------------+--------------------------+-----------+--------+---------+--------+------+---------+----+----------------------+\n",
            "| 20|Female|        Single|       Student|      No Income|             Post Graduate|          4| 12.9766|  77.5993|  560001|   Yes| Positive| Yes|                   Yes|\n",
            "| 24|Female|        Single|       Student| Below Rs.10000|                  Graduate|          3|  12.977|  77.5773|  560009|   Yes| Positive| Yes|                    No|\n",
            "| 22|  Male|        Single|       Student| Below Rs.10000|             Post Graduate|          3| 12.9551|  77.6593|  560017|   Yes|Negative | Yes|                   Yes|\n",
            "| 22|Female|        Single|       Student|      No Income|                  Graduate|          6| 12.9473|  77.5616|  560019|   Yes| Positive| Yes|                    No|\n",
            "| 22|  Male|        Single|       Student| Below Rs.10000|             Post Graduate|          4|  12.985|  77.5533|  560010|   Yes| Positive| Yes|                   Yes|\n",
            "| 27|Female|       Married|      Employee|More than 50000|             Post Graduate|          2| 12.9299|  77.6848|  560103|   Yes| Positive| Yes|                   Yes|\n",
            "| 22|  Male|        Single|       Student|      No Income|                  Graduate|          3|  12.977|  77.5773|  560009|   Yes| Positive| Yes|                    No|\n",
            "| 24|Female|        Single|       Student|      No Income|             Post Graduate|          3| 12.9828|  77.6131|  560042|   Yes| Positive| Yes|                   Yes|\n",
            "| 23|Female|        Single|       Student|      No Income|             Post Graduate|          2| 12.9766|  77.5993|  560001|   Yes| Positive| Yes|                   Yes|\n",
            "| 23|Female|        Single|       Student|      No Income|             Post Graduate|          4| 12.9854|  77.7081|  560048|   Yes| Positive| Yes|                   Yes|\n",
            "| 22|Female|        Single|       Student|      No Income|             Post Graduate|          5|  12.985|  77.5533|  560010|   Yes| Positive| Yes|                   Yes|\n",
            "| 23|  Male|        Single|       Student| Below Rs.10000|             Post Graduate|          2|  12.977|  77.5773|  560009|   Yes|Negative | Yes|                   Yes|\n",
            "| 23|  Male|        Single|       Student|      No Income|             Post Graduate|          5| 12.8988|  77.5764|  560078|   Yes| Positive| Yes|                   Yes|\n",
            "| 21|  Male|        Single|       Student|      No Income|                  Graduate|          4|  12.977|  77.5773|  560009|   Yes| Positive| Yes|                    No|\n",
            "| 23|Female|        Single|Self Employeed| 10001 to 25000|             Post Graduate|          5| 12.9438|  77.5738|  560004|   Yes| Positive| Yes|                   Yes|\n",
            "| 24|Female|        Single|       Student|      No Income|             Post Graduate|          6| 12.8893|  77.6399|  560068|   Yes| Positive| Yes|                   Yes|\n",
            "| 28|Female|        Single|      Employee| 25001 to 50000|             Post Graduate|          2| 12.9783|  77.6408|  560038|   Yes| Positive| Yes|                   Yes|\n",
            "| 23|Female|        Single|       Student|      No Income|                  Graduate|          3|  12.982|  77.6256|  560008|   Yes|Negative | Yes|                    No|\n",
            "| 25|  Male|        Single|       Student|      No Income|                  Graduate|          4| 12.8988|  77.5764|  560078|   Yes|Negative | Yes|                    No|\n",
            "| 21|Female|        Single|       Student| Below Rs.10000|             Post Graduate|          1| 12.9783|  77.6408|  560038|   Yes| Positive| Yes|                   Yes|\n",
            "+---+------+--------------+--------------+---------------+--------------------------+-----------+--------+---------+--------+------+---------+----+----------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.drop('Post_Graduate_customer').show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UrNMoyEU_4cB",
        "outputId": "fb0db0cd-e6f9-4f6f-aa2f-60a0d7441bf0"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+------+--------------+--------------+---------------+--------------------------+-----------+--------+---------+--------+------+---------+----+\n",
            "|Age|Gender|Marital Status|    Occupation| Monthly Income|Educational Qualifications|Family size|latitude|longitude|Pin code|Output| Feedback|_c12|\n",
            "+---+------+--------------+--------------+---------------+--------------------------+-----------+--------+---------+--------+------+---------+----+\n",
            "| 20|Female|        Single|       Student|      No Income|             Post Graduate|          4| 12.9766|  77.5993|  560001|   Yes| Positive| Yes|\n",
            "| 24|Female|        Single|       Student| Below Rs.10000|                  Graduate|          3|  12.977|  77.5773|  560009|   Yes| Positive| Yes|\n",
            "| 22|  Male|        Single|       Student| Below Rs.10000|             Post Graduate|          3| 12.9551|  77.6593|  560017|   Yes|Negative | Yes|\n",
            "| 22|Female|        Single|       Student|      No Income|                  Graduate|          6| 12.9473|  77.5616|  560019|   Yes| Positive| Yes|\n",
            "| 22|  Male|        Single|       Student| Below Rs.10000|             Post Graduate|          4|  12.985|  77.5533|  560010|   Yes| Positive| Yes|\n",
            "| 27|Female|       Married|      Employee|More than 50000|             Post Graduate|          2| 12.9299|  77.6848|  560103|   Yes| Positive| Yes|\n",
            "| 22|  Male|        Single|       Student|      No Income|                  Graduate|          3|  12.977|  77.5773|  560009|   Yes| Positive| Yes|\n",
            "| 24|Female|        Single|       Student|      No Income|             Post Graduate|          3| 12.9828|  77.6131|  560042|   Yes| Positive| Yes|\n",
            "| 23|Female|        Single|       Student|      No Income|             Post Graduate|          2| 12.9766|  77.5993|  560001|   Yes| Positive| Yes|\n",
            "| 23|Female|        Single|       Student|      No Income|             Post Graduate|          4| 12.9854|  77.7081|  560048|   Yes| Positive| Yes|\n",
            "| 22|Female|        Single|       Student|      No Income|             Post Graduate|          5|  12.985|  77.5533|  560010|   Yes| Positive| Yes|\n",
            "| 23|  Male|        Single|       Student| Below Rs.10000|             Post Graduate|          2|  12.977|  77.5773|  560009|   Yes|Negative | Yes|\n",
            "| 23|  Male|        Single|       Student|      No Income|             Post Graduate|          5| 12.8988|  77.5764|  560078|   Yes| Positive| Yes|\n",
            "| 21|  Male|        Single|       Student|      No Income|                  Graduate|          4|  12.977|  77.5773|  560009|   Yes| Positive| Yes|\n",
            "| 23|Female|        Single|Self Employeed| 10001 to 25000|             Post Graduate|          5| 12.9438|  77.5738|  560004|   Yes| Positive| Yes|\n",
            "| 24|Female|        Single|       Student|      No Income|             Post Graduate|          6| 12.8893|  77.6399|  560068|   Yes| Positive| Yes|\n",
            "| 28|Female|        Single|      Employee| 25001 to 50000|             Post Graduate|          2| 12.9783|  77.6408|  560038|   Yes| Positive| Yes|\n",
            "| 23|Female|        Single|       Student|      No Income|                  Graduate|          3|  12.982|  77.6256|  560008|   Yes|Negative | Yes|\n",
            "| 25|  Male|        Single|       Student|      No Income|                  Graduate|          4| 12.8988|  77.5764|  560078|   Yes|Negative | Yes|\n",
            "| 21|Female|        Single|       Student| Below Rs.10000|             Post Graduate|          1| 12.9783|  77.6408|  560038|   Yes| Positive| Yes|\n",
            "+---+------+--------------+--------------+---------------+--------------------------+-----------+--------+---------+--------+------+---------+----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "distinct_df = df.distinct()\n",
        "\n",
        "print('DataFrame aftee removing duplicates:')\n",
        "distinct_df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vANjvyj1_8fE",
        "outputId": "edda0966-887c-4986-e20e-2aca5fd19446"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "DataFrame aftee removing duplicates:\n",
            "+---+------+--------------+--------------+---------------+--------------------------+-----------+--------+---------+--------+------+---------+----+----------------------+\n",
            "|Age|Gender|Marital Status|    Occupation| Monthly Income|Educational Qualifications|Family size|latitude|longitude|Pin code|Output| Feedback|_c12|Post_Graduate_customer|\n",
            "+---+------+--------------+--------------+---------------+--------------------------+-----------+--------+---------+--------+------+---------+----+----------------------+\n",
            "| 26|Female|       Married|      Employee| 25001 to 50000|                  Graduate|          3|  13.014|  77.5658|  560012|   Yes| Positive| Yes|                    No|\n",
            "| 22|  Male|        Single|       Student|      No Income|             Post Graduate|          1| 13.0626|  77.5284|  560015|   Yes| Positive| Yes|                   Yes|\n",
            "| 23|Female|        Single|       Student|      No Income|             Post Graduate|          2|  12.977|  77.5773|  560009|   Yes| Positive| Yes|                   Yes|\n",
            "| 21|  Male|        Single|       Student|      No Income|                  Graduate|          2| 13.0012|  77.5995|  560046|    No|Negative |  No|                    No|\n",
            "| 26|  Male|        Single|      Employee| 25001 to 50000|             Post Graduate|          3|  13.014|  77.5658|  560012|   Yes| Positive| Yes|                   Yes|\n",
            "| 27|Female|        Single|       Student|      No Income|                      Ph.D|          5| 13.0503|  77.5529|  560013|    No|Negative |  No|                    No|\n",
            "| 28|  Male|       Married|      Employee|More than 50000|             Post Graduate|          1| 12.9925|  77.5633|  560021|   Yes| Positive| Yes|                   Yes|\n",
            "| 21|Female|        Single|       Student|      No Income|                  Graduate|          2| 13.0103|  77.5796|  560080|   Yes| Positive| Yes|                    No|\n",
            "| 22|  Male|       Married|Self Employeed| 10001 to 25000|                    School|          3|  12.982|  77.6256|  560008|   Yes|Negative | Yes|                    No|\n",
            "| 23|Female|        Single|       Student|      No Income|             Post Graduate|          4| 12.9854|  77.7081|  560048|   Yes| Positive| Yes|                   Yes|\n",
            "| 26|  Male|        Single|Self Employeed| 25001 to 50000|                  Graduate|          3| 12.9766|  77.5993|  560001|   Yes| Positive| Yes|                    No|\n",
            "| 23|Female|        Single|       Student|      No Income|             Post Graduate|          2| 12.9766|  77.5993|  560001|   Yes| Positive| Yes|                   Yes|\n",
            "| 24|  Male|        Single|       Student|      No Income|             Post Graduate|          5| 12.9621|  77.5376|  560104|   Yes| Positive| Yes|                   Yes|\n",
            "| 21|  Male|        Single|       Student|      No Income|                  Graduate|          5| 12.9783|  77.6408|  560038|   Yes| Positive| Yes|                    No|\n",
            "| 21|Female|        Single|       Student|      No Income|             Post Graduate|          3| 12.9149|  77.5635|  560070|   Yes| Positive| Yes|                   Yes|\n",
            "| 19|  Male|        Single|       Student|      No Income|                  Graduate|          2| 13.0019|  77.5713|  560003|    No|Negative |  No|                    No|\n",
            "| 24|Female|       Married|Self Employeed| 10001 to 25000|                  Graduate|          5| 12.9579|  77.6309|  560007|   Yes| Positive| Yes|                    No|\n",
            "| 22|Female|        Single|       Student|      No Income|                  Graduate|          3| 12.9343|  77.6044|  560029|   Yes| Positive| Yes|                    No|\n",
            "| 31|Female|       Married|    House wife|      No Income|                    School|          5| 13.0289|    77.54|  560022|   Yes| Positive| Yes|                    No|\n",
            "| 29|  Male|       Married|      Employee| 25001 to 50000|                  Graduate|          4| 12.9261|  77.6221|  560034|    No|Negative |  No|                    No|\n",
            "+---+------+--------------+--------------+---------------+--------------------------+-----------+--------+---------+--------+------+---------+----+----------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Creating dataframe from the scratch\n",
        "data = [('Alice', 34, 'Single'),\n",
        "        ('Bob', 45, 'Married'),\n",
        "        ('Charlier', 28, 'Single')]\n",
        "\n",
        "columns = ['Name', 'Age', 'MaritalStatus']\n",
        "df = spark.createDataFrame(data, columns)\n",
        "\n",
        "# Register the DataFrame as a temporary view\n",
        "df.createOrReplaceTempView('people')\n",
        "\n",
        "# Execute a SQL query using spark.sql()\n",
        "result = spark.sql('SELECT * FROM people WHERE Age > 30')\n",
        "\n",
        "# Show the result\n",
        "result.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VOLMEt0-ASvA",
        "outputId": "977301e0-56b7-4108-d2f6-1269520c084d"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+---+-------------+\n",
            "| Name|Age|MaritalStatus|\n",
            "+-----+---+-------------+\n",
            "|Alice| 34|       Single|\n",
            "|  Bob| 45|      Married|\n",
            "+-----+---+-------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.describe().show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C5JeYZNUASDY",
        "outputId": "b5e5ac48-8cdb-4be8-d1aa-e583152244cd"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+--------+------------------+-------------+\n",
            "|summary|    Name|               Age|MaritalStatus|\n",
            "+-------+--------+------------------+-------------+\n",
            "|  count|       3|                 3|            3|\n",
            "|   mean|    NULL|35.666666666666664|         NULL|\n",
            "| stddev|    NULL| 8.621678104251709|         NULL|\n",
            "|    min|   Alice|                28|      Married|\n",
            "|    max|Charlier|                45|       Single|\n",
            "+-------+--------+------------------+-------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(df.dtypes) # see the difference with pandas df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j9AjqhJiB2tb",
        "outputId": "0ef67600-ed64-4b9d-c0b8-c018c2d4c12e"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[('Name', 'string'), ('Age', 'bigint'), ('MaritalStatus', 'string')]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "filtered_df = df.filter((df['Age'] > 30) & (df['MaritalStatus'] == 'Single'))\n",
        "filtered_df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "59kUXNMVB6za",
        "outputId": "e7d5d904-ea78-4159-a64f-9c72cf4b2faa"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+---+-------------+\n",
            "| Name|Age|MaritalStatus|\n",
            "+-----+---+-------------+\n",
            "|Alice| 34|       Single|\n",
            "+-----+---+-------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "collected_data = df.collect()\n",
        "print(collected_data)\n",
        "# type(collected_data) # it is a list. Really? For big data, you want to see 1M full observations in a list"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9ABubLYliLHp",
        "outputId": "a0f99ec4-13df-437a-bebb-b60c60c91b53"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Row(Name='Alice', Age=34, MaritalStatus='Single'), Row(Name='Bob', Age=45, MaritalStatus='Married'), Row(Name='Charlier', Age=28, MaritalStatus='Single')]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a DataFrame\n",
        "data = [('Alice', 34, 'Single'),\n",
        "        ('Bob', 45, 'Married'),\n",
        "        ('Charlier', 28, 'Single'),\n",
        "        ('David', 37, 'Divorced'),\n",
        "        ('Eve', 52, 'Married')]\n",
        "\n",
        "columns = ['Name', 'Age', 'MaritalStatus']\n",
        "df = spark.createDataFrame(data, columns)\n",
        "\n",
        "# Use agg() to compute the count of each marital status\n",
        "marital_status_counts = df.groupBy('MaritalStatus').agg(count('*').alias('Count'))\n",
        "\n",
        "# Show the result\n",
        "marital_status_counts.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qlZ0rbJ7iTXj",
        "outputId": "4c7d7823-ade0-4ad2-9c37-59c520c6161a"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------+-----+\n",
            "|MaritalStatus|Count|\n",
            "+-------------+-----+\n",
            "|      Married|    2|\n",
            "|       Single|    2|\n",
            "|     Divorced|    1|\n",
            "+-------------+-----+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data1 = [('Alice', 34, 'Single'),\n",
        "        ('Bob', 45, 'Married'),\n",
        "        ('Charlier', 28, 'Single')]\n",
        "\n",
        "columns1 = ['Name', 'Age', 'MaritalStatus']\n",
        "df1 = spark.createDataFrame(data1, columns1)\n",
        "\n",
        "# Example of data for the second DataFrame\n",
        "data2 = [('Alice', 'New York'),\n",
        "         ('Bob', 'Los Angeles'),\n",
        "         ('Charlier', 'Chicago')]\n",
        "\n",
        "columns2 = ['Name', 'City']\n",
        "df2 = spark.createDataFrame(data2, columns2)\n",
        "\n",
        "joined_df = df1.join(df2, 'Name', 'outer')\n",
        "\n",
        "# Show the joined DataFrame\n",
        "joined_df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8W8BgwMQmPhc",
        "outputId": "bc699d7e-935f-4bb2-d275-5a0be794594b"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+---+-------------+-----------+\n",
            "|    Name|Age|MaritalStatus|       City|\n",
            "+--------+---+-------------+-----------+\n",
            "|   Alice| 34|       Single|   New York|\n",
            "|     Bob| 45|      Married|Los Angeles|\n",
            "|Charlier| 28|       Single|    Chicago|\n",
            "+--------+---+-------------+-----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Example of data for the first DataFrame\n",
        "data1 = [('Alice', 34, 'Single'),\n",
        "        ('Bob', 45, 'Married'),\n",
        "        ('Charlier', 28, 'Single')]\n",
        "\n",
        "columns1 = ['Name', 'Age', 'MaritalStatus']\n",
        "df1 = spark.createDataFrame(data1, columns1)\n",
        "\n",
        "# Example of data for the second DataFrame\n",
        "data2 = [('New York',),\n",
        "         ('Los Angeles',),\n",
        "         ('Chicago',)]\n",
        "\n",
        "columns2 = ['City']\n",
        "df2 = spark.createDataFrame(data2, columns2)\n",
        "\n",
        "# Use union() to combine the two DataFrames\n",
        "cross_join_df = df1.crossJoin(df2.select('*'))\n",
        "\n",
        "# Show the combined DataFrame\n",
        "cross_join_df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KER6W4d9Ec0o",
        "outputId": "a665989d-d40f-4a58-ceab-3a36114e1e95"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+---+-------------+-----------+\n",
            "|    Name|Age|MaritalStatus|       City|\n",
            "+--------+---+-------------+-----------+\n",
            "|   Alice| 34|       Single|   New York|\n",
            "|   Alice| 34|       Single|Los Angeles|\n",
            "|   Alice| 34|       Single|    Chicago|\n",
            "|     Bob| 45|      Married|   New York|\n",
            "|Charlier| 28|       Single|   New York|\n",
            "|     Bob| 45|      Married|Los Angeles|\n",
            "|     Bob| 45|      Married|    Chicago|\n",
            "|Charlier| 28|       Single|Los Angeles|\n",
            "|Charlier| 28|       Single|    Chicago|\n",
            "+--------+---+-------------+-----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Example data for the first DataFrame (small)\n",
        "data1 = [('Alice', 34),\n",
        "        ('Bob', 45),\n",
        "        ('Charlier', 28)]\n",
        "\n",
        "columns1 = ['Name', 'Age']\n",
        "df1 = spark.createDataFrame(data1, columns1)\n",
        "\n",
        "# Example data for the second DataFrame (large)\n",
        "data2 = [('Alice', 'New York'),\n",
        "         ('Bob', 'Los Angeles'),\n",
        "         ('Charlier', 'Chicago'),\n",
        "         ('David', 'Houston'),\n",
        "         ('Eve', 'San Francisco')]\n",
        "\n",
        "columns2 = ['Name', 'City']\n",
        "df2 = spark.createDataFrame(data2, columns2)\n",
        "\n",
        "broadcast_df1 = broadcast(df1)\n",
        "print(broadcast_df1.show())\n",
        "print(df1.show())\n",
        "\n",
        "joined_df = broadcast_df1.join(df2, 'Name', 'outer')\n",
        "\n",
        "# Show the joined DataFrame\n",
        "joined_df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TJG0YOFhGAS3",
        "outputId": "e2c09ec4-9b88-4a3b-d5ba-5f1e625dd9cc"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+---+\n",
            "|    Name|Age|\n",
            "+--------+---+\n",
            "|   Alice| 34|\n",
            "|     Bob| 45|\n",
            "|Charlier| 28|\n",
            "+--------+---+\n",
            "\n",
            "None\n",
            "+--------+---+\n",
            "|    Name|Age|\n",
            "+--------+---+\n",
            "|   Alice| 34|\n",
            "|     Bob| 45|\n",
            "|Charlier| 28|\n",
            "+--------+---+\n",
            "\n",
            "None\n",
            "+--------+----+-------------+\n",
            "|    Name| Age|         City|\n",
            "+--------+----+-------------+\n",
            "|   Alice|  34|     New York|\n",
            "|     Bob|  45|  Los Angeles|\n",
            "|Charlier|  28|      Chicago|\n",
            "|   David|NULL|      Houston|\n",
            "|     Eve|NULL|San Francisco|\n",
            "+--------+----+-------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_alias = df.alias('Person')\n",
        "df_alias.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uDuWXCxtIm-L",
        "outputId": "74363544-297f-403d-f329-7c120c1e347e"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+---+-------------+\n",
            "|    Name|Age|MaritalStatus|\n",
            "+--------+---+-------------+\n",
            "|   Alice| 34|       Single|\n",
            "|     Bob| 45|      Married|\n",
            "|Charlier| 28|       Single|\n",
            "|   David| 37|     Divorced|\n",
            "|     Eve| 52|      Married|\n",
            "+--------+---+-------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Example data\n",
        "data = [('Alice', 'Sales', 5000),\n",
        "        ('Bob', 'Sales', 6000),\n",
        "        ('Charlier', 'Marketing', 4000),\n",
        "        ('David', 'Marketing', 3000)]\n",
        "\n",
        "columns = ['Name', 'Department', 'Salary']\n",
        "df = spark.createDataFrame(data, columns)\n",
        "\n",
        "# Perform rollup aggregation across 'Department' and 'Name;\n",
        "rollup_df = df.rollup('Department', 'Name').sum('Salary')\n",
        "\n",
        "# Show the resulting DataFrame\n",
        "rollup_df.show()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C9QWEXKYJP2A",
        "outputId": "52b2bd90-9fbe-4b95-f187-bfa6f48ca064"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+--------+-----------+\n",
            "|Department|    Name|sum(Salary)|\n",
            "+----------+--------+-----------+\n",
            "|     Sales|     Bob|       6000|\n",
            "|      NULL|    NULL|      18000|\n",
            "|     Sales|   Alice|       5000|\n",
            "|     Sales|    NULL|      11000|\n",
            "| Marketing|Charlier|       4000|\n",
            "| Marketing|    NULL|       7000|\n",
            "| Marketing|   David|       3000|\n",
            "+----------+--------+-----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Example data\n",
        "data = [('Alice', 'Sales', 5000),\n",
        "        ('Bob', 'Sales', 6000),\n",
        "        ('Charlier', 'Marketing', 4000),\n",
        "        ('David', 'Marketing', 3000)]\n",
        "\n",
        "columns = ['Name', 'Department', 'Salary']\n",
        "df = spark.createDataFrame(data, columns)\n",
        "\n",
        "# Perform cube aggregation across 'Department' and 'Name;\n",
        "cube_df = df.cube('Department', 'Name').sum('Salary')\n",
        "\n",
        "# Show the resulting DataFrame\n",
        "cube_df.show()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MG0UolZbLMzS",
        "outputId": "d616acb7-122f-4602-9d3f-4b975651626a"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+--------+-----------+\n",
            "|Department|    Name|sum(Salary)|\n",
            "+----------+--------+-----------+\n",
            "|      NULL|   Alice|       5000|\n",
            "|     Sales|     Bob|       6000|\n",
            "|      NULL|     Bob|       6000|\n",
            "|      NULL|    NULL|      18000|\n",
            "|     Sales|   Alice|       5000|\n",
            "|     Sales|    NULL|      11000|\n",
            "|      NULL|Charlier|       4000|\n",
            "| Marketing|Charlier|       4000|\n",
            "| Marketing|    NULL|       7000|\n",
            "|      NULL|   David|       3000|\n",
            "| Marketing|   David|       3000|\n",
            "+----------+--------+-----------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Example data\n",
        "data = [('Alice', 'Sales', 5000),\n",
        "        ('Bob', 'Sales', 6000),\n",
        "        ('Charlier', 'Marketing', 4000),\n",
        "        ('David', 'Marketing', 3000)]\n",
        "\n",
        "columns = ['Name', 'Department', 'Salary']\n",
        "df = spark.createDataFrame(data, columns)\n",
        "\n",
        "# Perform pivot operation on 'Name' column\n",
        "pivot_df1 = df.groupBy('Department').pivot('Name').sum('Salary')\n",
        "# Show the resulting DataFrame\n",
        "pivot_df1.show()\n",
        "\n",
        "\n",
        "# Perform pivot operation on 'Department' column\n",
        "pivot_df2 = df.groupBy('Name').pivot('Department').sum('Salary')\n",
        "# Show the resulting DataFrame\n",
        "pivot_df2.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "waLNyKCVLv5h",
        "outputId": "e4890f32-429e-4148-8348-9bcb8e9173b7"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+-----+----+--------+-----+\n",
            "|Department|Alice| Bob|Charlier|David|\n",
            "+----------+-----+----+--------+-----+\n",
            "|     Sales| 5000|6000|    NULL| NULL|\n",
            "| Marketing| NULL|NULL|    4000| 3000|\n",
            "+----------+-----+----+--------+-----+\n",
            "\n",
            "+--------+---------+-----+\n",
            "|    Name|Marketing|Sales|\n",
            "+--------+---------+-----+\n",
            "|Charlier|     4000| NULL|\n",
            "|     Bob|     NULL| 6000|\n",
            "|   Alice|     NULL| 5000|\n",
            "|   David|     3000| NULL|\n",
            "+--------+---------+-----+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Example of data\n",
        "data = [('Alice', 34, 'Single'),\n",
        "        ('Bob', 45, 'Married'),\n",
        "        ('Charlier', 28, 'Single')]\n",
        "\n",
        "columns = ['Name', 'Age', 'MaritalStatus']\n",
        "df = spark.createDataFrame(data, columns)\n",
        "\n",
        "# Sort the DataFrame by the 'Age' column in ascending order\n",
        "sorted_df1 = df.sort('Age')\n",
        "sorted_df1.show()\n",
        "\n",
        "sorted_df2 = df.orderBy('Age')\n",
        "sorted_df2.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-lfHPBApM9mp",
        "outputId": "eea12d24-71b1-4aa5-d1e7-5ea66bd831b5"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+---+-------------+\n",
            "|    Name|Age|MaritalStatus|\n",
            "+--------+---+-------------+\n",
            "|Charlier| 28|       Single|\n",
            "|   Alice| 34|       Single|\n",
            "|     Bob| 45|      Married|\n",
            "+--------+---+-------------+\n",
            "\n",
            "+--------+---+-------------+\n",
            "|    Name|Age|MaritalStatus|\n",
            "+--------+---+-------------+\n",
            "|Charlier| 28|       Single|\n",
            "|   Alice| 34|       Single|\n",
            "|     Bob| 45|      Married|\n",
            "+--------+---+-------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Example of data\n",
        "data = [('Alice', 34, 'Single'),\n",
        "        ('Bob', 45, 'Married'),\n",
        "        ('Alice', 34, 'Single'),\n",
        "        ('Charlier', 28, 'Single')]\n",
        "\n",
        "columns = ['Name', 'Age', 'MaritalStatus']\n",
        "df = spark.createDataFrame(data, columns)\n",
        "\n",
        "# Drop duplicate rows considering all columns\n",
        "deduplicated_df = df.dropDuplicates()\n",
        "deduplicated_df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wO1Q5TzqOTAT",
        "outputId": "66d52afe-a7d8-4edd-ceeb-81938b991afc"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+---+-------------+\n",
            "|    Name|Age|MaritalStatus|\n",
            "+--------+---+-------------+\n",
            "|   Alice| 34|       Single|\n",
            "|     Bob| 45|      Married|\n",
            "|Charlier| 28|       Single|\n",
            "+--------+---+-------------+\n",
            "\n"
          ]
        }
      ]
    }
  ]
}