extract_links.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ca73080e-7ba3-4fac-8c3e-d7468e4014b7",
   "metadata": {},
   "source": [
    "# Extract workshop links from `_posts`\n",
    "Extracts workshop links from `_posts` and writes them to `links.csv` file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "1c6aa851-b919-4f0b-a3c8-c66ad60b87c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import markdown\n",
    "from lxml import etree\n",
    "from pathlib import Path\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "60ac038d-98b5-420a-ac21-de6d39adf2b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('_posts/2019-12-04-Rijkzwaan.md',mode='r') as file:\n",
    "    body = file.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "bca346e1-c3a1-4cbe-baba-63eb9cc5fa9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_links(filepath):\n",
    "    with open(filepath,mode='r') as file:\n",
    "        body = file.read()\n",
    "    doc = etree.fromstring(f'<div>{markdown.markdown(body)}</div>')\n",
    "    for link in doc.xpath('//a'):\n",
    "      yield str(link.get('href'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "0c99ed9b-5033-4c85-869f-e7e3bdfe86db",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_links(links):\n",
    "    return [link for link in links if 'eventbrite' not in link]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "564ae85e-90d0-4011-ab1a-04b105ee6ca8",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "pathlist = Path('_posts').glob('*')\n",
    "for path in sorted(pathlist):\n",
    "    path_in_str = str(path)\n",
    "    links = list(read_links(path_in_str))\n",
    "    links = filter_links(links)\n",
    "    slug = path_in_str.split('/')[1][:-3]\n",
    "    this_data = dict()\n",
    "    this_data['slug'] = slug\n",
    "    if len(links) > 0:\n",
    "        this_data['url'] = links[0]\n",
    "    if len(links) > 1:\n",
    "        this_data['other_url'] = links[1]\n",
    "    data.append(this_data)\n",
    "df = pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "6c3aa2e2-9604-4904-b94d-4929eafba6fc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>slug</th>\n",
       "      <th>url</th>\n",
       "      <th>other_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2016-01-25-Utrecht</td>\n",
       "      <td>https://github.com/esciencecenter-digital-skil...</td>\n",
       "      <td>http://mkuzak.github.io/2016-01-25-Utrecht/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-09-07-Xenon-RSE2017</td>\n",
       "      <td>https://rse.ac.uk/conf2017/</td>\n",
       "      <td>https://xenonrse2017.readthedocs.io/en/latest/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2017-09-08-Utrecht</td>\n",
       "      <td>https://esciencecenter-digital-skills.github.i...</td>\n",
       "      <td>https://esciencecenter-digital-skills.github.i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2017-11-06-Utrecht</td>\n",
       "      <td>https://esciencecenter-digital-skills.github.i...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2018-01-23-Wageningen</td>\n",
       "      <td>https://esciencecenter-digital-skills.github.i...</td>\n",
       "      <td>https://github.com/jhidding/stability</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>2022-05-30-dc-socsci-python-nlesc</td>\n",
       "      <td>https://esciencecenter-digital-skills.github.i...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>2022-06-21-ttt-nehterla</td>\n",
       "      <td>https://esciencecenter-digital-skills.github.i...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>2022-09-14-ds-rpackaging</td>\n",
       "      <td>https://esciencecenter-digital-skills.github.i...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>2022-09-26-ds-dl-intro</td>\n",
       "      <td>https://esciencecenter-digital-skills.github.i...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>2022-10-10-ds-parallel</td>\n",
       "      <td>https://esciencecenter-digital-skills.github.i...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>64 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 slug  \\\n",
       "0                  2016-01-25-Utrecht   \n",
       "1            2017-09-07-Xenon-RSE2017   \n",
       "2                  2017-09-08-Utrecht   \n",
       "3                  2017-11-06-Utrecht   \n",
       "4               2018-01-23-Wageningen   \n",
       "..                                ...   \n",
       "59  2022-05-30-dc-socsci-python-nlesc   \n",
       "60            2022-06-21-ttt-nehterla   \n",
       "61           2022-09-14-ds-rpackaging   \n",
       "62             2022-09-26-ds-dl-intro   \n",
       "63             2022-10-10-ds-parallel   \n",
       "\n",
       "                                                  url  \\\n",
       "0   https://github.com/esciencecenter-digital-skil...   \n",
       "1                         https://rse.ac.uk/conf2017/   \n",
       "2   https://esciencecenter-digital-skills.github.i...   \n",
       "3   https://esciencecenter-digital-skills.github.i...   \n",
       "4   https://esciencecenter-digital-skills.github.i...   \n",
       "..                                                ...   \n",
       "59  https://esciencecenter-digital-skills.github.i...   \n",
       "60  https://esciencecenter-digital-skills.github.i...   \n",
       "61  https://esciencecenter-digital-skills.github.i...   \n",
       "62  https://esciencecenter-digital-skills.github.i...   \n",
       "63  https://esciencecenter-digital-skills.github.i...   \n",
       "\n",
       "                                            other_url  \n",
       "0         http://mkuzak.github.io/2016-01-25-Utrecht/  \n",
       "1      https://xenonrse2017.readthedocs.io/en/latest/  \n",
       "2   https://esciencecenter-digital-skills.github.i...  \n",
       "3                                                 NaN  \n",
       "4               https://github.com/jhidding/stability  \n",
       "..                                                ...  \n",
       "59                                                NaN  \n",
       "60                                                NaN  \n",
       "61                                                NaN  \n",
       "62                                                NaN  \n",
       "63                                                NaN  \n",
       "\n",
       "[64 rows x 3 columns]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "2bfc7ae2-db8a-460a-9795-c37c44ea177d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('links.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4efc8e49-e86b-4100-91dd-ad2586ce2937",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}