[TP_1]_ANN_with_PyTorch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "63e677bff0f24a0bb9ac3651be6a8c8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dd470e06e581436f83c244489a82f50c",
              "IPY_MODEL_e63665ce65d443b394aab48cf074b11b",
              "IPY_MODEL_3cd19bf77cfb41a6a6963180ead88667"
            ],
            "layout": "IPY_MODEL_0228a76f7a3e4f0f8943d9dd1e2bddbe"
          }
        },
        "dd470e06e581436f83c244489a82f50c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e0494fbd06644440a5a854b7f0d2eb86",
            "placeholder": "​",
            "style": "IPY_MODEL_17bc0640b5ac471da08ebdf2711a2797",
            "value": "100%"
          }
        },
        "e63665ce65d443b394aab48cf074b11b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_53028024358d409b843cdd9ef46e38d5",
            "max": 9912422,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_656f0703c251403f86ad77873f889f68",
            "value": 9912422
          }
        },
        "3cd19bf77cfb41a6a6963180ead88667": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b5efc48cde0e4d3b8bec7cf9e9c30c2c",
            "placeholder": "​",
            "style": "IPY_MODEL_284307b1c28e49e29f671de968289982",
            "value": " 9912422/9912422 [00:00&lt;00:00, 25819706.90it/s]"
          }
        },
        "0228a76f7a3e4f0f8943d9dd1e2bddbe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e0494fbd06644440a5a854b7f0d2eb86": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "17bc0640b5ac471da08ebdf2711a2797": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "53028024358d409b843cdd9ef46e38d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "656f0703c251403f86ad77873f889f68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b5efc48cde0e4d3b8bec7cf9e9c30c2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "284307b1c28e49e29f671de968289982": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cc299d3273ef4dab9c30492e2e45f1cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_24c2ff0cfdce449f8a3cc8faf42f67f8",
              "IPY_MODEL_f70894021dbb40fba26ca434c9a75f70",
              "IPY_MODEL_b7bca870c30147fba2f41f61866da7d4"
            ],
            "layout": "IPY_MODEL_f2d904aa4cc44606a3693bc74f9ab6d3"
          }
        },
        "24c2ff0cfdce449f8a3cc8faf42f67f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b007181fdd9c45139cbf441a3e99edda",
            "placeholder": "​",
            "style": "IPY_MODEL_96517a86eccc492d92ed31c7c61bbf30",
            "value": "100%"
          }
        },
        "f70894021dbb40fba26ca434c9a75f70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_27111eac6bce439f98fe8d22505692a7",
            "max": 28881,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_79a9d621e3f845098edbd5e627ff6882",
            "value": 28881
          }
        },
        "b7bca870c30147fba2f41f61866da7d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2e22976459f242f784a4104c224bece0",
            "placeholder": "​",
            "style": "IPY_MODEL_bae7039ec0284cfeb929a532cc944fe5",
            "value": " 28881/28881 [00:00&lt;00:00, 884570.18it/s]"
          }
        },
        "f2d904aa4cc44606a3693bc74f9ab6d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b007181fdd9c45139cbf441a3e99edda": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "96517a86eccc492d92ed31c7c61bbf30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "27111eac6bce439f98fe8d22505692a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79a9d621e3f845098edbd5e627ff6882": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2e22976459f242f784a4104c224bece0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bae7039ec0284cfeb929a532cc944fe5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0e483e536e154561a9995c348a09cbc2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cbd3559e8171481782505aad5ca27f62",
              "IPY_MODEL_d1755af0c0d0470885d368291b15f541",
              "IPY_MODEL_8654aa132d91454badf1e37070527bfd"
            ],
            "layout": "IPY_MODEL_235584d9f4fd485aab129a94c707b2b6"
          }
        },
        "cbd3559e8171481782505aad5ca27f62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_83a6de6d2b754a1692060e9bc87a052c",
            "placeholder": "​",
            "style": "IPY_MODEL_b32d84979dae48d2842be38c5755d6d0",
            "value": "100%"
          }
        },
        "d1755af0c0d0470885d368291b15f541": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_36ae9643f1054b08a138e12faaf0b166",
            "max": 1648877,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d35037e50fc842c49248e63087026826",
            "value": 1648877
          }
        },
        "8654aa132d91454badf1e37070527bfd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f33e0bdd9cb94adb81a357ade7fa38d2",
            "placeholder": "​",
            "style": "IPY_MODEL_b88c6bf1546347ed9253b02fdf4db774",
            "value": " 1648877/1648877 [00:00&lt;00:00, 18138806.00it/s]"
          }
        },
        "235584d9f4fd485aab129a94c707b2b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "83a6de6d2b754a1692060e9bc87a052c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b32d84979dae48d2842be38c5755d6d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "36ae9643f1054b08a138e12faaf0b166": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d35037e50fc842c49248e63087026826": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f33e0bdd9cb94adb81a357ade7fa38d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b88c6bf1546347ed9253b02fdf4db774": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d0821a40e091483999e35c6af862faaa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5554de99683d45648649a7ce12c77c48",
              "IPY_MODEL_7c22e03c0c774797822b987eff4ff176",
              "IPY_MODEL_57eb81e848654cd1b787a9719183a1f8"
            ],
            "layout": "IPY_MODEL_8af11124d9a24feaaddf1c48c04a61dd"
          }
        },
        "5554de99683d45648649a7ce12c77c48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_392d0220caf94a10912a06c4a00c057e",
            "placeholder": "​",
            "style": "IPY_MODEL_09228659d1794370893c7e15c0f6aac3",
            "value": "100%"
          }
        },
        "7c22e03c0c774797822b987eff4ff176": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c2cb90a014874b499ea1a635ce72fd06",
            "max": 4542,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_23ab5dad117142248de1aadbe28f19ba",
            "value": 4542
          }
        },
        "57eb81e848654cd1b787a9719183a1f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_898e990b9ebd4c8984d1d6a20c449084",
            "placeholder": "​",
            "style": "IPY_MODEL_d29aaad5396548dfbafce6c7f4765a60",
            "value": " 4542/4542 [00:00&lt;00:00, 147296.00it/s]"
          }
        },
        "8af11124d9a24feaaddf1c48c04a61dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "392d0220caf94a10912a06c4a00c057e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "09228659d1794370893c7e15c0f6aac3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c2cb90a014874b499ea1a635ce72fd06": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "23ab5dad117142248de1aadbe28f19ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "898e990b9ebd4c8984d1d6a20c449084": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d29aaad5396548dfbafce6c7f4765a60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/KontainPluton/TPs_Artificial_Neural_Network/blob/main/%5BTP_1%5D_ANN_with_PyTorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ym3v3hBGDnIF"
      },
      "source": [
        "# PyTorch introduction\n",
        "\n",
        "\n",
        "**Objective** The goal of this practical session is to implement the simple fully functional artificial neural network (ANN) you've implemented in the previous session, but in a much more simple way using PyTorch. \n",
        "\n",
        "At this stage we will assume that you've fully understand what has been done in the first session, and that you've in particular understood the takeaway message proposed at the end of the session (we will quickly repeat some notions, but do not hesistate to read it again carefully if it's not the case). \n",
        "\n",
        "As we will see, the PyTorch framework will implicitly take care of numerous tricky details, in particular those related to gradient computation and backpropagation.   \n",
        "\n",
        "**Prerequisites**: introductory level in machine learning, basics of calculus and linear algebra, a first quick introduction to ANNs, basic Python skills, previous pratical sessions, and well, as usual, motivation!\n",
        "\n",
        "**Estimated time**: 8 hours (generous)\n",
        "\n",
        "****\n",
        "\n",
        "\n",
        "<img  width=\"200\" src=\"https://drive.google.com/uc?id=1_d9U9O9kUOvQTvm4vdpEUyn0deB2nBXI\"/>\n",
        "\n",
        "*PyTorch is an open source machine learning library based on the Torch library, used for applications such as computer vision and natural language processing primarily developed by Facebook's AI Research lab (FAIR). It is free and open-source software released under the Modified BSD license. Although the Python interface is more polished and the primary focus of development, PyTorch also has a C++ interface.*\n",
        "\n",
        "*A number of pieces of Deep Learning software are built on top of PyTorch, including Tesla Autopilot, Uber's Pyro, HuggingFace's Transformers, PyTorch Lightning, and Catalyst.*\n",
        "\n",
        "*PyTorch provides two high-level features:*\n",
        "\n",
        "* *Tensor computing (like NumPy) with strong acceleration via graphics processing units (GPU)*\n",
        "* *Deep neural networks built on a tape-based automatic differentiation system*\n",
        "\n",
        "source: https://en.wikipedia.org/wiki/PyTorch\n",
        "\n",
        "Official website: https://pytorch.org/ \n",
        "\n",
        "This tutorial is based on https://pytorch.org/tutorials/beginner/pytorch_with_examples.html. Several modifications have been made, and extra content added to match, as much as possible, with the content given in this course and your background. \n",
        "\n",
        "We will first briefly discuss important notions of the framework in order to introduce some concepts and to implement our neural network. Specific sessions will be next proposed in which you'll have the change to deeply analyse some of those notions. \n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u_jV-snuFVkV"
      },
      "source": [
        "## PyTorch: Tensors\n",
        "\n",
        "**PyTorch is heavily based on the notion of *Tensor*. \n",
        "You can consider that these Tensors are conceptually identical to numpy array (matrix)** for now, there are *just* n-dimensional arrays and, similarly to the  many operations as Numpy provides to operate on Numpy arrays, PyTorch provides numerous operations to operate on Tensors. Obviously PyTorch is not reinventing the wheel here, Tensor are supercharged n-dimensional arrays in the context of neural network development, just be patient.   \n",
        "\n",
        "You recall our first raw ANN implementation using Numpy. \n",
        "Let's do the same using PyTorch.\n",
        "\n",
        "We will still use the same labelling function (implemented using Numpy). \n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IhCrfiar-Rl2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "44d216b0-6d35-49a6-d037-d3cfaa4f0960"
      },
      "source": [
        "# -*- coding: utf-8 -*-\n",
        "import numpy as np\n",
        "import torch\n",
        "\n",
        "theta = 45\n",
        "\n",
        "# our labeling function\n",
        "def x_axis_theta_rotation(p):\n",
        "  \"\"\" return new (x,y,z) coordinates of the point after a theta-based rotation along the x-axis\n",
        "      Note: understanding this function is not important. \n",
        "      We could also add a random noise\n",
        "  \"\"\"\n",
        "\n",
        "  return np.array([\n",
        "    p[0],\n",
        "    p[1]*np.cos(p[0] * theta) - p[2]*np.sin(p[0] * theta),\n",
        "    p[1]*np.sin(p[0] * theta) + p[2]*np.cos(p[0] * theta)\n",
        "  ])\n",
        "\n",
        "def get_dataset(N, D_in, D_out):\n",
        "\n",
        "  \"\"\"Create N random input and associated label\"\"\"\n",
        "  x = np.random.randn(N, D_in)\n",
        "  y = np.array([x_axis_theta_rotation(xi) for xi in x]) \n",
        "\n",
        "  return (x,y)\n",
        "\n",
        "N = 1000 # size of the dataset\n",
        "D_in, D_out = 3, 3 # input and output dimensions, we name them to distinguish them hereafter\n",
        "\n",
        "x, y = get_dataset(N, D_in, D_out)\n",
        "x, y = torch.from_numpy(x).float(), torch.from_numpy(y).float()\n",
        "\n",
        "print(\"Data (first 10 elements)\")\n",
        "print(\"Inputs\\n\", x[:10,:])\n",
        "print(\"Labels\\n\", y[:10,:])\n",
        "\n",
        "print(\"\\nExample\")\n",
        "print(\"input 1: \", x[0])\n",
        "print(\"label 1: \", y[0])\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Data (first 10 elements)\n",
            "Inputs\n",
            " tensor([[ 0.6814, -0.4042,  0.2958],\n",
            "        [ 0.3772, -0.1664, -0.1145],\n",
            "        [ 0.5588, -0.1401, -1.9276],\n",
            "        [-0.2006, -0.7230, -0.6830],\n",
            "        [-1.1777,  1.8797,  0.9316],\n",
            "        [ 0.2266, -1.0920, -0.7031],\n",
            "        [ 0.1084,  0.7845,  0.2417],\n",
            "        [-0.2677, -0.0423,  0.2796],\n",
            "        [ 1.1464,  1.1791, -0.0476],\n",
            "        [-0.4697, -1.8787, -0.8839]])\n",
            "Labels\n",
            " tensor([[ 0.6814, -0.0934,  0.4920],\n",
            "        [ 0.3772, -0.0590,  0.1932],\n",
            "        [ 0.5588, -0.1135, -1.9294],\n",
            "        [-0.2006,  0.4025,  0.9095],\n",
            "        [-1.1777, -1.3521, -1.6041],\n",
            "        [ 0.2266,  0.2940,  1.2650],\n",
            "        [ 0.1084,  0.3665, -0.7345],\n",
            "        [-0.2677, -0.1758,  0.2215],\n",
            "        [ 1.1464,  0.3365,  1.1311],\n",
            "        [-0.4697,  0.5622,  1.9987]])\n",
            "\n",
            "Example\n",
            "input 1:  tensor([ 0.6814, -0.4042,  0.2958])\n",
            "label 1:  tensor([ 0.6814, -0.0934,  0.4920])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gs-FlRnO-SFa"
      },
      "source": [
        "As you can see, there is a bridge between Numpy and PyTorch. \n",
        "The output is pretty similar to what we got except that we are now dealing with Tensors. Note that the same syntactic sugar syntax can be used to manipulate our tensors (e.g. `x[:1,:]`). "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sivAleVaEFyL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "02ab048e-99a3-4175-f487-e8655e25e987"
      },
      "source": [
        "# -*- coding: utf-8 -*-\n",
        "\n",
        "import torch\n",
        "\n",
        "H = 64 # number of neurons in the hidden layer\n",
        "\n",
        "dtype = torch.float\n",
        "\n",
        "# Randomly initialize weights\n",
        "w1 = torch.randn(D_in, H, dtype=dtype)\n",
        "w2 = torch.randn(H, D_out, dtype=dtype)\n",
        "\n",
        "learning_rate = 1e-6\n",
        "\n",
        "for t in range(30001):\n",
        "    # Forward pass: compute predicted y\n",
        "    h = x.mm(w1) # x @ w1 (like np.dot)\n",
        "    h_relu = h.clamp(min=0)\n",
        "    y_pred = h_relu.mm(w2)\n",
        "\n",
        "    # Compute and print loss\n",
        "    loss = (y_pred - y).pow(2).sum().item()\n",
        "    if t % 1000 == 0:\n",
        "        print(t, loss)\n",
        "\n",
        "    # Backprop to compute gradients of w1 and w2 with respect to loss\n",
        "    grad_y_pred = 2.0 * (y_pred - y)\n",
        "    grad_w2 = h_relu.t().mm(grad_y_pred)\n",
        "    grad_h = grad_y_pred.mm(w2.t())\n",
        "    grad_h[h < 0] = 0\n",
        "    grad_w1 = x.t().mm(grad_h)\n",
        "\n",
        "    # Update weights using gradient descent\n",
        "    w1 -= learning_rate * grad_w1\n",
        "    w2 -= learning_rate * grad_w2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0 155201.40625\n",
            "1000 2581.572021484375\n",
            "2000 2258.8515625\n",
            "3000 2122.04833984375\n",
            "4000 2040.269775390625\n",
            "5000 1991.5140380859375\n",
            "6000 1957.214111328125\n",
            "7000 1932.2303466796875\n",
            "8000 1914.2841796875\n",
            "9000 1901.155029296875\n",
            "10000 1889.0750732421875\n",
            "11000 1878.1668701171875\n",
            "12000 1870.048583984375\n",
            "13000 1862.3287353515625\n",
            "14000 1855.537841796875\n",
            "15000 1849.213134765625\n",
            "16000 1842.78955078125\n",
            "17000 1836.79150390625\n",
            "18000 1831.572265625\n",
            "19000 1826.4666748046875\n",
            "20000 1821.822509765625\n",
            "21000 1817.405517578125\n",
            "22000 1813.10546875\n",
            "23000 1808.8836669921875\n",
            "24000 1805.5235595703125\n",
            "25000 1803.0594482421875\n",
            "26000 1800.482666015625\n",
            "27000 1797.9573974609375\n",
            "28000 1795.822265625\n",
            "29000 1793.76953125\n",
            "30000 1791.8878173828125\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S53rHff-Cc5_"
      },
      "source": [
        "Do not compare the results you obtain running the first notebook since the datasets are not the same.\n",
        "\n",
        "As you can see, the implementation is quite similar. \n",
        "We just had to use the proper functions to implement the operations we wanted to perform on our Tensors in both the forward and backward passes. We will not discuss them in detail now but it's quite intuitive `mm(...)` stands for matrix multiplication, `t()` for transpose...\n",
        "\n",
        "*****\n",
        "\n",
        "Let's now introduce a first very interesting feature of tensors. \n",
        "We will change the device on which will be performed our computations. \n",
        "\n",
        "We will used Graphic cards, GPUs ([Graphical Processing Units](https://en.wikipedia.org/wiki/Graphics_processing_unit)). These devices are very effective to operate on n-dimensional arrays, e.g. to perform linear algebra operations which are often used in our ANNs. \n",
        "\n",
        "<img  width=\"300\" src=\"https://drive.google.com/uc?id=1cxGXDY06sH9GRS1o8xOksTwOEPzicRvU\"/>\n",
        "\n",
        "Above a picture of several Nvidia v100 32G GPUs.\n",
        "\n",
        "First you have to specify you want to use a GPU using Colab. \n",
        "To do so, in the menu, select `Runtime/Change Runtime type` then select `GPU`. \n",
        "You'll need to reload some of the code you already executed since your notebook is now most surely running on a different computer. *As specified by the Colab interface, GPUs are sparse resources, only use them if you require them (limitations are applied)*.\n",
        "\n",
        "Tiny modifications will be made in our code to enable training on GPUs.\n",
        "Thoses modifications aim at making sure that the datastructures we are using during the forward and backward passes are located on the proper device. It would obviously make no sense to perform a matrix multiplication of two matrices located to different devices. \n",
        "\n",
        "\n",
        "We will therefore: \n",
        "* load the training data on the corresponding device. Tensors can simply be loaded to a device using `.to()`  \n",
        "* Make sure that the tensors related to our ANN's parameters are on GPU. To do so we can specify the device on which the tensor data have to be stored creating a tensor,e.g. to create a random tensor on a given device `dev` we will simply use `w1 = torch.randn(..., device=dev)`. \n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CQdkDr-zEXG9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d8b989ea-fc96-4f22-8fa4-a27578db6385"
      },
      "source": [
        "import datetime\n",
        "\n",
        "def run_training(x, y, epochs, device): \n",
        "\n",
        "  print(\"starting training on \", device)\n",
        "  start_time = datetime.datetime.now()\n",
        "\n",
        "  # Put our training data on the good device\n",
        "  x = x.to(device)\n",
        "  y = y.to(device)\n",
        "\n",
        "  H = 500 # number of neurons in the hidden layer\n",
        "  dtype = torch.float\n",
        "  # Randomly initialize weights\n",
        "  w1 = torch.randn(D_in, H, device=device, dtype=dtype)\n",
        "  w2 = torch.randn(H, D_out, device=device, dtype=dtype)\n",
        "\n",
        "  learning_rate = 1e-6\n",
        "  for t in range(epochs):\n",
        "      # Forward pass: compute predicted y\n",
        "      h = x.mm(w1) # x @ w1 also supported\n",
        "      h_relu = h.clamp(min=0)\n",
        "      y_pred = h_relu.mm(w2)\n",
        "\n",
        "      # Compute and print loss\n",
        "      loss = (y_pred - y).pow(2).sum().item()\n",
        "      if t % 5000 == 0:\n",
        "          print(\"{}\\t{:.3f}\".format(t, loss))\n",
        "\n",
        "      # Backprop to compute gradients of w1 and w2 with respect to loss\n",
        "      grad_y_pred = 2.0 * (y_pred - y)\n",
        "      grad_w2 = h_relu.t().mm(grad_y_pred)\n",
        "      grad_h = grad_y_pred.mm(w2.t())\n",
        "      grad_h[h < 0] = 0\n",
        "      grad_w1 = x.t().mm(grad_h)\n",
        "\n",
        "      # Update weights using gradient descent\n",
        "      w1 -= learning_rate * grad_w1\n",
        "      w2 -= learning_rate * grad_w2\n",
        "\n",
        "  run_time = datetime.datetime.now() - start_time\n",
        "  print(device, \"training time ~\", run_time.seconds, \"sec\\n\")\n",
        "  return run_time.seconds\n",
        "\n",
        "epochs = 5000\n",
        "\n",
        "device = torch.device(\"cpu\")\n",
        "run_time_sec_cpu = run_training(x, y, epochs, device)\n",
        "\n",
        "device = torch.device(\"cuda:0\") # Uncomment this to run on GPU\n",
        "run_time_sec_gpu = run_training(x, y, epochs, device)\n",
        "\n",
        "print(\"Speeding factor using GPU  x {}{:.2f}\".format(\"+\" if run_time_sec_cpu > run_time_sec_gpu else \"-\", run_time_sec_cpu / run_time_sec_gpu))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "starting training on  cpu\n",
            "0\t2973040.000\n",
            "cpu training time ~ 24 sec\n",
            "\n",
            "starting training on  cuda:0\n",
            "0\t2242825.000\n",
            "cuda:0 training time ~ 1 sec\n",
            "\n",
            "Speeding factor using GPU  x +24.00\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q4-yKzmJMvAT"
      },
      "source": [
        "Note that the model we are implementing is very tiny. You can augment the number of parameters increasing $H$. Do not exagerate tho, give it a try with $H=500$: keep in mind that this modification impacts the shapes of $W_{1}$ and $W_{2}$ that will be loaded on GPUs (a 32G GPU is today a big GPU).\n",
        "\n",
        "Speeding factor up to 50 can be obtained (yes, in one day you can train a model that would require 50 days training on CPU).\n",
        "\n",
        "This is already a very interesting feature of PyTorch Tensors!\n",
        "\n",
        "But there is more. \n",
        "\n",
        "*Please free the GPUs changing the runtime type to GPU*\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Ze6vFw4xemG"
      },
      "source": [
        "device = torch.device(\"cpu\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b2TSLBZkFbT9"
      },
      "source": [
        "## PyTorch: autograd\n",
        "\n",
        "Using PyTorch, it is possible to specify that we want the history of operations applied to a Tensor to be recorded. Recording these operations will enable the framework to automatically perform differentiation to compute the gradients we will need to perform backpropagation (`autograd`). \n",
        "\n",
        "We will use this powerful mechanism to automatically compute the gradient of the loss function with regard to the parameters. Only the Tensors related to the parameters will thus be concerned.  \n",
        "\n",
        "Telling PyTorch that we will need to compute the gradient of something (the loss) with regards to a Tensor is very easy. Creating the Tensor you just need to specify `requires_grad=True` (default is false). \n",
        "\n",
        "Creating a random Tensor for which the information required to compute the gradient will be stored is thus: `w1 = torch.randn(..., requires_grad=True)`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uGZv7GxwE1s_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "83e95d21-9b1d-4130-a67f-55af6e10d4e8"
      },
      "source": [
        "# Create random Tensors for weights.\n",
        "# Setting requires_grad=True indicates that we want to compute gradients with\n",
        "# respect to these Tensors during the backward pass.\n",
        "w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True)\n",
        "w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True)\n",
        "\n",
        "learning_rate = 1e-6\n",
        "for t in range(500):\n",
        "    # Forward pass: compute predicted y using operations on Tensors; these\n",
        "    # are exactly the same operations we used to compute the forward pass using\n",
        "    # Tensors, but we do not need to keep references to intermediate values since\n",
        "    # we are not implementing the backward pass by hand.\n",
        "    y_pred = x.mm(w1).clamp(min=0).mm(w2)\n",
        "\n",
        "    # Compute and print loss using operations on Tensors.\n",
        "    # Now loss is a Tensor of shape (1,)\n",
        "    # loss.item() gets the scalar value held in the loss.\n",
        "    loss = (y_pred - y).pow(2).sum()\n",
        "    \n",
        "    if t % 100 == 99:\n",
        "        print(t, loss.item())\n",
        "\n",
        "    # Use autograd to compute the backward pass. This call will compute the\n",
        "    # gradient of loss with respect to all Tensors with requires_grad=True.\n",
        "    # After this call w1.grad and w2.grad will be Tensors holding the gradient\n",
        "    # of the loss with respect to w1 and w2 respectively.\n",
        "    loss.backward()\n",
        "\n",
        "    # Manually update weights using gradient descent. Wrap in torch.no_grad()\n",
        "    # because weights have requires_grad=True, but we don't need to track this\n",
        "    # in autograd.\n",
        "    with torch.no_grad():\n",
        "        w1 -= learning_rate * w1.grad\n",
        "        w2 -= learning_rate * w2.grad\n",
        "\n",
        "        # Manually zero the gradients after updating weights\n",
        "        w1.grad.zero_()\n",
        "        w2.grad.zero_()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "99 5178.23046875\n",
            "199 3778.9248046875\n",
            "299 3423.142333984375\n",
            "399 3236.691650390625\n",
            "499 3101.42822265625\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9KB7EC2yS21R"
      },
      "source": [
        "You should be very pleased at this stage. \n",
        "No need to compute the gradient of the loss with regards to the parameters of the network anymore (and that was painful). This can now be automatically managed by the framework."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yGnbYR75wtFp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0e4683f2-822e-46b8-9095-8cb8129b7f67"
      },
      "source": [
        "# Create random Tensors for weights.\n",
        "# Setting requires_grad=True indicates that we want to compute gradients with\n",
        "# respect to these Tensors during the backward pass.\n",
        "w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True)\n",
        "w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True)\n",
        "\n",
        "print(\"Gradient w1: \", w1.grad)\n",
        "print(\"Gradient w2: \", w2.grad)\n",
        "\n",
        "learning_rate = 1e-6\n",
        "\n",
        "y_pred = x.mm(w1).clamp(min=0).mm(w2)\n",
        "loss = (y_pred - y).pow(2).sum()\n",
        "print(\"loss: \", loss)\n",
        "\n",
        "loss.backward()\n",
        "\n",
        "print(\"Gradient w1 (shape): \", w1.grad.shape)\n",
        "print(\"Gradient w2 (shape): \", w2.grad.shape)\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Gradient w1:  None\n",
            "Gradient w2:  None\n",
            "loss:  tensor(114628.4062, grad_fn=<SumBackward0>)\n",
            "Gradient w1 (shape):  torch.Size([3, 64])\n",
            "Gradient w2 (shape):  torch.Size([64, 3])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4pwnlLJE9bjU"
      },
      "source": [
        "From the [official documentation](https://pytorch.org/docs/stable/notes/autograd.html), you can read: \n",
        "\n",
        "*Autograd is reverse automatic differentiation system. Conceptually, autograd records a graph recording all of the operations that created the data as you execute operations, giving you a directed acyclic graph whose leaves are the input tensors and roots are the output tensors. By tracing this graph from roots to leaves, you can automatically compute the gradients using the chain rule.*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_rh9t5r_Fq9i"
      },
      "source": [
        "## PyTorch: nn\n",
        "\n",
        "We will keep simplifying the way we define our ANNs. Indeed, when building neural networks we frequently think of arranging the computation into layers, some of which have learnable parameters which will be optimized during learning.\n",
        "PyTorch provides the `nn` package which makes the definition of ANN easy.\n",
        "\n",
        "In PyTorch, the `nn` package provide higher-level abstractions over raw computational graphs that are useful for building neural networks (Keras for instance provides the same functionalities using TensorFlow). \n",
        "\n",
        "Indeed, the `nn` package defines a set of Modules, which are roughly equivalent to neural network layers. A Module receives input Tensors and computes output Tensors, but may also hold internal state such as Tensors containing learnable parameters. The `nn` package also defines a set of useful loss functions that are commonly used when training neural networks.\n",
        "\n",
        "In this example we use the nn package to implement our simple ANN:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kcXGlobQE9dp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d9935393-441f-414e-b5fc-bf9b3941aaeb"
      },
      "source": [
        "\n",
        "# Use the nn package to define our model as a sequence of layers. nn.Sequential\n",
        "# is a Module which contains other Modules, and applies them in sequence to\n",
        "# produce its output. Each Linear Module computes output from input using a\n",
        "# linear function, and holds internal Tensors for its weight and bias.\n",
        "model = torch.nn.Sequential(\n",
        "    torch.nn.Linear(D_in, H),\n",
        "    torch.nn.ReLU(),\n",
        "    torch.nn.Linear(H, D_out),\n",
        ")\n",
        "\n",
        "def train(model, epochs, log_frequency):\n",
        "\n",
        "  print(\"Training\")\n",
        "\n",
        "  # The nn package also contains definitions of popular loss functions; in this\n",
        "  # case we will use Mean Squared Error (MSE) as our loss function.\n",
        "  # Remember that we are dealing with a regression task.\n",
        "  loss_fn = torch.nn.MSELoss(reduction='sum')\n",
        "\n",
        "  learning_rate = 1e-4\n",
        "  for t in range(epochs+1):\n",
        "      # Forward pass: compute predicted y by passing x to the model. Module objects\n",
        "      # override the __call__ operator so you can call them like functions. When\n",
        "      # doing so you pass a Tensor of input data to the Module and it produces\n",
        "      # a Tensor of output data.\n",
        "      y_pred = model(x) # same as model.forward(x) (Pass forward)\n",
        "\n",
        "      # Compute and print loss. We pass Tensors containing the predicted and true\n",
        "      # values of y, and the loss function returns a Tensor containing the\n",
        "      # loss.\n",
        "      loss = loss_fn(y_pred, y)\n",
        "      if t % log_frequency == 0:\n",
        "          print(t, loss.item())\n",
        "\n",
        "      # Zero the gradients before running the backward pass.\n",
        "      model.zero_grad()\n",
        "\n",
        "      # Backward pass: compute gradient of the loss with respect to all the learnable\n",
        "      # parameters of the model. Internally, the parameters of each Module are stored\n",
        "      # in Tensors with requires_grad=True, so this call will compute gradients for\n",
        "      # all learnable parameters in the model.\n",
        "      loss.backward()\n",
        "\n",
        "      # Update the weights using gradient descent. Each parameter is a Tensor, so\n",
        "      # we can access its gradients like we did before.\n",
        "      with torch.no_grad():\n",
        "          for param in model.parameters():\n",
        "              param -= learning_rate * param.grad\n",
        "\n",
        "\n",
        "train(model, 30000, 5000)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training\n",
            "0 2990.069580078125\n",
            "5000 1783.3824462890625\n",
            "10000 1735.5933837890625\n",
            "15000 1753.14306640625\n",
            "20000 1729.34521484375\n",
            "25000 1811.726318359375\n",
            "30000 1744.028076171875\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xqBVCRniKGzc"
      },
      "source": [
        "We will later use numerous module and loss implementations exposed by the `nn` package. It will help us to define more complex ANNs.\n",
        "\n",
        "See how it is easy now to add several layers to our model. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "68tiuz47Kify",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "847ec4e9-5dc9-44d6-c9a4-39a7c9092081"
      },
      "source": [
        "model = torch.nn.Sequential(\n",
        "    torch.nn.Linear(D_in, H),\n",
        "    torch.nn.ReLU(),\n",
        "    torch.nn.Linear(H, H),\n",
        "    torch.nn.ReLU(),\n",
        "    torch.nn.Linear(H, H),\n",
        "    torch.nn.ReLU(),\n",
        "    torch.nn.Linear(H, D_out),\n",
        ")\n",
        "\n",
        "train(model, 30000, 5000)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training\n",
            "0 3058.288330078125\n",
            "5000 926.2432250976562\n",
            "10000 506.29962158203125\n",
            "15000 339.3179626464844\n",
            "20000 231.51913452148438\n",
            "25000 221.85214233398438\n",
            "30000 91.50235748291016\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dA5c0K_5L6CZ"
      },
      "source": [
        "You can also use fancy python constructs to create your network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xak_AVBNMBTO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f3febdf8-c671-4eda-8995-cabb625e0918"
      },
      "source": [
        "def buildFFN(nb_hidden_layers): # FFN: Feedforward network\n",
        "\n",
        "  assert nb_hidden_layers >= 2, \"nb_hidden_layers must be >= 2\"\n",
        "  \n",
        "  # defining the first hidden layer\n",
        "  layers = [\n",
        "            torch.nn.Linear(D_in, H),\n",
        "            torch.nn.ReLU()\n",
        "            ]\n",
        "\n",
        "  # adding any extra hidden layers\n",
        "  for i in range(nb_hidden_layers -2):\n",
        "    layers += [\n",
        "              torch.nn.Linear(H, H),\n",
        "              torch.nn.ReLU()\n",
        "              ]\n",
        "\n",
        "  # adding the output layer\n",
        "  layers += [torch.nn.Linear(H, D_out)]\n",
        "\n",
        "  model = torch.nn.Sequential(*layers) # The single star * unpacks the sequence/collection into positional arguments\n",
        "  print(model)\n",
        "\n",
        "  nb_trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad) # model.parameters() gives access to the model parameters.\n",
        "  print(\"number of trainable paramerters: \", nb_trainable_params)\n",
        "\n",
        "  return model\n",
        "\n",
        "H = 128 # number of neurons in each layer\n",
        "model = buildFFN(nb_hidden_layers = 10)\n",
        "\n",
        "train(model, 10000, 1000)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sequential(\n",
            "  (0): Linear(in_features=3, out_features=128, bias=True)\n",
            "  (1): ReLU()\n",
            "  (2): Linear(in_features=128, out_features=128, bias=True)\n",
            "  (3): ReLU()\n",
            "  (4): Linear(in_features=128, out_features=128, bias=True)\n",
            "  (5): ReLU()\n",
            "  (6): Linear(in_features=128, out_features=128, bias=True)\n",
            "  (7): ReLU()\n",
            "  (8): Linear(in_features=128, out_features=128, bias=True)\n",
            "  (9): ReLU()\n",
            "  (10): Linear(in_features=128, out_features=128, bias=True)\n",
            "  (11): ReLU()\n",
            "  (12): Linear(in_features=128, out_features=128, bias=True)\n",
            "  (13): ReLU()\n",
            "  (14): Linear(in_features=128, out_features=128, bias=True)\n",
            "  (15): ReLU()\n",
            "  (16): Linear(in_features=128, out_features=128, bias=True)\n",
            "  (17): ReLU()\n",
            "  (18): Linear(in_features=128, out_features=3, bias=True)\n",
            ")\n",
            "number of trainable paramerters:  132995\n",
            "Training\n",
            "0 3045.44482421875\n",
            "1000 1990.05322265625\n",
            "2000 1986.0855712890625\n",
            "3000 1869.8726806640625\n",
            "4000 1645.4398193359375\n",
            "5000 1845.59375\n",
            "6000 981.119140625\n",
            "7000 630.8112182617188\n",
            "8000 591.160888671875\n",
            "9000 392.5194396972656\n",
            "10000 181.07496643066406\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "smUQtrZJFB_q"
      },
      "source": [
        "## PyTorch: optim\n",
        "\n",
        "Up to this point we have updated the weights of our models by manually mutating the Tensors holding learnable parameters (with `torch.no_grad()` to avoid tracking history in autograd). \n",
        "\n",
        "This is not a huge burden for simple optimization algorithms like stochastic gradient descent, but in practice we often train neural networks using more sophisticated optimizers like AdaGrad, RMSProp, Adam, etc.\n",
        "\n",
        "The `optim` package in PyTorch abstracts the idea of an optimization algorithm and provides implementations of commonly used optimization algorithms.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gg_D5b65FxNl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "73e2af60-db85-46a1-ebcd-a521aa7c2aa8"
      },
      "source": [
        "\n",
        "def train_optim(model, epochs, log_frequency):\n",
        "\n",
        "  print(\"Training (nn.optim)\")\n",
        "\n",
        "  loss_fn = torch.nn.MSELoss(reduction='sum')\n",
        "\n",
        "  # Use the optim package to define an Optimizer that will update the weights of\n",
        "  # the model for us. Here we will use Adam; the optim package contains many other\n",
        "  # optimization algorithms. The first argument to the Adam constructor tells the\n",
        "  # optimizer which Tensors it should update.\n",
        "  learning_rate = 1e-4\n",
        "  optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
        "  model.train()\n",
        "  \n",
        "  for t in range(epochs+1):\n",
        "\n",
        "      y_pred = model(x)\n",
        "      loss = loss_fn(y_pred, y)\n",
        "\n",
        "      if t % log_frequency == 0:\n",
        "          print(t, loss.item())\n",
        "\n",
        "      # Before the backward pass, use the optimizer object to zero all of the\n",
        "      # gradients for the variables it will update (which are the learnable\n",
        "      # weights of the model). This is because by default, gradients are\n",
        "      # accumulated in buffers( i.e, not overwritten) whenever .backward()\n",
        "      # is called. Checkout docs of torch.autograd.backward for more details.\n",
        "      optimizer.zero_grad()\n",
        "\n",
        "      # Backward pass: compute gradient of the loss with respect to model\n",
        "      # parameters\n",
        "      loss.backward()\n",
        "\n",
        "      # Calling the step function on an Optimizer makes an update to its\n",
        "      # parameters\n",
        "      optimizer.step()\n",
        "\n",
        "H = 64\n",
        "model = buildFFN(nb_hidden_layers=2)\n",
        "train_optim(model, 30000, 5000)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sequential(\n",
            "  (0): Linear(in_features=3, out_features=64, bias=True)\n",
            "  (1): ReLU()\n",
            "  (2): Linear(in_features=64, out_features=3, bias=True)\n",
            ")\n",
            "number of trainable paramerters:  451\n",
            "Training (nn.optim)\n",
            "0 3160.486572265625\n",
            "5000 1808.784912109375\n",
            "10000 1652.245361328125\n",
            "15000 1542.7691650390625\n",
            "20000 1479.944091796875\n",
            "25000 1445.5252685546875\n",
            "30000 1425.3642578125\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LpnO9cIYpX1T"
      },
      "source": [
        "## Save / Load a model\n",
        "\n",
        "Details about model saving and loading will be discussed later.\n",
        "This will be used to load a train model to perform inference and to load reload a model and optimizer state to continue training an already trained model.  \n",
        "\n",
        "You can nevertheless already check the syntax in the official documentation   \n",
        "https://pytorch.org/tutorials/beginner/saving_loading_models.html"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EJJ-9jwdTQAi"
      },
      "source": [
        "# Exercise\n",
        "\n",
        "You'll be asked to create your first PyTorch ANN alone.\n",
        "\n",
        "You'll deal with a famous Handwritten digits dataset: http://yann.lecun.com/exdb/mnist/\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "63yiBbCgUWlk",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 457,
          "referenced_widgets": [
            "63e677bff0f24a0bb9ac3651be6a8c8e",
            "dd470e06e581436f83c244489a82f50c",
            "e63665ce65d443b394aab48cf074b11b",
            "3cd19bf77cfb41a6a6963180ead88667",
            "0228a76f7a3e4f0f8943d9dd1e2bddbe",
            "e0494fbd06644440a5a854b7f0d2eb86",
            "17bc0640b5ac471da08ebdf2711a2797",
            "53028024358d409b843cdd9ef46e38d5",
            "656f0703c251403f86ad77873f889f68",
            "b5efc48cde0e4d3b8bec7cf9e9c30c2c",
            "284307b1c28e49e29f671de968289982",
            "cc299d3273ef4dab9c30492e2e45f1cb",
            "24c2ff0cfdce449f8a3cc8faf42f67f8",
            "f70894021dbb40fba26ca434c9a75f70",
            "b7bca870c30147fba2f41f61866da7d4",
            "f2d904aa4cc44606a3693bc74f9ab6d3",
            "b007181fdd9c45139cbf441a3e99edda",
            "96517a86eccc492d92ed31c7c61bbf30",
            "27111eac6bce439f98fe8d22505692a7",
            "79a9d621e3f845098edbd5e627ff6882",
            "2e22976459f242f784a4104c224bece0",
            "bae7039ec0284cfeb929a532cc944fe5",
            "0e483e536e154561a9995c348a09cbc2",
            "cbd3559e8171481782505aad5ca27f62",
            "d1755af0c0d0470885d368291b15f541",
            "8654aa132d91454badf1e37070527bfd",
            "235584d9f4fd485aab129a94c707b2b6",
            "83a6de6d2b754a1692060e9bc87a052c",
            "b32d84979dae48d2842be38c5755d6d0",
            "36ae9643f1054b08a138e12faaf0b166",
            "d35037e50fc842c49248e63087026826",
            "f33e0bdd9cb94adb81a357ade7fa38d2",
            "b88c6bf1546347ed9253b02fdf4db774",
            "d0821a40e091483999e35c6af862faaa",
            "5554de99683d45648649a7ce12c77c48",
            "7c22e03c0c774797822b987eff4ff176",
            "57eb81e848654cd1b787a9719183a1f8",
            "8af11124d9a24feaaddf1c48c04a61dd",
            "392d0220caf94a10912a06c4a00c057e",
            "09228659d1794370893c7e15c0f6aac3",
            "c2cb90a014874b499ea1a635ce72fd06",
            "23ab5dad117142248de1aadbe28f19ba",
            "898e990b9ebd4c8984d1d6a20c449084",
            "d29aaad5396548dfbafce6c7f4765a60"
          ]
        },
        "outputId": "3780e65e-4baf-4459-ca79-8ece299cf40c"
      },
      "source": [
        "import torch\n",
        "from torchvision import datasets, transforms\n",
        "\n",
        "'''\n",
        "  This code is used to download the dataset which is already splitted\n",
        "  into a training  set and a testing set.\n",
        "  Note that we apply simple transformation to the data to convert them\n",
        "  into PyTorch Tensors and to normalize the pixel values entries in a \n",
        "  specific range of values. \n",
        "'''\n",
        "\n",
        "transform_actions = transforms.Compose(\n",
        "    [\n",
        "     transforms.ToTensor(),\n",
        "     transforms.Normalize((0.5,), (0.5,))\n",
        "    ]\n",
        ")\n",
        "\n",
        "mnist_trainset = datasets.MNIST(\n",
        "    root='./data', \n",
        "    train=True, \n",
        "    download=True, \n",
        "    transform = transform_actions\n",
        ")\n",
        "    \n",
        "mnist_testset = datasets.MNIST(\n",
        "    root='./data', \n",
        "    train=False, \n",
        "    download=True, \n",
        "    transform = transform_actions\n",
        ")\n",
        "\n",
        "\n",
        "print(\"Training set size: \", len(mnist_trainset))\n",
        "print(\"Testing set size : \", len(mnist_testset))\n",
        "\n",
        "# we define a loader and an iterator to process the training set\n",
        "trainloader = torch.utils.data.DataLoader(mnist_trainset, batch_size=64, shuffle=True)\n",
        "testloader = torch.utils.data.DataLoader(mnist_testset, batch_size=64, shuffle=True)\n",
        "\n",
        "mnist_trainset_iter = iter(trainloader)\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ./data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/9912422 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "63e677bff0f24a0bb9ac3651be6a8c8e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./data/MNIST/raw/train-images-idx3-ubyte.gz to ./data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to ./data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/28881 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cc299d3273ef4dab9c30492e2e45f1cb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./data/MNIST/raw/train-labels-idx1-ubyte.gz to ./data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to ./data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1648877 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0e483e536e154561a9995c348a09cbc2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./data/MNIST/raw/t10k-images-idx3-ubyte.gz to ./data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to ./data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/4542 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d0821a40e091483999e35c6af862faaa"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting ./data/MNIST/raw/t10k-labels-idx1-ubyte.gz to ./data/MNIST/raw\n",
            "\n",
            "Training set size:  60000\n",
            "Testing set size :  10000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xKi3upVW-1Tx"
      },
      "source": [
        "elem = next(mnist_trainset_iter)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batch, target = elem\n",
        "print(batch.shape)\n",
        "print(target.shape)\n",
        "print(batch[30])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9UoOQV4F6Z5P",
        "outputId": "630558d0-4a03-4519-876d-3205ad942603"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([64, 1, 28, 28])\n",
            "torch.Size([64])\n",
            "tensor([[[-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -0.8118, -0.0745,  0.4039,  1.0000,\n",
            "           0.9922,  0.6078, -0.1451, -0.8980, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -0.1608,  0.8667,  0.8980,  0.5137, -0.0588,\n",
            "          -0.0667,  0.3098,  0.8745,  0.7804,  0.1608, -0.9137, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -0.9608,  0.1843,  0.9686,  0.3176, -0.7647, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -0.8039,  0.0431,  0.9529,  0.7569, -0.9216,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -0.8275,\n",
            "           0.4196,  0.9922,  0.2471, -0.8353, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -0.2078,  0.9843, -0.8824,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -0.9294,  0.4902,\n",
            "           0.8118, -0.4196, -0.9529, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -0.1765,  0.9843, -0.8824,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -0.9294,  0.3412,\n",
            "          -0.6863, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000,  0.4118,  0.9922, -0.8824,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000,  0.4039,  0.9843, -0.8824,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000,  0.6078,  0.3333, -0.9922,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -0.2706,  0.9922, -0.3412, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000,  0.3412,  0.9216, -0.7490, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -0.9843, -0.1059,  0.9922, -0.2314, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -0.2549,  0.9843,  0.3647, -0.8902, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -0.4510,  0.9137,  0.5137, -0.8824, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -0.7098,\n",
            "           0.9059,  0.6157, -0.8902, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -0.8196,  0.8588,\n",
            "           0.8431, -0.6078, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -0.5373,  0.6784,  0.8980,\n",
            "          -0.4431, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -0.4510,  0.9059,  0.8431, -0.5137,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -0.6235,  0.7098,  0.9843, -0.4902, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -0.5373,  0.7647,  0.9843,  0.9216, -0.4980, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -0.4196,  0.8824,  0.2235, -0.6706, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000],\n",
            "         [-1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000,\n",
            "          -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000, -1.0000]]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HLZJIfW3ceko"
      },
      "source": [
        "Let's see what we are dealing with. \n",
        "\n",
        "Note that normally you'll have to read the documentation but here we will explore the data and associated labels. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nxWRQDANVSAn",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 407
        },
        "outputId": "b0816f6e-809f-43c9-a8d5-e8297b8aa1ae"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "images, labels = mnist_trainset_iter.next()\n",
        "\n",
        "print(\"Images shape: \", images.shape)\n",
        "print(\"Each batch contains {} images, for which {} channel of size {} x {} is defined \".format(*images.shape))\n",
        "\n",
        "# The following code is just used to plot\n",
        "figure = plt.figure()\n",
        "plt.subplots_adjust(top=5.2, bottom=4) \n",
        "num_of_images = 60\n",
        "for index in range(0, num_of_images):\n",
        "    plt.subplot(6, 10, index+1)\n",
        "    plt.axis('off')\n",
        "    plt.gca().set_title(labels[index].item())\n",
        "    plt.imshow(images[index].numpy().squeeze(), cmap='gray_r')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Images shape:  torch.Size([64, 1, 28, 28])\n",
            "Each batch contains 64 images, for which 1 channel of size 28 x 28 is defined \n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 60 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAFjCAYAAACe+NhqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9d3hU17m+fa9RmZE0qqiCCgiECqJIIEASBgSIZtMxYGMbl3NCiuPEieMk5+SynRMff47T47gHY5sY2xhjmukdgYRoQgIVBGoI9YbQSJq6vj+E5kc1RTMjcrzv65oLGI32+7Bn7Wev9a53rS2klCgoKCgoOAZVbwtQUFBQ+C6hmK6CgoKCA1FMV0FBQcGBKKaroKCg4EAU01VQUFBwIIrpKigoKDgQxXQVFBQUHIhNTVcI0XbdyyyEeNOWMe5Cyz4hROdVWop6QcN9cT6EELFCiD1CiEtCiHNCiHmO1nCdniVCiAIhhE4IcV4I8UAvaOgvhNgihGgWQtQIIf4hhHB2sIZnhRDHhBB6IcRHjox9Ey3/EkJUCyFahRBnhRD/8V3VIYRQCyFWCCHKhRCXhRA5QogZNgsgpbTLC9ACbcB4e8W4Tfx9wH/0Ruz76XwAzsBZ4GeAEzAJ0AGDe+k8pAPlwFi6bvr9gH69oGML8BGgAYKBPOA5B2uYD8wF3gE+6o3v4yotQwD1lb/HADXAyO+iDsADeAXof6WNPgRcBvrb4vj2TC8sAOqAg3aM8e9Eb52PGKAv8BcppVlKuQc4BDzuYB3d/Bb4HylllpTSIqW8KKW82As6BgBrpJSdUsoaYBtdF7zDkFKuk1KuBxodGfcWWs5IKfXd/7zyGvhd1CGl1EkpX5FSll1po5uBUmCkLY5vT9NdBnwir9w6eon/TwjRIIQ4JISY2Is64P44H90IIN7hQYVwAkYBAVfSHJVXhvVujtYC/BVYIoRwF0L0A2bQZbzfWYQQbwsh2oFCoJqu0cB3VsdVeoKAwcAZWxzPLqYrhIgAJgAf2+P4d8gvgUi6hq/vA5uEEA6/c0Ovn48iunrYvxBCuAghpl7R4t4LWoIAF2Ah8AAwAkgAftMLWg7Q1bNtBSqBY8D6XtBx3yCl/CHgSdd3sw7Qf/tv/N/WASCEcAE+BT6WUhba4pj26uk+DmRIKUvtdPzbIqU8IqW8LKXUSyk/pmtIPbOX5PTa+ZBSGunKGz5IV37s58AauozG0XRc+fNNKWW1lLIB+DMO/l6EECq6erXr6Mrf+QO+wO8dqeN+5EoKKgMIBX7wXdZxpZ2sAgzAs7Y6rr1M9wl6t5d7MyRdw+reoFfPh5QyV0o5QUrZR0o5ja4RQHYv6Gimy+yvTrH0RrrFDwgH/nHlptwIrKT3bsr3I870Qk73JvSKDiGEAFbQNTpbcKXzYhNsbrpCiBS6hvRf2vrYd6HBRwgxTQihEUI4CyGWAuPphZzdfXI+hl05F+5CiBeAELpm7nuDlcCPhRCBQghf4HlgsyMFXOlhlwI/uNI+fOjKuec6UseV2Bq6qkqcuturIzVc0RF4pYxPK4RwEkJMAx4Bdn8XdVzhHSAWmCWl7Ljdh+8KO5RbvAescmSJx000BABH6SrzaAGygPRe0nI/nI8/AM10laxtBQb1ohYX4O0r30sN8HdA0ws6RtBVVtgMNNCVcglysIZX+H8z9N2vV3rhXAQA+698J610lc/953dYR8SV76LzyjXT/Vpqi+OLK0EUFBQUFByAsgxYQUFBwYEopqugoKDgQBTTVVBQUHAgiukqKCgoOBDFdBUUFBQcyO1qAnujtOFmCxgUHdei6LgWRceN3C9aFB3XofR0FRQUFByIYroKCgoKDkQxXQWHc+DAAWbNmoWbmxtBQUH8+Mc/7m1JCgoOQzFdBYdiNBo5cOAAe/fuRa/XYzKZcHJy6m1Z31na29spKirin//8J08//TRDhw4lJCSE6dOns2LFCioqKrBYLA7RYjAY2LlzJ4888gihoaF4e3szYMAAnn32WQoLbbKr4h2h1+s5fPgw8+fPx8fHBx8fH6Kjo3nxxRc5e/ZszwPcZp1wb2AzHXV1dfLw4cPynXfekc8//7z87LPPekVHD/k/o6Ozs1O+9957csiQIVKlUkm1Wi3nzZsni4uLHaIjKytLfv/735fR0dEyICBALlmyRGZmZt7df8IGOm6FxWKRRUVF8ne/+5389a9/LY1G473quCMtf/7zn+WUKVNkeHi49Pb2lm5ubtLZ2dn63Xh7e8uJEyfK3bt33+l/4Z7PSX19vXz55Zfl4MGDpUajkUIICUiVSiW9vLzkokWLZEVFhd11XLhwQf7ud7+T8fHxUq1WW/fEUKlU0sfHR6alpclVq1ZJs9l8rzpuW71wSy5cuMBnn33GF198gcFgYMKECZw7d46CgoLuTSPo2h0NRo0axRtvvMHAgfbboa28vJw//OEPuLq6MmDAAAoKCjh69CjV1dXodDoGDx7M6NGj7RZf4fYcOnSIr776iuLiYgBiY2OZP38+kZGRdotpNBqprKzk73//O1u2bEGv1xMcHIy3tzeFhYVkZWUxduxYu8W/G0pKSnj33XdZsWIFo0aNwmw24+xsn03HcnNzWbNmDTk5OZhMJmtvtvvaNRqNGI1GcnJy+OMf/0hLSwvz58+3i5ZuPbt376akpASz2YyUEm9vb9RqNc3NzRw4cIB33nmH1157zW4a8vPzeeWVV9i9ezeXL1/G2dmZKVOmMH/+fAwGA//61784fPgwdXV16HQ6li9ffk9x7vkbLSkp4bPPPiM/Px8pJRcuXECv16PX33yT9y+//JJf/epX9xruthQXF3PkyBGKi4txdXVFr9fj7OzM4MGDSUlJYerUqaSkpNgt/quvvsrGjRtpbW0lLCyMtLQ0+vbtS3h4OCaTiaqqqht+x9fXl2nTpqHRaOyiSUrJxx9/zKpVq6isrCQyMpLBgwcTGhpKcnIyw4cPx9PT0y6xr0en07Fjxw5OnjyJyWTC29ubKVOmMHfuXFQq+2S5jEYjf/vb31i9ejUdHR1Mnz6dOXPmEBkZydatW/nnP//Jxx9/jJSSRx99lKCgILvouBNaWlrYvn07a9aswWw2ExgYaLfzAvD6669z5swZ9Hq9tXMEkJCQwMKFC9FoNKxZs4Zjx46RmZmJVqu1q+lWVlbS2NiIxWLB3d2doUOHsnTpUoxGI6+//joNDQ2cOHHCbvEB/vKXv7Bz507a29tJS0vjqaeeIiUlBV9fX6SUpKens3HjRv70pz/x3nvvMWjQICZPnnzXce7ZdL29vXFxcQHAbDZz+fJloKt3e31PV6/X09TUdK+h7giDwYBerycgIICkpCTCw8OJj48nMTGRPn364OXlhZubfR7FdfnyZfLz8ykqKqK1tZXy8nLy8vJwdXXFy8sLs9mMTqe74ffUajVr167llVdesfkowGw288033/Dhhx+SnZ2NXq+noqKCrKws1Go13t7eREREMH78eGbNmkVoaCh9+vSxqYar+eKLL9i6dSstLS0AjBgxgunTp6PVau0Sr6qqihdeeIGDBw8ydOhQli1bRnJyMkFBQajVapKSkti7dy9bt27ljTfeYMeOHSxdutRqOI6mvr6e06dPU11dDUBRUREHDhwgJSXFLu22oaEBg8FwzXtBQUEsWLCAp59+GmdnZ7y8vCgvL6exsZELFy5QWVlJaGiozbUATJo0CXd3dxoaGoiMjCQyMhJ/f3+2b9+OSqXCbDbfoNfWLF68GIPBwKBBg5g5cyYxMTF4eHhYfx4TE8PFixfx8PCgtLSUr776yrGmO3jwYF5//XU2btxIYWEher0elUpFcnIyfn5+bN68mYMHD94wbLE3SUlJLF++nMGDB6PVatFqtdfcye3Be++9R2ZmJh0dHQgh0Ov11NfXA1BTU4OUErPZfI0OKSVCCHQ6HXFxcfz617+2qSadTsfnn3/OsWPHrKOPzs5OOjs7AaitrbXeHA4cOMBzzz3HQw89ZFMN3WRnZ7Nhwwbr0HH06NH8x3/8B2PGjLFLvMrKSl588UW2bt3K/PnzeeaZZxg6dOg1bSEuLo5f/epXpKamsnfvXnJzc3nttdfYsWMHM2bM4MEHH8TLy8su+m5GaWkpp0+fxmKx4OPjg6urK2+++SZFRUX88Ic/tHm8WbNmERMTw+DBgxk8eDDu7u6o1WrCwsLw8vJi9+7dfPLJJzQ3N9OnTx9mzpyJv7+/zXV007dvX6ZPn47JZMLNzQ21Wo3RaKS9vR2DwUC/fv2YNGmS3eIDpKSkMHDgQDw8PPD19bV2KrtRqVSo1WrrTaC5ufme4tyz6bq7u5OamkpkZCStra1WUwkICECtVqPT6cjMzMRgMKBWq+3aiwKs5h4fH09cXBwBAQF2jXc1x48fp6GhAbPZjFqtRqvVEhsbS0tLC+7u7gQHB1NVVUVxcTE6nQ6z2Qx0GW97ezulpbZ/dFp7ezvV1dV0dHTwxBNPkJqaSktLCyUlJZw5c4bTp0/T0tJCTU0Nrq6u99yAbsfly5f5+uuvOX78uNXwR48ezZQpU+zSyzWbzbz66qts27aN5ORkli9fzvDhw1Gr1dd8zt3dnWHDhhEREcHkyZM5c+YM69evZ9euXRQXF2M2m3nsscdsru9mnDt3jm3btpGXl4dGoyEpKYmf/exnSClxcnJCp9Nd0+OyBQ8//DCdnZ14eXnh5eV1Te7YYrFgMpmoq6tDr9dbr197jgBUKtU17cFoNHLkyBG2bt2KyWRi6NChzJ49227xoatNDBgwwK4xoAemC13D4/79+9/wfmdn5zWNPDAw0GGTFZ2dnRiNNnuc0R3H7O7Jjx49mkcffZRhw4bR2dmJi4sLHh4etLW1kZOTw5dffsmhQ4eArvMXGxvLokWLbK4pLy+P5uZmRowYwbx585g0aRJ6vZ6WlhZqa2vZs2cP77//PhcvXsRgMFBXV2dzDdDV0z9x4gQNDQ1YLBaSk5OZMmWK3XpNVVVV7Nixw3qzGTJkyA2G242zszN9+vShT58++Pn5UVxczJYtWygvL7eOVOyNlJIzZ86QmZlJa2srUVFRzJ49m7S0NKArNefq6mrzuMHBwTd932KxcPHiRfLz82lpaUEIQWBgIFOnTrW5hpthMBioqalh27ZtrF27lpMnTxIZGcmyZcuIjo52iIY7QaVS4e5+bw/UtsvUaEFBATk5OZjNZrRaLTExMcTGxtojlJXLly9jNBo5deoUu3btwtnZ2ZpHDgwMZNSoUXaZJc/JyaG4uBiDwYCUkr59+zJhwgRiYmJu+KyXlxeZmZnW0hG1Ws3QoUMZN26czXVlZGRQW1trHUZ2D5UDAgLo378/VVVV1h63yWSisbHR5hoAOjo6aG1txWg04ufnx/Tp0xk9erTdJok8PT1ZvHgxb731FmvWrKGjo4MxY8YwcODAW5qvTqfj6NGjbN++HbVazdixY0lMTLSLvutpamri6NGjFBYW4uzsbO3RdWu9lWZb09bWxoULFzh79iz79+9nx44dtLe3ExcXx6xZsxg0aJDdNbS2trJu3TrrhGtJSQkqlYqYmBjGjx/vsHNxK9rb26msrOTy5cv4+fkxZcqUezqOXUw3OzubjIwMzGYzTk5OeHh42OVufTXFxcVcunSJ7OxsampqMBgMtLW1WXNk48eP55FHHiEpKcmmOd6WlhZ0Op21p+vv73/LYVhzczMXLlwAuiYZNRoNAwcOtMuwraamBpPJREJCAv369bO+39TUxOHDh1m7di1NTU14eHgwYsQIkpOTba4BuvKrHR0dSCkJCwtj6NCh1tSPlBK9Xk91dTXFxcUEBgYSHx/fozIpHx8fnnnmGXQ6HevXr6e0tJRBgwYRExPDsGHD8PHxITg4mP79++Pp6YnJZCI3N5cvvvgCvV7PT3/6Ux544AGGDRtmq1Pwrezdu5ft27fT0tLCkCFDePDBBwkPD3dIbOgapW3fvp0jR45QUlJCeXk5ZWVl1NXV0bdvX9LT01m4cKFDtJSVlbFx40ZraR+Aq6srtbW1FBUVERIS4hAdt+LChQvs27cPk8nEiBEj7mkSDexkutXV1VRWViKEwNXVFQ8PD7vfpWpqaujo6KClpQUpJYGBgYSEhODs7MyFCxdYvXo1ly5dwsnJiZEjR9osbnR0NM888wxVVVVYLBYeeuihW+avW1tbqaurs1Z4eHt7262Mzc3NDScnJ8rLy2lqasJgMFBQUEBGRgbbtm3jxIkTuLq6kpqayrPPPsv48ePtouPMmTPWioVRo0YRHR2NSqWioaGBkydPcvz4cc6ePUtZWRlBQUFMnjyZJUuW9CjfGxkZyXPPPUd8fDy5ubkUFRVx6NAhDh8+bG0bw4cPZ/jw4bi4uLBlyxaqqqpYvnw5c+fOtfv8QzfV1dXs2bOH06dP4+Xlxbhx40hPT3dI7G5aWlpYu3YtmzdvRqfTWVNzQgg6OzspLCxk8+bNtLW1kZCQYNfVg97e3iQnJ6NWqzEYDNY0x6lTp/jkk0/o168fUVFRdov/bXR0dHDixAl27tyJVqslNTWVwMDAezqW3R/33D2bX1tba9ckdVhYmHV2eubMmYwbN47AwEBcXV0pKCjg66+/ZseOHWg0GgIDAwkLC7NJ3JCQEJYvX27N63aXJF1P9+REd4/YyckJLy+vm6YhbEH//v1xc3Pjq6++orOzE61Wy6lTpzh58iQXL160DmWffvppZs6caZfhvk6no6CggEuXLiGEYODAgYSEhFBSUsLmzZvZsGEDOTk5VlN2dnbm7NmzPZ5kU6lUDBo0iEGDBlFZWUlBQQEtLS0UFRVRWVlJeXk569atY+fOnTg7O1NcXMzw4cOZNWuWwwxXp9OxefNmDh8+jF6vJy4ujrS0NIf35gwGAzqdjs7OTkwmE1qtFnd3dywWC52dnWzbto0jR45w4sQJFi5cSHJyst00RkRE8PjjjzN9+nSMRiPFxcWsXr2a7du3s2PHDiIjI/nNb35jl9iAdeLw4sWLuLm54e3tjaenJ1JKjh8/zrp166itrWXixIn33MsFO5hua2srra2tVnO5dOkSBw8exMvLiyeeeMJuJpOenk59fT0uLi7MnTuXxMREq/mlpKQwYMAA/vKXv7BlyxYGDhzI888/bzOjuZNKiUuXLlFRUcGlS5cA8PDwICYmxm61w4mJifj6+nLy5EneffddpJTX1DmGhIQwZ84cpk6darf8amNjI2VlZXR0dODi4oKzszNCCDIyMnjrrbcoLi6+JtVjMploaWmhvb3dZhpCQ0OvqS1tamqisLCQ/fv3c/DgQfLy8qiqqiI4OJgTJ06QnJyMt7e3zeLfivPnz7Nx40bOnDlDQEAAkydPJiUlxa4LIm6Gl5cXM2bMwN3dnZaWFkJDQ+nXr5/V9DIyMqipqeHrr7+mrKyMF198kblz59pNT3BwsHWSLy4uDiklRUVFFBcXs23bNp555hmbm35paSlnzpzh5MmTtLS0cP78ebRaLQEBAQQHB+Pk5ERGRgZ79+61Vie1t7eTn5+PTqfDzc0NFxcXPD096du3723j9dh06+vraWxspLy8nMuXL1tnq7vp7OykoKAAIQRjxoyxm+l2T5Sp1Wrc3d2vuZi1Wi2TJk2isbGRn/zkJ3zxxRfMnz/fIeUh3dTX11NUVGSdFff29mbkyJF2WxwQHx/P0KFDKSsro7Oz01ry09jYSHNzMwMHDuTBBx/Ex8fHLvGhqxfV0dGB2WwmJCSEwMBAhBCUlpZSUVFxw+dVKhWurq52W/oK4OfnR0pKCkOGDGHChAl8/fXXrF27lry8PF577TWWLVvGtGnT7LYIALrawtatWyksLMRkMjF8+HC7x7wVPj4+PP7446Snp3P58mWCgoIIDAzEZDJZR4h79+7l+PHjFBQUsGXLFqKjo+0+MQ5Y5zxiYmI4e/Ysly5dsnlut7Kykg8++IBNmzZRUVFhzfPrdDpruZxGo0Gn01k7DxkZGbS1tREcHExTUxO+vr6o1WpCQkJ4+OGHiYuL+9aYPWrdVVVVrF+/npMnT7J//37rjLjJZEIIYTU+T09PEhMT7b73gZ+f3y1/ptVqCQ0NxcnJifr6ek6ePOlQ021ubqa2ttZh8Xx8fFiyZAleXl5cunSJwYMH09raypYtW+jo6CAwMPCec1J3ire3NyEhIWg0Gvr27UtwcDBms5n29vYbdq5SqVT4+PgwevRou94IrtaWkpJCnz59uHTpEitWrCA7O5vKykrq6upYvnz5t7annnDkyBE+//xzzp07R0hICCkpKcTExNh9Ec+t0Gg0N5R+dqefoqKiSE5O5r//+785efIkmzZtol+/frz88ss21dBdv97da+xGSmltKy4uLvj6+to07u7du1m5ciXu7u6kpaWRkJCAXq+3dh6LiopobGxErVYTFBSEq6srTk5O5Ofnk5WVhYuLC3379sXZ2RlfX19iY2Pta7rZ2dn861//orS0FIvFgoeHxzW9G+gqOB4+fDhz587t9dnH3kKn05GVlcXhw4etF5Zarbar6alUKmbMmMGECRNwdnbG1dWVd955h6amJoKCghg9erTdZ8kDAgKIj48nKyvLOgPt5uZ2Q01wt+Gmp6fzwgsv2P1mcDWDBg0iKSmJNWvW4OnpiVqt5v333ycoKIjFixfbfFFCe3s7hw8fpqamBo1Gw0MPPcTixYuvqTCxJ2azGYvFYk313A61Wk1wcDB+fn5YLBY6OjqsKTJboNPpKCkp4fTp03h4eJCUlGT1CSklTU1NVFdXWyeehw8fbrPY0JVacHFx4Xvf+x5PPvmkdf+NlpYWXn/9dSoqKtBoNCQkJDB27Fi0Wi0ajQaz2UxpaSlubm6kp6ejVqut7f129Mh0Bw0axLx582hoaLC+V1JSwpEjR6isrLR+Zvny5cycObMnoXpM91Lc7uW39hzCXs+5c+c4duzYNftPhISEMG3aNLvH7i7gNplMtLa20t7ezqBBgxgxYoTdY0PX9+/l5UVxcTErV65ErVZz/vx5TCYT0GW4Xl5epKen81//9V931GhtiUqlIiwsjLi4ODQaDTNnzuStt97ijTfeICoqigceeMCm8Y4ePcrevXtpbGwkNjaWqVOnMnjwYJvGuBWNjY2cO3eOy5cvExwcjIeHh/Va8PT0tJZ1dhtyW1sbtbW17Nq1i+3bt+Pq6kpYWJhNa3aPHDnCK6+8wtGjR5k9ezZhYWGEhIRgNpupqanh2LFjnD17Fo1GY7dVpiEhIaSlpdGnTx+klDQ3N7N7924yMjKwWCykp6fz4x//2GYVPj1ynvj4+Bsukk2bNllLxqCrqmDcuHEONbmb0dHRYd3kw9XV1aG97paWFusyYeiaRAsPD3fYTDl0XXBVVVV0dnYSHBxs1202r2bEiBHWIWFOTo71fSklGo2GoKAgJkyYwAsvvOBww4Wu6pqkpCTmzZvHxo0bCQ0NZd68eaxatYqtW7eSkpJiszIpi8XCF198QWFhIWq1mlmzZjFmzBiHbeK+fv16fv/731NeXk5SUhJ+fn6oVCr8/PwYNWoUffv2xcnJybrvwNGjR9m8eTP79u1DpVIRFBTE8uXLbboXxJ49eygqKqKzs/OaFZM1NTWsW7eOTz75hM7OThISEnjiiSdsFvdqysrKyM3NpX///rS3t/Pll1/y4YcfUldXx8yZM/nhD39IamqqzeLZ1AlNJhP19fXWEqDeors0y8nJyVoTW1lZSVZWFkajEX9/f5KSkhym5+zZsxQWFlo1RUVF8cgjjzgsPnTllBsbG/H09CQuLo6IiAiHxHVxcSEgIAAPDw86Ojqsm4a4ubmRmJjI9773PdLT0+02oXgn9OnTh9jYWFasWMGaNWtYunQpH374IZs2beK555675ZLZu6V7rwu9Xk9ISAijR4+2WeninWAwGKwVLN0rI6HrxvPxxx8DXaWMgYGBuLi4WCc7u9M/Y8aMsfmikaSkJL788kvq6uooLCxkxYoVuLi4kJOTY31aRFRUFIsWLbJLDbOzszMmk4mPP/4Ys9nMhg0byMzMxNXVlaeffprvf//7Nu+g2NR0a2tr2b9/P/n5+bY87F1z7tw5GhoaCA4Oxs3NjdbWVnbu3Mnq1avx9/e36V3rdhgMBkpKSigrK0MIgVqtZvjw4Q5Ptxw9epTc3FyGDBlCamqqwyZtoqOj+cc//sGrr77Kzp07rTe8sWPHkpyc3GvF7tczatQonnrqKd59910aGxvx8fGhoaGBkpISm5nu+fPnOXfuHB0dHXh7e9utXPBWPP300wCsXLmSyspKOjs7rSk3i8WCwWDAZDJRXV1tXdQEWNM/P//5z20+GomNjSUiIoLa2lrKysooLi7uerrClZRHYmIizzzzDPPmzbPLAqvIyEi8vLzIysri0KFDqNVqEhISePLJJ5k1a5ZdRsQ2Nd2MjAwKCwtxcnKyDqV7Y0Z23759vPXWW+h0Opydnamrq8NisaBWq0lPT+enP/2pw7Ts3buXrKwsazWHSqVy+DPBLBYL+fn5FBcXs2TJEoYMGeKw2CqVivDwcN5//32HxbwXgoODmTVrFidPnmTDhg1IKQkPD7epMe7bt4/m5mZcXV2ZM2eOw9MparWaH/zgBzz66KMYjUZyc3M5efIkra2t1NTUkJ2dTXl5uXUHtu7RYHh4OEuXLrXLcvXBgwfz3nvvsXbtWg4cOMCZM2dobm4mPj6eJ598ktmzZ9t1x8BFixZRX1/P22+/TV1dHWlpafz0pz9lzJgxdltFa1PTHTlyJHPnzqW9vd2a/Pby8nJ4wff48eM5fPgwGzdutD74cODAgTz++OM8+uijNuu53AnXP03Dw8PD4U8o6F5Tb7FYrCUvCjcSGxvL//7v/xIeHs65c+dYuHAhCQkJNjv+okWLOHTokHVVU29V83Qv/pg0aZLd96i9EwYMGMAvfvELfvGLXzg8trOzMz/5yU/4yU9+4riYtjzYoEGD+PWvf02/fv04ePAgc+bMsZZTOJK4uDg++eQTh8a8U6Kjox2eWujevDwqKorU1NRefSzN/Uz3UuU33njDLsePiIhg/fr1djm2wr8PdikpeOKJJ+w20/jvxpAhQ4iJieHo0aPEx3GxC4AAACAASURBVMezZMkShy7KuJrU1FQmTJjQK7EVFBS6ELd5jI5jnrFzLTdLAis6rkXRcS2Kjhu5X7QoOq5/01HPLlNQUFBQAMfOcCkoKCh8x1FMV0FBQcGBKKaroKCg4EAU01VQUFBwIIrpKigoKDgQxXQVFBQUHIhiugoKCgoORDFdBQUFBQeimK6CgoKCA1FMV0FBQcGBKKaroKCg4EAU01VQUFBwIIrpKigoKDgQxXQVFBQUHIhNTVcI0V8IsUUI0SyEqBFC/EMI0SvPXhdCLBFCFAghdEKI80KIB3pBw7NCiGNCCL0Q4iNHx79Kh1oIsUIIUS6EuCyEyBFCzOglLfdFGxFCxAoh9gghLgkhzgkh5vWChvulfbRd9zILId78Duuw6/Vi657u20AdEAKMACYAP7RxjNsihEgHfg88BXgC44ESR+sAqoBXgQ97IfbVOAMX6Po+vIHfAGuEEP17QUuvt5ErJr8B2Az4Ad8D/iWEGOxIHdwn7UNKqe1+AcFAB/Dld1UHdr5ebG26A4A1UspOKWUNsA1w3KNn/x+/Bf5HSpklpbRIKS9KKS86WoSUcp2Ucj3Q6OjY1+nQSSlfkVKWXTkfm4FSYGQvyLkf2kgM0Bf4i5TSLKXcAxwCHnekiPulfVzHArpuige/qzrsfb3Y2nT/CiwRQrgLIfoBM+i6qByGEMIJGAUEXBk2Vl4ZwtruWdr/5gghgoDBwJleCN/rbeQWCMCxz0S/P1kGfCJ7/5Ey94sOm18vtjbdA3T1WlqBSuAY4OjHnwYBLsBC4AG6hrAJdA0RvvMIIVyAT4GPpZSFvSDhfmgjRXT1on4hhHARQkylayjp7mAd9xVCiAi6zsPHig6rFptfLzYzXSGEiq4eyzrAA/AHfOnKrTqSjit/vimlrJZSNgB/Bhz73PP7kCvf0SrAADzbS/F7vY1IKY3AXOBBoAb4ObCGrpvAd5nHgQwpZamiw37Xiy17un5AOPAPKaVeStkIrMTBZielbKbr4rl6WNLrQ5TeRgghgBV0jQQWXDEeR3NftBEAKWWulHKClLKPlHIaEAlkO1rHfcYT3Ae9S+4DHfa8Xmxmuld6lKXAD4QQzkIIH7ryMrm2inEXrAR+LIQIFEL4As/TNVPtUK6cBw3gBDgJITS9VUIHvAPEArOklB23+7A9uJ/aiBBi2JXvw10I8QJd1RQfOVjDfdM+hBApQD96p1rgvtOBPa8XKaXNXnTlT/cBzUADXUO2IFvGuEMdLnSVJrXQNXz8O6DpBR2v0NXLvvr1Si/oiLgSuxNou+q1tBe03C9t5A9XNLQBW4FB39X2cUXLe8Cq3oh9v+mw9/UirgRRUFBQUHAAyjJgBQUFBQeimK6CgoKCA1FMV0FBQcGBKKaroKCg4EAU01VQUFBwILerCeyN0gZxk/cUHdei6LgWRceN3C9aFB3XofR0FRQUFByIYroKCgoKDuQ7ZboGg4GMjAwmTZqEm5sbbm5u9O3bl2PHjvW2NLuwZcsWVq9eTVlZWW9LUVBQuEJv7QPgcAwGA9nZ2bz66qtkZGRgNHbtX6HX6/m/tCqvqKiIH/zgB5w5c4b29nbMZjNqtRpn566vOiYmhieeeII5c+YQGBjYy2oVAF577TV+//uujdYWLFjAhx/29oNGvttIKSkoKCArK4tDhw7R0NCAv78/CQkJzJs3j379+vU8wLe87giLxSJramrkV199JV9++WU5Z84cOXv2bLly5UpZVVUlpZSyrKxMPvvss3LAgAHy+eefl1u2bJE6ne5mh7tnHbciNzdX/upXv5KpqalSrVZLIYR0c3OTaWlpcs2aNbK+vt4hOqqrq+Vrr70mY2Nj5W9/+9s7/bW70vHiiy9Kf3//69fzW18uLi7Sx8dHjh8/Xn722Wd3I79H56Ompka++OKLMioqSgYGBsrAwEAZFBRkfSUkJMh//vOfdtdhQ2ymo6CgQL711lty6tSpMjQ0VP7ud7/rqY5/+3PSGzrMZrP85ptv5NKlS2X//v2lp6endHNzkx4eHlKr1UpfX1+5aNEief78+Z7osI3pbtiwQaalpUl/f3/p5eUlNRqN1Gg0MiAgQD7yyCOysLBQ5uXlyQkTJkiVSiU9PT1lWFiYfOmll+5U6D1hNBrl4cOH5eOPPy59fHyshuvi4iITExNlWVmZ7OjokBaLxa46uikuLpY/+tGPpEajkbNmzZLNzc138mt3paO2tlb+9re/lU899ZSMioqSLi4uNzVfV1dXmZCQIFetWnWn8u/pfBgMBvnpp5/K2bNny+DgYOns7CyFEDe8tFqtXLRokd10XE1dXZ1cuXKlnDp1qpwzZ47csWPH3R7CJjq6MZlMsri4WP7kJz+RQgg5ceJEeebMmZ7o+Lcyu/tBh8VikStWrJAjRoyQUVFR8rHHHpNvvvmm3LVrlywoKJC7du2SS5YskQEBAXLevHmytLT0XnX0PL2Qm5vLu+++S2ZmJnq9/pqf6fV6Ll68SGNjI0OGDGHp0qUcOHCAtrY2jEYjRUVFPQ3/rWRnZ/O3v/2NrVu30tbWZn3f09OTCRMmEB4eTte2mfbn4sWLfP7553z11VcYDAYuXLhAbm4u48ePt2mcgIAAnnvuOQwGA/v27cPNzQ1fX1/KysooKiqitLSU48ePU1hYSH5+Pu+99x79+/dn3LhxNtXRzcaNG3nzzTfJy8vDaDSSkJDAxIkTCQkJAaC2tpZ9+/aRm5tLZ2enXTRczddff80HH3zAyZMnaW5uRghBYWEh8+fP5/nnnycgIMDuGq7HycmJ5uZmKioqkFJSU1PDqVOniIuLs1vMlpYWpJR4enpiMBjYv38/LS0t6PV66uvraWpq4syZM5SVlWEwGHB1dWXEiBH89a9/xc/Pz266eosjR47wwQcfoNVq+eUvf8m4cePw9va2puYiIyMRQtDY2EhBQQF5eXn079//nmL1yHSNRiP79++nsLAQvV5PSkoKoaGhVFRUUFpaSlNTE5cuXaKyspLhw4fj5eV1ze+7u9vn6Sj5+fkcOnSIjRs3Wk0ewNvbm9GjR/Pggw/y4IMPOsxwAcrLyzl69CgNDQ0AWCwWzGazzeMIIfDx8QFg5syZODs74+zszIgRI+js7ESn07Ft2zb++Mc/cu7cOU6fPs3q1avtZrp5eXlcuHCBjo4OUlNT+fGPf8yECRNwcXGxGl5FRQU5OTk0NjZy9uxZBg+2z0N5v/rqK/7+979z9OhRXF1dGTVqFEIITp8+za5du0hKSmL27NnU1taSn5/PlClT7KLjZphMJus8Q2hoKNOmTbPp8btv9Dt37qS2tpbdu3djNBoZOXIkVVVVFBQU0N7ejpQSo9GIRqNBq9XS2tpKRUUFzs7OGI1GmpqaemS6p0+fpra2FovFgr+/P7GxsXR0dHD27FlaW1tv+junTp1i9erV17zn4+PDb3/7Wx544IF71nI1DQ0NxMXF8fDDDzNu3Dg8PDyu8QdXV1d8fX1xd3fHbDbf0MG8G3pkunl5eXz99ddUV1fTp08f5s+fz8yZMzGZTBQUFPDpp59SV1eHu7s7x48f5+2337b+B5KSknj2Wds+MUZKSWFhIe+88w47duygqqqKtrY2hBBWM5o4cSJLly61GpOjMBgMdHR0YLFYCA0NZcGCBSQmJto1plarvebvWq3WOiGQmJhIQ0MDbW1tZGZmkpmZSXJyss01dHZ2Wm8uCQkJjBw50jqB19raSkNDA62trRiNRmprazl58qTdTHf//v3k5uYSERHBokWLrMa2fft2tm7dyp///GcOHjzIqVOnCAoKcqjpdtO/f38eeughm/cmL1y4wH/+539SWlpKZ2cnLi4uBAUFcf78efR6PbGxscTExBAQEIAQAn9/fwYNGkR2djY//elPEUKg0WhwcXG5Zw2vvvoqhw8f5sKFC0gpCQwMZOTIkeh0OnJzc2lqarrp77W1tVFZWYkQoms/2iv6Ll26dM9ariclJYUhQ4YQFBRkt85gNz0y3SNHjlBSUoJer2fYsGHEx8cTHR2NEIIBAwag0WjIzMzk3LlzbNu2jezsrqeh+Pj48L3vfY+hQ4fa5D/RTWFhIX/84x/ZsmULDQ0NmEwmoKuHO3jwYNLS0li6dCl9+vSxadzb0drayunTpzl79iwAkZGRPPbYY3h7eztURzexsbHMnz+fs2fPkpOTQ0tLCxcv2ucJ9cnJyWzZsoW6ujpycnJ46623mDx5Mp6enmzZsoXDhw9TUFCAs7Mzfn5+djPcnJwccnNzaWtrY8qUKSxZsoRBgwYBEB4eTkxMDJ999hmrVq1CrVY73HDr6+uprKwkPj6eRYsW2fz4KpUKHx8fxo8fT0JCAuHh4QQHB+Pq6gqARqPB19cXjUYDYH2/oKAAADc3N2JiYggKCrpnDStWrKC+vp729nYAzp8/T35+Pkajkba2NgwGg9VYgWtMtvu97j9tjZ+f321vdGazGYPB0ONYPTLd5uZmq4ja2lpKSkq4dOkSPj4+uLq6otVqqaioYOPGjdbcUN++ffnRj37EjBkzenTXvJ7Tp0/z3nvvsWXLFmpra63v+/v7M336dBYvXkxcXBzh4eE2i3knSCnZu3cvn3/+OTU1NXh4eDBgwADCwsIcquNqDAYD9fX11lSHlNIuqQ6ApKQkEhMTuXDhAjk5OZw9e5YDBw7g4uJCWVkZzc3N+Pr6smDBAoYPH05MTIxddKxbt46ioiJGjhxJeno6AwYMsLY/JycnamtrKSws5NKlS4waNYoFCxbYRcetuHjxIufPnyc2Ntaa77YlwcHB/OY3v8HNzY3AwEA8PDxQq9W3TLHpdDr27NnDRx99BHSZtqenp9WU74WamhoMBgNSSlQqFZ2dndZ/A1YtV2u6/r3uUauj6ejooLi4mIKCAvz9/YmKirrnY9msTre6uppPPvkEi8XCkCFDKC4uZvv27WRkZFBbW4tarWbo0KEsXLiQJ5980qa9zbNnz/LBBx+wYcMGq5EAjB8/ntmzZzNmzBhGjRplbTANDQ2sWbOGS5cu4ebmxqJFi+jbt6/N9FxNXV0dGRkZnDp1ynrTiY+Pt9bNOhqz2UxeXh6bN2+23pxcXV3x9PS0S7zAwECmTJnCxYsXOXbsGHV1dda4QghCQkJYuHAhy5Ytw9fXFzc3N7voKCwspKWlhaeeeooRI0agVqsBqKqq4quvvmLlypWcP38eT09PYmJiGDhwoF103Erb0aNHUalUdjFc6Oqp3k06q76+noMHD3LkyBGklNZeck/45S9/ya5duzAajTftxXbTp08fEhMTrd+BwWDgzJkzfP3111RWVuLs7Iyvr69DU4QVFRXs378fk8lEenq6dZR0L/Toyh8zZgxHjhxh//79tLa2cuLECS5fvkxwcDDV1dWUlZXR3t5OUFAQs2bNYurUqYwdO9amDaugoIC//e1vVhO5OqUwefJkli1bhlarRa1WU1VVxZ49e9izZw8ZGRm0tbWhVqspLCzk3XfftZmmbnQ6nTWnaTab0Wq1JCUlMWXKlF65W5vNZnJzc/niiy84cuQIRqMRX19fJk2axLBhw+wS09nZmbS0NOrr67l48eINkyVarZaYmBiGDRuGk5OTXTQUFRVRXV1N3759SUxMJCgoCIPBwIkTJ9iwYQPbt2/n9OnTeHh4kJKSwiOPPGI3LdfT0NDAhg0b2L17N8HBwXbr6d8tZWVlZGVlYTAY8PX1Zdq0aaSnp/fomN///vdJTk6+7ajKy8uLQYMGERwcDHTdAPR6vXXySq1WExcXZ7dU1PV0m35mZiahoaHMnj0bDw+Pez5ej0336aefpqqqipycHKu4M2fOXPO5wMBAHnroIR588EGbNubi4mLefvtt1qxZQ3NzM9D1hUVERJCSksKUKVPQaDSUlZVRXV1NXl4eK1asIDf32ofPfvHFFzY33ZKSElpbW62zwVJKPDw8iImJsWsp0K3Q6XTk5eXx2WefsWnTJlpaWtBoNAwbNoxHH32U0NBQu8W2WCzU1dWh0+lu+FljYyOZmZmkpKTYzfirqqqsZp+ZmUlDQwMdHR1kZmZy4MABGhsbsVgsBAcHM2/ePCZMmGAXHTejpKSEw4cP09LSQlpamt2qSO4Gk8lEVVUVpaWlCCEIDg5mxowZPcrnQleKo9tI7xSdTkdmZiZffvmldRSr0WhITEx02IrKixcvcujQIS5fvsyMGTMYMmRIj47XI9Otq6vDbDbj5OT0rT03i8WCwWCwmeE2NzeTmZnJrl27rGmC7vghISE88sgjzJkzB71ez+eff86pU6c4d+4cVVVV5OXl3VAKkpqaahNd3VRWVlJSUoKPjw/FxcXWGtSgoCBiYmIcnlpob2/n4MGDrF69mj179lBfX4+/vz8xMTEsW7bMrlUU7e3tfP7556xduxaj0cjEiROJioqiubmZI0eOUFFRwb59+4iKimLgwIE96kHciv79+9OvXz8yMzP59NNPrZNEQUFBhIWF0dHRQWdnJ5GRkYwbN876c0dQUlJCRUUFffv2Zfz48cTGxjos9rdpys7OprW1FXd3d6Kjo+1eaXMrysvLWb9+PRkZGQgh0Gq1jB07tse97uvR6/W0tbVhMplQq9VotVqcnZ3p6Ojg6NGjZGRkEBQUZJ0E7gn3dPVbLBa2b9/Ojh07OHHiBGfPnrUOGZycnNBqtUgprb2LhoYGDh48SEJCgk1yZRcuXOD111+noKDAWuQNXb3ctLQ0a9naqlWr+Oabb6irq7tleYmbmxuPP/54jzVdTUdHB6GhoXh5ebF+/XpOnz4NdN3po6OjbRrrdjrOnz/PiRMnWLt2LQcPHsTNzY2pU6cydOhQEhISmDFjxjWlZbaksbGRvXv3smrVKlpbW5k3bx4LFy4kMTGRmpoaVqxYwdq1a6mvr+fw4cMsWrTILkPGAQMGsHDhQpydnTlz5gze3t7079+flJQU6urqqKysxM/Pj/HjxzNgwACbx78V5eXlZGVlUVFRQWJiYo/yhLbCZDJx4sQJdu/eTVtbG7GxsSxevLhXJn6NRiPnzp3j5MmTCCFwdnYmNDSUpUuXMmbMGJvF0ev1ZGZmcuTIES5fvoynpycDBw4kOjqa+vp6Nm/ejE6nY8GCBT3u5cI9mG5nZyeZmZm89NJL5OXlWasXnJycCAsLY+DAgQwcOJCamhr27t3L5cuXqa2tZdu2bSQlJdnEdEtKSsjNzb0hPxgbG8vEiRNpbm5m9erVrF+/nvr6+pv2woUQeHp6MmjQIJv3LrpnNouLizl37hwtLS0EBAQQGRlp9zIxk8mEwWCgrKyMnJwcdu7cyf79+6msrESlUpGQkMD8+fNJTU21btzR2dmJq6srHR0dQFfv3xaVJd3pn9LSUqZMmcKyZcsYM2YMKpUKb29vnnrqKXQ6HR988AFVVVXU1dXZLU+3aNEiYmNjOXbsGCEhIcTExNDQ0MBHH32E2WwmNTWVuXPn9mh2/m7JyckhKysLvV5PTExMj2bEbYHFYqG0tJTs7GwqKysBGDhwoMMrObopKSnh4MGD1NbWIqXEzc2NuLg4UlJSbBonOzubdevWUVlZicVioaGhAYPBQHR0NK2trZw/f5709HSeeuopm0y437XpNjY28j//8z+cPn36mpq1qKgoHn30UWbMmEFYWBgnT56ks7OT3bt3W+vbLl++3GPB0LWU82YrQrRaLfv376esrIwTJ05QX19/y2N4eXkxbdo0pkyZ0uNc1c0wm80UFhZazW706NE88sgjditZq62tpb6+3lrvuXv3bnbv3k1NTY11clGj0Vh7n6WlpURERKBSqdBqtfTr14+CggLrzXPy5Mk91pSXl0dpaak1px8TE4NK1bWbqJOTE9HR0YwePZoPPvigx7Fuh1arJTk52boApKmpie3bt3PgwAH69evH9OnTHZ5r755ojY6OZsKECXarXLhT2tvb2blzJ9988w1tbW307duXIUOG2LS0806prq7myy+/5NNPP6Wmpga1Wk3fvn0ZNWqUTa+h9vZ23nvvPXx9fXnppZcIDQ0lOzubP/3pT3zxxRcYjUbS0tJ46KGH7nnZ7/XctemazWYqKiqshqvRaIiIiOAHP/gBDz/8sDVRPm7cOAoKCti/fz9msxmdTkdJSYlNRK9fv57Ozs4berDdRnMrnJycUKlUuLu7k5aWxm9+8xvi4+Ntoul6Lly4wDfffEN+fj4hISFMmTKF1NRUq+nYksbGRj766COysrIoKyujsLDwpvsYdHZ2kpube81EohCCPn36MGjQILKysqxLho8ePdojTXV1dXz22Wc0Njbywx/+kPT0dHx9fW/4nJub2w3Lw+2NxWKhuLiYgwcP0tDQwJQpU5g4caJDNdTX13P8+HGqqqpYuHChzXtv90JFRQUZGRmcO3cOLy8v0tPTWb58ucN16PV6tmzZwqeffkp1dTVCCPz8/Jg8eTJJSUk2jVVSUkJGRgYvvPACERER+Pr6MmbMGOLi4jh27BhGoxEPDw+blqfdten6+vqSmJhonY2OiIjgxRdfZMyYMbi4uGAymazDlPPnz1tzvd3rtm2Bh4fHXfWaXVxccHd3JygoiPDwcPr168fPf/5zuxkuwObNm9mzZw86nY6HH36YOXPm2MVwoWtTmddee+2Wa9dvR1NTEydOnMDPzw8PDw+bbDyza9cuysrK8PDwYOLEiTfsQWo0GmlsbMRoNFp7EI4q06qpqWHnzp3k5eUxcOBAxo8fb9fqjetpampi9erVfPPNN7S2thIeHm63OvE7RafTsX37do4cOYKLiwv9+/cnNTWViIgIh2s5deoUe/fupaqqCiEEarWa+Ph4nnnmGRISEmwaKz8/n87OTjo7O7FYLLS2trJp0yays7Nxd3fHzc2NiooKMjMziYqKskkH4a5N19PTk+XLl5Obm0tJSQl9+vQhMjKSTZs2ERQUxNixY6mtreWrr77i66+/xmQyIYSgX79+zJw5s8eCAaZPn86uXbswGAw0NzdbNwq5Hq1Wi4eHB8HBwdb62MWLF9tEw7fRPRlRWVmJr68vUVFRd10qczfk5+ejVqtRqVS4urpa87NXnxcPDw/c3NyuMX5XV1d8fHzQaDS4u7uTkpJCXFycTWbvvby8UKvVdHR03DAiaW9vp6CggA0bNvDll19SWVnJxIkT7bZA43o2bNjAJ598QnV1NdOmTXN4L3f9+vW8/fbbnD17luHDh98XE2jZ2dls2rSJ0tJSwsPDGTVqFJGRkQ7XodPpWLlyJZ9++qn1vbCwMObOnWtzwwWsq/IOHTrEmDFjKCoq4q233kJKyQsvvEBHRwerV69m1apV9OnTh3nz5qHRaKyloPdyrdxT9cKUKVOIjIykoqKCqqoqDh48yI4dOygpKbHuTNTc3GzNu/r4+JCcnGyTPCHA66+/ztatW9Hr9fzzn/+kqqrqpgXXaWlpTJw4kbCwMJKSkvD397dJ/NtRUlJCXV0dUkqioqKIiYmx20orgD/84Q+YzWZ27tzJgAED6NevHydOnLDWLgMkJiaSlJR0TaWCv78/Y8eO7flO+Ddh+vTpvP322xw5coTjx48THR2Nv78/7e3t7N69mw8//JBjx45hsVhISkriZz/7mV1HHldTXV1Nc3Mz0dHRDu/l6nQ6cnJyqK+vR6PRMG3aNLtsNHQ3GI1GTp06RXl5OdC1y9msWbMcWq/czf79+zl9+rT1Rt1dkzt9+nS7xIuPjyc4OJicnByee+45ampqCAkJ4dlnn2Xu3Ll0dHSgUqn44IMP+Pvf/46HhwcJCQkYDAYuXrzIqFGj7nqDnHsuGB0zZgzHjx+nrKyMl1566aafcXJywsfHhwceeIAnnnjCZnmRwMBAli1bBsADDzxg3ZLuevr169crkxPZ2dmUl5fj7u7Ogw8+6JCL6s9//rPdY9wN5eXltLW10d7ezpYtW6zbBW7bto2srCzMZjMRERHMnDmThx9+2Oa10reio6ODjo4OnJycGD9+vM22BrxT1qxZw9atW2lububhhx9m0aJFdrnp3Q25ubns3r2bCxcuoFariY6OJj4+3m7psFthNps5f/48NTU1XZt9OzszceJEfvGLX9it1z1w4EBeeeUV/vrXv1JcXMyECRN49tlnSUpKsi4Vf/LJJzGZTPzjH/9g8eLFaDQa/Pz88PLy4v3337/7PPOtdje/3W7r5eXlMjExUarVaqlSqaxPAFCpVFKtVkutVivj4+PlG2+8IZuamu5kl/Vv2229N7hnHfn5+XLGjBnS09NTjh8/Xq5bt04aDAaH67Axd/0EiyeeeEL6+flJlUplfbm6ukqNRiOHDh0q//a3v8nLly/bVcf1HD58WKalpcmYmBj56aef3m3sHutYtWqVTEhIkN7e3nfz1I671XHH56S1tVV+//vflz4+PtLV1VXOnj1bHjx40JZa7pj8/Hz52GOPSU9PTymEkGFhYfLll1++W/+4Jx1ms1maTKZbPUVGNjY2yrVr18rFixfL5ORk+fzzz8uSkpJ70XHvPd3w8HBeeuklNm/ezM6dO61L9MLCwhg3bhwzZ84kNTXVuj/nd4nueuHz589b99E1Go29UnrTWwQGBvL73/8eIQTbt2+3biQ/YsQIJk+ezKRJk+5paNZTujd1d3Z2dnhPDuCxxx7jsccec3jcW3Hw4EGOHTtGa2srERERzJ07t9eWImdlZXHy5EnrJLlWqyUsLOymVS+25nZtwc/PjwULFtikZrlH61HnzJnDnDlzeizi/yIvvvgiL774Ym/L6FWCg4OtWwPeLyQlJZGamsrFixdtVnf570xmZqZ1IcSIESN6dRnytm3bKCgosHbSRo4cyaRJk3pNj734zjyCXUGhm5dffrm3Jdw3DB48GH9/f7RaLfPmzWPkyJG9piUqKoqAgADr1p8hISEOXZbtKIT89p3Y7bNN+7dzs1yEouNaFB3Xoui4ryblPQAAIABJREFUkftFyx3raGpq4i9/+QsrV65kwIAB/PKXv+Shhx5yuA4bctPv5namq6CgoKBgQxw/k6CgoKDwHUYxXQUFBQUHopiugoKCggNRTFdBQUHBgSimq6CgoOBAFNNVUFBQcCCK6SooKCg4EMV0FRQUFByIYroKCgoKDkQxXQUFBQUHopiugoKCggNRTFdBQUHBgSimq6CgoOBAFNNVUFBQcCCK6SooKCg4EJuarhCivxBiixCiWQhRI4T4hxDC4U+nEELECiH2CCEuCSHOCSHmOVrDVVr2CfH/s/feYXFd5/r2vei9CtERSIBQRahaqKMuy5atYqsrcWQ7dpTkJHHO+U7ixLEcn5zjOHaKLcddtnrvCFXUECC6ACEQgqGJ3mGYYQb27w/EfGqWZTGzIc6+r2suWwMz78Paez97rXe9a22hEUK03H7l9pIONyHEASFEqxCiSAixspd0bBVClAshmoQQeUKI9b2ko0+cq7e1hNw+R7b2RvzbGnq9PYQQG4QQyUIIrRBis5yxvwlTHBtj93Q3AVWANzAKmAa8auQYD+X2iXIIOAq4AS8BW4UQoXLquIcNkiQ53H4N7iUNHwLtgCewCvhICDGsF3T8CQiUJMkJeBr4oxCiN54R0+vn6h18CCT1Uuxu+kJ73AL+CHwhc9yHYfRjY2zTDQJ2S5KkkSSpAogB5L6wwwAf4H1JkjokSToLxAFrZNbRZxBC2ANLgN9JktQiSdIl4DC90CaSJGVLkqTt/uft1yC5ddA3zlWEEMuBBuCM3LHvodfbQ5Kk/ZIkHQRq5Yz7TZjq2BjbdP8KLBdC2AkhfIH5dB283kYAw3sx/p+EEDVCiDghxPReiB8K6CVJyrvjvQx6wWQAhBCbhBBq4DpQDkT3goxeP1eFEE7ARuCXcsb9Bnq9PfoSpjw2xjbdC3RdyE1AKZAMHDRyjG8jl65h0q+FEJZCiDl0DZXsZNbRzX8BAwFf4BPgiBBC7p6dA13H5E4aAUeZdQAgSdKrt2NPAfYD2od/wiT0hXP1LeBzSZJKZY77IPpCe/QlTHZsjGa6Qggzuu6M+wF7oB/gCvyfsWI8CpIk6YBngCeBCuBXwG66TiTZkSQpUZKkZkmStJIkfUVXqmOBzDJaAKd73nMCmmXWYeB26ucS4Ae8ImfsvnCuCiFGAbOA9+WK+RAtvd4efQlTHxtj9nTdgADgg9sGUwt8ifwGgyRJVyVJmiZJkrskSXPp6mlekVvHNyDxzY/NNhV5gIUQIuSO98KBbJl1PAgL5M/p9oVzdToQCBQLISqA14AlQohUGTV00xfaoy8xHRMeG6OZriRJNUAh8IoQwkII4QKsA64aK8ajIoQYKYSwuZ2feo2uGdnNvaDDRQgx97YWCyHEKmAqMufKJElqpasXs1EIYS+EmAQsArbIqUMI0V8IsVwI4SCEMBdCzAVWIPMkUh85Vz+h62Yz6vbrn8AxYK6MGoA+0x7cjm0DmAPm3deNnBpuY9JjY+yc7mJgHlAN5AM64BdGjvEorKFrgqYKmAnMvmPGXE4s6SqBqQZqgJ8Cz9wzoSUXrwK2dLXJDuAVSZLk7ulKdKUSSoF64F3gPyRJOiyzDujlc1WSJLUkSRXdL7pSQBpJkqrl0nAPfeHafR1oA/4/YPXt/39dZg0mPzZCkiRjfI+CgoKCwiOgLANWUFBQkBHFdBUUFBRkRDFdBQUFBRlRTFdBQUFBRhTTVVBQUJCRb6uB643ShgctHFB03I2i424UHffTV7QoOu5B6ekqKCgoyIhiugoKCgoyopjuvwkdHR20tbXR0tJCW1sbHR0dD/19nU4nkzIFhQezefNmhgwZwoYNG6it7RNb7BoFk5huRUUFW7ZsYeHChfj4+DBo0CBeffVVqqt7a4Xjvzd5eXn88Y9/JDIykqCgIMaNG8dLL73Enj17aG9vv+/3tVotH3/8sdHil5WV8d577zF+/HgWLFjAe++9R3p6utG+/3FQq9Xk5eXxwQcfMH/+fFauXMmFCxd6TU91dTXvv/8+M2bM4KOPPuo1HX2F69evc+HCBXJzc8nMzCQvT96V83/5y18YPnw4v/rVrygtNe4GhUbdTKKhoYEdO3bw9ddfk5+fT0tLC1qtFiEEMTExhIWF8bOf/cyYIe8jKyuLo0ePUlxcTH5+PuXl5cyaNYsXX3yRoUOHmjT2g2hra6O8vJyGhoZv/V0zMzMCAwNxcXExWvxr167xf//3fxw9epSmpib0ej11dXUUFBRw4cIFkpOTeeWVVwgMDLzrc2q12ijxS0pKeP3114mJiaGurg5zc3Pi4+N59913mTNnDq+99hrDh8u/v/z58+f55z//SUJCAo2NjZiZmREfH09gYCADBgxg0qRJTJ48maCgIGxsbEyu59q1a5w4cYLExERcXFxYtGgRPj4+Jo3Z2dnJ3r172blzJ9euXTMcc0tLS0MbzJo1i4kTJ2JlZWVSLfei1+vR6/XY2NgQFhbGmDHyPdGpsbGRq1evUlBQgCRJWFgYd88do31bZ2cn0dHRfPXVV6SlpdHe3o4QXZN3kiRRU1NDQkKCyU03MTGR6Ohohg8fzsyZM7l+/Tru7u44Osq7X3daWhppaWlkZmaSkpJCSUnJfb8jSZKhjQCEEISEhODu7s7SpUtZvHhxjzSUlJTw7rvvcvjw4btMv7Ozk7a2NgoKCti7dy9ubm78/Oc/N4m55Ofnc+3aNSRJIioqCh8fH/Lz87l69Spnz55lyJAhvWK6U6ZMoba2lsrKSoQQzJ07l6qqKq5du8a+ffs4fvw4Hh4erFu3jp/+9KcmNd5bt24RHR3NhQsX0Gg0ZGdnc+zYMV588UWTxQQ4d+4cH3zwAcnJydjY2DBixAj8/f1paGhApVLx6aefsm3bNkaNGsXKlSt55plnjG5A30RrayvNzc14eHgQEhIiq+knJydTWFiIh4cHgwcPxsPDw/CzhoYGSkpKGDx48GNrMloLZmVlsW/fPjIyMrC1tWXQoEFotVoKCwsBcHBwMHlPs6KignPnztHY2MiECRNYuHAhTU1NSJKEu7u7SWPfSUlJCX/961+JjY2ltbUVtVqNVtu1ydmdN6J7/21paUlYWBjh4eEMGTKkRxra2trYv38/ly5dorGx0fC+vb09jo6OqNVqmpqaKCkpYffu3UycOJGpU6f2KOaDGDVqFM8//zzp6eksXryYCRMmEBcXx4cffkhWVtYjjQBMgYODA+7u7tjb2xMcHMyGDRtob2+ntbWVyspKEhMTef/999m0aRMeHh784Ac/MImO0tJSdu/eTXR0NBqNBgBHR0dCQkK+5ZM959ChQ2RnZzN16lTWr19PREQE1tbW6PV6VCoVe/fuZf/+/Zw7d47y8nJcXV2ZOXOmyXUB3Lhxg+vXr9OvXz8GDZJ3u+WbN29SXV1NREQEI0eOxNzcHICmpiaOHj3Kli1bePHFF1m6dOljfb9RTLekpIRNmzZx7tw5AFauXMnatWu5ceMG27Zt4/Lly1hbW5vc+E6fPk1GRgZ6vR4HBwc6Ozs5ceIEWVlZLF26lOnTp5s0PnTdfP785z8bepd3muqIESOYMWMGkiTh4eHBjBkz7vu8u7s7rq6uuLm59UjHvn372Lp1K8XFxdy5k9ycOXP44Q9/SE1NDadOnWLHjh0UFBSwc+dOxo0bh62tLRYWFsyePbtH8btxcXFh7dq1PPnkk3h6elJSUsLZs2e5du0azs7OeHt7GyXO46BSqairq8Pf359+/foZ3h84cCAhISF4eHjw9ttvc+XKFZOYbltbGwcPHmTz5s3k5+cbjpOdnR2enp5Gj3cvEyZM4NSpU9TW1uLs7HyXuXl7ezNo0CCioqL44osvSE1N5dq1a7KZbnFxMYWFhUycOPGunqapSUpK4vDhw5SUlDB58uS7rsOqqioSEhK4ePEilpaWLFy48LFGQD023ba2Nj799FOOHTtGc3MzzzzzDOvWrWPs2LGEhobS2NhIYmIiQgiT58by8/OpqalhwoQJ+Pn5ERMTw9///nfUajVeXl6ymK61tTXl5eW0trYSFhaGpaUlxcXFhh734MGDWbBgATY2Nnh5eZlEQ2VlJQcOHCAnJ8fQwwYYPXo0y5YtY9asWWg0Gry8vCguLiYxMZELFy6QkpLC5MmTMTc373FPuxshBP3796dfv36o1WrOnj3LqVOnaGlpITIyksjISKPE+a5UVFRw/fp1hBD4+/vf9TNzc3P69+/PjBkz2LlzJxcuXKCoqIgBAwYYLb5arTbMf+Tl5d2VjjMzM5NlGD937lwOHz5MbGwsp06dIjg4mKCgIKDrPPb396etrQ0PDw+srKywt7c3uSaAgoIC8vLy0Gq1srVFN2fOnCEzM9NwDXSfGzqdjpycHM6dO4dGoyE3N5fz588zd+5339e8x39Nc3MzSUlJVFZW4u/vz7x58wxd8paWFsrKytBoNLi6ujJixIiehnsoGo2Gjo4OnJ2daWlpISkpiRs3bmBtbU1paSnt7e0mzw35+fnh7u6Oubk5QUFBLFiwAAsLC9LT02lvbyc4OPi+SStjUldXx/vvv098fLxhYsTFxYXRo0ezevVqZs6cia2tLba2tjzxxBO8/PLLlJWVUVpayp49exg7diw2NjZGvUE2NzeTlpbG6dOnOXLkCGq1mmeffZYf/ehHRjP370pdXR3V1dUIIbC1tb3v5x0dHdTX1yNJElVVVXelaHpKW1sbf/3rX9m6dSuFhYU4OzszevRotFotsbGxVFdXk5KSYvIUg7u7O+vXr6eoqIgDBw7g6+vLqlWrDDfI9PR0du7cSXZ2Nk8//TSzZs0yqZ5umpqaaG7uenyfp6cnoaGhssStqqoiNTWV2tpaPD098fHxwc6u63m2Op2OiooKbt68iRACa2vrxx6l9dh09Xo9Wq0WvV5PcHAwwcHB2NracuvWLfbt28eBAwfw8PDgmWeeISwsrKfhHoqLiws2NjakpKTQ1tZGXl4eHR0dtLe3U1paSkFBgck12NraYmVlhbm5ObW1tQwYMIBp06YxdepUWltbDT0JU9De3s7WrVvZvXs3lZWVSJKEs7Mzzz33HMuWLWPUqFF3DaNdXV0ZN24cERERHDlyhHPnzlFfX2/0If/BgwfZtWsXqamp1NTUYGNjg7m5Ofb29piZ9U6puEqlorS0lM7OTjQaDZWVlRQXF3Pz5k2ys7PR6XTk5eVx/fp1wsPD8fPzM0rctrY2tm/fzpYtW8jNzcXf35+nn36aqKgoLl26xLlz5wwVL3IwadIkXnnlFd5//32+/vprPD09Wbx4MZIkYW1tTUBAAFVVVdjb21NfX09AQIAsurpxcHDocartUWhvb2fv3r1cvXoVrVbLhAkT7vIKSZLQarWGEktbW9vHvhn02HSdnZ0JCwsjPT0dvV5vmBk/e/YsmzdvprKykieffJLnnnsOBweHnoZ7KOPHjycmJobk5GRKS0sNSfjKykqampqoqakxafxuAgICcHR0JC8vj8zMTCIjIwkLC0OSJENS3hTk5uayY8cOysrK6OzsxNnZmeeff54XX3yRkSNHPrCX7+Pjw+LFi0lKSqK2tvaudISxuHLlCnFxcYZJs46ODi5cuICLiwvW1tZEREQYPebDqKysJCMjg7KyMpqbm9myZQtnz56lqqqKmpoampqa8PHxwcHBgeXLlxMVFdXjC1+n05Gbm8uZM2fYtm0bKpUKf39/Vq1axcqVK3FyciI1teu5h3q9nra2NmP8qd+Kra0tCxcupKGhgc8//5w9e/bg5eWFp6cnWVlZFBQUUFhYSEVFBWPGjCE8PFwWXQBOTk6yTYDX19cTHR1NUVERQ4YMYeHChQQHBxt+3tTURFVVleEatra2fuzRYI9N197enrlz5xIfH8+NGzc4ffo0ubm57Nu3j5KSEmbMmMELL7xw1x9gKsaMGcPatWsNw7IhQ4ZQXl7Onj17TB77Tp5++mnKyso4deoUp0+fJjAwkLlz55r0BOro6GDfvn1cv36d9vZ2zMzMmDNnDi+88ALh4eFYWlo+8HNOTk6MHTsWLy8vSkpKKC4uxt/fn/j4eCZPnmwUbePGjQMwzM6XlJSQkZHB4cOHcXBwwMnJyaQz1Hq9ntraWm7cuEFmZiY5OTlcuXKF2tpaJEmiuLgYvV6Ph4cHI0aMwNvbm4CAADw8PAgODjZKzz86OppDhw5x/vx5ysrKCA4OZsmSJaxcuZLBgwejUqloaWkBuupE09PTKSwsNOnIqBs3NzeeeeYZsrKyOHXqFO+88w79+vUjNzeX/Px8dDods2bNMvkosRuNRkN7eztubm4mm/e4l/3795OVlYVWq8Xa2pqioiLi4uLo7OykpaWF0tJSLl26BHRVl/QkVWqUDHVkZCRz585l69at7NixAwsLC8rLyxk7diyvvPIK06ZNM2kPrxtnZ2eWLFnClClTMDMzw8nJib179xqKrE3d0+6m22QGDBjAhQsX2Lp1K7W1tTz55JMMHDjQJDHVajXHjh0zXLjTpk1j/fr1jBgx4hsNtxtHR0dGjhxJfn4+ycnJTJgwgcuXLxvNdJ988kmmT59OZ2cnAJmZmXz00UecOHGCgwcP4u/vz6uvvmqUWPfS2NhIXFwcly9fJisri7q6OoQQlJWV4erqyowZM5g0aRJ+fn54enoSFBSEvb29USdvYmNj+dvf/kZCQgJtbW2GyUXoMuNjx45RWVlJfHw80JUDv3jxIh999BE/+MEPZFnU032DOX78ODExMTg5OeHi4oKnpyfjxo1jzZo1stVTd484rKysZFmYUlBQwLZt26ioqEAIQX5+Pps3b+bEiRNIkkRTUxNqtdpQ092dlntcjHJmeXh4sHz5ctLT0zlx4gTQNfsZGBjIqFGjZDHcbpydnXF2dgagtraWsrIyGhoa6Nevn9Hycg9Cr9cTHx9PeXk548ePZ/To0QQEBDB06FB27NjBli1baGhoYM2aNSaZSDty5Ag3b95Er9djZmbG008/zZgxYwwTAQ/DxsaGkSNHsnv3bqMveYSuCZs7e/n29vYUFhYahq0pKSnodLpvvTl8V+rr6zl27Bhbt25Fq9USFBTErFmz8PHx4auvvkKtVrN69WqioqKMGvdevv76a5KSkgw9feha5nrt2jUaGhruSul0VzBUVlayfft2wzlkSnQ6HSkpKVy/fh2dToeVlRUREREsWLAANzc3xo0bx8iRI02q4U4qKiqoqanB2toaa2trk8dLS0szTLT7+vri4uKCpaUllpaW6PV6JEkylPNZWlri4+PToxuQ0W7n7u7u2NnZIYRAkiTa29vJzMzk7NmzLF682OgX1KNQX19PdXU1dnZ2BAYG3jWJZGzS0tL405/+RH19PW+++SY+Pj6GSQlPT08++ugj9u/fj5WVFS+//LJRl/oCvPPOO4YZX2dnZ3x8fB7aS9BqtbS0tFBXV0dycjJxcXFYWFjIUjfr4uJCVFQUN2/eZPv27VRXV1NaWmr0ofSZM2f48MMPcXV15dVXX2X27Nm4uLjQ0tLCqVOnKCoqkuW8dHFxwc3NDXd3dzQaDTqdDhsbG5ydnQ034Lq6OkpLSw2VPhEREYSGhpp8KTBAamoqH3zwAampqXh4eGBnZ4evry9Lly6VJb1xLzU1NdTW1jJs2DBZ6pWtrKwYMWIEo0ePZvTo0fj5+WFnZ0dAQABarZaysjIyMzPZtWsXDQ0NDB8+vEe5baOYrkaj4cyZM9y8eRMvLy+sra2pqakhNTWVXbt2ERUVJWuBczdZWVmkp6fj5eVl8nzUP//5T7Kysli7di2DBg0y9O7Nzc2ZOnUqWq2Wt956i127dhESEsLTTz9t1CFsWVkZer0e6MrTarVasrOzga7Jku4bYWdnJ52dnVRXV5OXl4dKpSImJob8/Hy8vLyYNGmS0TR9ExYWFnR2dqJWq2lvb6e+vp6qqiqjXuAVFRV89dVX6PV6/uM//oMpU6Y8sDRMDlavXo2jo6OhAkCr1eLv709gYCD29vZotVpOnjzJ9u3baWtrIzg4mDfffPOu9I4kSajVaqPXyjY1NfHFF1+QlpZGVFQUkZGRxMbGkpGRQVxcnOym29jYiEqlor6+HhcXF1l8Izw8nF/96leMGjXqGztDu3fvZvPmzdjZ2eHt7d2jHniPr3qNRkNcXByfffYZ1dXVLF++HH9/fw4dOsTFixepqanhxo0b9OvX7659BuQgMzOTtLQ0Ro0aZfJ8bmpqKmZmZgQFBeHk5HRfKdTIkSOZM2cOf/vb39i8eTPDhg1j8ODBJtFSWVnJO++8Y0gt9O/fH0mSqKuro6Ojg87OTmpqalCpVHd9zsbGxqQpGOgyj8rKSo4fP86JEyewsrIiPDzc6LWYx44dIz09nV/84heGlXbd6PX6b93a0piMGTPmoRu2lJaWcubMGRoaGrCxscHb2/u+FFRnZydlZWVGb6fU1FSuXr3KE088wU9+8hMGDx6MTqcjKyuLa9euyVLbfifXrl2juLgYa2trXF1dZdkzJSAg4FtL4ZqamgxrEXqa8uix6WZmZvL2229z48YNVq5cycsvv8ygQYOwsbEhOzub7Oxsdu3axdixY2XfqaijowO9Xo+1tbXJTTc4OJiYmBjefPNNJEli5syZhnQLdF00AwYMwMPDg+TkZLZt28bGjRuNFt/Hx4eGhgb0ej0ajYasrKxH/qyZmRnu7u5ERESYfNVRS0sLx44dY/v27VRVVTFp0iSeffZZXF1djRonIyMDNzc3Fi9efN93V1ZWUl9fj42NzSPlvE2NSqXi5s2bSJJEQEAA8+bNu+/mZ6oVnZmZmdTW1jJjxgzDjPzw4cMZNGgQubm5lJSUyLr3Qfex8fHxYfLkySbrmDwujo6OPe7998h09Xo9e/fuJScnh6lTp7Jq1SqCg4PR6XQEBQUxatQokpOT0Wq1hgS9XEiSZJgt9/f3N3l94ezZs8nJyaGgoIAPP/yQK1eu4ObmdtfeC0VFRTQ3N1NTU0NycrJR4//Xf/0XGzdupKioCK1We9d+C9+EhYUFDg4O+Pr6MmfOHF566SXc3d1pb283es4ZunaOSkpK4sCBA2RlZREcHMyiRYtMtp4/NDT0gedcWloaZWVlzJkzR7YyqG9Cr9eTk5NDbm4uZmZmREREsHbt2vt+z8zMzCQLE7rPz+4FThYWFnh6euLl5UVCQgIFBQWymW5jYyMJCQkUFhYyZMgQ2RdifBMajYbW1lagq7yup1UcPTLdmzdvcuHCBVpbW5k6dSrW1takp6dTUlLCkSNHyMrKYvDgwSxcuFC2ddvdNDY20tzcbNi9y9Q9mjVr1mBmZsbBgwfJyclh165dhic03Dn7KYTAx8fH6PuDrlixAhsbGzZv3mxYUdXW1oZOpzPENjMzw8bGBgsLC0Ppy8yZM1m1ahUjRowwGJSVlRXLly83qj61Ws3hw4f55JNPSElJISAggFWrVrF69WqjxrmT7tWS3VtodnZ20tTURGJiIl5eXkycOLHXe7oqlYrU1FTKyspwcHDAx8dH1mvF2dkZKysrMjIyKCwsJCQkBFtbW+zs7Ay7rsmBXq9n27ZtHDhwgNraWsO+xn2B7Oxs4uLi7rqOe0KPTHfPnj0UFRUhSRJXrlxhz549htIYS0tLvLy8iIqKkm1nojvJysoiNzcXS0tLg8mYEltbW9avX8/69etJT0/n66+/5tKlS5SXl99VKmRtbc2UKVNYtWqVUeMLIVi8eDFz586lra0NSZK4ePEi165dM5Qk+fj4MHbsWEO+sLun+6AZfCcnJ6Po6ujooLm5mYMHD/L555+TmZlJv379ePHFF/nxj39s9LRCN3Z2dsTHx3P8+HGeffZZHB0dqaqq4p///CfHjx9n6dKljBw5UvZ5hntJS0sjOTkZjUZDeHi4bPsbdDN79mz27dvHqVOnmDhxIkFBQeh0OvR6Pc3NzSYpIXwQ+fn57Nu3jxs3bhAaGsqsWbN6bV+OByGEQAiBubl5j5eu98h0s7KyaG1tpbW1lW3btmFhYYG5uTkuLi4MGzaMl156ieXLl8uey72T7sfTyNl7GDVqFKNGjaKqqorKysq76jCtrKzo37+/yVba2NvbG/7WxYsX93gj9J6SmZnJZ599xv79+6mvr2fatGm89NJLzJw501BPbQrWr1/PpUuX2LRpE1evXsXGxsawGmzFihWsWbOmV7eV7MbMzMxwEbe0tMi2VL0bLy8vwsPDSU1N5dixY/j7+1NbW0tiYiIhISHMnz9fFh1mZmb4+/szdOhQFixYwPjx42WJ+yiYmZlhbm6Oo6Mjnp6ePe6Q9Mh0X3zxRXQ6HZcvX6atrY3Ro0czffp0pk2bRkREhNF6S49D98ncXXbSGz2a/v37G1Ye/bvy7rvvcvz4cdzc3HjhhRdYt26dLBt0d5ddvf766+zcuRMhBNOmTeO3v/0tM2bMkHVT+4cxcOBAhg0bRmVlJRMnTmTatGmya1i/fj25ubkcOnSI6OhoJEnCxcWFl19+2WQrKO8lNDSUzZs3yxLru9K/f38mTpyIra0tS5cu7XHttPiWHEXPExjfnQe543fWUVBQwNtvv41Wq+WNN954nAvdKDqMgKLjbhQdd/NNvYnvpOXWrVvs37+fL7/8krq6OlavXs2Pf/xjfH19e6qlr7RJX9Hx/TVdI6DouBtFx930ZR3Qd7QoOu6hdzYzVVBQUPg35dt6ugoKCgoKRkTp6SooKCjIiGK6CgoKCjKimK6CgoKCjCimq6CgoCAjiukqKCgoyIhiugoKCgoyopiugoKCgowopqugoKAgI4rpKigoKMiIYroKCgoKMqKYroKCgoKMKKaroKCgICOK6SooKCjIiGK6CgoKCjKimK6CgoKCjBjVdIUQG4QQyUIIrRBiszG/+zG0LBdC5AghWoUQN4UQU3pBQ19qj3NCCI0QouX2K7eXdLh/9ATLAAAgAElEQVQJIQ7cPi5FQoiVvaRjqxCiXAjRJITIE0Ks7yUdfaI9bmvpC9dMn2gPIUSgECJaCFEvhKgQQnwghOjRMyW7McqX3MEt4I/AXMDWyN/9yAghZgP/BzwPXAF667GvfaI97mCDJEmf9bKGD4F2wBMYBRwTQmRIkpQts44/AT+SJEkrhAgDzgkh0iRJSpFZR59ojz50zfSJ9gA2AVV0tYMLcAp4Ffh7T7/YqD1dSZL2S5J0EKg15vc+Bm8CGyVJSpAkqVOSpDJJksrkFtGH2qNPIISwB5YAv5MkqUWSpEvAYWCN3FokScqWJEnb/c/br0FyauhL7UEfuGb6WHsEAbslSdJIklQBxADDjPHF37ucrhDCHBgLeAgh8oUQpbeHBn2hp9nb/EkIUSOEiBNCTO+F+KGAXpKkvDvey8BIJ/N3RQixSQihBq4D5UC0zBL6RHv0oWumT7THbf4KLBdC2AkhfIH5dBlvj/nemS5dwxJLYCkwha4hSgTwem+K6gP8FzAQ8AU+AY4IIWTt2QEOQNM97zUCjjLrAECSpFdvx54C7Ae0D/+E0ekr7dFXrpm+0h4AF+gy+yagFEgGDhrji7+Pptt2+7//kCSpXJKkGuA9YEEvaup1JElKlCSpWZIkrSRJXwFxyN8mLYDTPe85Ac0y6zAgSVLH7WGsH/CKzOH7Snv0lWumT7SHEMKMrl7tfsAe6Ae40pXz7jHfO9OVJKmerjvTnY85Vh55fD8SIGSOmQdYCCFC7ngvHJB7kuRBWCBzTpc+0h596JrpE+0BuAEBwAe3Oym1wJcY6SZk7JIxCyGEDWAOmAshbIxVZvEd+RL4qRCivxDCFfgFcFRuEX2lPYQQLkKIud3xhRCrgKkYKUf1qEiS1EpX72GjEMJeCDEJWARskVPH7fNiuRDCQQhhLoSYC6wAzsipo6+0x216/ZrpK+1xu6dfCLxy+3pxAdYBV40VwGgv4A/8/zPB3a8/GDPGI+qwpKvkowGooKvMw6YXdPSV9vAAkugapjUACcBsuXXc1uJGV26sFSgGVvZSe5y/3RZNQCbw4r9re9zW0Veumb7SHqOAc0A9UAPsBjyN8d3idgAFBQUFBRn43uV0FRQUFPoyiukqKCgoyIhiugoKCgoyopiugoKCgowopqugoKAgI99WM9obpQ0PKthXdNyNouNuFB3301e0KDru4Xvf0925cyehoaFERkZy5oyste8KCv+yNDY2sm/fPqZPn46fnx8bN27sbUnfG77XpqvT6SgsLOTGjRt3Fj0rKCg8hIaGBmJiYnj//fe5ePEijo6ORERE9Las7w0mNd2jR4+yYMEC/vznP8tuePn5+bz11lt8+umnWFlZ4e/vT1hYmKwaFB5MYWEhv/zlLwkJCeEPf/gDOp2utyUp3KapqYmYmBj+8Y9/cOXKFQICAnjhhReYMkX2h0h8bzHpPgCSJKFSqTh58iRLly4lKCjIlOHuoqKigrS0NFQqFaGhocycORMfHx/Z4j+I6Oho3njjDdzc3Ni4cSMTJkzoVT29gUaj4fDhw+zZs4eysjJ2796Nj48P69evx8zMNH2A/Px8Lly4QENDA+Xl5WRnZ+Po6IhWq8XOzo4lS5YwZcoU+vfvb5L4/0qo1Wpyc3PJzc1Fr9fj4uLCsGHDcHZ27m1psqJSqdi9ezd79uyhqKgIACEEERERbNy4kfHjxz/2d5vUdIcOHcqsWbM4dOgQ+/fv51e/+pUpw91FZmYm2dnZSJKEj48Pw4YNM9lF/Sg0NDSQlZVFdnY2vr6+pKen/9uZrk6nY+vWrXz22WfcunULSZK4ceMGX3zxBQEBAcybN8+o8Xbv3s27775LeXk5ra2tdHZ20tHRgU6nQwjRtQ5eCJKSkvjhD3/Ib37zG6PG/1ejtbWVo0ePsnfvXurq6vDw8GDq1KlMmDABIeTbkE6r1XLz5k1Onz5NSkoK7u7uREZGMm7cOGxtbU16c2xtbeXIkSN8/PHHpKenA2BtbY2rqyvW1tbk5ORw5MiRvmu6tbW1qFQqnJ2dCQ4ONmWou4iPjycmJoZbt27h6urKhAkTGDt2rGzxH0Rrayv19fXodDpsbW3x8vLqVT1yo9Fo+Oyzz/jkk0/Izc1l0KBBBAcHU1BQwNWrV/n73/+Or68vI0aMMFrM5uZmiouLqaysfOjvqVQqLly4wIIFCxg1apTR4j8Kzc3N5Ofnk5CQQFpaGkVFRahUKqysrHB3d8fNzY2f/exnTJ8+3eRa0tLSOHr0KHl5eZibmzNmzBjWr1+Pm5ubyWPfydWrV/nggw+IiYmhra0Nc3Nzdu7ciZOTE0OGDOGtt95i+PDhRo+r1Wr57W9/y8GDB7G1teW5555j0aJFhIWFoVarOXr0KH/+859JSUmhtrYWd3f3x4pjMtPt6OigrKyMvLw8+vfvL2s+NTY2lqSkJLRaLYsXL2b9+vXY2NjIFv9BSJJEZ2cnAA4ODgQGBvaqHrm5dOkS+/fvJzc3F2traxYsWMD69etJSEjgL3/5C3Fxcfz+979n48aNRjPeoKAgbG1tEUIwYcIEfHx8sLCwYOzYsbi4uLB3715OnjyJXq+ntbWVlpYWo8R9FBobGzl8+DC7d+8mLy8PGxsbPD098ff3Z+bMmQghUKvVxMfH85vf/AYvLy+WL1/Oc889ZxI98fHxfPHFFyQmJqLT6RgwYABRUVGEhYWZtJebk5PD0aNHOX36NDU1NUBXB0WtVjNw4EAGDera4riyspKEhATKysqoqanhtddeY9asWdjb2xtNy6ZNmzhy5Aiurq5s2LCBOXPm4OHhgY2NDVlZWeTm5iJJEl5eXj3yE5OZbnV1Nenp6dTX1xMZGYmnp6epQt1FXl4eaWlp1NZ2PQvSy8urTxhcfHw8hw8fxsLCAl9fX/z9/WWJ29jYSHNzM87OzkiSRGNjI+bm5jg6OuLoeP9TULRaLbW1tVRXV1NZWUlVVRWrV6/ukYbk5GQ++eQTUlJSaG9vZ/78+Tz//PMMGTIELy8v1Go1v/71r4mLi+PLL7/k3XffNUoqaNy4cXz66ae0trbi7++Po6MjZmZmODs7k5mZibm5eY9jfBeSkpLw8PAgPz+fr776CpVKhaurK6tXr2bSpEn0798fe3t7w3HR6/UsW7aMsrIy3nnnHTZu3IinpyfTpk0zqq6rV6/y9ddfc+rUKWpqavDy8mLx4sUsWbIES0tLo8a6k4yMDP7nf/6HxMREWltb8fb2xsHBAWtra4YOHcpTTz1l+FsrKyvZs2cPX3zxBcnJyfzud78jNTWVX/7yl0bJN1dXV3Pq1Cmam5t57bXXWLx4MS4uLoYbjq2tLQ4ODtjb2zNw4EBsbR//8XEmM93CwkLS0tKwtbUlLCzsgRe4KTh58iQZGRm0t7czadIkoqKiZL+47qWtrY0bN25w8+ZNgoODmT9/vsknJjo6Orh48SJ79+7Fw8ODiIgIrl27RkJCAlqtFltbW1xcXGhtbcXe3p7W1lZ0Oh06nY7W1lZDz8vJyalHptvR0cHBgwe5ePEiGo2GJ598kh//+MeMGjUKc3Nz3N3dCQsLw97envr6es6cOUNWVhYjR47scRs4Ojoybdo0Ojs7sbS0NBh5eno6O3fuJCWl62nrXl5eTJo0idDQ0B7H/Cb+93//l+jorudeNjU1odVqeeGFF5gzZw6enp7069cPC4v7L0cvLy9CQkIwMzNj3rx5vPvuu0Y13ebmZk6dOsW5c+eorKzEwcGBRYsWsX79egICAowW517a29vZuXMnFy5coKWlhVWrVrFo0SJDvtba2hovLy/69esHgJ+fn8FDPv30U3JycmhubsbS0pLXX+/5o9xiY2O5efMmrq6uhIaG4urqetfPu+cDzM3NsbOz61GnwCSmq9frUalUqFQqQkJCmD59uizGV1hYyPnz5ykpKaFfv37MmTOHiRMnmjzuo+jKy8tDp9Ph7+9PZGSkydqjubmZ8+fPc/nyZVQqFf369WP48OH4+/tjZ2dHUFAQnZ2dtLS00NjYaBjKdXZ2MmxY10NXHR0dCQwMxN3dHTs7ux7piY2N5cyZM9TW1jJ48GCWLVvG5MmTDT0FIQQhISGsXr2af/zjHxQXF7N9+3ajmC5wV0+tubmZtLQ0vvrqK6Kjo6mursbJyYk5c+awevVqwwVubH7zm9+wY8cO1qxZQ11dHfv37zfUjffv3/9b8/uWlpaMGzeOpUuXcvLkSaNqu3LlCjExMahUKoQQjB8/nkWLFhEaGmrSa/b69eucOHGCuro6fvjDH/Liiy8yfPhwrK2tH/j7VlZWhIaGsnz5cjIyMoiNjeXWrVvs37+fZcuWMXjw4B7pCQkJwdXVlRs3bvDhhx+Sk5PDtGnTGDx4MI2NjZw4cYK4uDjCwsKYOnVqj2IZ3XT1ej03b94kMTGR5uZm5s+fb5Kk94PIyMggPz8fjUZDWFgYQ4cOlX0S4EHk5+eTm5tr6D2aMtWyadMmUlNT8fPzY8GCBYSHh+Pv72/IQZmbmyOEQKPR0NLSQkdHB21tbajVajw8PICuoZSTk1OPh/g1NTVs376d69ev4+fnx+rVq4mKisLBweGu3/P29mbmzJns2LGD5uZmQw/UmKhUKg4fPszx48dJSUmhpqYGSZLo6Ojg1q1bpKamYmlpiY+Pj1FHZSqVip07dzJs2DBWrFiBpaUlfn5+HDhwgF27dpGYmHjXqMfd3Z2BAwei1+tpaGigo6ODmTNnMm7cOKZMmcLBg0Z5IC0AZWVlHDx4kKtXr6LVagkKCmL+/PlMmDDB5J2k5ORkbt26hY+PD0uWLGHEiBFYWVk99DNWVlYMGzaMDRs2oFKpDAuf/vu//5v9+/f3SM/QoUP5z//8T3bv3k1iYiJZWVmcPXuWyZMnG0YDAwcONNwceoLRTbe+vp5jx45x4sQJfH19mTRpUo97S49CTU0NFy5c4NatW5iZmREaGkpgYCCdnZ1otVq0Wi0VFRWUlpYycuRI2XLMer2egoICCgsLsbOzM+StTMWRI0eora1l4MCBODs7I4TAwsLivsS/g4ODSXXo9XpOnjzJ5cuXkSSJyZMnM2vWLHx9fe/7XWtra0OeOzs7G41GY1QtSUlJbN26lZMnT6JSqe76frVaTVJSEtXV1QwdOpTw8HBmzZrFmDFjjBJ7y5Yt1NTU8KMf/YjAwEBsbW1Zs2YNQ4YMIT09nVu3bqHVdj35vampiba2NpydnbGyssLT0xM7OztcXV0No8fw8HCj6AKIi4vj0qVL1NfXY2lpycSJE5k6daphaF1UVERsbCySJBEeHo69vT1ubm6Gm/Pj0tDQwMmTJ2ltbSUqKopBgwZ9q+F2Y29vz8yZM/n5z3/OG2+8QWNjI6dOneqRHug6B5988kkGDBjA+fPn2b17N2fOnCE9PR1LS0sCAwN55ZVXmDp1ao8n5Y1uujU1NSQnJ9Pc3MySJUt6VM/2XTh9+jSxsbHU19cTEhLCjBkz8PHxIT4+nnPnzlFTU0NVVRU1NTVMnjyZp59+WpbyIJ1OR1VVFXV1dQwbNozRo0ebdHJi0aJFlJaWYmdnR2ZmJteuXcPNzc0wnHd2diYgIAA/P7/HLnl5FGpra9mzZw+lpaVERkby7LPPMmjQoAf2noUQ2NjY4OLiYhItO3bsYOfOnVRVVd33s+7JxYyMDK5du8bly5fJyMhgxYoVPPXUUz2OvXfvXszNzRkxYoThGPj6+uLr68vUqVOpra2lvb0d6Jq112q1uLm5GXKa9vb2SJJEUVERJ06cYP78+T3WBF0pr+PHj1NUVIRer2fw4MFERUURGhpKdXU1aWlpnD59mpiYGHQ6HUOHDsXDw4NFixYxd+7cHo2CvvzyS+Li4tBoNAQEBHznSSkHBwfmzJnDX/7yFzQaDZMmTXpsLXdibW3NmDFj8PHxISsri4yMDAoLC3F2dmbevHmMHz/eKFVQRjVdvV5PTk4OhYWFhIaGMmPGDNnqUVNSUigrK0On0xEREYGzszNHjx7l2LFjnD9/nsbGRgAsLCy4ceMGlpaWsphuUVERxcXF6HQ6goKCTL4g4qWXXqK6uprm5mbq6uqorq5GrVZTV1dHXV0d7e3txMfH4+LiwuTJk4mIiOjRTOw3cePGDbKysmhvb2fq1KlMnDjxG021o6ODhoYGSktLMTMzM2lKyNnZGVtbW/z9/fHx8cHa2hqdTkdJSQmZmZkUFRVRU1NDXV0dw4cP7/Eqylu3bhEYGPjAnpyrq+t9EzYPQq1WEx0dTVVVFQsXLuyRnm5SU1NJSUmhpaUFa2trnnjiCYKDg7l27Rrnz58nJiaGq1ev0tjYiF6vJzc3FwcHB9RqNSNGjMDPz++xY3/++edUVlYaSii/C1qtluvXrxMbG0tLSwteXl5GX3SVkZFBQUEBwcHBODg4UFFRwa1bt1CpVEbxDKOabnNzMwkJCVy/fp25c+eadPbzYVRWVrJ7925SU1MpLi7G1tYWX19fzM3NaWlpob6+3rC0z5RIksSVK1e4evUqLi4uhISE4O3tbdKYhYWF+Pr6MnDgQENvRJIkmpqaqKyspKSkhIyMDJKTk8nNzWXdunU9nhi4F7VazZkzZ2hsbGTy5MlMmDDhodUaer2eqqoq8vPzcXBwMHrqZ8WKFbS3t6NSqQgKCsLb25vQ0FCCg4OxtbWlvb2djIwMtm/fTlJSEjU1NWRnZ5OammqUpeu+vr6PPbpRq9UcP36cr7/+mmeffZbJkyf3WI9Op+Py5ctUVFSg1+uxsLCgpaWFU6dOcf36dRITE6mursbR0ZGBAwei0WgoLy+npaWF06dP89prr/XIdKuqqtDr9Tg6OhIaGvpI6ceOjg5qa2tJS0tj27ZtXLp0CbVaTWRkJLNnz35sLfeSk5PDF198QXl5OStXriQsLIwdO3aQlJTE8ePHGTp06COnQr4Jo5pufn4+WVlZtLS0YGNj88ASGFOgVqupra01bJwSGxsLdA0XBg8ezPjx4wkMDKS0tJTz589TX18vS565traW9PR0iouLGTFiBFOmTPnG2VljcfDgQby9vRk7diyhoaE4ODgghMDZ2RlnZ2dCQ0OZNGkS0dHRfPTRR1y4cMHoppudnc3BgwdpaGhg1apVTJgw4aG96aamJvLz85EkCXNzc6PnmseNG0f//v1pbm7G09MTNze3+yaKBg0ahIuLC2+//TY1NTXodDqampqMquO70tLSwsmTJ3nnnXcwNzfntddeM8r3qtVq4uLiqK+vB7pWCx4/fpyYmBi0Wi2Ojo488cQTjB8/nqCgIFQqFZ988gkNDQ1YWFgYLS1la2uLn5/fNw7ZJUmitbUVlUpFbm4uN2/e5MSJE8TGxmJhYcGwYcNYvHixUbR0s2vXLuLj45kyZQpPP/00wcHBVFZWkpKSQlxcHEuXLiUkJKRHMYzqipmZmahUKhwcHPDx8TFZju5empqaqK6uvm8CZvDgwaxbt47p06dTU1PD/v37qaurIzg4mHHjxplUk16vJz4+nqSkJJqamhgyZIgs5Wu+vr5s376d8+fP89RTTzFu3Di8vLywtrbGzMwMMzMzQ+rB1tbWJJsAnTt3juLiYsNk0LflwQoLC9m3bx/QVa4WGRlpdE0DBgx46M+trKxwcHAw+m54NjY2VFdXo9frH/kznZ2dVFdXc/LkSTZt2oSZmRk//elPGThwoFE0SZKERqOho6PD8J5arTbonTJlChs2bCAyMhJJkoiOjsbS0tJwQ+xpdUf3goP6+noOHDiAo6MjgwcPxtraGgsLCywtLWlvb6ekpITk5GSOHz/OqVOn0Gg0qNVq3N3dGTlyJKtWrWLVqlU90nInOp2OlJQU6uvrmTZtGkOHDsXa2prx48czZswY8vPziY+P7zum297eTkVFBWq1muDgYMLDwx8pX2UM3Nzc8Pf3x9bW1mC8NjY2TJ8+nREjRnDkyBGio6PJzc3Fz8+PtWvXGv0OeS/dZSZZWVnY2Njg5eVl0omrbp5//nmKioo4efIkf/3rX5kwYQLjx4/H29sbR0dHJEkiISGB48eP4+vra5It+zIyMtBqtYSEhODp6fnQEY9GoyE/P5/09HSsra0JCwtj0aJFRtf0bZSXl3Px4kVyc3OBLmMwRtlUZGQkhw8fJjMz07Ck9GG0tbWRn5/PsWPH+OSTTwgLC+PVV181Wi4Xus7NOw33TpycnAgMDMTa2pr09HSKior4+uuvaWhowMPDgzlz5vR4tObm5kZ9fT16vZ6jR49SV1fHlClTcHNzw93dHW9vb5qamjh06BC7d++msbERT09PXFxccHFxYfz48fzwhz80ylD/TiorK2ltbcXDwwNPT0/D3xkYGMjw4cOJi4vjypUrhtK/x8VopltWVkZKSgqtra2sXLnS6EsVH0b35iB3ngwuLi7cunWL999/n9jYWMzNzQkJCWHJkiU8++yzJh/mS5KEVqtFp9MxcOBA2Tb8cXJy4g9/+APz5s3jwIEDXL58mZiYGNzc3PD29qa1tZXm5mamT5/OCy+80OO79r20traSk5MDwPTp0/H29v7GmW6tVkt6ejrR0dEIIRg4cCDPPPOMyao7JEmivr6e9vZ2nJycsLOzQ6/XU19fT2xsLDt27KChoQFLS0s8PT2NMgr4wx/+QEZGBn/6059wcnJixIgRODg43GXonZ2dtLW10djYSGpqKlu2bCE1NZV58+bx1ltvGf1m3b2pv7m5+X3mW11dzccff8zmzZuBrlyqVqulX79+PPfcc7z55ps9vnZWrlzJ0aNHKS4upqqqij179rBnzx6ga0QSFhaGhYUFhYWFdHR0MHfuXJYsWYK5uTne3t4mGQlBV65ZrVYzduzYu+aj3NzcCAkJwcLCgpycHMrLy3s0X2UU09Xr9aSlpVFcXIyDgwPBwcGy76I1atQoBgwYQH19vaEmd+/evdjY2ODu7s7o0aNZv3498+fPN2nJVjf29vZ4e3vj5OTEsGHDjLp71rdhYWHB5MmTmTx5MhkZGSQlJdHY2IilpSW2trYMHz6cCRMmmGSry7S0NMrLy/Hw8GDt2rXfaFxqtZqUlBQ++ugjDhw4QFBQEOvXr2f58uVG19RNZWUlO3bsoLS0lDlz5hAeHk5lZSXHjx9n27Zt5OTkYGNjw6BBg1i7di2zZs3qccwhQ4awYsUKtm7dynvvvcfs2bOZOHEi/fr1QwhBZ2cntbW15ObmkpCQwOXLl7G1tWXjxo2sWLHCCH/1/bi4uDBp0iTa2tpobm4GukYcbW1tBpPV6/XY2Nhgb2+Pn58fs2fP5ve//z1OTk49jv+73/2OtWvXsmPHDvbt20dDQwNNTU1oNBpqa2u5dOmSIWf70ksvsXDhQqOlVh6Gl5cXbm5uhr1H2tvbsbKywsrKiiFDhjB+/HiKi4u5fv1675tudXU1X375JSkpKQwePNioXf5H5dlnn0Wv1/Pxxx9TUFBgeD8iIoIlS5YwceJEw45FcpCTk0NKSgptbW2MGTPGqEXt34Xw8HBZY7e2thou2O7Vb/ei0Wg4deoU7733HomJiQQHB7Nu3TpWrFhh0nKxzZs3s2nTJsOqNz8/P0M1R1NTE05OTowfP56f/OQnRn08zRtvvMHQoUN55513eP311wkICDCsEmxqaqKoqIiGhgZGjhzJyy+/zLRp00x6ro4ePZr33nuPQ4cOGba9vHr1KmlpaYbSSkdHR0aOHMnUqVMZPnw4TzzxhFFv0gMGDODXv/4169atQ6fTcfjwYUOJIXSV082dO5cZM2bI0kkC8PHxITIyki+++IKdO3fi6OhoyOs6OzszYMAArl69SnZ2NnPmzHnsOEYxXSEElpaWWFpaGjYblhszMzOef/55nn/+edljPwi1Wm3YOMYYS2r/Veg22u4nd3RvVHJnD+rMmTP8/e9/JyMjg3HjxvHyyy+zcuVKk7dRbW0tWq2WhoYGzp8/b3jfwsICDw8PJk2axIYNG4iKijJ67GXLljF9+nTOnTtHQkKCoTLC2dmZWbNmMW/ePNmWy0PXSGzlypWyxXsQ3ekCgA0bNvSqlm5eeOEFCgoKOHToEImJiURFRTF+/Hiqq6tJSEjAxsamx3t0GMV0vby8eOWVV2hubsbR0VG2O1NfJjc3l9raWmbNmtUnNt2Ri4CAAHx9fVGpVHz55ZfY29vj6elJaWkpJ06cICcnh/r6eurr6xk+fDivvPKKbBf/L37xC6qqqjh8+DBtbW2GzsKQIUNYs2YNS5YsMWkdtYeHB8uWLWPZsmUmi6HQM3x8fHjvvffw8vJi69atfP7553zyySdA143qqaee6vETTsS3lMj0lWfF/8vp2LJlCzt27GDlypU93Y/2e9EeRqRHOrp3PaurqzPMhE+ePPlxei99uT2g72j5l9Sh0WhITEwkJiaG69evo9PpCA0NZfHixd9lgcoDj41iut+MouNuFB1305d1QN/Roui4h3+PRKOCgoJCH0ExXQUFBQUZ+bb0goKCgoKCEVF6ugoKCgoyopiugoKCgowopqugoKAgI4rpKigoKMiIYroKCgoKMqKYroKCgoKMKKaroKCgICOK6SooKCjIiGK6CgoKCjKimK6CgoKCjCimq6CgoCAjiukqKCgoyIhiugoKCgoyopiugoKCgowopqugoKAgI0Y1XSHEViFEuRCiSQiRJ4RYb8zv/w46Wu55dQgh/tELOoYIIc4KIRqFEPlCiGfl1nCPnhAhhEYIsbWX4veV49IndNzW4iaEOCCEaBVCFAkhZH9ErxBigxAiWQihFUJsljt+H9Rh0vPDKE8DvoM/AT+SJEkrhAgDzgkh0iRJSjFynIciSZJD9/8LIRyACmCPnBqEEBbAIeCfwGxgGnBECBEhSVKenFru4EMgqZdi94nj0pd03OZDoB3wBEYBx4QQGZIkZcuo4RbwR2AuYCtj3D6pw9Tnh1F7upIkZUTpKmwAACAASURBVEuSpO3+5+3XIGPGeAyWAFXARZnjhgE+wPuSJHVIknQWiAPWyKwDACHEcqABONMb8R9Abx2XPqNDCGF/O/7vJElqkSTpEnAYmc8RSZL2S5J0EKiVM25f1XEPRj8/jJ7TFUJsEkKogetAORBt7BjfkXXA11LfeC6RAIbLHlQIJ/h/7Z13fFTnlfe/V3XURiNUEU0SCARIIAkQWDSBKaaYYhsDDphgYycub5xkU17HWWezeTdxnN3YiW1sY8AF8NIxCBAdBEgCCVBBQkK9ozpqozbtef8QmqUakGZGZHO/n898jKfc89Mt557nPOc5l38Hfm5t29/D43Jc+lLHcEB/x8gnHRjdB1pk7o3Zzw+zO10hxOuAGzAV2At0fv8vLIckSUPoGtZ/3Qfmr9N1h/ylJEn2kiTNuanFuQ+0/AHYJIQo7wPbd9HHx+Vx0uEKNN/xXhNd149MH2Op88Mi1Qs3h9PngYHAa5aw8ZCsBs4LIYqsbVgIoQOWAAvoygn9C7ATsKrjkyQpHJgFfGBNuw+gz47LY6ZDAyjveE8JtPSBFpm7scj5Ye6JtHttvy9zui8C7/WVcSFEBl13SgAkSUrE+lFVDBAAlEqSBF3Rla0kSaOEEJFW1tJNnx6XW+hrHbmAnSRJwUKIvJvvjQWsOYkmc38scn6YLdKVJMlHkqQVkiS5SpJkK0nSXGAlfTRxI0lSNDCAvpuVRpKkMZIkKSRJcpYk6RdAf+ArK8vYQNeNL/zm6zPgEF0zxFbncTguj4sOIUQrXSm4f5ckyUWSpMnAYmCLNXVIkmQnSZICsKXrhqy4WX1jVR4XHTe1WOz8MGd6QdCVSigHGoD/BH4qhDhgRhuPwhpgrxCiL4dqq+maTKwBngRm31LdYRWEEG1CiKruF11D2g4hRK01ddzC43BcHicdr9NVHlUD/DfwmpXLxQB+C7QD/xdYdfPfv7WyhsdJB1jw/JD6fvJYRkZG5p8HeRmwjIyMjBWRna6MjIyMFZGdroyMjIwVkZ2ujIyMjBWRna6MjIyMFXlQDVxflDZI93hP1nE7so7bkXXczeOiRdZxB3KkKyMjI2NFZKcrIyMjY0VkpysjIyNjRWSn+09MRkYGv/vd78jOzu5rKTKPITqdjoMHD/LMM89QVGT9RmxHjx5l5syZREREsHHjRqvbtxQWaSZhNBppbm4mLS2NpKQkWltbAfDx8eGFF17Ay8vLEmZlHpHq6mr279+Pn58fI0eO7Gs5Mo8ZSUlJfPDBBwwcOJBBgwZZ1XZjYyOJiYmcP38egI0bN+Ln58fChQutqsMSPJLTLSkpwdXVlYqKCjQaDWq1mvj4eNzd3QkPD6e8vJxr167R2NhIUVERubm5dHZ20t3foX///jg7O7NunfWeV2k0GsnJyeHkyZN4eXmxcuVKq9kG6Ozs5MKFCxw9epTU1FQ0Gg0xMTE8+eSTxMTEWNS2wWCgpaUFhUKBQqG463Oj0Uhrayt1dXUW1fGwGI1G6urqOH/+PH5+fkRHR1vNdkFBAZs2bSIxMZEXXniBNWvW4OjoaBXbRqORjo4OtFqt6T1JknB0dLzncbMGFRUVHD9+HLVazbp167Czs16zLyEESUlJnDlzBp1OhyRJNDQ0UFNTYzUNlZWVrF+/nvLyctasWcOMGTPMtu1H2pOrVq1CkiRqa2tpb29Hr9fT2tqKjY0NTk5O6HQ6Ojs7kSQJhUKBVqulufl/GuO7uLjQ1tZmNvHdFBYW0tDQgLOzM1qtltbWVvLy8igoKCAnJ4fU1FRaW1tZsGABzz//PLa2tmbXcCft7e2cO3eOrVu3cv78edRqNR0dHRiNRrKyskhJScHDw4OxY8daTENCQgJxcXHMmTOH6dOnY2NzdzbJzc2NsLAwi2l4WLRaLVlZWWzYsIFDhw4xYMAAfve73/HUU09ZzKYQAp1OR319PXFxcXz11VfU19djNBpZsWKFVZxuY2Mjhw8fZuvWreTl5ZneVygUBAcHM336dJ577jn8/f252Q/ZKmRmZnLo0CEUCgXDhg2zml2A2tpaTpw4QXJysuk9IQRGo9Fq9jdu3Mi3337L0KFDMRgMd32nO2CBrmvoUXgkp5uVlUVbWxs6nY47u5Pp9Xrs7OwIDw9n+fLljBw5kgMHDrBhwwba2tqQJAl3d3dCQkIeSeDD8Mknn3Dy5EkUCoVJX1tbGxqNhvb2doxGI1FRUSxevNgqDreuro5PP/2UPXv2UFBQgJOTEzNmzGDMmDFkZmZy+PBhUlJS+Oqrr/jgA8s90KGxsZH4+Hj8/f2ZMGECrq6ut32Wl5eHq6sr06dP/56t9I79+/dTVVXF0KFDiYiIwNPTk/r6etLS0mhqaiIiIgKlUsnevXv561//Sk1NDc3NzdTW1vL222/3yul2R493nqtNTU1cunSJ9PR0MjMzyc3Npb6+nurqapydnRkwYAAODg69/dMfCltbW7y9vZk2bRojR46kuLiY5uZmnJycuHLlChcuXGDfvn2m4fWtx9BStLS0kJOTQ2FhIZMmTcLf39/iNm8lPj6eM2fO0NHRAYBKpWLq1KnMnWudFtCff/45X375JeXl5dja2nLq1Cm0Wi1+fn6MGTOGmpoatm/fzrFjx/Dw8OCDDz7Az8/vobf/SE7X1tYWBwcHdDodnp6exMTEEBwcjJOTE2PHjsXX1xc3Nzfc3NzIysqisLCQzs5ObG1tGTZsGL/4xS+YNGnSI++EB/Hss8/i4eHBkCFDUCgU6HQ6AHJycjhw4AANDQ2maM/S1NTUsH79erZt20Z5eTkLFy5kxYoVjB49Gg8PD3Jzc3F0dGTPnj0Wn8BKT0+nubkZFxeXu5xIeXk5iYmJ2NjY4OHhYTabOp2O69ev880331BeXs7Vq1fRaDT069cPf39/nJ2daWtro7KyEq1WS//+/XF0dCQ/P5/r169ja2uLJEl4eHiwfPnyHus4f/48W7ZsIScn57ZhO3QFCE1NTTQ3N6PRaEwRC4C/vz/PPfecRaLc7vTJlStXKC0tZdSoUYSEhDBu3DjGjBmDXq83jSBtbGzIz89n06ZNnD17lhdffJFly5bxs5/9zOy67qS6upqCggJ8fX2ZMWMGPj4+Frd5p321Wg2AUqlk7ty5/OQnP2HAgAEWtWswGPj0009N165er6esrIwvv/ySvXv3olAocHd3p7Ozk/Lyctrb21m2bBkqleqR7DyS033vvfewtbWlubkZX19fRo0ahSRJFBcXY29vT3V1NfHx8eTk5JCTk8P169exsbEhMjKSX//610ybNg2l8s5HQvWecePGERQUhIuLC7a2tqbIxtfXl0uXLtHW1oaXl9cjDwN6wtatW9mxYwf19fWsXLmSdevWER4ejpOTE5IkYWNjw9ixY9m1a9ddzsCcZGdnc/78eZycnOjfvz/29va3fV5ZWUlqaqrZJ0j27NnD5s2buXr1Kq2traaRRmVlJdnZ2dja2mIwGNDr9QDk5+dja2t7276ws7MjNDSUtWvX9ljHhQsXiI2Npaqq6rZI18bGBqVSaXKqjo6OGI1G2tvbUSqVjBs3jilTpph9KJ+YmEhiYiLnzp2juLgYjUaDt7c3q1atYtmyZfj6+t71myFDhjBw4EDi4+P54x//yIYNG5gwYQJTpkwxq7ZbaWlp4dixY8THxzN58mSef/75u84dSyOEMB0zJycnhg0bxqhRo+6ZHjMXBoOBM2fOsGPHDgoLC5EkCTc3N4KDg/H19aW2tpaqqiqys7PRarXY2dkxcuRIli1b9sh590dyus899xw2Njbo9XqKi4s5fvw4GRkZlJaW0t7eTmdnJ2q1moaGBtMd297eHkdHR7y8vMwaUd2Ko6PjPcN7lUqFi4uLRWzei8zMTGJjYykvL2fp0qW88sorREREmA5KW1sbKSkpxMXFWVxLQkIChYWFhIWF4evre5cT6ejoQAhh1nzd+fPn+fDDD0lLSzM5USEEjo6OhISE0K9fv7t+o9PpqKyspKioCCEEtra2jB49ml/+8pf3dEQPi8FgwMXFBV9fX0JDQwkMDMTR0RFnZ2dCQkLw9vYGIDU1lR07dpCVlYWLi8ttn5kDIQQnTpzgm2++ITk5Ga1Wy4gRIwgKCsLT05ORI0feNxhwdHQkLCwMT09P8vLy2LBhA+vXr7eo09VoNBQUFODi4sKSJUsYMmSIxWw9DCNGjGDmzJkWd/wpKSls2LCBzMxMtFot4eHhPP/880yePNk0F5WVlcU333xDUlISKpWK+fPnExUV9ci2Hsnpuru7m/79zjvvcPToUaqrq2lvb78tmuj+tyRJ6PV6srOz+etf/0pxcTELFiywmPO9E1tbW6vNunZ0dLBx40YyMjLw8/Pj6aefJjw83ORwKyoqOHToEN9++y0XL168743CXJw9exYvLy8WL15MQEDAPb/j7OzMwIEDzWZz06ZNpKeno9PpcHBwwMXFhaeffpohQ4YwduxY0zD11vMjKyuLjRs3UlRUhJ2dHYGBgfziF7/odSpowYIFhISEoNfr6d+/P56entjZ2WFvb4+XlxfOzs7o9XoaGxtNNwiFQoGXl5dZI6rk5GT+8pe/UFRUREhICIsXL2b06NF4e3vj5OSESqXC2dn5vr+XJAlfX19WrVrFF198wenTp82m7U6EECQkJHD27FkGDx7M6NGjLRpd3ovKykoKCwtpamrCwcGBQYMGMXz4cIvabGtr4/PPP+fEiRN0dnayYsUKnn32WaZMmWK6RltbWykrK6O2thYnJyfGjx/PD37wgx6NnnvskZKSkiguLkYIgSRJDB48mFGjRjFgwADT0K2uro6srCzKy8s5cuQIZWVlFBQUMH/+fCZMmNBT04+EtR5HVFJSwpkzZ2hoaGD69OkEBQWZyrFu3LjBkSNH2L17N1lZWRiNRoKCgiw2M19TU0NeXh4jR44kKirqrpucwWCgs7MTZ2dnBg8ebDa7tbVdj12zsbFh8ODBvPzyyyxYsMA0yrlXnrS1tZWmpiYkScLOzo7+/fuzcOHCXk9khYaGEhoa+r3fqaio4MqVK1RWVgLg7e1NZKR5H5B8+fJlUlJSiImJ4dVXX2Xq1KmPPBlma2trGiU0NTWZVd+tqNVqkpKSqKurY9GiRRYNCu5HVVUVJSUltLa24u/vT1hYmMWDtNraWi5duoRarWbq1KmsWbOG6dOn4+TkBHT5kGvXrrFnzx5KS0sJDAzk6aefZtSoUT2y12Onu2rVKioqKgAoKipi4sSJREdH4+fnZ7pg1Go1165d49q1a5w/f5709HQ+//xz8vLy+MUvfkF4eHhPzd+G0WgkPT2d6upqHBwc8PPzw2g0cunSJSoqKqivr+fChQumC/FRE98PQ1NTkyniv3HjBtu3bzfVydbV1ZGTk2O6SQUFBbFu3TrmzJljdh0AFy9epLGxkREjRuDp6YkkSRiNRtMEUk5ODvHx8aa8lLl48cUXCQ0NRafTMWTIkO9dCCOEoLS0lMTERGpqalCpVEyZMoX58+dbLSV09epVLly4QGtrK66uroSEhPT4QrofpaWlGAwGRo4cyfjx4x/J4RqNRpqamqiqqiI+Ph6DwWCxhUVCCOLj40lISCAkJIRZs2ZZNTXXTWVlJTU1Neh0Ojw8PBg0aND3jgTMgbOzM8OGDcPT05O1a9cyYcIEk8PV6/WUlJQQGxvL2bNnGTBgAC+99BLz5s3rcSVUj53uK6+8gsFgQAhBXl4eAQEBd+XgAgICGDt2LHV1dYSFhfHRRx9x4cIFDh48iJeXF++9955Zir9PnTrF1q1bTUPUgQMHmnRdv36dhoYG4uLiqK+vZ8aMGcybN8/sK7AGDhzI5MmTqampITU1lezsbGxsbHBzcyMgIIDBgwfT0dFBVVUV48aNY926dWbNHd5KQkICjY2NlJSUcPToURwdHWlpaUGtVlNbW0tOTg5paWkEBwebdSJtyZIlzJkzByEEdnZ233vRVlRUsGfPHvbs2UNbW5spj2uuG/GD6J6XKC0tBWDQoEHMmDHD7I4mNzfXVE3zqLS0tHD06FGOHTtGdnY2o0aNYvHixWbV143RaCQtLY28vDxWrVrF6NGjLWLn+2hvbyctLY2SkhKgazFVYGCgxe16e3vz8ssvY2dnR1RU1G1zD/X19Rw9epQDBw7Q2dnJ7NmzWblyZa+umx473VurECZOnHjf79na2uLr68vcuXNpamqiurqa4uJiDh8+zOrVqxk3blxPJZgoLCykvr4ejUZjmikXQtDY2EhTUxNubm64urqSmZlJc3MzYWFhZne6/v7+/PjHP0apVFJZWYmdnR1+fn4MGjSIwMBArl27Rl5eHt7e3owbN85iDhe6clRarZbY2FhOnToFdEXiLS0tphxdR0cHY8aMMauTcXBweOi0wLVr19i/f7+pbM7Dw8OiE0R30tDQQEVFBS0tXU/YHjRoEJMmTTJ7HbdarX7kon6DwUB1dTXHjh3jyy+/JDExkWHDhvHWW2+xYsUKs+qDrii3qqqKmpoa+vXrR2BgoFXqge/k+vXrXLhwgaqqKiRJYtiwYVa7Cd9reXF7eztJSUns2bOHoqIixo4dy5IlS/D09OyVrR45XY1GQ2FhIUqlEn9//4e60Pr168eCBQtISkqiqKjIVF5mDqe7ZMkSQkNDqa6uRqvV0tLSYrpzHz58GEdHRxYuXMjAgQNRKBQWWwUWFRVFUFCQqTZZpVKhUCgoLi7mxIkTlJWVMXXqVGbOnGkR+93MmjWL/Px8srKy6OzsxM3NDX9/f3x8fBg0aBAlJSVkZGQwbNgwq5cDGQwGGhsbyc7Opri4GDs7Ozw8PKwS0dxKfn4+V69epbm5GTs7O5RKpUVyhxEREVy+fNkUVbu7u3/v9dLS0kJWVhbHjh1j27ZtNDQ0MGnSJJ5//vke1YQ+DI2NjcTGxpKWlkZkZCSTJk2y+gQadJU5lpeXm1aAOTo6mob51qazs5PMzEz27t3LxYsXCQoKYtWqVcybN6/X6Y4eOd3k5GQ2b95M//79mTZtGhMmTMDb2/uBUYKLi4spZ6bVarly5UpPzN+Fj4/PPQu4z549S3FxMRUVFYwfP94qfRfuzLkZDAZSU1NJSUnBzs6OIUOGMGLECItqmDdvHkqlkosXL2JnZ8eAAQNwdnYmMDAQpVLJpk2baGlpYcGCBVZbedVNSUkJJ0+e5PDhw9TW1tKvXz/mzZvH6tWrrabBYDCQnp5Obm4u0LWMc8CAARbJYS5btoxTp05x+vRpVCoVy5cvN80rdF8vBoOByspKSkpKKCgoYP/+/aSmpuLl5cWTTz7JokWLmDt3rsWWAZeVlbF//34aGhr48Y9/bLVJ7lsRQtDR0WGq3+5LdDodOTk57Ny5k3PnzuHk5MS8efNYuXKlWWr9e+R0jx8/zqFDh2hubmbHjh28+uqrLFmyhGHDhn1vjrZ7PTl03Um6h77/m9FoNCQnJ5ORkYGXlxfDhw+3yAKRW7G3tycmJuaeDXWSkpJIS0vD29vbon0f7kSn01FWVsb27dv55ptvyMvLM02e/fCHP2TatGlW09LU1ERBQYGp2sLDw4OhQ4daZEg9adIkXnrpJbZu3cr27dvJy8tj0aJFhIaGmiKmtrY2jh8/TlxcHHq9HoVCwVNPPcWiRYuYPHmyxc+X5uZmJElizJgxjBgxwipL5e+kvb2dgoKC21ai3Vqiak2uX7/O9u3b2bNnDxqNhtmzZ7NgwQKzHYceOd2goCCUSiXNzc2Ul5fzu9/9jpycHFavXs2oUaNwd3fH2dn5thpZg8FAWVkZsbGxQJdjsHT9XTfdTXD6grKyMoqLi9Hr9YSHh1u0gcvDUFpaSl5eHqGhoVa9uNRqNR9//DE7d+6krq4OBwcHoqOjefPNN81epvUg6urqKCkpobGx0eK2bGxs+MlPfsLo0aP59ttvOXPmDO+++y4ODg63DeElSSI4OJjZs2cTFRVFdHS0xfOqRqMRjUZDRkYG7u7uPPvss4wfP96iNu+HRqMhLS2NqqoqAIKDgwkICKC9vd2qKYaWlhZ27NjBli1baGxsZPny5abjZy565HSXLVvG+fPn2b9/P83NzQgh2L17NwkJCUydOpXp06cTFhbGgAEDsLe3p729ndraWo4cOcKJEyeArtVib7zxhtn+kO/D0dHRKkuA76StrY2TJ0+SnJyMi4sLYWFhREREWF1HN93LXRUKBUOHDrVaJGE0GqmoqKC4uBi1Wo1Wq8XHx4eIiAgiIiKs2jawL5AkiVmzZhEZGcm2bdvYtWsX1dXVps9dXFyYNGkSK1euJDo62mo3Q7VazenTp9m1axdTp05l4sSJVs/xdyNJ0m3pk+7a2VGjRln1pnz69GkOHTpEdXU1kydPZtmyZWYfEfbobFepVLzyyiuUlZWRnZ1NS0sLWq2WkpISSkpK2LFjB56enoSHh9O/f38yMjLIz89Ho9FgMBhwc3Nj8uTJFu8aZGNjg42NDe3t7VaJam5FCMGlS5c4ePAghYWFeHl59Vlv1G60Wi3l5eUYjUZCQ0Ot4uyEEDQ1NfHll18SHx9vWn48aNAgFAoFx44ds+osdV/Sr18/3njjDdauXXtbu8DuVqjWzq/Hx8fz+9//HicnJ6ZNm2b1RuW3cmu/BegakRUWFlptcRN0pZ1OnTpFeXm5qfe2JSa9e3zVTZkyhb1795KUlMSuXbtITU2lqqoKjUaDVqulqqqKI0eOmL5va2uLvb09Hh4exMTE8Mknn1g8V6VSqfDw8EChUFg90m1qauK7777j0qVL2NjYEBERYdWSqHvR3XDH0dHRaoXvzc3N5ObmUlFRgV6vx9bWFldXV/R6PW1tbQQGBt6zJ8P/VmxsbPqkHOtOhBBcvHiRuro6hg0b1uejDRsbGxwcHEwNkezt7QkNDTVLddPDYDAY+PLLL9m/fz/19fVERUUxZMgQi9wIe7WnVSoV8+bNY+7cudTU1JCenk58fDwpKSnk5uaaJioUCgWDBg0iKiqKCRMmsHTpUovWqXbj6emJt7c3AQEBVjt43VRVVZGbm2taGWapu+aj0N3GcciQIb2uNXxYEhMT+c1vfkNmZibQlctfunQpv/rVryzSW/lhsLGxMbWQtGYk9TjR2tqKXq9HpVKxaNGiPjsW3bi6ujJx4kTTDTo0NNSqaYXs7GwOHTpEVVUVgwcP5plnnrFcc//usP4+rx7R1tYmEhMTxZ///Gfx5z//WZw6dUrU19c/7M/NpiM/P1+89dZb4q233hJ6vf5Rf94rHVu2bBFhYWECEGvXrhW5ubmPat8sOsxIj3QcOXJETJgwQTg5OQkvLy/xxhtviKysLKvruBWj0Sg++ugjERAQICRJEsHBweKjjz6yug4zYdZr1wJa/iF0zJgxQ9jb2ws7Ozvx3nvvidraWkvpsMyDKZ2cnHjiiSd44oknLLH5h6atrQ2VSoVSqaStrc2qKYaQkBD8/f1RKBTMnTvXrI1l/pEIDQ3l3XffpbGxEXd3d8aOHdvn+0KSJFasWEFVVRU7d+7kiSee6JPaVJnHhzfeeAO1Wk1kZCQLFiyw6EhQEt8/vOqLsde9KsBlHbcj67gdWcfdPC5aZB13YP21fjIyMjL/xDwo0pWRkZGRMSNypCsjIyNjRWSnKyMjI2NFZKcrIyMjY0VkpysjIyNjRWSnKyMjI2NFZKcrIyMjY0VkpysjIyNjRWSnKyMjI2NFZKcrIyMjY0VkpysjIyNjRWSnKyMjI2NFZKcrIyMjY0VkpysjIyNjRWSnKyMjI2NFzO50JUlaIUlStiRJrZIkFUiSNNXcNh5Cw1ZJkm5IktQsSVKuJEnrrK3hpo6RkiSdkiSpSZKkfEmSlvaFjptaAiRJOixJUoMkSVWSJH0sSZLVn0YoSVI/SZL23Tw/SiRJesHaGm7qOCNJUockSZqbr+t9oEFzx8sgSdJH1tZxh6bgm/tl6z+7Dkv5MrM6XUmSZgN/BtYCbsA0oNCcNh6SPwEBQgglsAj4f5IkWfXJlDcd2n7gINAPeBXYKknScGvquIX1QA3QHwgHpgOv94GOTwAt4Av8APhUkqTRfaAD4E0hhOvN1whrG7/FtivgB7QDu6yt4w4+AVL6WAP0sQ5L+jJzR7q/B/5dCHFBCGEUQlQIISrMbOOBCCGyhBCd3f978zXUyjJCAH/gAyGEQQhxCkgAVltZRzeBwE4hRIcQogo4AljV2UmS5AI8C/yrEEIjhDgPHKDv9snjxLN03RTP9ZUASZJWAI3Ayb7S8BjpsJgvM5vTlSTJFhgPeN8cSpffHMI6mcvGI+pZL0lSG5AD3AAO94WOO5CA0D6y/SGwQpIkZ0mSBgDz6HK81mQ4oBdC5N7yXjpWdv638CdJkuokSUqQJCmmjzR0swb4RvTRo1wkSVIC/w78vC/sP046LO3LzBnp+gL2wHPAVLqGsBHAb81o46ERQrxO17BgKrAX6Pz+X5id63RFLr+UJMlekqQ5dA3pna2so5uzdDm3ZqAcuAR8Z2UNrjft30oTXcfJ2vwaCAIGABuAWEmSrD0aAkCSpCF0nRtf94X9m/wB2CSEKO9DDY+LDov6MnM63fab//1ICHFDCFEH/BWYb0Ybj8TNYf15YCDwmpVt64AlwAKgCvgXYCddDs+qSJJkQ1dUuxdwAbwAD7pyVtZEAyjveE8JtFhZB0KIi0KIFiFEpxDia7pSP311rq4GzgshivrCuCRJ4cAs4IO+sP+46cDCvsxss9dCiAZJksq5/VHHj8tTL+2wfk4XIUQGXREMAJIkJdI30Uw/YDDw8c1cd6ckSV8C/w/4lRV15AJ2kiQFCyHybr43Fsiyoob7Ibj/48wtzYvAe31kGyAGCABKJUmCrhGJrSRJo4QQkf9sOizuy4QQZnvRlYtJAXzoiqTOAX8wp42H0OADrODmAQPmAq3AImvquKllDKCgK6XwC6AIcLS2fOocuAAAHoVJREFUjptaCoH/S9cNSAXsA77tAx3bgf+mK+KeTFd6YbSVNahunheKm/vjBzfPkeF9sD+ib9p264vz4qYGZ7qqJ7pf/wnsBrz/GXXc1GIxX2buOs0/0DV0zQU66BpO/4eZbTwIQVcq4TO60iclwE+FEAesrAO6ho3r6MoPnQNmi/+pqrA2z9A1mfZrwACcAn7WBzpeBzbTle+uB14TQlg70rWnK8oPoWtf5ABLxO0TfNZiDbBXCGH1FEs3Qog2oK37/yVJ0gAdQojaf0YdN7GYL5NuenUZGRkZGSsgLwOWkZGRsSKy05WRkZGxIrLTlZGRkbEistOVkZGRsSKy05WRkZGxIg8qGeuL0oZ7FajLOm5H1nE7so67eVy0yDruQI50ZWRkZKyI2Z1uQUEBP/3pT/H09CQwMJC//OUv5jYhIyMjY3Vyc3P5y1/+wt///vdebcfsTre4uJisrCwaGhpwcHBAqbyzv4mMjIzMPxbJycn86le/4v333yctLY2Wlp4vIDTrMuDr16/z1Vdfcf78eRwdHZk4cSJz5841p4mHQq1Wk5KSwtGjRzl16hTFxcXcbKBBQEAAP/rRj3B1deXJJ5+kf//+VtfXl5SWlhIXF8fp06dxdHRkyZIlLFy4EHt7e4vb1mq1xMfH88knn3D27FkkSWL48OGMGzcOSZKYNWsWTz/9NDY2ctZL5vGhsbGRpKQkEhIScHJyYvjw4bi59aIb6QOaMzwS27ZtE+Hh4cLJyUk888wzIjExURgMhkfdTK90vP322yI8PFwMHDhQuLi4CHt7eyFJkullb28vlEql8PHxEc8884yorKy0iA4zYhYdnZ2d4ttvvxWTJ08WKpVKODg4CEdHRxEeHi6+++47i+toaWkR77zzjhgyZIhQKBSm4+Hg4CBcXFyEi4uLGD58uFi/fr3Q6XQW02FGHmcdPdbS2Ngo/va3v4nRo0eLF198UeTl5fVWi0WoqKgQoaGh4mc/+5lFdej1enHq1Cnx1FNPCXt7ezFr1qxH2Sf3bqZzvw8eVWhCQoJ4+umnhZOTk5g2bZo4cOCA0Gq1j7KJ7xP60KxYsULY29sLOzs7YWNjI1QqlYiJiRHPPfecGDx4sHBzcxOSJAkbGxvh7u4uli5dahEd3TQ1NYmUlBTx2WefiTfffFO88847IjU19VE20WsdnZ2d4sMPPxTDhg0T7u7u4vnnnxexsbHi3XffFUqlUixbtkw0NjZaTEdpaal45ZVXhLe3t+m4dDtdGxsb08ve3l6EhYWJS5cuWUSHmXmcdfRYy+HDh8W0adOEJEli5MiRYvPmzb3V8kgYDAZRWVkpampq7vud1tZWsXPnTqFSqcR7771nER3dxMfHi8WLFwuFQiEiIiLE3r17hV6vf9ifW67LWFtbG7GxsaSkpKBSqZg/fz5Tp061ypD1TsLCwkhISKCiooLg4GBeffVVli9fjq2tLTdu3CA1NZWUlBS2b99Oc3MzV65csZiW2tpatm/fzrfffktBQQGdnZ1ERUWxcOFCi9m8FxcuXCApKYmhQ4eyevVqYmJiiIqKQqVScfToUS5fvszJkyd55plnzG67s7OT/fv3c+7cORoaGjAajTg7O+Pp6YmNjQ319fVoNBoADAYDFRUV7Nmzh3HjLP8cUaPRiNFoRKfT0d7e/r3ftbe3792Q8iZtbW38+c9/prGxkQkTJuDj40P//v0ZNGgQTk5O2NraUl1dzfXr13FycuKJJ57otc2HpbCwkMOHD5OamooQgsrKSi5fvszatWutpiErK4stW7YQFhbG6tV3PzrPaDRSWFjIH//4R0aOHMlzzz1nER16vZ6EhAQ2bdpEUlIStra2hISEMGbMGGxtbXu1bbM43ZSUFBITE1Gr1SxZsoQnn3wSd3d3c2z6kXnllVdYsmQJWq0WhUKBr68vHh4eAHh7e+Pm5sb169dpa+vqIKfVas2uoaamhjNnznDgwAESEhJoamrC3d2dyMhIli1bxqhRo8xu8350O34/Pz/+5V/+hcDAQFxdXVEoFLi5uaFSqSgtLaW21jLd83bt2sXXX39NcXExRqMRV1dXli9fzurVq6moqODTTz/l/Pnzpu/rdDrKysrQarU4ODiYVUtJSQkeHh4olUqam5uJj48nISGBvLw88vPzTXn/boQQSJKEJEl4eXmxfPly1q1b1ysNGo2GK1eukJiYyL59+7Czs8PR0RGlUom3tzdOTk6Ul5fT0tLC1KlTrep09+zZw+7du3F1dcXX15eKigrKy8vRaDS4urpaRUNJSQn5+fkEBATc9ZkQgvLycj7++GOqqqr4r//6L4YMGWIRHcXFxWzdupXY2Fg6OjqYNWsWb731FoMHD+71ts3idLOzs7lx4wbe3t7MnDmT0aNH33UCWwtvb2+8vb3vet9oNFJfX8+FCxc4cOAAer0eOzs7s+zEbnQ6HfHx8WzevJlLly6h1WoZM2YMCxYsYOzYsXh4eGAwGDh79ixOTk5MmzbN4qOBzz//nNjYWNasWcPQoUPp16+f6bP29nbUajUuLi6EhISY3XZmZia7d+/m+vXraLValEol06dP57nnnmPSpElcunQJZ+fbHxnX0dHBpUuXyMzMJDLSPA8L0Ol0xMXFcfz4cVatWsWIESOIjY1l9+7ddHR0MHHiRGbPnk1RURGlpaV33YiNRiOZmZns2rWr107Xw8ODiRMnkp6eTktLC35+fqabQH5+Plqtls7OTnx9fQkKCuqVrYdFCEFSUhKnT5+msbGRxYsX4+/vz4YNG6iuriY7O5sJEyZYXEdlZSUnTpwgNzeXWbNm3fW5Wq1m//79HDhwgJEjRzJnzhzs7MzdEhzKy8vZvHkzR48epampiTFjxjBnzhyEEBw6dAilUomfnx8jRozoUdTba8VCCPLy8mhoaGDEiBEEBgbi5NQnDwC+LwaDgbq6OuLi4nj//fcpLCxEqVSycuVK5s8332Ox1Go1GzZs4OjRo7i4uLBmzRqWLl1KcHAwSqWSiooKtm/fzrZt2/D390ev11u8uiMnJ4fKykoyMjLIyckhOjoa6Kqn3rFjB2VlZUydOpXRo837QF61Ws1f//pXEhMTTUP3p59+mpdffpmIiAjs7e0JDAzk1VdfZdSoUZw7d468vDyam5spKSnhT3/6E19++aVZIqyLFy+yfv16cnJymDt3LhEREXh7ezNp0iSGDh3KpEmTUCqVtLS00NraitFovO33RqMRtVptlgvc3t4ePz8/nJ2diY6OZsWKFQwcOJCOjg4qKio4dOgQsbGxTJ8+neXLl/fa3sNQVVXF1q1bSUpKIjo6mh/+8IeUlZUBXemh7vSPJdHr9Vy8eJGLFy+i0WjuCkaam5s5efIk69evRwjBqlWr8PT0tIiWs2fPcvToUaqqqgCorq5m165d7Nmzh/r6euzs7PDy8mL8+PF4eHiwcOFCQkJCHrrqptdnUV5eHpmZmTQ3NxMaGmqxcL+nqNVqkpOTOXLkCKdPnyYnJwcHBwdCQkJYs2YNY8aMMZutCxcukJaWhlar5ec//zlr1qxh4MCB6HQ6Ll++zK5du9i7dy+FhYU0NzdTWlpqNtv3Y+LEiVy8eJGrV6+SmZlJVFQUFRUVbN26lb179+Ln58fq1atNKRhzcfz4ceLj4015XKVSyfjx4wkPDzfVbnt7ezNr1iwiIiJYuHAhycnJfPrpp5SVlXHu3Dni4uJYtmxZr3Q0NjZy8OBB0tLSiIyMpH///jg4OBAVFcWoUaNQqVQmPSqVqtd/98Og1+uRJIlx48YxY8YMUyruypUrdHZ24uPjw4wZM8w6CrsfVVVVfP755xw+fJjRo0fz6quvEh0dzcWLFxk5ciQGg8HiGgwGA3v37mXTpk2UlJSwYMECJk2adNt3SktL2b59OyUlJUydOpWFCxeadTRtMBgoLS3l6tWr7Nu3j6KiIvR6PQD19fU0NjZiMBhM79nb25OZmYmjoyNnzpzhpZdeIiYmBi8vrwfa6rXTTUxMpKioiMGDBzNlyhQGDRrU2032Go1GQ3Z2NqdPnyY1NZXi4mIKCgqoq6tjwIABrFq1ihkzZhAWFmbWqLyystJ00URGRuLr60t5eTmnT58mNjaWa9eu0draaroj3hlRWYLFixeTmJjIwYMHuXz5MoGBgVy4cIFvvvkGg8HAkiVLmDZtWq8nB24lOzubLVu2UFNTY3K4K1euJCYm5rbI1dbWFjc3N9zc3Bg8eDBDhgyhpKSEzz//nLa2NjIzM3vldBsbG/n444/Zv38/TU1NKJVKMjIyqKioAMDHxwd7e3scHBxQKBS9/rsflurqavr378+IESNMk3M6nY5r166RmprKqFGjmD59+gO20ntqamrYsGEDW7ZswcPDgzVr1jBr1iyUSiVubm54e3tz48YNizve/fv389lnn5GcnMzQoUOZNWsWw4cPN32en5/P119/zYULFwgNDeWnP/0pPj4+ZrN/48YN9u3bx7lz5ygoKKCkpOS26F6v1+Pk5ISLiwtqtRqtVotOp6O6utr0+4aGBvR6PStWrHigvV453e4cTE1NDVFRUQQFBd2Vo+vs7KSwsJCLFy9SX1/PsGHDGDduHAMHDuyN6ftSUFBAXFwcp06d4tKlS6ZhkkKhICIigrVr17J48WKL3ByCgoLo168fFRUVbNq0ibi4OBobGykoKMDJyYkVK1bQ1NTEZ599hkKhsEokM3jwYBYvXsz169c5e/YsxcXFFBUVYTQaefnll1m5cqXZJz3Pnz9PWlqaabLSz8+PefPmERIScl/nbmNjg6enJ6GhoUBX2kqn0/VKx549e9iyZQvFxcXo9XouX75MaWkpfn5+prytl5cXgYGBxMTEMGHChLvOX0sQFhbGoEGDCA0NNd2A1Wo1ubm5tLS0MHTo0HtOJJmTmpoaNm7cyNdff41Go2Ht2rXMnz/fNGTvjiIbGxvJyclh1qxZdHR0kJWVRXV1tdnSctXV1ezYsYOUlBRaW1tRKBRoNBoSExMpLS2lqamJvLw8vvvuO3Q6HZMnT2bmzJlmsd1tf/369ezevZuioiLTeWFvb8+AAQNwd3fHycmJiIgIxowZQ2VlJR0dHWg0GpKSkrh+/To6nY60tDTy8/MfymavnG5ycjJpaWno9XoiIyMZMGDAbZ83NjaSnJxMbGwsJ0+epKGhgWHDhjF37lx++9vf9sb0fSkvL+fw4cOcPn0arVZrOnkkScLX15fo6GiLReMRERG88sorHD9+nOrqaqqrq00TJ1OmTGH69OmcO3eOL774Ajc3N0aMGGERHXcyceJEwsLC2LlzJwUFBaZo/6WXXrJIOqi0tJS2tjbT7H90dDTDhw9/YDWCJEnY2dkhhECv11NeXo7RaOzxCrWjR4/S2trKqFGj8PDwICAgAA8PD4YMGUJzczMXLlzg6NGjGI1GioqKTGkHc0b99yImJgYhxG0laHl5eVy5cgUfHx/Gjx9vkQmibmpqati8eTMbN26krq6O+fPnM2/ePNP129nZSVNTE52dnTQ0NHDs2DHs7OyoqqpCrVabzekZjUZ27txJcnKyKbIsLS1l27Zt2NnZUV5eTmtrKxqNhs7OTqZNm8ayZctwdHQ0i32AgwcPsnXrVsrLy2+L6AcMGMBzzz1HZGQkjo6ODBs2jKCgIDQaDXq9ntbWVs6fP8+XX35JQkLCI9ns1ZG9fPkydXV1+Pv7Ex0djb+/v+mziooKDh8+zN69e7ly5QparRZfX1+Sk5MRQljM6Q4aNIgJEyaYLlyj0UhtbS2FhYVkZ2dz8OBBs82K34mPjw8vvvgikZGR1NbWYjQa8fT0ZMiQIfj4+NDe3k5ZWRmOjo6MHDkSPz8/i+i4lZqaGi5cuEBJSQl6vR69Xo+7uzujRo2yiMPV6XRkZmaaIgYfHx/mzp37yDc6g8FAamoqOp2uxxfZrFmzGD9+PIMHD6Zfv34MHToUd3d33Nzc6OzsZOzYscTGxpKYmEh8fDzDhg0jMjLS4k731gqSbnJzc7l69SqTJk1i8uTJFrOdlpbGwYMHTfnT0NBQwsPD0Wg0xMXFmfKXV65cobCwkLa2NpKTkyktLcXb25s5c+YQExNjFi06nY59+/aZ0j3QlWPunsDqxt7entGjR7N69WrTRLA5yMjIYMuWLVRVVd3mcF1cXHjqqaf44Q9/eFd5p4uLC9B1fhqNRvbt2/fIdnvldIuKimhrazNFud0zjhUVFezatYtt27bR2NjIuHHjCA4ORqFQ8OGHH1o0RxQUFGTKTXUnvfPz89m1axeJiYkcO3aM119//aES3j3B1dX1rkmAbrKzs7l48SI+Pj7Mnj3bokNZo9FIVlYWcXFx7N69m+rqasaMGUNjYyOlpaVcunSJpUuXmj2XqdVquXLlCu3t7UiSxNixYwkJCXmov9VoNN42OulewdNTXn311ft+5ujoyKRJk+jfvz/u7u58/PHHZGVlodfrzRpJPQx6vZ6amhqam5sZPHiwxVJv6enpfPDBB8TFxVFXV2fav0lJSaSmplJSUsKNGzfQaDS0tLSg0+mwt7fHy8uLRYsWMWnSJGbMmGG2eRAhBAaDwXSMFQoFrq6u2NjY0NLSYqp68ff355lnnmHOnDlmsdvNF198weXLl3F2dsbJyckUxapUKqZPn37PkajBYKCyspKkpCQyMjK4ePGiqY575MiRD2W3V063rq4OnU6HSqUynajdDvebb77BwcGBN954g6lTp+Lp6cmBAwdMQ8ie0tbWxsWLFzEYDIwbNw6VSnXXLGZQUNBtNY7dw7WcnBxTM5x58+b1WENP0Gq15OTkkJ6eTmBg4H0dszmoq6vj8uXLfPvttxw/fhwXFxeWLFlCTEwMiYmJfP7551y7do2KigqGDh1qMR2AqT75QbS1tZGRkWFaIWhnZ8f48eMtWsfs7e2NUqkkMTERIQRardYqk5t30tDQgFqtRqlU4uPjY7GGPxs3bmTXrl23rb7Lzc2ltrYWDw8PFAqFKQXT1NRETk4OHR0dTJs2jXfffdfsKQ87OzsWL15sGgX269eP/v3709raSnx8PJmZmfTr14958+ZZpEQsNjaWtrY2QkND0ev1XL9+3ZTKuHHjBk1NTaZRSVNTE7m5uRQXF5Oens62bdtMk2oTJ04kJibmoW8KZt2LDQ0N7Nu3j82bN6PT6XjppZdYt24ddnZ2pKWlceTIEZycnAgLC+uxjSNHjvCHP/wBvV7Phx9+yJQpUx4YmbS0tFBTU4MQAo1Gw7Vr16zudBsaGsjMzKSpqYmoqCiLObuysjL279/P+vXrqaqqIjw8nKVLl/L0008TEBCAjY0NJ06coL6+nvz8fIs7XZVK9cBcbnt7O+np6fztb3/jxIkTQNcFGRQUZNGhvtFopKqqirKyMjw8PBg+fLhVqxi6qaqqorKyEn9/f9NEoiWorq5mwIAB6HQ6PD09cXJyMkVoISEhKJVKBg4cSGBgIBkZGbz//vtcuXIFNzc3i+SY7ezsePPNN/nBD36ATqfD2dkZGxsbjh8/zpUrV1AoFEyePJnVq1cTGBhodvuenp5UVlaalqZ3n2sNDQ3s3bsXOzs706KhwsJCvvvuO1JSUujo6KCjowOlUklkZCRvv/02U6ZMeeiRq1n3ZE5ODocOHSIvL48VK1Ywb948DAYD2dnZ7N27lzNnzhAaGsrrr7/eYxvvvPMO+fn5uLi4UFtb+8BURWdnJwkJCXz99ddUVlYSEBBgtXrMW6msrKSgoIAhQ4YwdepUi63Y27VrF++//z5NTU3MmzeP3/zmN0RGRpqip+DgYCIjIzl16hRpaWnMnDnTrNGkjY0N/fr1o6qqCr1ez7Vr1ygtLcXDw+Mu56vX61Gr1aSnp7N7925OnjyJRqPBzs6OgQMH9toBGQwG8vPzcXR0xNPTExcXF9N+6OzsNNUDp6Wl4efnR2RkZJ/0C1Gr1ajVavz8/Cx6E3zttddMKZTIyEi8vLzw9PSkX79+d/3dt+4rS+Lg4ICvry/QdRNMSUnh66+/5uLFi4wZM4Znn33WYnMwr732Grt27aKtrQ0PDw/c3NxobW019V1IS0sz7Zfu/hwGgwEHBweCgoKYOXMmy5cvZ/r06Y90PffK6d4rgnF2dsbX15eBAwdia2vLuXPn2LBhA+fOncPb25tnn322VxdTSUkJBoMBDw8P07DgXjPcQgjq6urIyMggLi6O0tJSlEolY8eOZcqUKT223xMMBgNXrlwhPj6eJ554gqioKIvZunHjBkIIVCoVCxcuvKv0KDg4mOjoaPbu3cuxY8dYsWKFWSfUHB0dmTVrFsXFxTQ3N7Nv3z7c3Nx44YUXCA4ONp0zra2tlJWVkZSUxK5du0hOTga6op/AwED+z//5PyxatKhXWhobG/n973+PUqkkOjqaoKAgUz7yxo0bHDlyhEOHDtHa2srixYstdnE/iJqaGtRqNREREXdVAJmTGTNmMGPGDIttv7fU1NTw3XffkZGRgb+/PytXrmTFihUWy7G/+OKLLF++HCEE165d47//+785cuQIJSUlaLXa22p1bW1tsbe3R6VSERwcbPptTxYV9crpBgQE4OzsTHNzM2q1Gnd3d8aOHYuzszNKpZK4uDh27txJQkICnp6ePPXUU7z11lu9MWlacNDU1MTGjRsJDw9n3Lhx+Pr64uTkhE6no7W1lZaWFnbv3s2mTZsoLS3FxcWFmJgY/vSnP1mtVKub7hrM2tpaHB0dLVoO9Ktf/YqKigqOHj3KZ599hlarZfbs2QwYMAAHBweMRiNOTk6oVCqTLnM6XRsbG6ZMmcLRo0e5fv067e3tfPXVV6SmpjJt2jRT0/iMjAxOnz5NfX09ra2tQJfDDQgI4M033zSlpXpDZ2cnLi4unD17lj179txW92tvb2+auFm2bBk///nPrVJNcifdXbOqqqrw8fExzY4/Luj1ejo6OnpVuvcwCCE4fPgw+/fvR6vV8sYbb7B27VqLTmo6ODiYgoBJkyYxfvx4QkND2bRpExkZGXR0dABdgYSfnx/BwcGEh4ezaNEiJk+e3OPRaq/O6lmzZpGenk5+fj4HDx6kpaWFnJwc6urqbuscFRoayoIFC3jttdd6nTP7z//8T/71X/+V8vJyPvnkE5ydnVGpVMybN4+xY8dSWVlJfHy8qbBao9GYJghmzpxpdYcLXXnWoqIiVCoVgwYNwtbW1lTDam68vb35j//4D/r160dsbCy//OUvmT9/PqtXr2b48OFUVFRw7Ngxbty4QWhoqEWK8BcuXEhDQwO///3vKS8vp7Ozk+TkZFM0C5hmrLv3gb29PcHBwfzoRz/iRz/6kVmG+f7+/nzxxRekp6fz3Xff3dZJrTt/GhgYiL+/v8XW8T+IpqYm6uvr6d+//0PPflsDW1tbHBwcaGlp4dq1azQ2Nt6z1M1cqNVqTp8+TWVlJdHR0UydOvWejassiZ2dHS+//DLDhg3jo48+IjU1FYDhw4cze/Zspk+fzvjx43t/3d6v0a54vJoy30ZycrJ46aWXhJ+fn1CpVMLR0dHUCPvW5tgODg4iICBAzJs3T/z9738XLS0tZtXxsFy+fFksW7ZMBAYGivfee0/cuHFD1NfXW1SHwWAQX3zxhYiIiBAKhUKoVCoxaNAgoVKphEKhEFOnThU7d+58lD/jkXQ0NDSIlStXCh8fH2Fvb39bs/Jbj5O9vb1wc3MTkydPFqdOnXrQUyMeWYcFMYuO9PR08fzzz4tFixaJy5cvm0tHr/dJaWmp+PWvfy0AER4eLq5evdpTLQ/F3/72NxEYGCjGjh0r9u3bJ4xGY8+E91KHmbFcE3NrM2HCBEJCQpgxYwbt7e18/fXXprZ4txISEsJPf/pTnn/++T5S+j9IkkRjYyPfffcdlZWVrF692qKRg42NDevWrSMoKIjNmzdz/vx5Ghsbsbe354UXXuDf/u3fLNonQ6VS8cEHH7B582a2bdtGZWXlPb/n7+/P0qVLefHFFy1erfA4cuHCBVJTU3nyySctXknyKHh7ezNmzBj8/PxobW0lLS3NYpUV3bWv3b1AJk6c2GetYa3BP6TTBXBzc2PVqlVAV+Pyx5nuZuEtLS3k5uYSHh5ukRKYezFz5kyzrlV/FHx9fXn77bd5++23+8T+PwJNTU20tLTg6uraZ43/74VCoWDKlCn8/Oc/p6ioyOINeGxsbJgzZw4LFiz4X/+w2H9Yp/uPRHBwMC+//DI6nQ6FQsEf/vCHPsshyjyePI6R3eDBg/nlL39pcTu2trb88Y9/tLidxwXZ6VqJqKgoi5aKyfxjIkkSwcHBt7UylPnfjSS+f217zxe+95x73fJlHbcj67gdWcfdPC5aZB13vvkApysjIyMjY0Ysv85PRkZGRsaE7HRlZGRkrIjsdGVkZGSsiOx0ZWRkZKyI7HRlZGRkrIjsdGVkZGSsyP8HpYPMQQ6Q3OIAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hQFg8Qgbc9-p"
      },
      "source": [
        "Thus, as you can see, we are dealing with inputs that are represented as 28 x 28 matrices. Note that in this case only one channel is provided for each image (black/white image). Dealing with color image you'd have three channel (RBG).\n",
        "\n",
        "Since we are dealing with an image dataset, the batch structure is therefore:\n",
        "\n",
        "`number_of_batch_elements x number_of_channels x image_height_pixels x image_width_pixels` \n",
        "\n",
        "Since we are dealing with a single channel, it could have been `number_of_batch_elements x image_height_pixels x image_width_pixels`\n",
        "\n",
        "As you can see, each image represent an single handwritten digit. The aim is to develop a predictor that will be able to predict to which written digit is associated each image (a numerical value among $[0, 1, ..., 9] $).\n",
        "\n",
        "We therefore are dealing with a classification task, and in particular a multiclass single label classification task, i.e. each entry is associated to a single class (label) from a set of classes. \n",
        "\n",
        "You're asked to develop and train an ANN to tackle this task. \n",
        "Do not look at solutions on the web, you'll find plenty of models implemented with PyTorch that are based on more advanced concepts. It will not be very useful for you and you're able to this by yourself, trust us. \n",
        "\n",
        "Good luck. \n",
        "\n",
        "A few tips are provided below with additional information following on some technical parts: \n",
        "\n",
        "* For a given input, the output of your network will be a probability distribution that will express the probability that the input is associated to each of the potential classes considered in your multiclass setting. To do so you'll use the softmax function that is implemented in PyTorch (see below) https://pytorch.org/docs/stable/generated/torch.nn.Softmax.html\n",
        "\n",
        "* You'll have to consider a new loss since you're now dealing with a classification setting. This loss will be used to favour the output probability distrutions that are closer to the expected one (the one with the full mass associated to the correct label). Select the Cross Entropy Loss (see below for more information) https://pytorch.org/docs/stable/generated/torch.nn.CrossEntropyLoss.html\n",
        "\n",
        "* Try a simple network first. Try representing your images as vectors of values (stored into a Tensor), as we did for our first ANN. To do so, you'll have to `flatten` the input, i.e. convert a Tensor into a n-dimensional Tensor (a verctor). You can do this directly in your network, no need to preprocess your data. Have a look to `torch.nn.flatten` https://pytorch.org/docs/stable/generated/torch.nn.Flatten.html. This layer will be use to convert a $[-0.5, 0.5]^{1 \\times 28 \\times 28}$ tensor into a $[-0.5, 0.5]^{784}$ tensor ($28 \\times 28 = 784$)    \n",
        "\n",
        "* Contrary to what we did so far, you'll not be able to process the full training dataset at each optimizer step (remember, this is the step in which you finalize the backpropagation updating the trainable parameters using the computed gradients). Consider the following terminology:\n",
        "  * a batch, a set of inputs for which you'll perform the forward pass.\n",
        "  * an epoch: the number of batch iterations required to process the whole training dataset.\n",
        "  * You can process several batches before performing the backpropagation, i.e. perform `loss.backward()` and then update the gradients (this is called gradient accumulation), but that not important for you now. Consider that you perform a step for each batch you process. \n",
        "\n",
        "* Rember that what we want is to be good on the test set. \n",
        "\n",
        "Some additional information is provided below. \n",
        "\n",
        "**Softmax function** \n",
        "\n",
        "The softmax function can be used to transform a vector of real-values $ \\mathbf{z} \\in \\mathbb{R}^{|\\mathcal{C}|}$ into a vector $\\mathbf{z'} \\in [0,1]^{|\\mathcal{C}|}$ such as $\\sum_i^{|\\mathcal{C}|} \\mathbf{z'}_i = 1$. \n",
        "It is often used as the last activation function of a neural network to normalize the output of a network and to obtain a probability distribution over predicted output classes. \n",
        "\n",
        "Considering a muticlass prediction setting with a set of classes $\\mathcal{C}$, the network will internaly predict a vector $\\mathbf{z} \\in \\mathbb{R}^{|\\mathcal{C}|}$ that will be converted using the softmax into a probability distribution $\\mathbf{z'} \\in [0,1]^{|\\mathcal{C}|}$ with $\\sum_i^{|\\mathcal{C}|} \\mathbf{z'}_i = 1$. \n",
        "\n",
        "The final prediction will then be based on a decision rule exploiting $\\mathbf{z'}$, e.g in a single label setting the prediction will be defined as $\\hat{y} := \\text{argmax}_{i \\in 1,\\ldots,|\\mathcal{C}|} \\mathbf{z'}_i$\n",
        "\n",
        "$\\text{softmax}(\\mathbf{z})_i = \\frac{e^{z_i}}{\\sum_{j=1}^{|\\mathcal{C}|} e^{z_j}} \\text{ for } i = 1, \\dotsc , |\\mathcal{C}| \\text{ and } \\mathbf z=(z_1,\\dotsc,z_{|\\mathcal{C}|}) \\in \\mathbb{R}^{|\\mathcal{C}|}$\n",
        "\n",
        "In practice, the softmax can be seen as a soft [argmax](https://en.wikipedia.org/wiki/Arg_max) (differentiable argmax)\n",
        "\n",
        "**Cross entropy**\n",
        "\n",
        "First recall the log loss considered to evaluate a probabilistic binary classifier. \n",
        "\n",
        "For a given input observation we expect a prediction $y \\in \\{0,1\\}$ (label is true or false). \n",
        "The classifier will compute the probability $p \\in [0,1]$ that the input is labelled by the true label: $p = \\hat{\\mathbb{P}}[y=1]$.\n",
        "\n",
        "The loss for this entry can be defined by: \n",
        "\n",
        "$l(p,y) = −(y log(𝑝)+(1−𝑦)log(1−𝑝))$\n",
        "\n",
        "In a *similar fashion*, the cross entropy can be used to consider multiclass settings. It can be used to compare two probability distributions $p$ and $q$ over the same underlying set of events [(source)](https://en.wikipedia.org/wiki/Cross_entropy). Considering two discrete probability distributions $p$ and $q$ over a support $\\mathcal{X}$ the cross entropy $H(p,q) \\in [0,+\\infty]$ is defined by:\n",
        "\n",
        "$H(p,q)=-\\sum _{x\\in {\\mathcal {X}}}p(x)\\,\\log q(x)$. \n",
        "\n",
        "With $\\mathbf{y} \\in [0,1]^{|\\mathcal{C}|}$ the expected distribution and $\\hat{\\mathbf{y}} \\in [0,1]^{|\\mathcal{C}|}$ the predicted one.\n",
        "\n",
        "\n",
        "$H(\\mathbf{y},\\hat{\\mathbf{y}}) = -\\sum _{i\\in {1,\\ldots,|\\mathcal{C}|}} \\mathbf{y}_i\\,\\log \\hat{\\mathbf{y}}_i$.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vXRhmk77uVAu"
      },
      "source": [
        "# Resources \n",
        "\n",
        "* [Free Book: Deep Learning with PyTorch](https://pytorch.org/assets/deep-learning/Deep-Learning-with-PyTorch.pdf)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Model (Classification problem)"
      ],
      "metadata": {
        "id": "EE4wks75S6db"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def buildFFN(nb_hidden_layers): # FFN: Feedforward network\n",
        "\n",
        "  assert nb_hidden_layers >= 2, \"nb_hidden_layers must be >= 2\"\n",
        "  \n",
        "  # defining the first hidden layer\n",
        "  layers = [\n",
        "            torch.nn.Flatten(),\n",
        "            torch.nn.Linear(D_in, H),\n",
        "            torch.nn.ReLU()\n",
        "            ]\n",
        "\n",
        "  # adding any extra hidden layers\n",
        "  for i in range(nb_hidden_layers -2):\n",
        "    layers += [\n",
        "              torch.nn.Linear(H, H),\n",
        "              torch.nn.ReLU()\n",
        "              ]\n",
        "\n",
        "  # adding the output layer\n",
        "  layers += [\n",
        "            torch.nn.Linear(H, D_out),\n",
        "            torch.nn.Softmax(dim=1)\n",
        "            ]\n",
        "\n",
        "  model = torch.nn.Sequential(*layers) # The single star * unpacks the sequence/collection into positional arguments\n",
        "  print(model)\n",
        "\n",
        "  nb_trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad) # model.parameters() gives access to the model parameters.\n",
        "  print(\"number of trainable paramerters: \", nb_trainable_params)\n",
        "\n",
        "  return model\n",
        "\n",
        "########################################\n",
        "\n",
        "D_in = 28 * 28 # (= 784) An image is 28*28 pixels with only one channel \n",
        "D_out = 10 # 10 classes {0,1,2,3,4,5,6,7,8,9}\n",
        "\n",
        "H = 64# Number of neurons\n",
        "J = 3 # Number of hidden layers\n",
        "\n",
        "learning_rate = 1e-4 # Learning rate (êta)\n",
        "\n",
        "model = buildFFN(J)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OopFoyOqNy-o",
        "outputId": "e0afdc9f-f992-4a44-a84a-5bc80e7dbdcb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sequential(\n",
            "  (0): Flatten(start_dim=1, end_dim=-1)\n",
            "  (1): Linear(in_features=784, out_features=64, bias=True)\n",
            "  (2): ReLU()\n",
            "  (3): Linear(in_features=64, out_features=64, bias=True)\n",
            "  (4): ReLU()\n",
            "  (5): Linear(in_features=64, out_features=10, bias=True)\n",
            "  (6): Softmax(dim=1)\n",
            ")\n",
            "number of trainable paramerters:  55050\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Training"
      ],
      "metadata": {
        "id": "VjmMEuurS3A_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def train(model, epochs, log_frequency, device):\n",
        "\n",
        "  AccuracyForEachIteration = []\n",
        "\n",
        "  model.to(device)\n",
        "\n",
        "  loss_fn = torch.nn.CrossEntropyLoss(reduction='sum')\n",
        "\n",
        "  # Use the optim package to define an Optimizer that will update the weights of\n",
        "  # the model for us. Here we will use Adam; the optim package contains many other\n",
        "  # optimization algorithms. The first argument to the Adam constructor tells the\n",
        "  # optimizer which Tensors it should update.\n",
        "  optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
        "  \n",
        "  for t in range(epochs):\n",
        "\n",
        "    model.train()\n",
        "\n",
        "    for batch_id, [batch_x,batch_y] in enumerate(trainloader):\n",
        "\n",
        "      batch_x = batch_x.to(device)\n",
        "      batch_y = batch_y.to(device)\n",
        "\n",
        "      y_pred = model(batch_x)\n",
        "      loss = loss_fn(y_pred, batch_y)\n",
        "\n",
        "      # Uncomment if you want to print loss\n",
        "      #if t % log_frequency == 0 and batch_id % log_frequency == 0:\n",
        "        #print(t, batch_id, loss.item())\n",
        "\n",
        "      # Before the backward pass, use the optimizer object to zero all of the\n",
        "      # gradients for the variables it will update (which are the learnable\n",
        "      # weights of the model). This is because by default, gradients are\n",
        "      # accumulated in buffers( i.e, not overwritten) whenever .backward()\n",
        "      # is called. Checkout docs of torch.autograd.backward for more details.\n",
        "      optimizer.zero_grad()\n",
        "\n",
        "      # Backward pass: compute gradient of the loss with respect to model\n",
        "      # parameters\n",
        "      loss.backward()\n",
        "\n",
        "      # Calling the step function on an Optimizer makes an update to its\n",
        "      # parameters\n",
        "      optimizer.step()\n",
        "\n",
        "    model.eval()\n",
        "\n",
        "    correctPredictions = 0\n",
        "    total = 0\n",
        "\n",
        "    for batch_test_id, [batch_test_x,batch_test_y] in enumerate(testloader):\n",
        "\n",
        "      batch_test_x = batch_test_x.to(device)\n",
        "      batch_test_y = batch_test_y.to(device)\n",
        "\n",
        "      y_test_pred = model(batch_test_x)\n",
        "\n",
        "      _, index = torch.max(y_test_pred,1)\n",
        "      \n",
        "      total += y_test_pred.size(0)\n",
        "      correctPredictions += (batch_test_y == index).sum().item()\n",
        "\n",
        "    AccuracyForEachIteration = np.append(AccuracyForEachIteration,(100 * correctPredictions / total))\n",
        "\n",
        "    print(\"Iter :\",t,\" Correct predictions:\", correctPredictions,\" /\",total)\n",
        "    print(\"Accuracy: {:.3f}%\\n\".format(100 * correctPredictions / total))\n",
        "\n",
        "  plt.plot(np.arange(0,epochs),AccuracyForEachIteration,color='red')\n",
        "  plt.xlabel('iteration')\n",
        "  plt.ylabel('accuracy')\n",
        "  plt.show()\n",
        "\n",
        "########################################\n",
        "\n",
        "ITERATIONS = 10\n",
        "DISPLAY_LOSS = 100\n",
        "\n",
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "train(model,ITERATIONS,DISPLAY_LOSS,device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 800
        },
        "id": "Npb1aGowSpLZ",
        "outputId": "6b45f7e8-925b-40ca-d416-3bbaca8ee8b2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter : 0  Correct predictions: 7465  / 10000\n",
            "Accuracy: 74.650%\n",
            "\n",
            "Iter : 1  Correct predictions: 8371  / 10000\n",
            "Accuracy: 83.710%\n",
            "\n",
            "Iter : 2  Correct predictions: 8975  / 10000\n",
            "Accuracy: 89.750%\n",
            "\n",
            "Iter : 3  Correct predictions: 9049  / 10000\n",
            "Accuracy: 90.490%\n",
            "\n",
            "Iter : 4  Correct predictions: 9106  / 10000\n",
            "Accuracy: 91.060%\n",
            "\n",
            "Iter : 5  Correct predictions: 9150  / 10000\n",
            "Accuracy: 91.500%\n",
            "\n",
            "Iter : 6  Correct predictions: 9191  / 10000\n",
            "Accuracy: 91.910%\n",
            "\n",
            "Iter : 7  Correct predictions: 9188  / 10000\n",
            "Accuracy: 91.880%\n",
            "\n",
            "Iter : 8  Correct predictions: 9224  / 10000\n",
            "Accuracy: 92.240%\n",
            "\n",
            "Iter : 9  Correct predictions: 9248  / 10000\n",
            "Accuracy: 92.480%\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de7xUdb3/8dcbUBE0IEDlIgJKKlGiZ7xLat7NvJB5KSvrGHpSUzsdL3VKszRK007nlL9Iq/M7+TNGBLUsRc1rKrpBvCB6VJCbiFsRVBC5fX5/rLXbs2E2DDCLtffM+/l4zGPPunzXfPY8YL33+n7XRRGBmZnZmjrkXYCZmbVNDggzMyvLAWFmZmU5IMzMrCwHhJmZldUp7wKqqVevXjFw4MC8yzAzazcmT578VkT0LrespgJi4MCBNDQ05F2GmVm7IWlWa8vcxWRmZmU5IMzMrCwHhJmZleWAMDOzshwQZmZWVqYBIekCSc9LmibpwnTeNZJelPSspAmSurfS9jVJz0maKsmnJpmZbWaZBYSkYcDXgX2APYDjJO0C3AsMi4hPAv8LXLaOzRwaEcMjopBVnWZmVl6W10HsDkyKiKUAkh4CRkbET0vWeQI4OcMazMxqw+rVsHgxvPVWy1djY7L84our/pFZBsTzwFWSegIfAMcCa3YVfQ0Y20r7ACZKCuDXETGm3EqSRgGjAAYMGFCNus3MshUBS5c27+DX3OmvGQBvvQVvvw2rVpXf3g47tK+AiIjpkn4CTASWAFOBf/x2kr4LrARubmUTB0XEPEnbAfdKejEiHi7zOWOAMQCFQsFPPzKzzW/58sp28qWvZcvKb6tDB+jZE3r3hl69YLfdkp+lr6ZlTa+uXTP5tTK91UZE3ATcBCDpamBu+v5M4DjgsGjlkXYRMS/9+aakCSRjGWsFhJlZpt5/H2bPhlmzkp+lr7lzk539u++23r579+Ydef/+MHx46zv6Xr2S9Tu0jRNMMw0ISdulO/gBwEhgP0lHAxcDBzeNT5Rp1xXoEBHvpe+PBK7MslYzq0OrV8Mbb6y98y+dfuedlm06dkx29AMGwL77Jjv5cjv6Xr2SI4Ettsjnd6uCrG/Wd1s6BrECODciFkn6L2Arkm4jgCci4hxJfYEbI+JYYHtgQrq8E/D/IuLujGs1s1qzZMnaf/WX7vznzoUVK1q26dYt2fkPGAAHHAA77dQ8PWAA9O2bhEQdyLqLaUSZebu0su7rJAPZRMQMklNjzaytWLo02bkuX57sIDt02Lifpe+TPwI3zurVsGBB6zv/2bOTgd1SHTpAv37Jjn7//Vvu+Jte3bpt2vdUQ2rqdt9mtgkikh3ujBnw6qstf86YAfPnZ/O5GxMyq1cn9Sxf3nJb227b/Bf/vvs27/Sb5vXtC52826uUvymzevLhh/Daa62HwNKSYUEp6WvfeWc45hgYPBgGDYLOnZMd9KpV1fu5oW0A+vRZu/unW7dNOyqxFhwQZrUkIulWWXPH3/R+7txknSZduiQ7/p13hiOOaH4/eDAMHAhbbZXbr2L5c0CYtTcrViT9662FwJqnXPbpk+zwDz20ZQAMHgzbb++/uK1VDgiztmjZsmRn//LLyeuVV5IAePXVJBxKr6jdaquk62fwYBgxomUIDBqUHCWYbQQHhFleli+HmTObQ6D0NXt2y66gnj1hl11gv/3gi19sGQJ9+7aZC6ustjggzLK0YkUyKFwuBGbNah5wBejRA4YMgQMPhDPPTN43vXr0yOs3sDrmgDDbVCtXJjv70u6gpvczZ7bsDvrIR5Id/r77whlntAyBnj3z+x3MynBAmFVi1SqYM6f8kcDMmS2vxt1mm2SHv9decMopLUOgd28PClu74YAwKxWRnAo6dWryeuYZmDYtGTAuvSirS5dkTOATn4CRI1uGgM8MshrhgLD6tWIFTJ/eHARNobBwYfM6Q4bAxz8Oxx+fBEJTCPTt6xCwmueAsPqwaFFzCDT9nDat+aigc+fkaOBzn0tuxzx8eDK97bb51m2WIweE1ZaIZMB4zaOC115rXme77ZIAuPDC5jAYMsT36DFbg/9HWPv14Yfwwgstg2Dq1OS5vZB0AX3sY8kZQ2ef3RwGO+yQb91m7YQDwtqHt99OgqA0DF54ITnFFJJB409+Ek4/vTkIhg3L7FGMZvXAAWFtz+LF8Pe/wxNPNIfBnDnNy/v0SQLgM59Jfu6xRzKAXCcPcTHbXBwQlr/GRnjkEXj44eT1zDPJFcYdOiQPbB8xojkI9tgjOY3UzDLngLDNb9685jB4+OGkqwiSM4n23x++9z341KeSsQN3EZnlxgFh2YpILjIrDYQZM5Jl224LBx0EX/pSEgiFAmy5Zb71mtk/ZBoQki4Avg4I+E1E/FzSR4GxwEDgNeCUiHinTNuvAP+eTv4oIv47y1qtSiKSi89KA2HevGRZz55Jd9F558HBByeDyj611KzNyux/p6RhJOGwD7AcuFvSn4FRwP0RMVrSpcClwCVrtP0ocDlQAAKYLOnOckFiOVu1Cp59Fh56KAmDRx6Bt95KlvXpkwTBpz6VvHbf3belNmtHsvzzbXdgUkQsBZD0EDASOAE4JF3nv4EHWSMggKOAeyNiYdr2XuBo4JYM67VKLF8Okyc3Hx08+mjzE8wGDYLjjmsOhMGDfTsKs3Ysy4B4HrhKUk/gA+BYoAHYPiLmp+u8AZQ7JaUfUHJeI3PTeWuRNIrkqIQBAwZUp3Jr9sEHMGlSEgYPPQSPP57Mg+SI4PTTkzAYMQJ23DHfWs2sqjILiIiYLuknwERgCTAVWLXGOiEpyrXfgM8ZA4wBKBQKm7QtIxlDuP9++NvfklB48snkpnZScqrpqFFJIBx0UHLLCjOrWZmOEEbETcBNAJKuJjkSWCCpT0TMl9QHeLNM03k0d0MB9CfpirKs3XhjEgKdOiVnFV10UTKOcMAB0L173tWZ2WaU9VlM20XEm5IGkIw/7AcMAr4CjE5/3lGm6T3A1ZKanrN4JHBZlrUayb2NrrwyCYOJE30Nglmdy/ocw9vSMYgVwLkRsUjSaKAo6Z+BWcApAJIKwDkRcVZELJT0Q+CpdDtXNg1YW4ZuvDF5WM7vf+9wMDMUUTvd9oVCIRoaGvIuo31atgx23jl5PfSQzz4yqxOSJkdEodwyX6Vkid/8Bl5/Hf7wB4eDmQHgq5YsOW316qvhkEPg0EPzrsbM2ggfQRj8+tfwxhswdmzelZhZG+IjiHq3dCmMHg2f/nRyfYOZWcpHEPXuhhtgwQIYNy7vSsysjfERRD1bsgR+8hM44ojkymgzsxIOiHr2q18lT3P7wQ/yrsTM2iAHRL16/3346U/hqKOSp7iZma3BAVGv/uu/kuc2+OjBzFrhgKhH770H11wDxx6bPPfZzKwMB0Q9+s//hIUL4Yor8q7EzNowB0S9WbwYrr0WPvtZ2HvvvKsxszbMAVFvfvELeOcdHz2Y2Xo5IOrJokVw3XVwwgmw1155V2NmbZwDop78/OdJSPjowcwq4ICoF++8A9dfDyNHJs+WNjNbDwdEvbj+enj3Xbj88rwrMbN2wgFRDxYuTLqXTj4ZPvnJvKsxs3Yi07u5SroIOAsI4Dngq8C9wLbpKtsBT0bEiWXarkrbAMyOiOOzrLWm/exnya01fPRgZhsgs4CQ1A/4JjA0Ij6QVAROi4gRJevcBtzRyiY+iAh3lm+qt95KTm095RQYNizvasysHcm6i6kTsLWkTkAX4PWmBZI+AnwauD3jGurbz36W3Nb7+9/PuxIza2cyC4iImAdcC8wG5gOLI2JiySonAvdHxLutbKKzpAZJT0haqwuqiaRR6XoNjY2NVau/JjQ2JrfVOO00GDo072rMrJ3JLCAk9QBOAAYBfYGuks4oWeV04JZ1bGKniCgAXwB+LmnncitFxJiIKEREoXfv3lWqvkZccw188IGPHsxso2TZxXQ4MDMiGiNiBTAeOABAUi9gH+Cu1hqnRyBExAzgQWDPDGutPQsWwC9/CV/4Auy2W97VmFk7lGVAzAb2k9RFkoDDgOnpspOBP0fEsnINJfWQtFX6vhdwIPBChrXWnp/+FJYtg+99L+9KzKydynIMYhIwDphCcrpqB2BMuvg01uheklSQdGM6uTvQIOkZ4AFgdEQ4ICr1xhtwww3wpS/Bxz6WdzVm1k4pIvKuoWoKhUI0NDTkXUb+LrooGZx+6SXYuezQjZkZAJImp+O9a/GV1LXm9deTo4cvf9nhYGabxAFRa0aPhlWr4N//Pe9KzKydc0DUknnzYMwYOPNMGDw472rMrJ1zQNSSH/84OXr47nfzrsTMaoADolbMmQO/+Q187WswcGDe1ZhZDXBA1Iqrr4YIHz2YWdU4IGrBrFlw001w1lkwYEDe1ZhZjXBA1IKrrgIJvvOdvCsxsxrigGjvZs6E3/0Ovv516N8/72rMrIY4INq7q66Cjh3hssvyrsTMaowDoj179VX4/e/h7LOhX7+8qzGzGuOAaM9+9CPYYgu49NK8KzGzGuSAaK9eeQX+53/gnHOgT5+8qzGzGuSAaK9++EPYcku45JK8KzGzGuWAaI9eegn+8Af4xjdghx3yrsbMapQDoj364Q+hc2e4+OK8KzGzGuaAaG+mT4dbboHzzoPttsu7GjOrYQ6I9ubKK2HrreHf/i3vSsysxlUUEJLGS/qMJAdKnqZNg7Fj4fzzoVevvKsxsxpX6Q7/V8AXgJcljZa0ayWNJF0kaZqk5yXdIqmzpN9Lmilpavoa3krbr0h6OX19pcI6a9uVV0LXrvDtb+ddiZnVgYoCIiLui4gvAnsBrwH3SXpM0lclbVGujaR+wDeBQkQMAzoCp6WL/y0ihqevqWXafhS4HNgX2Ae4XFKPDfzdastzz0GxCBdcAD175l2NmdWBiruMJPUEzgTOAp4G/oMkMO5dR7NOwNaSOgFdgNcr/LijgHsjYmFEvJN+xtGV1lqTfvAD+MhH4FvfyrsSM6sTlY5BTAAeIdnJfzYijo+IsRFxPrBNuTYRMQ+4FpgNzAcWR8TEdPFVkp6VdL2krco07wfMKZmem84rV9soSQ2SGhobGyv5ddqfZ56B225Ljh4++tG8qzGzOlHpEcQvImJoRPw4IuaXLoiIQrkGaZfQCcAgoC/QVdIZwGXAbsDewEeBTboUOCLGREQhIgq9e/felE21XT/4AXTrBhddlHclZlZHKg2IoZK6N01I6iHpG+tpczgwMyIaI2IFMB44ICLmR+JD4HckYwxrmgfsWDLdP51Xf55+GiZMSMKhR30Pw5jZ5lVpQHw9IhY1TaTjAl9fT5vZwH6SukgScBgwXVIfgHTeicDzZdreAxyZBlEP4Mh0Xv254gro3h0uvDDvSsyszlQaEB3THToAkjoCW66rQURMAsYBU4Dn0s8aA9ws6bl0Xi/gR+k2C5JuTNsuBH4IPJW+rkzn1ZfJk+HOO5OB6W7d8q7GzOqMImL9K0nXADsBv05nnQ3MiYh/zbC2DVYoFKKhoSHvMqrnuOPgscfgtdeSM5jMzKpM0uTWxpI7VbiNS0hC4V/S6XuBG6tQm7XmySfhrruSR4o6HMwsBxUFRESsBm5IX7Y5XHFFckHc+efnXYmZ1amKAkLSEODHwFCgc9P8iBicUV317fHH4a9/hdGjYdtt867GzOpUpYPUvyM5elgJHAr8X+APWRVV9664IrkZ37nn5l2JmdWxSgNi64i4n2RQe1ZEXAF8Jruy6thjj8HEicnDgLYpe5G6mdlmUekg9Yfprb5flnQeyUVr3ntl4fLLkwcBfWN91yGamWWr0iOIC0juw/RN4J+AMwDfgrvaHnkE7rsPLrkkua23mVmO1nsEkV4Ud2pEfBt4H/hq5lXVq8svh+23h3POybsSM7P1B0RErJJ00OYopq5NnQoPPADXXQdduuRdjZlZxWMQT0u6E7gVWNI0MyLGZ1JVPRo7Fjp2hC9/Oe9KzMyAygOiM/A28OmSeUFyh1bbVBHJ0+IOP9xPizOzNqPSK6k97pClKVNgxgz47nfzrsTM7B8qvZL6dyRHDC1ExNeqXlE9KhZhiy3gpJPyrsTM7B8q7WL6c8n7zsBJVP58aVuXpu6lI47wA4HMrE2ptIvpttJpSbcAj2ZSUb156qnkdt5XXJF3JWZmLVR6odyahgDbVbOQutXUvXTCCXlXYmbWQqVjEO/RcgziDZJnRNimaOpeOuqo5LGiZmZtSKVdTL7ndBYmTYI5c+Dqq/OuxMxsLRV1MUk6SVK3kunukk7Mrqw6USzCVlvB8cfnXYmZ2VoqHYO4PCIWN01ExCLg8vU1knSRpGmSnpd0i6TOkm6W9FI677eStmil7SpJU9PXnRXW2X6sXp0ExNFH+5GiZtYmVRoQ5dZbZ/eUpH4kd38tRMQwoCNwGnAzsBvwCWBr4KxWNvFBRAxPX7X3J/bjj8O8eXDKKXlXYmZWVqUB0SDpOkk7p6/rgMkVtOsEbC2pE8ntwl+PiL9ECngS6L9xpbdzTd1Ln/1s3pWYmZVVaUCcDywHxgJ/BJYB63weZkTMA64FZgPzgcURMbFpedq19CXg7lY20VlSg6Qn1jXeIWlUul5DY2Njhb9OzlavhltvhWOP9TOnzazNqvQspiXApRuyYUk9gBOAQcAi4FZJZ0RE07OsfwU8HBGPtLKJnSJinqTBwN8kPRcRr5apbQwwBqBQKKx1O5A26e9/h/nz4dRT867EzKxVlZ7FdK+k7iXTPSTds55mhwMzI6IxIlaQ3Pn1gLT95UBv4FutNU6PQIiIGcCDwJ6V1NoujB0LW28Nn/Fjvc2s7aq0i6lXeuYSABHxDuu/kno2sJ+kLpIEHAZMl3QWcBRwekSsLtcwDaCt0ve9gAOBFyqstW1btQrGjUvCYRs/1tvM2q5KA2K1pAFNE5IGUuburqUiYhIwDpgCPJd+1hjg/wDbA4+np7B+P91mQdKNafPdSQbGnwEeAEZHRG0ExCOPwIIFPnvJzNq8Su/m+l3gUUkPAQJGAKPW1ygiLmft6yXKfmZENJCe8hoRj5GcBlt7isXkkaLHHpt3JWZm61TpIPXdkgokofA0cDvwQZaF1aSVK+G225JTW7t2zbsaM7N1qvRmfWcBF5BcszAV2A94nJaPILX1eeghePNNdy+ZWbtQ6RjEBcDewKyIOJTkjKJF625iaykWkyOHY47JuxIzs/WqNCCWRcQyAElbRcSLwK7ZlVWDmrqXjj8+OcXVzKyNq3SQem56HcTtwL2S3gFmZVdWDXrgAXj7bXcvmVm7Uekg9Unp2yskPQB0o/VbZFg5xWJyW42jj867EjOzilR6BPEPEfFQFoXUtBUrYPz45LGinTvnXY2ZWUU29pnUtiHuvx8WLnT3kpm1Kw6IzaFYTB4KdOSReVdiZlYxB0TWli+HCRPgxBOT5z+YmbUTDois3XcfLFrk7iUza3ccEFkbOxa6d4cjjsi7EjOzDeKAyNKHH8Ltt8NJJ8GWW+ZdjZnZBnFAZGniRHj3XXcvmVm75IDIUrEIPXrAYYflXYmZ2QZzQGRl2TK44w4YORK22CLvaszMNpgDIit33w3vvQennpp3JWZmG8UBkZViEXr2hEMPzbsSM7ONkmlASLpI0jRJz0u6RVJnSYMkTZL0iqSxksqe3iPpsnSdlyQdlWWdVffBB3DnnfC5z0GnDb7dlZlZm5BZQEjqB3wTKETEMKAjcBrwE+D6iNgFeAf45zJth6brfhw4GviVpI5Z1Vp1f/0rLFnis5fMrF3LuoupE7C1pE5AF2A+yWNKx6XL/xs4sUy7E4A/RsSHETETeAXYJ+Naq6dYhN694eCD867EzGyjZRYQETEPuBaYTRIMi4HJwKKIWJmuNhfoV6Z5P2BOyXRr67U9S5bAn/4EJ5/s7iUza9ey7GLqQXIkMAjoC3Ql6S6q9ueMktQgqaGxsbHam99wf/kLLF3q7iUza/ey7GI6HJgZEY0RsQIYDxwIdE+7nAD6A/PKtJ0H7Fgy3dp6RMSYiChERKF3797Vq35jFYuw/fYwYkTelZiZbZIsA2I2sJ+kLpIEHAa8ADwAnJyu8xXgjjJt7wROk7SVpEHAEODJDGutjvffh7vuSrqXOrafMXUzs3KyHIOYRDIYPQV4Lv2sMcAlwLckvQL0BG4CkHS8pCvTttOAIkmg3A2cGxGrsqq1au66KznF1d1LZlYDFBF511A1hUIhGhoa8itg5Eh44gmYM8dHEGbWLkiaHBGFcst8JXW1vPdeMkD9+c87HMysJjggquVPf0qe/+DuJTOrEQ6IaikWoV8/2H//vCsxM6sKB0Q1vPtucnuNz38eOvgrNbPa4L1ZNdxxByxf7u4lM6spDohqKBZhwADYb7+8KzEzqxoHxKZatAjuuSfpXpLyrsbMrGocEJvqjjtgxQp3L5lZzXFAbKpiEXbaCfbeO+9KzMyqygGxKRYuhIkTk6MHdy+ZWY1xQGyK22+HlSvh1FPzrsTMrOocEJuiWITBg2GvvfKuxMys6hwQG+vtt+G++9y9ZGY1ywGxsSZMgFWrfPaSmdUsB8TGKhZhl11g+PC8KzEzy4QDYmM0NsLf/ubuJTOraQ6IjTF+fNK95LOXzKyGOSA2RrEIu+4Kn/hE3pWYmWXGAbGhFiyABx9095KZ1bxOWW1Y0q7A2JJZg4HvA/sDu6bzugOLImKtkV5JrwHvAauAla09M3WzGz8eVq/22UtmVvMyC4iIeAkYDiCpIzAPmBARP29aR9LPgMXr2MyhEfFWVjVulLFjYffd4eMfz7sSM7NMba4upsOAVyNiVtMMSQJOAW7ZTDVsuvnz4eGHk8Fpdy+ZWY3bXAFxGmsHwQhgQUS83EqbACZKmixpVGsbljRKUoOkhsbGxiqV24rbboOI5NkPZmY1LvOAkLQlcDxw6xqLTmfdRw8HRcRewDHAuZI+VW6liBgTEYWIKPTu3bsqNbeqWIRhw2Do0Gw/x8ysDdgcRxDHAFMiYkHTDEmdgJG0HMRuISLmpT/fBCYA+2Rc57rNmwePPurBaTOrG5sjIModKRwOvBgRc8s1kNRV0rZN74EjgeczrXJ9xo1LupccEGZWJzINiHTnfgQwfo1Fa41JSOor6S/p5PbAo5KeAZ4E7oqIu7Osdb2KRdhjj+QCOTOzOpDZaa4AEbEE6Flm/pll5r0OHJu+nwHskWVtG2TOHHjsMbjqqrwrMTPbbHwldSXGjUt++uwlM6sjDohKFIuw554wZEjelZiZbTYOiPWZNQueeMKD02ZWdxwQ63NrevmGA8LM6owDYn2KRSgUYPDgvCsxM9usHBDrMnMmPPWUjx7MrC45INalqXvJZy+ZWR1yQKzL2LGwzz4wcGDelZiZbXYOiNa88gpMmeLuJTOrWw6I1rh7yczqnAOiNcUi7L8/DBiQdyVmZrlwQJTzv/8LU6e6e8nM6poDopxiMfl58sn51mFmliMHRDnFIhx4IPTvn3clZma5cUCsafp0eO45OPXUvCsxM8uVA2JNt94KEnzuc3lXYmaWKwfEmopFGDEC+vbNuxIzs1w5IEpNm5a8fPaSmZkDooVi0d1LZmapzAJC0q6Sppa83pV0oaQrJM0rmX9sK+2PlvSSpFckXZpVnf8QkQTEwQfDDjtk/nFmZm1dZgERES9FxPCIGA78E7AUmJAuvr5pWUT8Zc22kjoCvwSOAYYCp0samlWtADz/PLz4os9eMjNLba4upsOAVyNiVoXr7wO8EhEzImI58EfghMyqg+TooUMHGDky048xM2svNldAnAbcUjJ9nqRnJf1WUo8y6/cD5pRMz03nrUXSKEkNkhoaGxs3rrqI5Nbehx4K2223cdswM6sxmQeEpC2B44H09qjcAOwMDAfmAz/blO1HxJiIKEREoXfv3hu3kWeegZdf9tlLZmYlNscRxDHAlIhYABARCyJiVUSsBn5D0p20pnnAjiXT/dN52SgWoWNHdy+ZmZXYHAFxOiXdS5L6lCw7CXi+TJungCGSBqVHIKcBd2ZSXdPZS4cdBr16ZfIRZmbtUacsNy6pK3AEcHbJ7J9KGg4E8FrTMkl9gRsj4tiIWCnpPOAeoCPw24iYlkmRS5fCIYfA4Ydnsnkzs/ZKEZF3DVVTKBSioaEh7zLMzNoNSZMjolBuma+kNjOzshwQZmZWlgPCzMzKckCYmVlZDggzMyvLAWFmZmU5IMzMrCwHhJmZlVVTF8pJagQqvaX4mnoBb1WxnPbM30VL/j5a8vfRrBa+i50iouydTmsqIDaFpIbWriasN/4uWvL30ZK/j2a1/l24i8nMzMpyQJiZWVkOiGZj8i6gDfF30ZK/j5b8fTSr6e/CYxBmZlaWjyDMzKwsB4SZmZVV9wEh6WhJL0l6RdKledeTJ0k7SnpA0guSpkm6IO+a8iapo6SnJf0571ryJqm7pHGSXpQ0XdL+edeUJ0kXpf9Pnpd0i6TOeddUbXUdEJI6Ar8EjgGGAqdLGppvVblaCfxrRAwF9gPOrfPvA+ACYHreRbQR/wHcHRG7AXtQx9+LpH7AN4FCRAwjeTTyaflWVX11HRDAPsArETEjIpYDfwROyLmm3ETE/IiYkr5/j2QH0C/fqvIjqT/wGeDGvGvJm6RuwKeAmwAiYnlELMq3qtx1AraW1AnoAryecz1VV+8B0Q+YUzI9lzreIZaSNBDYE5iUbyW5+jlwMbA670LagEFAI/C7tMvtRkld8y4qLxExD7gWmA3MBxZHxMR8q6q+eg8IK0PSNsBtwIUR8W7e9eRB0nHAmxExOe9a2ohOwF7ADRGxJ7AEqNsxO0k9SHobBgF9ga6Szsi3quqr94CYB+xYMt0/nVe3JG1BEg43R8T4vOvJ0YHA8ZJeI+l6/LSkP+RbUq7mAnMjoumIchxJYNSrw4GZEdEYESuA8cABOddUdfUeEE8BQyQNkrQlySDTnTnXlBtJIuljnh4R1+VdT54i4rKI6B8RA0n+XfwtImruL8RKRcQbwBxJu6azDgNeyLGkvM0G9pPUJf1/cxg1OGjfKe8C8hQRKyWdB9xDchbCbyNiWs5l5elA4EvAc5KmpvO+ExF/ybEmazvOB25O/5iaAXw153pyExGTJI0DppCc/agzkR8AAAIcSURBVPc0NXjbDd9qw8zMyqr3LiYzM2uFA8LMzMpyQJiZWVkOCDMzK8sBYWZmZTkgzMqQ9Fj6c6CkL1R5298p91lmbY1PczVbB0mHAN+OiOM2oE2niFi5juXvR8Q21ajPLEs+gjArQ9L76dvRwAhJU9P7/3eUdI2kpyQ9K+nsdP1DJD0i6U7SK4wl3S5pcvrMgFHpvNEkdwCdKunm0s9S4pr0+QLPSTq1ZNsPljyL4eb06l2zTNX1ldRmFbiUkiOIdEe/OCL2lrQV8HdJTXfx3AsYFhEz0+mvRcRCSVsDT0m6LSIulXReRAwv81kjgeEkz1rolbZ5OF22J/BxkltK/53kqvdHq//rmjXzEYTZhjkS+HJ6K5JJQE9gSLrsyZJwAPimpGeAJ0huCjmEdTsIuCUiVkXEAuAhYO+Sbc+NiNXAVGBgVX4bs3XwEYTZhhFwfkTc02JmMlaxZI3pw4H9I2KppAeBTXkk5Ycl71fh/7u2GfgIwmzd3gO2LZm+B/iX9LboSPpYKw/O6Qa8k4bDbiSPcG2yoqn9Gh4BTk3HOXqTPMHtyar8FmYbwX+FmK3bs8CqtKvo9yTPZR4ITEkHihuBE8u0uxs4R9J04CWSbqYmY4BnJU2JiC+WzJ8A7A88AwRwcUS8kQaM2Wbn01zNzKwsdzGZmVlZDggzMyvLAWFmZmU5IMzMrCwHhJmZleWAMDOzshwQZmZW1v8HBg/hP5SRkJwAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}