GPT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "V100",
      "authorship_tag": "ABX9TyNx6RsjI1cUEqTajaFYLibc",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "82453ec5ee5b4a7ab3f1b5d8c7d137ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e3b255ca57054758aee7088affa72937",
              "IPY_MODEL_eddb55cecd7a4bff9d02ba65cfb9ece6",
              "IPY_MODEL_0ad1af3da4034b4bb4933de754506bc1"
            ],
            "layout": "IPY_MODEL_c23fae9292004be1811a2f4ce0525321"
          }
        },
        "e3b255ca57054758aee7088affa72937": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5d4cc58dac80429b9adc78127c926d4c",
            "placeholder": "​",
            "style": "IPY_MODEL_28b90497cc534cc4b1451a625ecd55bb",
            "value": "Dl Completed...: 100%"
          }
        },
        "eddb55cecd7a4bff9d02ba65cfb9ece6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77569cecd1934e30a93136bc6dbcc4b0",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ea804e26dd3e46e1ab251f8d3cb0cdc3",
            "value": 1
          }
        },
        "0ad1af3da4034b4bb4933de754506bc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7e774a6fac934eb88beda0716a028683",
            "placeholder": "​",
            "style": "IPY_MODEL_03bf5613823b46edbfaa1c9d3f7d2a1c",
            "value": " 1/1 [06:22&lt;00:00, 31.58s/ url]"
          }
        },
        "c23fae9292004be1811a2f4ce0525321": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5d4cc58dac80429b9adc78127c926d4c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "28b90497cc534cc4b1451a625ecd55bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "77569cecd1934e30a93136bc6dbcc4b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "ea804e26dd3e46e1ab251f8d3cb0cdc3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7e774a6fac934eb88beda0716a028683": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "03bf5613823b46edbfaa1c9d3f7d2a1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cf2a404c8b9c49248fca3274d53d6272": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_76448977c9f844be9e10e1a30d6ea250",
              "IPY_MODEL_c2ec16612f2c4911ad723656d92eb17b",
              "IPY_MODEL_c0e736d4625d4b3ea27c0f4a3b833608"
            ],
            "layout": "IPY_MODEL_385d17961de24c099ca93a286c538c7c"
          }
        },
        "76448977c9f844be9e10e1a30d6ea250": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_512cc2fa88264041a96dfc83759b3c7c",
            "placeholder": "​",
            "style": "IPY_MODEL_816d37bccfab4fabb499cbf2e16251b2",
            "value": "Dl Size...: 100%"
          }
        },
        "c2ec16612f2c4911ad723656d92eb17b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d6de34d454b84564a8b107db6f2712ad",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b61044f88ea84e0f8981750c0eaaff3b",
            "value": 1
          }
        },
        "c0e736d4625d4b3ea27c0f4a3b833608": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c87d9991f6d14bd7a2f019ddff9fbdc8",
            "placeholder": "​",
            "style": "IPY_MODEL_ce79337495ca4ed9a0c2b3e40b2132d7",
            "value": " 2996/2996 [06:22&lt;00:00, 101.96 MiB/s]"
          }
        },
        "385d17961de24c099ca93a286c538c7c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "512cc2fa88264041a96dfc83759b3c7c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "816d37bccfab4fabb499cbf2e16251b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d6de34d454b84564a8b107db6f2712ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "b61044f88ea84e0f8981750c0eaaff3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c87d9991f6d14bd7a2f019ddff9fbdc8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce79337495ca4ed9a0c2b3e40b2132d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ecc68fc2ddb7453d84f6f08e49c0fab1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_485dfe1b43db4807b472f1be7131f0d7",
              "IPY_MODEL_a095a043c0bd45fa871071c38c63b024",
              "IPY_MODEL_53bd66fe90244bf1809e6aa0f793d7b2"
            ],
            "layout": "IPY_MODEL_ce744d3b0d3c436b98a870428df460c8"
          }
        },
        "485dfe1b43db4807b472f1be7131f0d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cdf5e310eb2d45089cd4edce932452ec",
            "placeholder": "​",
            "style": "IPY_MODEL_72d03b0862fc41c1872514813014f10b",
            "value": "Extraction completed...: 100%"
          }
        },
        "a095a043c0bd45fa871071c38c63b024": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f24d1bbe1bd1491fbbc19cde92e8565f",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6410ef9362f646ababb252f9b5da979c",
            "value": 1
          }
        },
        "53bd66fe90244bf1809e6aa0f793d7b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f90cdcefe17e49ff91be7d3dfd56b1cd",
            "placeholder": "​",
            "style": "IPY_MODEL_76ef1beee8544edbbf1b57cb1db02ec9",
            "value": " 1/1 [06:22&lt;00:00, 382.46s/ file]"
          }
        },
        "ce744d3b0d3c436b98a870428df460c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cdf5e310eb2d45089cd4edce932452ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "72d03b0862fc41c1872514813014f10b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f24d1bbe1bd1491fbbc19cde92e8565f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "6410ef9362f646ababb252f9b5da979c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f90cdcefe17e49ff91be7d3dfd56b1cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "76ef1beee8544edbbf1b57cb1db02ec9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "048ad5ecd7c44ae3b37977268a998628": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_61a6c8584c324be593f598a4d6cec70d",
              "IPY_MODEL_31c20e32f7bd4442ae95a3a7bdee0a42",
              "IPY_MODEL_68df84bada634adf84bfc9e833d098a6"
            ],
            "layout": "IPY_MODEL_8ad067436f764ec9a6d42aa8779323a4"
          }
        },
        "61a6c8584c324be593f598a4d6cec70d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1eae80a5ba4541bea2dcfb3c5fcf919e",
            "placeholder": "​",
            "style": "IPY_MODEL_366dd58c884046b19406e3ef74fb2a5e",
            "value": "Generating splits...: 100%"
          }
        },
        "31c20e32f7bd4442ae95a3a7bdee0a42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c54b4aaafd074a4887c93e04f8bae41b",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_818c22457f7f4f93b261426b91dfde7a",
            "value": 1
          }
        },
        "68df84bada634adf84bfc9e833d098a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_872efebaa375427eaef3139980e84bdd",
            "placeholder": "​",
            "style": "IPY_MODEL_398680c0b9044f9e86a50a221a0f5795",
            "value": " 1/1 [22:02&lt;00:00, 1322.75s/ splits]"
          }
        },
        "8ad067436f764ec9a6d42aa8779323a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "1eae80a5ba4541bea2dcfb3c5fcf919e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "366dd58c884046b19406e3ef74fb2a5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c54b4aaafd074a4887c93e04f8bae41b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "818c22457f7f4f93b261426b91dfde7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "872efebaa375427eaef3139980e84bdd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "398680c0b9044f9e86a50a221a0f5795": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5dfecdd93bb34eb3b85415a348972b56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dba7003f50ca4c9e8f3187c1a2faf9a1",
              "IPY_MODEL_6ba1f78f752e42f984b8592f51ee19d3",
              "IPY_MODEL_41354c8686c64bbb9ded756d385c0095"
            ],
            "layout": "IPY_MODEL_e8a5243f3611486b8df334da1b74c7d2"
          }
        },
        "dba7003f50ca4c9e8f3187c1a2faf9a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c78578e655c84d7dbfa6c4cf40a6cb11",
            "placeholder": "​",
            "style": "IPY_MODEL_6340e4c9c9e04bc5b13f0a8f0dfbff50",
            "value": "Generating train examples...: 100%"
          }
        },
        "6ba1f78f752e42f984b8592f51ee19d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_912399ebb8a9464084cc624839d443da",
            "max": 3848330,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9c60fb41aa104ab292aaf826205db5fa",
            "value": 3848330
          }
        },
        "41354c8686c64bbb9ded756d385c0095": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_745af5b58389489f91664c1fbfe1502d",
            "placeholder": "​",
            "style": "IPY_MODEL_a6767891668942b78f831e0037e3bb7c",
            "value": " 3847065/3848330 [19:52&lt;00:00, 3029.03 examples/s]"
          }
        },
        "e8a5243f3611486b8df334da1b74c7d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "c78578e655c84d7dbfa6c4cf40a6cb11": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6340e4c9c9e04bc5b13f0a8f0dfbff50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "912399ebb8a9464084cc624839d443da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c60fb41aa104ab292aaf826205db5fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "745af5b58389489f91664c1fbfe1502d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a6767891668942b78f831e0037e3bb7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f2cfc549ddca4ef1a0e8cad7d23c0fe8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dbbfd166f76b4a8198c55a3880ad299f",
              "IPY_MODEL_4f2d0ed3a5174815b4447f555521aedf",
              "IPY_MODEL_e3ac88f1e5424faf85336e1bb9295c36"
            ],
            "layout": "IPY_MODEL_0f7054aacd1449ff83a1bf15672219e1"
          }
        },
        "dbbfd166f76b4a8198c55a3880ad299f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d164507a4f824a4d96c177caaa3698fb",
            "placeholder": "​",
            "style": "IPY_MODEL_b6b8f297f6c847e4b6179901033e0f13",
            "value": "Shuffling /root/tensorflow_datasets/reddit/1.0.0.incompleteXKQW6K/reddit-train.tfrecord*...: 100%"
          }
        },
        "4f2d0ed3a5174815b4447f555521aedf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f000ecd3bf524fa3bb4c0a05f9301936",
            "max": 3848330,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6b9ca5a683274f78aa14697881ebd9f4",
            "value": 3848330
          }
        },
        "e3ac88f1e5424faf85336e1bb9295c36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8eee3ad39c524c5085dd85c704215dbd",
            "placeholder": "​",
            "style": "IPY_MODEL_9c90cc55c0284d119afc4b3e052e61a6",
            "value": " 3844583/3848330 [02:09&lt;00:00, 31596.80 examples/s]"
          }
        },
        "0f7054aacd1449ff83a1bf15672219e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "d164507a4f824a4d96c177caaa3698fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b6b8f297f6c847e4b6179901033e0f13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f000ecd3bf524fa3bb4c0a05f9301936": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6b9ca5a683274f78aa14697881ebd9f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8eee3ad39c524c5085dd85c704215dbd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c90cc55c0284d119afc4b3e052e61a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AdamClarkStandke/GenerativeDeepLearning/blob/main/GPT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Generative Pre-trained Transformer (GPT)\n",
        "\n",
        "---"
      ],
      "metadata": {
        "id": "GmDQN4FP0-Yc"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "This is a basic generative pre-trained transformer (GPT) used to generate text. As David Foster states in his book [Generative Deep Learning - 2nd Edition](https://www.amazon.com/Generative-Deep-Learning-Teaching-Machines/dp/1098134184/ref=asc_df_1098134184/?tag=hyprod-20&linkCode=df0&hvadid=632163212339&hvpos=&hvnetw=g&hvrand=18153244410548753671&hvpone=&hvptwo=&hvqmt=&hvdev=c&hvdvcmdl=&hvlocint=&hvlocphy=9006620&hvtargid=pla-1852750701094&psc=1&mcid=e1431dcee2ae37808a9d62e277627154&gclid=CjwKCAiAp5qsBhAPEiwAP0qeJhrNONFHcyhwaOsNRPnS2wgtXzPPZGM9Wm7zaXDn87j1IGA0UT5sfRoCz7cQAvD_BwE):\n",
        "\n",
        "\n",
        "> In this chapter, we are going to delve into how modern text generation models make use of the *Transformer* architecture to reach state-of-the-art performance on text generation challenges. In particular, we will explore a type of autoregressive model known as the generative pre-trained transformer (GPT), *which powers OpenAI’s GPT-4 model*, widely considered to be the current state of the art for text generation.\n",
        "\n",
        "Furthermore, as the authors of [Improving Language Understanding\n",
        "by Generative Pre-Training](https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf) state:\n",
        "\n",
        "> Our training procedure consists of two stages. The first stage is learning a high-capacity language model on a large corpus of text. This is followed by a fine-tuning stage, where we adapt the model to a discriminative task with labeled data.\n",
        "\n",
        "Accordingly, *this notebook will focus on the first stage* of the OpenAI paper in which unsupervised learning was used to pre-train a general-purpose *decoder transformer.* This includes implementing the following components found in (most) state of the art GPTs:\n",
        "\n",
        "*   Multi-Head Attention\n",
        "*   Causal Masking\n",
        "*   Transformer Block\n",
        "*   Positional Encoding\n",
        "\n",
        "I will be using the [reddit dataset](https://www.tensorflow.org/datasets/overview) to train the language model. This corpus contains preprocessed posts from the Reddit dataset. The dataset consists of 3,848,330 posts with an average length of 270 words for content, and 28 words for the summary.\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "j7GdunMU1Igt"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "ClP9KseeNHaZ"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import json\n",
        "import re\n",
        "import string\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import layers, models, callbacks, losses, preprocessing\n",
        "import tensorflow_datasets as tfds\n",
        "from tensorflow.keras.layers import TextVectorization"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "These are the default hyperparameters (that can be changed) used to train\n",
        "the language model where:\n",
        "\n",
        "VOCAB_SIZE: is the number of words that make up the dictionary\n",
        "MAX_LEN: size of the max sentence length that can be embedded\n",
        "KEY_DIM: embedding size for the query and key tensors found in the attention head\n",
        "N_HEADS: number of attention heads\n",
        "N: number of transformer blocks to use\n",
        "FEED_FORWARD_DIM: number of hidden units in the feed-forward layers of the transformer block\n",
        "EPOCHS: the total number of times to cycle through the entire training set\n",
        "'''\n",
        "\n",
        "VOCAB_SIZE = 30000\n",
        "MAX_LEN = 80\n",
        "EMBEDDING_DIM = 1024\n",
        "KEY_DIM = 1024\n",
        "N_HEADS = 12\n",
        "N = 1\n",
        "FEED_FORWARD_DIM =  2048\n",
        "SEED = 42\n",
        "BATCH_SIZE = 64\n",
        "EPOCHS = 5"
      ],
      "metadata": {
        "id": "VH2-ASv0YFxa"
      },
      "execution_count": 90,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Loading, Pre and Post-processing Reddit Dataset\n",
        "\n",
        "Basically I followed the great tutorial [Load Text](https://www.tensorflow.org/tutorials/load_data/text#download_more_datasets_using_tensorflow_datasets_tfds) to load the reddit dataset. As detailed below after the dataset was downloaded and extracted it was saved in the directory /root/tensorflow_datasets/reddit/1.0.0. If you are using Colab to download this dataset you will need to have at least 51 GB of System RAM otherwise you will not be able to download the dataset with the tfds.load function.\n",
        "\n",
        "I decided to sample 1 million posts from the normalized body of the Reddit dataset and then batch the sample with the BATCH_SIZE hyperparamter and randomly shuffle the elements of the dataset with a buffer size of 1000. The raw text was converted into tokens using  [TextVectorization](https://www.tensorflow.org/api_docs/python/tf/keras/layers/TextVectorization). After doing so the vocab which maps tokens to words was generated and the training set was created by shifting the Y dataset by one token."
      ],
      "metadata": {
        "id": "623X_BAxnZY_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ds = tfds.load('reddit')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168,
          "referenced_widgets": [
            "82453ec5ee5b4a7ab3f1b5d8c7d137ff",
            "e3b255ca57054758aee7088affa72937",
            "eddb55cecd7a4bff9d02ba65cfb9ece6",
            "0ad1af3da4034b4bb4933de754506bc1",
            "c23fae9292004be1811a2f4ce0525321",
            "5d4cc58dac80429b9adc78127c926d4c",
            "28b90497cc534cc4b1451a625ecd55bb",
            "77569cecd1934e30a93136bc6dbcc4b0",
            "ea804e26dd3e46e1ab251f8d3cb0cdc3",
            "7e774a6fac934eb88beda0716a028683",
            "03bf5613823b46edbfaa1c9d3f7d2a1c",
            "cf2a404c8b9c49248fca3274d53d6272",
            "76448977c9f844be9e10e1a30d6ea250",
            "c2ec16612f2c4911ad723656d92eb17b",
            "c0e736d4625d4b3ea27c0f4a3b833608",
            "385d17961de24c099ca93a286c538c7c",
            "512cc2fa88264041a96dfc83759b3c7c",
            "816d37bccfab4fabb499cbf2e16251b2",
            "d6de34d454b84564a8b107db6f2712ad",
            "b61044f88ea84e0f8981750c0eaaff3b",
            "c87d9991f6d14bd7a2f019ddff9fbdc8",
            "ce79337495ca4ed9a0c2b3e40b2132d7",
            "ecc68fc2ddb7453d84f6f08e49c0fab1",
            "485dfe1b43db4807b472f1be7131f0d7",
            "a095a043c0bd45fa871071c38c63b024",
            "53bd66fe90244bf1809e6aa0f793d7b2",
            "ce744d3b0d3c436b98a870428df460c8",
            "cdf5e310eb2d45089cd4edce932452ec",
            "72d03b0862fc41c1872514813014f10b",
            "f24d1bbe1bd1491fbbc19cde92e8565f",
            "6410ef9362f646ababb252f9b5da979c",
            "f90cdcefe17e49ff91be7d3dfd56b1cd",
            "76ef1beee8544edbbf1b57cb1db02ec9",
            "048ad5ecd7c44ae3b37977268a998628",
            "61a6c8584c324be593f598a4d6cec70d",
            "31c20e32f7bd4442ae95a3a7bdee0a42",
            "68df84bada634adf84bfc9e833d098a6",
            "8ad067436f764ec9a6d42aa8779323a4",
            "1eae80a5ba4541bea2dcfb3c5fcf919e",
            "366dd58c884046b19406e3ef74fb2a5e",
            "c54b4aaafd074a4887c93e04f8bae41b",
            "818c22457f7f4f93b261426b91dfde7a",
            "872efebaa375427eaef3139980e84bdd",
            "398680c0b9044f9e86a50a221a0f5795",
            "5dfecdd93bb34eb3b85415a348972b56",
            "dba7003f50ca4c9e8f3187c1a2faf9a1",
            "6ba1f78f752e42f984b8592f51ee19d3",
            "41354c8686c64bbb9ded756d385c0095",
            "e8a5243f3611486b8df334da1b74c7d2",
            "c78578e655c84d7dbfa6c4cf40a6cb11",
            "6340e4c9c9e04bc5b13f0a8f0dfbff50",
            "912399ebb8a9464084cc624839d443da",
            "9c60fb41aa104ab292aaf826205db5fa",
            "745af5b58389489f91664c1fbfe1502d",
            "a6767891668942b78f831e0037e3bb7c",
            "f2cfc549ddca4ef1a0e8cad7d23c0fe8",
            "dbbfd166f76b4a8198c55a3880ad299f",
            "4f2d0ed3a5174815b4447f555521aedf",
            "e3ac88f1e5424faf85336e1bb9295c36",
            "0f7054aacd1449ff83a1bf15672219e1",
            "d164507a4f824a4d96c177caaa3698fb",
            "b6b8f297f6c847e4b6179901033e0f13",
            "f000ecd3bf524fa3bb4c0a05f9301936",
            "6b9ca5a683274f78aa14697881ebd9f4",
            "8eee3ad39c524c5085dd85c704215dbd",
            "9c90cc55c0284d119afc4b3e052e61a6"
          ]
        },
        "id": "CVW-M6RWYaf0",
        "outputId": "4577750b-fe37-41af-d013-6517bda27437"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset 2.93 GiB (download: 2.93 GiB, generated: 18.09 GiB, total: 21.01 GiB) to /root/tensorflow_datasets/reddit/1.0.0...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Dl Completed...: 0 url [00:00, ? url/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "82453ec5ee5b4a7ab3f1b5d8c7d137ff"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Dl Size...: 0 MiB [00:00, ? MiB/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cf2a404c8b9c49248fca3274d53d6272"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Extraction completed...: 0 file [00:00, ? file/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ecc68fc2ddb7453d84f6f08e49c0fab1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating splits...:   0%|          | 0/1 [00:00<?, ? splits/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "048ad5ecd7c44ae3b37977268a998628"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train examples...:   0%|          | 0/3848330 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5dfecdd93bb34eb3b85415a348972b56"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Shuffling /root/tensorflow_datasets/reddit/1.0.0.incompleteXKQW6K/reddit-train.tfrecord*...:   0%|          | …"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f2cfc549ddca4ef1a0e8cad7d23c0fe8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset reddit downloaded and prepared to /root/tensorflow_datasets/reddit/1.0.0. Subsequent calls will reuse this data.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x=ds[\"train\"]"
      ],
      "metadata": {
        "id": "dJEPqtJASwFg"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_text = x.map(lambda text: text['normalizedBody'])"
      ],
      "metadata": {
        "id": "5ZTCdjSgUGop"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_text_2 = train_text.take(1000000).batch(BATCH_SIZE).shuffle(1000) # take 1mil samples from approx. 4 million Reddit comments for training"
      ],
      "metadata": {
        "id": "FDkGsRY84oG6"
      },
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vectorize_layer = TextVectorization(max_tokens=VOCAB_SIZE, output_mode='int',output_sequence_length=MAX_LEN+ 1)"
      ],
      "metadata": {
        "id": "K4bbU57-gyyd"
      },
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vectorize_layer.adapt(train_text_2)"
      ],
      "metadata": {
        "id": "xnNnV54HhC-w"
      },
      "execution_count": 73,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vocab = vectorize_layer.get_vocabulary()"
      ],
      "metadata": {
        "id": "dQnHQ8QfBSdL"
      },
      "execution_count": 74,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Display some token:word mappings\n",
        "for i, word in enumerate(vocab[:20]):\n",
        "    print(f\"{i}: {word}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HktOYO6XBUqQ",
        "outputId": "2d7ff3c8-119d-4b19-d69c-53c46a8e3eda"
      },
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0: \n",
            "1: [UNK]\n",
            "2: the\n",
            "3: i\n",
            "4: to\n",
            "5: and\n",
            "6: a\n",
            "7: of\n",
            "8: that\n",
            "9: in\n",
            "10: it\n",
            "11: my\n",
            "12: is\n",
            "13: for\n",
            "14: was\n",
            "15: you\n",
            "16: with\n",
            "17: but\n",
            "18: me\n",
            "19: this\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def prepare_inputs(text):\n",
        "    text = tf.expand_dims(text, -1)\n",
        "    tokenized_sentences = vectorize_layer(text)\n",
        "    x = tokenized_sentences[:, :-1]\n",
        "    y = tokenized_sentences[:, 1:]\n",
        "    return x, y\n",
        "\n",
        "train_ds = train_text_2.map(prepare_inputs)\n",
        "train_ds = train_ds.cache().prefetch(tf.data.AUTOTUNE)"
      ],
      "metadata": {
        "id": "lfSKUcz7nNuQ"
      },
      "execution_count": 76,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "example_input_output = train_ds.take(1).get_single_element()"
      ],
      "metadata": {
        "id": "r-7pJOSwAtJ-"
      },
      "execution_count": 77,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Example Input\n",
        "example_input_output[0][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z-hovGmdA5Ob",
        "outputId": "ca247bd0-5f94-478d-9b96-e309e38da40c"
      },
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(80,), dtype=int64, numpy=\n",
              "array([    3,    21,    66, 13429,   497,     2,   357,   129,     5,\n",
              "        1583,  2587,    73,  4713,     9,    11,  2627,     4,    47,\n",
              "          11,   302,   346,   248,  3745,  9232,    37,    21,   519,\n",
              "        1209,     2,   461,     7,    57,    37,   114,    10,    48,\n",
              "         138,     5,     3,    21,    66,  5916,     5,   599,   110,\n",
              "          73,    85, 13429,   416,  1828,    50,    66,  1468,   634,\n",
              "         246,   135,    11,  3745,  2180,    30,     2,  7351,    13,\n",
              "        6032,     3,    21,   274,  1214,     5,   958,    17,     3,\n",
              "         113,    47,    91,   616,     8,   534,     2,   201])>"
            ]
          },
          "metadata": {},
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Example Output (shifted by one token)\n",
        "example_input_output[1][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cs9Enw_mBBI7",
        "outputId": "13c3442a-1241-4b23-9b51-963180b5dd08"
      },
      "execution_count": 79,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(80,), dtype=int64, numpy=\n",
              "array([   21,    66, 13429,   497,     2,   357,   129,     5,  1583,\n",
              "        2587,    73,  4713,     9,    11,  2627,     4,    47,    11,\n",
              "         302,   346,   248,  3745,  9232,    37,    21,   519,  1209,\n",
              "           2,   461,     7,    57,    37,   114,    10,    48,   138,\n",
              "           5,     3,    21,    66,  5916,     5,   599,   110,    73,\n",
              "          85, 13429,   416,  1828,    50,    66,  1468,   634,   246,\n",
              "         135,    11,  3745,  2180,    30,     2,  7351,    13,  6032,\n",
              "           3,    21,   274,  1214,     5,   958,    17,     3,   113,\n",
              "          47,    91,   616,     8,   534,     2,   201,    57])>"
            ]
          },
          "metadata": {},
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Creating Casual Mask\n",
        "\n",
        "As David Foster states in his book [Generative Deep Learning - 2nd Edition](https://www.amazon.com/Generative-Deep-Learning-Teaching-Machines/dp/1098134184/ref=asc_df_1098134184/?tag=hyprod-20&linkCode=df0&hvadid=632163212339&hvpos=&hvnetw=g&hvrand=18153244410548753671&hvpone=&hvptwo=&hvqmt=&hvdev=c&hvdvcmdl=&hvlocint=&hvlocphy=9006620&hvtargid=pla-1852750701094&psc=1&mcid=e1431dcee2ae37808a9d62e277627154&gclid=CjwKCAiAp5qsBhAPEiwAP0qeJhrNONFHcyhwaOsNRPnS2wgtXzPPZGM9Wm7zaXDn87j1IGA0UT5sfRoCz7cQAvD_BwE):\n",
        "\n",
        "\n",
        "> [W]e want our GPT model to be able to handle a group of query vectors in parallel (i.e., a matrix)...we need to apply a mask to the query/key dot product, to avoid information from future words leaking through. This is known as *causal masking.*\n",
        "\n",
        "The casual_attention_mask function implements *causal masking*. The function takes in the batch_size, the source and destination sequence lengths, and the data type. In step 1 the destination sequence length's shape is changed from (seq_length,) to (seq_length, 1) while in step 2 the source sequence lengh's shape is not changed. In step 3 and 4 a matrix of booleans of shape (seq_length, seq_length) is created.In step 5 the mask is reshaped to have a 3D shape of (Batch_size,seq_length seq_length). In steps 6 and 7 a concatenation is done to create a 1D array with the values of [batch_size, 1, 1] and tiled to produce the final 3D casual mask of shape (batch_size, seq_length, seq_length).\n",
        "\n"
      ],
      "metadata": {
        "id": "14qRUAh5pk1o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def causal_attention_mask(batch_size, n_dest, n_src, dtype):\n",
        "    i = tf.range(n_dest)[:, None] # 1\n",
        "    j = tf.range(n_src)    # 2\n",
        "    m = i >= j - n_src + n_dest # 3\n",
        "    mask = tf.cast(m, dtype) # 4\n",
        "    mask = tf.reshape(mask, [1, n_dest, n_src]) # 5\n",
        "    mult = tf.concat(\n",
        "        [tf.expand_dims(batch_size, -1), tf.constant([1, 1], dtype=tf.int32)], 0\n",
        "    ) # 6\n",
        "    return tf.tile(mask, mult) # 7"
      ],
      "metadata": {
        "id": "sNnFppSTmYXH"
      },
      "execution_count": 85,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Creating Transformer Block\n",
        "\n",
        "As David Foster states in his book [Generative Deep Learning - 2nd Edition](https://www.amazon.com/Generative-Deep-Learning-Teaching-Machines/dp/1098134184/ref=asc_df_1098134184/?tag=hyprod-20&linkCode=df0&hvadid=632163212339&hvpos=&hvnetw=g&hvrand=18153244410548753671&hvpone=&hvptwo=&hvqmt=&hvdev=c&hvdvcmdl=&hvlocint=&hvlocphy=9006620&hvtargid=pla-1852750701094&psc=1&mcid=e1431dcee2ae37808a9d62e277627154&gclid=CjwKCAiAp5qsBhAPEiwAP0qeJhrNONFHcyhwaOsNRPnS2wgtXzPPZGM9Wm7zaXDn87j1IGA0UT5sfRoCz7cQAvD_BwE):\n",
        "\n",
        "\n",
        ">  A Transformer block is a single component within a Transformer that applies some *skip connections*, *feed-forward (dense) layers*, and *layer normalization* around the *multihead attention layer*...[where] [i]n Keras, we can build a *MultiHeadAttention layer that concatenates the output from multiple attention heads*, allowing each to learn a distinct attention mechanism so that the layer as a whole can learn more complex relationships.\n",
        "\n",
        "The Transformer block is one of the most important aspects of today's language models. Before Transformer Blocks were introduced the previous method of using bidirectional LSTMs was the state of the art for text generation. It was not til the Transformer block was introduced that allowed for large language models to take happen. In the paper [Improving Language Understanding\n",
        "by Generative Pre-Training](https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf) the authors used 12 of these transformers for each layer (i.e. a 12-layer decoder-only transformer). To speed up training I used 1 Transformer and most of the base model parameters as detailed by [Attention Is All You Need](https://arxiv.org/pdf/1706.03762.pdf) for the other hyperparamters.  \n"
      ],
      "metadata": {
        "id": "Li6MajyHwHW-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class TransformerBlock(layers.Layer):\n",
        "    def __init__(self, num_heads, key_dim, embed_dim, ff_dim, dropout_rate=0.1):# defining sublayers of the TransformerBlock\n",
        "        super(TransformerBlock, self).__init__()\n",
        "        self.num_heads = num_heads\n",
        "        self.key_dim = key_dim\n",
        "        self.embed_dim = embed_dim\n",
        "        self.ff_dim = ff_dim\n",
        "        self.dropout_rate = dropout_rate\n",
        "        self.attn = layers.MultiHeadAttention(\n",
        "            num_heads, key_dim, output_shape=embed_dim\n",
        "        ) # multi-head Attention layer\n",
        "        self.dropout_1 = layers.Dropout(self.dropout_rate)\n",
        "        self.ln_1 = layers.LayerNormalization(epsilon=1e-6)\n",
        "        self.ffn_1 = layers.Dense(self.ff_dim, activation=\"relu\")\n",
        "        self.ffn_2 = layers.Dense(self.embed_dim)\n",
        "        self.dropout_2 = layers.Dropout(self.dropout_rate)\n",
        "        self.ln_2 = layers.LayerNormalization(epsilon=1e-6)\n",
        "\n",
        "    def call(self, inputs):\n",
        "        input_shape = tf.shape(inputs)\n",
        "        batch_size = input_shape[0]\n",
        "        seq_len = input_shape[1]\n",
        "        causal_mask = causal_attention_mask(\n",
        "            batch_size, seq_len, seq_len, tf.bool\n",
        "        ) # casual attention mask\n",
        "        attention_output, attention_scores = self.attn(\n",
        "            inputs,\n",
        "            inputs,\n",
        "            attention_mask=causal_mask,\n",
        "            return_attention_scores=True,\n",
        "        ) # multi-head Attention\n",
        "        attention_output = self.dropout_1(attention_output)\n",
        "        out1 = self.ln_1(inputs + attention_output) # 1st skip connection and layer normilizations\n",
        "        ffn_1 = self.ffn_1(out1) # feed-forward network\n",
        "        ffn_2 = self.ffn_2(ffn_1)\n",
        "        ffn_output = self.dropout_2(ffn_2)\n",
        "        return self.ln_2(out1 + ffn_output)# 2nd skip connection and layer normilizations"
      ],
      "metadata": {
        "id": "5BmH3Wwo1buR"
      },
      "execution_count": 86,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Creating Token and Positional Embedding\n",
        "\n",
        "Because the orderining of words in a sentence is important for semantic meaning and predicting the next word in a sequence, a token and position embedding was used.\n",
        "\n",
        "As David Foster states in his book [Generative Deep Learning - 2nd Edition](https://www.amazon.com/Generative-Deep-Learning-Teaching-Machines/dp/1098134184/ref=asc_df_1098134184/?tag=hyprod-20&linkCode=df0&hvadid=632163212339&hvpos=&hvnetw=g&hvrand=18153244410548753671&hvpone=&hvptwo=&hvqmt=&hvdev=c&hvdvcmdl=&hvlocint=&hvlocphy=9006620&hvtargid=pla-1852750701094&psc=1&mcid=e1431dcee2ae37808a9d62e277627154&gclid=CjwKCAiAp5qsBhAPEiwAP0qeJhrNONFHcyhwaOsNRPnS2wgtXzPPZGM9Wm7zaXDn87j1IGA0UT5sfRoCz7cQAvD_BwE):\n",
        "\n",
        "> [I]n the multihead attention layer, there is nothing that cares about the ordering of the keys. The dot product between each key and the query is calculated in parallel, not sequentially...We use a technique called positional encoding when creating the inputs to the initial Transformer block. Instead of only encoding each token using a token embedding, we also encode the position of the token, using a position embedding."
      ],
      "metadata": {
        "id": "fsucmRfPB-vR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class TokenAndPositionEmbedding(layers.Layer):\n",
        "    def __init__(self, max_len, vocab_size, embed_dim):\n",
        "        super(TokenAndPositionEmbedding, self).__init__()\n",
        "        self.max_len = max_len\n",
        "        self.vocab_size = vocab_size\n",
        "        self.embed_dim = embed_dim\n",
        "        self.token_emb = layers.Embedding(\n",
        "            input_dim=vocab_size, output_dim=embed_dim\n",
        "        )\n",
        "        self.pos_emb = layers.Embedding(input_dim=max_len, output_dim=embed_dim)\n",
        "\n",
        "    def call(self, x):\n",
        "        maxlen = tf.shape(x)[-1]\n",
        "        positions = tf.range(start=0, limit=maxlen, delta=1)\n",
        "        positions = self.pos_emb(positions)\n",
        "        x = self.token_emb(x)\n",
        "        return x + positions"
      ],
      "metadata": {
        "id": "NGpZZMoTGr8j"
      },
      "execution_count": 87,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Constructing the Model"
      ],
      "metadata": {
        "id": "z92Pu30fG_Ap"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "inputs = layers.Input(shape=(None,), dtype=tf.int32)\n",
        "# Token and POsitional Embedding\n",
        "x = TokenAndPositionEmbedding(MAX_LEN, VOCAB_SIZE, EMBEDDING_DIM)(inputs)\n",
        "# 6-layer Transformer decoder\n",
        "for _ in range(N):\n",
        "  x = TransformerBlock(N_HEADS, KEY_DIM, EMBEDDING_DIM, FEED_FORWARD_DIM)(x)\n",
        "# output\n",
        "outputs = layers.Dense(VOCAB_SIZE, activation=\"softmax\")(x)\n",
        "gpt = models.Model(inputs=inputs, outputs=outputs)\n",
        "gpt.compile(\"adam\", loss=[losses.SparseCategoricalCrossentropy(), None])\n",
        "gpt.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mcWQER_SHEg3",
        "outputId": "01fbca50-cf0a-4a94-8341-4639cea4654c"
      },
      "execution_count": 92,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_6\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " input_7 (InputLayer)        [(None, None)]            0         \n",
            "                                                                 \n",
            " token_and_position_embeddi  (None, None, 1024)        30801920  \n",
            " ng_6 (TokenAndPositionEmbe                                      \n",
            " dding)                                                          \n",
            "                                                                 \n",
            " transformer_block_24 (Tran  (None, None, 1024)        54571008  \n",
            " sformerBlock)                                                   \n",
            "                                                                 \n",
            " dense_56 (Dense)            (None, None, 30000)       30750000  \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 116122928 (442.97 MB)\n",
            "Trainable params: 116122928 (442.97 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Training GPT\n",
        "\n",
        "I trained GPT using one V100 GPU for five epochs. It took approximately 8 hours to train the model during which the intial Sparse Categorical Cross-Entropy loss of 7 was able to be minimized to 4. Obviously this end result is far from being globally optimal and reaching a global minimum of 0. Because of this fact the text generated by the model (as seen in the next section) is not always semantically correct.\n",
        "\n",
        "In order to easily improve the text generation process some of the following changes should be tried:\n",
        "\n",
        "1.   use the full Reddit dataset instead of a sub-sample as I did\n",
        "2.   increase the VOCAB_SIZE\n",
        "3.   *train for at least 100 epochs*\n",
        "4.   increase the number of Transformer blocks and the number of attention heads\n",
        "5.   choose a different learning rate for the Adam algorithm\n",
        ""
      ],
      "metadata": {
        "id": "ukI6yNDIvsbB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a TextGenerator checkpoint\n",
        "class TextGenerator(callbacks.Callback):\n",
        "    def __init__(self, index_to_word, top_k=10):\n",
        "        self.index_to_word = index_to_word\n",
        "        self.word_to_index = {\n",
        "            word: index for index, word in enumerate(index_to_word)\n",
        "        }\n",
        "\n",
        "    def sample_from(self, probs, temperature):\n",
        "        probs = probs ** (1 / temperature)\n",
        "        probs = probs / np.sum(probs)\n",
        "        return np.random.choice(len(probs), p=probs), probs\n",
        "\n",
        "    def generate(self, start_prompt, max_tokens, temperature):\n",
        "        start_tokens = [\n",
        "            self.word_to_index.get(x, 1) for x in start_prompt.split()\n",
        "        ]\n",
        "        sample_token = None\n",
        "        info = []\n",
        "        while len(start_tokens) < max_tokens and sample_token != 0:\n",
        "            x = np.array([start_tokens])\n",
        "            y = self.model.predict(x, verbose=0)\n",
        "            sample_token, probs = self.sample_from(y[0][-1], temperature)\n",
        "            info.append(\n",
        "                {\n",
        "                    \"prompt\": start_prompt,\n",
        "                    \"word_probs\": probs,\n",
        "                }\n",
        "            )\n",
        "            start_tokens.append(sample_token)\n",
        "            start_prompt = start_prompt + \" \" + self.index_to_word[sample_token]\n",
        "        print(f\"\\ngenerated text:\\n{start_prompt}\\n\")\n",
        "        return info\n",
        "\n",
        "    def on_epoch_end(self, epoch, logs=None):\n",
        "        self.generate(\"did you\", max_tokens=20, temperature=1.0)\n",
        "\n",
        "# Tokenize starting prompt\n",
        "text_generator = TextGenerator(vocab)\n",
        "gpt.fit(\n",
        "    train_ds,\n",
        "    epochs=EPOCHS,\n",
        "    callbacks=[text_generator],\n",
        ")\n",
        "# Save the final model\n",
        "gpt.save(\"/content/gpt\")"
      ],
      "metadata": {
        "id": "fqhOU54SLS2z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Generating Text\n",
        "\n"
      ],
      "metadata": {
        "id": "Mk5tNdMdG-9P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# generating text using the prompt: should I go\n",
        "text_generator.generate(\"should I go\", max_tokens=20, temperature=1.0)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UIw_GargDkpH",
        "outputId": "c9900872-e158-4053-fd43-c8ea5cfb3e6c"
      },
      "execution_count": 96,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "generated text:\n",
            "should I go down let away with mild munchies and hello im a real lightweight and looking for a lightweight\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'prompt': 'should I go',\n",
              "  'word_probs': array([3.9252677e-04, 2.4097566e-02, 7.1504386e-03, ..., 1.2247952e-07,\n",
              "         2.1028454e-07, 1.1844580e-07], dtype=float32)},\n",
              " {'prompt': 'should I go down',\n",
              "  'word_probs': array([1.5663232e-03, 1.8423527e-02, 7.0624501e-02, ..., 4.6719705e-07,\n",
              "         2.5653665e-07, 1.5967171e-07], dtype=float32)},\n",
              " {'prompt': 'should I go down let',\n",
              "  'word_probs': array([2.1322555e-06, 9.7094597e-03, 3.7058368e-02, ..., 1.5525318e-08,\n",
              "         3.6275036e-08, 7.6346582e-09], dtype=float32)},\n",
              " {'prompt': 'should I go down let away',\n",
              "  'word_probs': array([3.8629423e-05, 3.1404667e-02, 6.4428546e-02, ..., 2.7698659e-08,\n",
              "         5.3843504e-08, 3.9898399e-08], dtype=float32)},\n",
              " {'prompt': 'should I go down let away with',\n",
              "  'word_probs': array([3.1660475e-05, 2.9596783e-02, 2.1106137e-01, ..., 2.2095466e-07,\n",
              "         1.5075646e-07, 6.4391621e-08], dtype=float32)},\n",
              " {'prompt': 'should I go down let away with mild',\n",
              "  'word_probs': array([1.3377670e-05, 1.3419777e-01, 4.4285282e-03, ..., 8.5880092e-06,\n",
              "         1.0465599e-06, 9.2442917e-07], dtype=float32)},\n",
              " {'prompt': 'should I go down let away with mild munchies',\n",
              "  'word_probs': array([1.8811961e-04, 1.8034020e-02, 1.6273420e-02, ..., 6.2276780e-07,\n",
              "         2.0340201e-06, 8.2538868e-08], dtype=float32)},\n",
              " {'prompt': 'should I go down let away with mild munchies and',\n",
              "  'word_probs': array([4.1557405e-06, 2.4552921e-02, 9.6323024e-03, ..., 2.3379329e-07,\n",
              "         1.8464252e-06, 5.4472608e-08], dtype=float32)},\n",
              " {'prompt': 'should I go down let away with mild munchies and hello',\n",
              "  'word_probs': array([5.2165593e-05, 8.4496960e-02, 9.5529556e-03, ..., 9.8473706e-08,\n",
              "         1.6006588e-07, 5.7068110e-07], dtype=float32)},\n",
              " {'prompt': 'should I go down let away with mild munchies and hello im',\n",
              "  'word_probs': array([6.4272535e-06, 1.4345978e-02, 1.0348479e-02, ..., 1.6322074e-08,\n",
              "         3.4057432e-08, 1.5993955e-08], dtype=float32)},\n",
              " {'prompt': 'should I go down let away with mild munchies and hello im a',\n",
              "  'word_probs': array([1.6195293e-06, 5.5960178e-02, 6.9604255e-05, ..., 1.8285023e-08,\n",
              "         1.6477056e-06, 1.4109723e-06], dtype=float32)},\n",
              " {'prompt': 'should I go down let away with mild munchies and hello im a real',\n",
              "  'word_probs': array([5.7309267e-06, 1.3774723e-01, 3.5062185e-05, ..., 1.1499954e-08,\n",
              "         7.3456732e-07, 4.4824387e-07], dtype=float32)},\n",
              " {'prompt': 'should I go down let away with mild munchies and hello im a real lightweight',\n",
              "  'word_probs': array([2.4114944e-04, 2.8388212e-02, 4.2330697e-03, ..., 4.8272950e-09,\n",
              "         1.3459568e-08, 8.3711080e-07], dtype=float32)},\n",
              " {'prompt': 'should I go down let away with mild munchies and hello im a real lightweight and',\n",
              "  'word_probs': array([6.9902688e-07, 2.1044938e-02, 5.1127225e-03, ..., 4.4711652e-09,\n",
              "         3.6775873e-08, 8.6825423e-08], dtype=float32)},\n",
              " {'prompt': 'should I go down let away with mild munchies and hello im a real lightweight and looking',\n",
              "  'word_probs': array([5.1028806e-06, 2.3015747e-03, 1.7783129e-04, ..., 6.6538930e-10,\n",
              "         9.1262509e-09, 3.1250508e-08], dtype=float32)},\n",
              " {'prompt': 'should I go down let away with mild munchies and hello im a real lightweight and looking for',\n",
              "  'word_probs': array([5.31914338e-06, 8.00962746e-03, 1.13392742e-02, ...,\n",
              "         1.44319205e-08, 8.47131432e-09, 1.45342893e-09], dtype=float32)},\n",
              " {'prompt': 'should I go down let away with mild munchies and hello im a real lightweight and looking for a',\n",
              "  'word_probs': array([2.6530740e-06, 4.4299055e-02, 6.0073697e-05, ..., 4.1821391e-08,\n",
              "         1.4938807e-06, 7.3759423e-08], dtype=float32)}]"
            ]
          },
          "metadata": {},
          "execution_count": 96
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# generating text using the prompt: this guy\n",
        "text_generator.generate(\"this guy\", max_tokens=20, temperature=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CwKKC20KEhWc",
        "outputId": "704bf8f1-52ec-4a53-ad39-6ed523940b08"
      },
      "execution_count": 98,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "generated text:\n",
            "this guy has never experienced a really big breakup for the first week of university he came out of nowhere\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'prompt': 'this guy',\n",
              "  'word_probs': array([4.17970004e-05, 1.12000285e-02, 1.00543676e-03, ...,\n",
              "         1.57740523e-08, 2.12044256e-07, 1.86644940e-08], dtype=float32)},\n",
              " {'prompt': 'this guy has',\n",
              "  'word_probs': array([1.8237852e-06, 4.9863472e-03, 1.2692408e-02, ..., 5.0823417e-09,\n",
              "         5.2507780e-09, 1.6172116e-08], dtype=float32)},\n",
              " {'prompt': 'this guy has never',\n",
              "  'word_probs': array([3.7056052e-06, 1.7005111e-03, 1.9142291e-04, ..., 2.8955099e-10,\n",
              "         3.3867982e-09, 2.5159232e-09], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced',\n",
              "  'word_probs': array([2.0916112e-05, 9.3607400e-03, 3.1279765e-02, ..., 4.1899160e-09,\n",
              "         4.2624958e-08, 1.8873907e-07], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced a',\n",
              "  'word_probs': array([1.8404297e-06, 2.8257452e-02, 3.7658425e-05, ..., 3.5909318e-09,\n",
              "         5.4782652e-07, 2.5327545e-06], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced a really',\n",
              "  'word_probs': array([5.8553519e-06, 2.0628784e-02, 2.6965744e-04, ..., 1.2991196e-09,\n",
              "         3.4576223e-07, 1.2204497e-07], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced a really big',\n",
              "  'word_probs': array([1.7200424e-06, 2.4539281e-02, 1.2370148e-04, ..., 5.6302167e-09,\n",
              "         3.2452913e-08, 2.7087475e-07], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced a really big breakup',\n",
              "  'word_probs': array([7.0210983e-05, 1.6720347e-03, 7.6076342e-03, ..., 2.5036342e-10,\n",
              "         2.1638838e-08, 8.8840387e-08], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced a really big breakup for',\n",
              "  'word_probs': array([1.7198744e-06, 2.4543167e-03, 5.5426236e-02, ..., 2.9350697e-10,\n",
              "         8.1042542e-09, 2.2303448e-08], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced a really big breakup for the',\n",
              "  'word_probs': array([2.6212663e-07, 2.6742832e-03, 1.3139359e-04, ..., 5.7115168e-10,\n",
              "         3.7906851e-08, 1.6940451e-08], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced a really big breakup for the first',\n",
              "  'word_probs': array([4.2580273e-06, 4.6317148e-04, 7.8849160e-05, ..., 3.0970872e-11,\n",
              "         4.0584500e-09, 1.5331708e-09], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced a really big breakup for the first week',\n",
              "  'word_probs': array([3.3897668e-05, 4.2409386e-04, 1.9506214e-03, ..., 4.9730325e-10,\n",
              "         7.0623805e-09, 1.5652558e-09], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced a really big breakup for the first week of',\n",
              "  'word_probs': array([2.4460985e-07, 4.8548440e-03, 4.8879769e-02, ..., 2.0869528e-10,\n",
              "         5.8895702e-09, 5.9239889e-09], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced a really big breakup for the first week of university',\n",
              "  'word_probs': array([3.9064485e-05, 3.1773669e-03, 8.5679144e-03, ..., 4.4544466e-09,\n",
              "         4.8055615e-08, 2.2960567e-08], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced a really big breakup for the first week of university he',\n",
              "  'word_probs': array([1.4912806e-06, 2.4086167e-03, 6.9893584e-05, ..., 8.7190850e-09,\n",
              "         5.6682332e-08, 4.6329416e-09], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced a really big breakup for the first week of university he came',\n",
              "  'word_probs': array([1.2016901e-06, 5.7806855e-04, 6.2961003e-04, ..., 1.8658106e-11,\n",
              "         3.0789753e-09, 5.7025566e-09], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced a really big breakup for the first week of university he came out',\n",
              "  'word_probs': array([6.3342522e-06, 1.1447599e-03, 3.8810812e-03, ..., 1.9288865e-10,\n",
              "         1.9601163e-08, 1.7447107e-08], dtype=float32)},\n",
              " {'prompt': 'this guy has never experienced a really big breakup for the first week of university he came out of',\n",
              "  'word_probs': array([3.6534928e-07, 4.6441755e-03, 1.8786083e-01, ..., 7.9599910e-10,\n",
              "         1.1084671e-08, 1.2583838e-08], dtype=float32)}]"
            ]
          },
          "metadata": {},
          "execution_count": 98
        }
      ]
    }
  ]
}