load_ner_model_from_hf_to_elastic.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": " Load NER from Hugging Face into Elasticsearch",
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "46e37a87884b4bb6ae5120f8a2152995": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_93049e0ffb9e4d64af271461b4e3a356",
              "IPY_MODEL_54dc699466b141c9bc6c27b4860c160e",
              "IPY_MODEL_62eaacf1d0c54d8494f013f959a396ec"
            ],
            "layout": "IPY_MODEL_d92d9bc76ef449a7b002fec279bc164c"
          }
        },
        "93049e0ffb9e4d64af271461b4e3a356": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d5d4272035ec489f881384e51b7ab383",
            "placeholder": "​",
            "style": "IPY_MODEL_33017035f21e4a3584bb86951ef6ede8",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "54dc699466b141c9bc6c27b4860c160e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_07a0b6587a2e4052af2203f25f506171",
            "max": 39,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2c6132ac37624d90a51485a95ab627a1",
            "value": 39
          }
        },
        "62eaacf1d0c54d8494f013f959a396ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_86976b6d8f654503b5daa4498fedb52c",
            "placeholder": "​",
            "style": "IPY_MODEL_57df67fac6274ae9b60567e1352550f2",
            "value": " 39.0/39.0 [00:00&lt;00:00, 995B/s]"
          }
        },
        "d92d9bc76ef449a7b002fec279bc164c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d5d4272035ec489f881384e51b7ab383": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "33017035f21e4a3584bb86951ef6ede8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "07a0b6587a2e4052af2203f25f506171": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2c6132ac37624d90a51485a95ab627a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "86976b6d8f654503b5daa4498fedb52c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "57df67fac6274ae9b60567e1352550f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bac587aca60d4ac2a3f98111b230bd03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_620f86cb69c54168946e7c13d0f873aa",
              "IPY_MODEL_98238564651b42838ee9d8a477039720",
              "IPY_MODEL_fe1c6467cdc440739f8524045accaebd"
            ],
            "layout": "IPY_MODEL_c2e2b98f1991452baf551b5cf81e95b6"
          }
        },
        "620f86cb69c54168946e7c13d0f873aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f6d221d18917448aa4ab7fd398f01c63",
            "placeholder": "​",
            "style": "IPY_MODEL_8e0f4128a37e4ad7a9f8c87ea30b0b00",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "98238564651b42838ee9d8a477039720": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_aa7c0c6df06749288cb968df8867956b",
            "max": 1257,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_63153c51050247d9b5646a6a6b18cc64",
            "value": 1257
          }
        },
        "fe1c6467cdc440739f8524045accaebd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a2219f35345449119c8a0671c6a41eed",
            "placeholder": "​",
            "style": "IPY_MODEL_9c5d35ee2faf46b5b142ed530b0a33d4",
            "value": " 1.26k/1.26k [00:00&lt;00:00, 31.4kB/s]"
          }
        },
        "c2e2b98f1991452baf551b5cf81e95b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f6d221d18917448aa4ab7fd398f01c63": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8e0f4128a37e4ad7a9f8c87ea30b0b00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "aa7c0c6df06749288cb968df8867956b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "63153c51050247d9b5646a6a6b18cc64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a2219f35345449119c8a0671c6a41eed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c5d35ee2faf46b5b142ed530b0a33d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5a5df53f4fda479f871fb445e76ceef6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_647ab851389d4efc8e3e197f33a119ec",
              "IPY_MODEL_94fd836a4156496cb2c0d7d5ccc8bd5b",
              "IPY_MODEL_6f4995647ebb40c683cb7409629253f8"
            ],
            "layout": "IPY_MODEL_7cb2e44a0aa3406586a759273f27f64f"
          }
        },
        "647ab851389d4efc8e3e197f33a119ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4a25cbf6e89a483f93b62b9416d80143",
            "placeholder": "​",
            "style": "IPY_MODEL_e714b9390d224ad2abdd996142584f4e",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "94fd836a4156496cb2c0d7d5ccc8bd5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3fceffe019734418988381a1e3ac52a3",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_dee008a902fb408dac6e03b62e0e7bf7",
            "value": 231508
          }
        },
        "6f4995647ebb40c683cb7409629253f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7d74e313d2d0432a924711d0a44f91d2",
            "placeholder": "​",
            "style": "IPY_MODEL_5adaaae9a3dd413b8e74ce8b26b1565d",
            "value": " 232k/232k [00:00&lt;00:00, 673kB/s]"
          }
        },
        "7cb2e44a0aa3406586a759273f27f64f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4a25cbf6e89a483f93b62b9416d80143": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e714b9390d224ad2abdd996142584f4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3fceffe019734418988381a1e3ac52a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dee008a902fb408dac6e03b62e0e7bf7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7d74e313d2d0432a924711d0a44f91d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5adaaae9a3dd413b8e74ce8b26b1565d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0a636987fd574185b8092b60950a6035": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_71244cee5c3c4f4ca72d063829ef4f1f",
              "IPY_MODEL_c32508c3257a43ab9e08b4184d06dd11",
              "IPY_MODEL_32156e2b1f2a4111a38b2aa025676e92"
            ],
            "layout": "IPY_MODEL_d9848f03b4a543639b63aec58178c738"
          }
        },
        "71244cee5c3c4f4ca72d063829ef4f1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0ac7c7719c724624999bc774e09296eb",
            "placeholder": "​",
            "style": "IPY_MODEL_28c82cbb8a7d4386a973d2f5e87d1602",
            "value": "Downloading (…)cial_tokens_map.json: 100%"
          }
        },
        "c32508c3257a43ab9e08b4184d06dd11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_faba45369def4b1fb2422a95d56baf65",
            "max": 112,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_43d3bad483714bccaaeda45e69ae88ee",
            "value": 112
          }
        },
        "32156e2b1f2a4111a38b2aa025676e92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d7ff56381ab94c85808a195e32dffa12",
            "placeholder": "​",
            "style": "IPY_MODEL_adfcae49f35845e79ea2a9917c08f4fe",
            "value": " 112/112 [00:00&lt;00:00, 4.87kB/s]"
          }
        },
        "d9848f03b4a543639b63aec58178c738": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0ac7c7719c724624999bc774e09296eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "28c82cbb8a7d4386a973d2f5e87d1602": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "faba45369def4b1fb2422a95d56baf65": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "43d3bad483714bccaaeda45e69ae88ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d7ff56381ab94c85808a195e32dffa12": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "adfcae49f35845e79ea2a9917c08f4fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f98ebf3e924048ce9a29ee2bc5884ca2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9e324c25d15249daa4eb680fb4fabaac",
              "IPY_MODEL_af6c2c88c4694654adefdfd6586baea8",
              "IPY_MODEL_209c8c4defb846cf8a73bca233078cd2"
            ],
            "layout": "IPY_MODEL_8586cb3311fb4e1da46e5a39c902b360"
          }
        },
        "9e324c25d15249daa4eb680fb4fabaac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5b1b3218a3d44adfa119ae42ba9adefd",
            "placeholder": "​",
            "style": "IPY_MODEL_3f5727f64e8a449b936e682477b91354",
            "value": "Downloading (…)&quot;pytorch_model.bin&quot;;: 100%"
          }
        },
        "af6c2c88c4694654adefdfd6586baea8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1152adca330c4341b5ba095a6d064aaa",
            "max": 438006919,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2d34f185fc77409cbe5e2feecbf8851c",
            "value": 438006919
          }
        },
        "209c8c4defb846cf8a73bca233078cd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_899101235b69405ea95ba9f12869e600",
            "placeholder": "​",
            "style": "IPY_MODEL_23bf3111d57e42c399ae84e5bf2e9a21",
            "value": " 438M/438M [00:02&lt;00:00, 203MB/s]"
          }
        },
        "8586cb3311fb4e1da46e5a39c902b360": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5b1b3218a3d44adfa119ae42ba9adefd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3f5727f64e8a449b936e682477b91354": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1152adca330c4341b5ba095a6d064aaa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2d34f185fc77409cbe5e2feecbf8851c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "899101235b69405ea95ba9f12869e600": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "23bf3111d57e42c399ae84e5bf2e9a21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f581bdfba88f477d8b8e9317ca0d5070": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4a13764a9ea04ae9b42be117234210f1",
              "IPY_MODEL_06e5b6b643214b3bb83b73e64003ef61",
              "IPY_MODEL_d8b1c932fd714a9bb77e0e23a4fa8c00"
            ],
            "layout": "IPY_MODEL_8d2062873e92461c9e48373126de67bb"
          }
        },
        "4a13764a9ea04ae9b42be117234210f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e745a647d3f54d6fa9c6f6e924017876",
            "placeholder": "​",
            "style": "IPY_MODEL_7644ac4714bc4ed891f275218cb17797",
            "value": "100%"
          }
        },
        "06e5b6b643214b3bb83b73e64003ef61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0cd1f50e92b0418d81608597ed20e194",
            "max": 104,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5a2f0b6243ee4ccfac442a953ebe0553",
            "value": 104
          }
        },
        "d8b1c932fd714a9bb77e0e23a4fa8c00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c78997ab7d4e4123b934bc494a4a4b68",
            "placeholder": "​",
            "style": "IPY_MODEL_7b54dabfdf0046faaa672129a3bf3e66",
            "value": " 104/104 [02:01&lt;00:00,  1.16s/ parts]"
          }
        },
        "8d2062873e92461c9e48373126de67bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e745a647d3f54d6fa9c6f6e924017876": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7644ac4714bc4ed891f275218cb17797": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0cd1f50e92b0418d81608597ed20e194": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5a2f0b6243ee4ccfac442a953ebe0553": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c78997ab7d4e4123b934bc494a4a4b68": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7b54dabfdf0046faaa672129a3bf3e66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jeffvestal/elastic_jupyter_notebooks/blob/main/load_ner_model_from_hf_to_elastic.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Loading a NER Model from Hugging Face into Elasticsearch\n",
        "\n",
        "This code will show you how to load a Named Entity Recognition (NER) model into elasticsearch.\n",
        "\n",
        "First we will configure our python environment\n",
        "\n",
        "1. Set up our python environment\n",
        "2. Load the chosed model from HF into Elasticsearch\n",
        "3. Deploy and start the model on ML nodes\n",
        "4. Test using the NER model\n",
        "\n",
        "\n",
        "### Elastic version support\n",
        "Requires Elastic version 8.0+ with a platinum or enterprise license (or trial license)\n",
        "\n",
        "You can set up a [free trial elasticsearch Deployment in Elastic Cloud](https://cloud.elastic.co/registration)."
      ],
      "metadata": {
        "id": "6xoLDtS_6Df1"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "---\n",
        "# Setup\n",
        "---\n",
        "---\n",
        "This section will set up the python environment with the required libraries"
      ],
      "metadata": {
        "id": "DgxCKQS7mCZw"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Install and import required python libraries"
      ],
      "metadata": {
        "id": "Ly1f1P-l9ri8"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Elastic uses the [eland python library](https://github.com/elastic/eland) to download modesl from Hugging Face hub and load them into elasticsearch"
      ],
      "metadata": {
        "id": "MJAb_8zlPFhQ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rUedSzQW9FIF"
      },
      "outputs": [],
      "source": [
        "pip install eland"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pip install elasticsearch"
      ],
      "metadata": {
        "id": "NK3Wx1I199yB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pip install transformers"
      ],
      "metadata": {
        "id": "cEfiiFXakzdP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pip install sentence_transformers"
      ],
      "metadata": {
        "id": "I20mDmJboKZw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pip install torch==1.11"
      ],
      "metadata": {
        "id": "uqcpWrbkBEB9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pathlib import Path\n",
        "from eland.ml.pytorch import PyTorchModel\n",
        "from eland.ml.pytorch.transformers import TransformerModel\n",
        "from elasticsearch import Elasticsearch\n",
        "from elasticsearch.client import MlClient"
      ],
      "metadata": {
        "id": "wyUZXUi4RWWL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Configure elasticsearch authentication. \n",
        "The recommended authentication approach is using the [Elastic Cloud ID](https://www.elastic.co/guide/en/cloud/current/ec-cloud-id.html) and a [cluster level API key](https://www.elastic.co/guide/en/kibana/current/api-keys.html)\n",
        "\n",
        "You can use any method you wish to set the required credentials. We are using getpass in this example to prompt for credentials to avoide storing them in github."
      ],
      "metadata": {
        "id": "r7nMIbHke37Q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import getpass"
      ],
      "metadata": {
        "id": "Xsd2m7HoTCLm"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "es_cloud_id = getpass.getpass('Enter Elastic Cloud ID:  ')\n",
        "es_api_id = getpass.getpass('Enter cluster API key ID:  ') \n",
        "es_api_key = getpass.getpass('Enter cluster API key:  ')"
      ],
      "metadata": {
        "id": "SSGgYHome69o",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "01af50a5-ad72-4458-c5c4-9223c056fb7f"
      },
      "execution_count": 8,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter Elastic Cloud ID:  ··········\n",
            "Enter cluster API key ID:  ··········\n",
            "Enter cluster API key:  ··········\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Connect to Elastic Cloud"
      ],
      "metadata": {
        "id": "jL4VDnVp96lf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "es = Elasticsearch(cloud_id=es_cloud_id, \n",
        "                   api_key=(es_api_id, es_api_key)\n",
        "                   )\n",
        "es.info() # should return cluster info"
      ],
      "metadata": {
        "id": "I8mVJkKmetXo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "---\n",
        "# Loading a model from Hugging Face model hub\n",
        "---\n",
        "---"
      ],
      "metadata": {
        "id": "pOuF_1VnmVU-"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The Elastic Stack machine learning features support transformer models that conform to the standard BERT model interface and use the WordPiece tokenization algorithm.\n",
        "\n",
        "A current list of supported architectures can be viewed in the\n",
        "[Elastic NLP Model Support Docs](https://www.elastic.co/guide/en/machine-learning/current/ml-nlp-model-ref.html)\n"
      ],
      "metadata": {
        "id": "NwIItJyhoWeT"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Download NER model from Hugging Face\n",
        "\n",
        "For this example we will be using the [dslim/bert-base-NER-uncased](https://huggingface.co/dslim/bert-base-NER-uncased) model\n"
      ],
      "metadata": {
        "id": "10VvWJ87alld"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Download the Model\n",
        "Here we specify the model id from Hugging Face. The easiest way to get this id is clicking the copy the model name icon next to the name on the model page. \n",
        "\n",
        "When calling `TransformerModel` you specify the HF model id and the task type. You can try specifying `auto` and eland will attempt to determine the correct type from info in the model config. This is not always possible so a list of specific `task_type` values can be viewed in the following code: \n",
        "[Supported values](https://github.com/elastic/eland/blob/15a300728876022b206161d71055c67b500a0192/eland/ml/pytorch/transformers.py#*L41*)"
      ],
      "metadata": {
        "id": "uBMWHj-ZmtvE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "hf_model_id='dslim/bert-base-NER-uncased'\n",
        "tm = TransformerModel(hf_model_id, \"ner\") # 'ner'"
      ],
      "metadata": {
        "id": "zPV3oFsKiYFL",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177,
          "referenced_widgets": [
            "46e37a87884b4bb6ae5120f8a2152995",
            "93049e0ffb9e4d64af271461b4e3a356",
            "54dc699466b141c9bc6c27b4860c160e",
            "62eaacf1d0c54d8494f013f959a396ec",
            "d92d9bc76ef449a7b002fec279bc164c",
            "d5d4272035ec489f881384e51b7ab383",
            "33017035f21e4a3584bb86951ef6ede8",
            "07a0b6587a2e4052af2203f25f506171",
            "2c6132ac37624d90a51485a95ab627a1",
            "86976b6d8f654503b5daa4498fedb52c",
            "57df67fac6274ae9b60567e1352550f2",
            "bac587aca60d4ac2a3f98111b230bd03",
            "620f86cb69c54168946e7c13d0f873aa",
            "98238564651b42838ee9d8a477039720",
            "fe1c6467cdc440739f8524045accaebd",
            "c2e2b98f1991452baf551b5cf81e95b6",
            "f6d221d18917448aa4ab7fd398f01c63",
            "8e0f4128a37e4ad7a9f8c87ea30b0b00",
            "aa7c0c6df06749288cb968df8867956b",
            "63153c51050247d9b5646a6a6b18cc64",
            "a2219f35345449119c8a0671c6a41eed",
            "9c5d35ee2faf46b5b142ed530b0a33d4",
            "5a5df53f4fda479f871fb445e76ceef6",
            "647ab851389d4efc8e3e197f33a119ec",
            "94fd836a4156496cb2c0d7d5ccc8bd5b",
            "6f4995647ebb40c683cb7409629253f8",
            "7cb2e44a0aa3406586a759273f27f64f",
            "4a25cbf6e89a483f93b62b9416d80143",
            "e714b9390d224ad2abdd996142584f4e",
            "3fceffe019734418988381a1e3ac52a3",
            "dee008a902fb408dac6e03b62e0e7bf7",
            "7d74e313d2d0432a924711d0a44f91d2",
            "5adaaae9a3dd413b8e74ce8b26b1565d",
            "0a636987fd574185b8092b60950a6035",
            "71244cee5c3c4f4ca72d063829ef4f1f",
            "c32508c3257a43ab9e08b4184d06dd11",
            "32156e2b1f2a4111a38b2aa025676e92",
            "d9848f03b4a543639b63aec58178c738",
            "0ac7c7719c724624999bc774e09296eb",
            "28c82cbb8a7d4386a973d2f5e87d1602",
            "faba45369def4b1fb2422a95d56baf65",
            "43d3bad483714bccaaeda45e69ae88ee",
            "d7ff56381ab94c85808a195e32dffa12",
            "adfcae49f35845e79ea2a9917c08f4fe",
            "f98ebf3e924048ce9a29ee2bc5884ca2",
            "9e324c25d15249daa4eb680fb4fabaac",
            "af6c2c88c4694654adefdfd6586baea8",
            "209c8c4defb846cf8a73bca233078cd2",
            "8586cb3311fb4e1da46e5a39c902b360",
            "5b1b3218a3d44adfa119ae42ba9adefd",
            "3f5727f64e8a449b936e682477b91354",
            "1152adca330c4341b5ba095a6d064aaa",
            "2d34f185fc77409cbe5e2feecbf8851c",
            "899101235b69405ea95ba9f12869e600",
            "23bf3111d57e42c399ae84e5bf2e9a21"
          ]
        },
        "outputId": "930de58b-303f-43e9-d158-232a6d337cf7"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/39.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "46e37a87884b4bb6ae5120f8a2152995"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/1.26k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bac587aca60d4ac2a3f98111b230bd03"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5a5df53f4fda479f871fb445e76ceef6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0a636987fd574185b8092b60950a6035"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)\"pytorch_model.bin\";:   0%|          | 0.00/438M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f98ebf3e924048ce9a29ee2bc5884ca2"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Set and confirm the model ID\n",
        "To make the name compatible with elasticsearch, the '/' is replaced with '__'\n",
        "\n"
      ],
      "metadata": {
        "id": "sX-9dHuDmwgX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "es_model_id = tm.elasticsearch_model_id()\n",
        "es_model_id"
      ],
      "metadata": {
        "id": "XkIQBBCbdqvQ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "7e9d1487-63c3-446b-eec1-14b7c108ece9"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'dslim__bert-base-ner-uncased'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Export the model in a TorchScrpt representation which Elasticsearch uses"
      ],
      "metadata": {
        "id": "p0L2cfYwbIld"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tmp_path = \"models\"\n",
        "Path(tmp_path).mkdir(parents=True, exist_ok=True)\n",
        "model_path, config, vocab_path = tm.save(tmp_path)"
      ],
      "metadata": {
        "id": "GsSpvvP-nbCK"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load the model into Elasticsearch\n",
        "Model should not already exist in elasticsearch"
      ],
      "metadata": {
        "id": "k1a_yNo6ba2E"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ptm = PyTorchModel(es, es_model_id)\n",
        "ptm.import_model(model_path=model_path, config_path=None, vocab_path=vocab_path, config=config) "
      ],
      "metadata": {
        "id": "Z4QD71Apnj4j",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "f581bdfba88f477d8b8e9317ca0d5070",
            "4a13764a9ea04ae9b42be117234210f1",
            "06e5b6b643214b3bb83b73e64003ef61",
            "d8b1c932fd714a9bb77e0e23a4fa8c00",
            "8d2062873e92461c9e48373126de67bb",
            "e745a647d3f54d6fa9c6f6e924017876",
            "7644ac4714bc4ed891f275218cb17797",
            "0cd1f50e92b0418d81608597ed20e194",
            "5a2f0b6243ee4ccfac442a953ebe0553",
            "c78997ab7d4e4123b934bc494a4a4b68",
            "7b54dabfdf0046faaa672129a3bf3e66"
          ]
        },
        "outputId": "bfb2dd1b-21ab-4e05-a7ff-4de4c3ce198b"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/104 [00:00<?, ? parts/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f581bdfba88f477d8b8e9317ca0d5070"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "---\n",
        "# Starting the Model\n",
        "---\n",
        "---"
      ],
      "metadata": {
        "id": "4UYSzFp3vHdB"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## View information about the model\n",
        "This is not required but can be handy to get a model overivew"
      ],
      "metadata": {
        "id": "wQwfozwznK4Y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "m = MlClient.get_trained_models(es, model_id=es_model_id)\n",
        "m.body"
      ],
      "metadata": {
        "id": "b4Wv8EJvpfZI",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bfbb7177-4acb-47af-b6ea-a718ed4ed035"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'count': 1,\n",
              " 'trained_model_configs': [{'model_id': 'dslim__bert-base-ner-uncased',\n",
              "   'model_type': 'pytorch',\n",
              "   'created_by': 'api_user',\n",
              "   'version': '8.6.0',\n",
              "   'create_time': 1675887938334,\n",
              "   'model_size_bytes': 0,\n",
              "   'estimated_operations': 0,\n",
              "   'license_level': 'platinum',\n",
              "   'description': \"Model dslim/bert-base-NER-uncased for task type 'ner'\",\n",
              "   'tags': [],\n",
              "   'input': {'field_names': ['text_field']},\n",
              "   'inference_config': {'ner': {'vocabulary': {'index': '.ml-inference-native-000001'},\n",
              "     'tokenization': {'bert': {'do_lower_case': True,\n",
              "       'with_special_tokens': True,\n",
              "       'max_sequence_length': 512,\n",
              "       'truncate': 'first',\n",
              "       'span': -1}},\n",
              "     'classification_labels': ['O',\n",
              "      'B_MISC',\n",
              "      'I_MISC',\n",
              "      'B_PER',\n",
              "      'I_PER',\n",
              "      'B_ORG',\n",
              "      'I_ORG',\n",
              "      'B_LOC',\n",
              "      'I_LOC']}},\n",
              "   'location': {'index': {'name': '.ml-inference-native-000001'}}}]}"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Deploy the model\n",
        "This will load the model on the ML nodes and start the process(es) making it available for the NLP task"
      ],
      "metadata": {
        "id": "oMGw3sk-pbaN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "s = MlClient.start_trained_model_deployment(es, model_id=es_model_id)\n",
        "s.body"
      ],
      "metadata": {
        "id": "w5muJ1rLqvUW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "18c630f1-4b24-408e-a20c-4482fd5a2d9e"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'assignment': {'task_parameters': {'model_id': 'dslim__bert-base-ner-uncased',\n",
              "   'model_bytes': 435663717,\n",
              "   'threads_per_allocation': 1,\n",
              "   'number_of_allocations': 1,\n",
              "   'queue_capacity': 1024,\n",
              "   'cache_size': '435663717b',\n",
              "   'priority': 'normal'},\n",
              "  'routing_table': {'dDqmmPySSKuH8d7HcF54uA': {'current_allocations': 1,\n",
              "    'target_allocations': 1,\n",
              "    'routing_state': 'started',\n",
              "    'reason': ''}},\n",
              "  'assignment_state': 'started',\n",
              "  'start_time': '2023-02-08T20:28:50.914490234Z',\n",
              "  'max_assigned_allocations': 1}}"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Verify the model started without issue"
      ],
      "metadata": {
        "id": "ZytlELrsnn_O"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "stats = MlClient.get_trained_models_stats(es, model_id=es_model_id)\n",
        "stats.body['trained_model_stats'][0]['deployment_stats']['nodes'][0]['routing_state']"
      ],
      "metadata": {
        "id": "ZaQUUWe0Hxwz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6cfe3bb0-9666-4e83-dd36-9f33849195c2"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'routing_state': 'started'}"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "---\n",
        "# Generating Entities\n",
        "---\n",
        "---\n",
        "There are two main ways NER modles, and the entities they generate, are used in elasticsearch\n",
        "\n",
        "1. Calling the `_infer` endpoint with an input string and getting the entities back in the response. The entities are not stored in elastic (directly) and can be used for tasks such as enhancing follow up queries.\n",
        "2. Passing a field from a document on ingest to a model with an ingest pipeline. The resulting entites are stored along with the document when it is indexed in elasticsearch.\n",
        "\n",
        "For this example, we will be taking the first approach calling `_infer` to test the NER model works with our data."
      ],
      "metadata": {
        "id": "8M-4Fc-zn7D_"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Setting up the index in elasticsearch to store vectors\n",
        "\n"
      ],
      "metadata": {
        "id": "qpm3jgJOzlxi"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Generate Vector for Query\n",
        "\n",
        "Before we can run a kNN query, we need to convert our query string to a vector."
      ],
      "metadata": {
        "id": "6Hu2n4bmGYkG"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Creating a sample string"
      ],
      "metadata": {
        "id": "XXAQXIS0pvfR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "docs =  [\n",
        "    {\n",
        "      \"text_field\": \"Hi, my name is Jeff Vestal. I live in Chicago and I work at Elastic\"\n",
        "    }\n",
        "  ]"
      ],
      "metadata": {
        "id": "wBNV7q5Dwlz6"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We call the `_infer` endpoint supplying the model_id and the doc[s] we want generate entities for. "
      ],
      "metadata": {
        "id": "T5_BYcpvrLsb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "response = MlClient.infer_trained_model(es, model_id=es_model_id, docs=docs, )"
      ],
      "metadata": {
        "id": "ZsWg7XPSGbiu"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The vector for the first doc can be accessed in the response dict as shown below"
      ],
      "metadata": {
        "id": "CdC3PkTyrZEV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "doc_0_vector = response['inference_results']\n",
        "doc_0_vector"
      ],
      "metadata": {
        "id": "Rle3J5mJXbdf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b8f34bfa-402d-46c5-ccf9-f29a8e550efe"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'predicted_value': 'Hi, my name is [Jeff Vestal](PER&Jeff+Vestal). I live in [Chicago](LOC&Chicago) and I work at [Elastic](ORG&Elastic)',\n",
              "  'entities': [{'entity': 'jeff vestal',\n",
              "    'class_name': 'PER',\n",
              "    'class_probability': 0.9976065299030981,\n",
              "    'start_pos': 15,\n",
              "    'end_pos': 26},\n",
              "   {'entity': 'chicago',\n",
              "    'class_name': 'LOC',\n",
              "    'class_probability': 0.9977499555665439,\n",
              "    'start_pos': 38,\n",
              "    'end_pos': 45},\n",
              "   {'entity': 'elastic',\n",
              "    'class_name': 'ORG',\n",
              "    'class_probability': 0.9957462750425009,\n",
              "    'start_pos': 60,\n",
              "    'end_pos': 67}]}]"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    }
  ]
}