finance_bert_monthly_data_engineering.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyMvo8tSmHDee6Lir4TMrxTo",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "6f5731be38894a5c9738d3e9aefa4b07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6ea10a8536514ee78d5152f42bbe915a",
              "IPY_MODEL_3596645b05804b90b93e58ed3200fad0",
              "IPY_MODEL_fd50cc1261514ec09f4025fc480e6a93"
            ],
            "layout": "IPY_MODEL_98cacf4f1f7a4f67a1a3b427b4e0cef9"
          }
        },
        "6ea10a8536514ee78d5152f42bbe915a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_94d7e0862eef4282a6a24450f768cd53",
            "placeholder": "​",
            "style": "IPY_MODEL_184cc6a699d54fc8a658553dacc34b61",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "3596645b05804b90b93e58ed3200fad0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ef4be62bfcd047f19adadbbc4cfa9409",
            "max": 252,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f553d1fc6ad6448bbb627a6f092c07c5",
            "value": 252
          }
        },
        "fd50cc1261514ec09f4025fc480e6a93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f841ecd585e74d028472d881ec628358",
            "placeholder": "​",
            "style": "IPY_MODEL_8d3ad8e0a8514298abd41418033470b4",
            "value": " 252/252 [00:00&lt;00:00, 15.4kB/s]"
          }
        },
        "98cacf4f1f7a4f67a1a3b427b4e0cef9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "94d7e0862eef4282a6a24450f768cd53": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "184cc6a699d54fc8a658553dacc34b61": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ef4be62bfcd047f19adadbbc4cfa9409": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f553d1fc6ad6448bbb627a6f092c07c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f841ecd585e74d028472d881ec628358": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d3ad8e0a8514298abd41418033470b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d86d06f47a7948208da4636192b98d09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1c4a460434d141bdbd382e399b532f4e",
              "IPY_MODEL_ce5f24fb982644cbb557b9577732dc6f",
              "IPY_MODEL_8a8eaf9c894e41dbaf4941fcc99e3512"
            ],
            "layout": "IPY_MODEL_a06aad81a5b74a1bbebcebe03426eb2c"
          }
        },
        "1c4a460434d141bdbd382e399b532f4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eea5a581f3d147e0adc84a6c76603b47",
            "placeholder": "​",
            "style": "IPY_MODEL_ded1874727b643d4be877b108c623b2c",
            "value": "vocab.txt: 100%"
          }
        },
        "ce5f24fb982644cbb557b9577732dc6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fb9221cc11094d889ec892e9dbe3c2a3",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_71589bc3e22c4d1f9b80866e4d6b5cec",
            "value": 231508
          }
        },
        "8a8eaf9c894e41dbaf4941fcc99e3512": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b803f8139c6645f095ce080fc3245764",
            "placeholder": "​",
            "style": "IPY_MODEL_301adffc637c42c7a57476b371aa3062",
            "value": " 232k/232k [00:00&lt;00:00, 617kB/s]"
          }
        },
        "a06aad81a5b74a1bbebcebe03426eb2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eea5a581f3d147e0adc84a6c76603b47": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ded1874727b643d4be877b108c623b2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fb9221cc11094d889ec892e9dbe3c2a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71589bc3e22c4d1f9b80866e4d6b5cec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b803f8139c6645f095ce080fc3245764": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "301adffc637c42c7a57476b371aa3062": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "babb14c4355a4f48b19c78effaa1ac18": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b2d83a05c84a410b8716d1f1458b2355",
              "IPY_MODEL_28f6de94b497429fb3a3b8b0200ae6c4",
              "IPY_MODEL_ea6b739651fc42ee8011b1b16ed7a70a"
            ],
            "layout": "IPY_MODEL_4a03c614df474c08991042ed2d3523f9"
          }
        },
        "b2d83a05c84a410b8716d1f1458b2355": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f07e928d3cf74eeaa925e8f6646d66fe",
            "placeholder": "​",
            "style": "IPY_MODEL_b9ee6323db8245c482d9f3f288f96320",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "28f6de94b497429fb3a3b8b0200ae6c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_20a66bccb930424ea58a008f5383ced2",
            "max": 112,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_563855d76cd84683bacf4c3429dad1d4",
            "value": 112
          }
        },
        "ea6b739651fc42ee8011b1b16ed7a70a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1d14a9a005154750af17573f7b6102d1",
            "placeholder": "​",
            "style": "IPY_MODEL_8a8f21d7c2c9423790e99141af8181b2",
            "value": " 112/112 [00:00&lt;00:00, 8.39kB/s]"
          }
        },
        "4a03c614df474c08991042ed2d3523f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f07e928d3cf74eeaa925e8f6646d66fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b9ee6323db8245c482d9f3f288f96320": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "20a66bccb930424ea58a008f5383ced2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "563855d76cd84683bacf4c3429dad1d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1d14a9a005154750af17573f7b6102d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8a8f21d7c2c9423790e99141af8181b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ae306c07391448b5a62f8386804dbb8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_204abcf278c24e73806b706fdf35cec2",
              "IPY_MODEL_b90674bec8e64cd59ed476d7977a0370",
              "IPY_MODEL_61cd6642972745f399be0892fd414788"
            ],
            "layout": "IPY_MODEL_a0967881414d4efa990427f51950bf81"
          }
        },
        "204abcf278c24e73806b706fdf35cec2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7eb327d14858488582586c8dd2d22462",
            "placeholder": "​",
            "style": "IPY_MODEL_6e8c786692b94920998e37f142ba871c",
            "value": "config.json: 100%"
          }
        },
        "b90674bec8e64cd59ed476d7977a0370": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c1098df9c0e640eb88699752ea341e8b",
            "max": 758,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_54ed606860b841cab49f2712082a0ffc",
            "value": 758
          }
        },
        "61cd6642972745f399be0892fd414788": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cfae771ecf2f4062bde2798c995339f2",
            "placeholder": "​",
            "style": "IPY_MODEL_19ebcf3c4f7340a99bd4f93e7dba69ae",
            "value": " 758/758 [00:00&lt;00:00, 60.2kB/s]"
          }
        },
        "a0967881414d4efa990427f51950bf81": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7eb327d14858488582586c8dd2d22462": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6e8c786692b94920998e37f142ba871c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c1098df9c0e640eb88699752ea341e8b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "54ed606860b841cab49f2712082a0ffc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cfae771ecf2f4062bde2798c995339f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "19ebcf3c4f7340a99bd4f93e7dba69ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gitHubAndyLee2020/FinanceBERT/blob/main/finance_bert_monthly_data_engineering.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### FinanceBERT Monthly Data Engineering"
      ],
      "metadata": {
        "id": "gXnp9gbL0Ge_"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Daily Financial News for 6000+ Stocks\n",
        "\n",
        "- https://www.kaggle.com/datasets/miguelaenlle/massive-stock-news-analysis-db-for-nlpbacktests"
      ],
      "metadata": {
        "id": "YRQrViutB0XW"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data Uploading\n",
        "\n",
        "- Run this section to get `training_data.jsonl` file.\n",
        "- Skip this section if you already have `training_data.jsonl` file."
      ],
      "metadata": {
        "id": "Sksa7wRMsbS_"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9VSCt8L5_o5o",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "outputId": "b1c17fec-f6df-4627-a777-5bac7e65030b"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-f5b4d2ca-3dce-42d2-ba81-1dd58eac692e\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-f5b4d2ca-3dce-42d2-ba81-1dd58eac692e\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving daily-financial-news.zip to daily-financial-news.zip\n"
          ]
        }
      ],
      "source": [
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip daily-financial-news.zip"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yZpL6bBNR3V9",
        "outputId": "0cd34057-e696-4590-d6a8-f7cbf2d458eb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  daily-financial-news.zip\n",
            "  inflating: analyst_ratings_processed.csv  \n",
            "  inflating: raw_analyst_ratings.csv  \n",
            "  inflating: raw_partner_headlines.csv  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!rm daily-financial-news.zip"
      ],
      "metadata": {
        "id": "AuVNLnfhR8MY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "analyst_ratings_processed = pd.read_csv('analyst_ratings_processed.csv')\n",
        "raw_analyst_ratings = pd.read_csv('raw_analyst_ratings.csv')\n",
        "raw_partner_headlines = pd.read_csv('raw_partner_headlines.csv')"
      ],
      "metadata": {
        "id": "dqEdodJeR9yj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data Examination\n",
        "\n",
        "- Run this section to get `training_data.jsonl` file.\n",
        "- Skip this section if you already have `training_data.jsonl` file."
      ],
      "metadata": {
        "id": "Z645Zu2wsepa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "analyst_ratings_processed.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "eMTOO9EISdzc",
        "outputId": "5e449a7b-be50-47dc-9c97-da927397bf0e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0                                              title  \\\n",
              "0         0.0            Stocks That Hit 52-Week Highs On Friday   \n",
              "1         1.0         Stocks That Hit 52-Week Highs On Wednesday   \n",
              "2         2.0                      71 Biggest Movers From Friday   \n",
              "3         3.0       46 Stocks Moving In Friday's Mid-Day Session   \n",
              "4         4.0  B of A Securities Maintains Neutral on Agilent...   \n",
              "\n",
              "                        date stock  \n",
              "0  2020-06-05 10:30:00-04:00     A  \n",
              "1  2020-06-03 10:45:00-04:00     A  \n",
              "2  2020-05-26 04:30:00-04:00     A  \n",
              "3  2020-05-22 12:45:00-04:00     A  \n",
              "4  2020-05-22 11:38:00-04:00     A  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-fcd4cb66-439c-4798-bf1d-f4c79fcaa748\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>title</th>\n",
              "      <th>date</th>\n",
              "      <th>stock</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.0</td>\n",
              "      <td>Stocks That Hit 52-Week Highs On Friday</td>\n",
              "      <td>2020-06-05 10:30:00-04:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.0</td>\n",
              "      <td>Stocks That Hit 52-Week Highs On Wednesday</td>\n",
              "      <td>2020-06-03 10:45:00-04:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2.0</td>\n",
              "      <td>71 Biggest Movers From Friday</td>\n",
              "      <td>2020-05-26 04:30:00-04:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3.0</td>\n",
              "      <td>46 Stocks Moving In Friday's Mid-Day Session</td>\n",
              "      <td>2020-05-22 12:45:00-04:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4.0</td>\n",
              "      <td>B of A Securities Maintains Neutral on Agilent...</td>\n",
              "      <td>2020-05-22 11:38:00-04:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fcd4cb66-439c-4798-bf1d-f4c79fcaa748')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-fcd4cb66-439c-4798-bf1d-f4c79fcaa748 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-fcd4cb66-439c-4798-bf1d-f4c79fcaa748');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c4185a33-de8c-447c-9a92-4d0e65b2aebc\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c4185a33-de8c-447c-9a92-4d0e65b2aebc')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c4185a33-de8c-447c-9a92-4d0e65b2aebc button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(analyst_ratings_processed)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i-0XIRmDbLNb",
        "outputId": "054a71e5-a8fa-4528-c10c-6454737fd886"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1400469"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "raw_analyst_ratings.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "HA9A_giASfxn",
        "outputId": "b9c6efe2-a0a6-465f-931e-a6617e943e91"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0                                           headline  \\\n",
              "0           0            Stocks That Hit 52-Week Highs On Friday   \n",
              "1           1         Stocks That Hit 52-Week Highs On Wednesday   \n",
              "2           2                      71 Biggest Movers From Friday   \n",
              "3           3       46 Stocks Moving In Friday's Mid-Day Session   \n",
              "4           4  B of A Securities Maintains Neutral on Agilent...   \n",
              "\n",
              "                                                 url          publisher  \\\n",
              "0  https://www.benzinga.com/news/20/06/16190091/s...  Benzinga Insights   \n",
              "1  https://www.benzinga.com/news/20/06/16170189/s...  Benzinga Insights   \n",
              "2  https://www.benzinga.com/news/20/05/16103463/7...         Lisa Levin   \n",
              "3  https://www.benzinga.com/news/20/05/16095921/4...         Lisa Levin   \n",
              "4  https://www.benzinga.com/news/20/05/16095304/b...         Vick Meyer   \n",
              "\n",
              "                        date stock  \n",
              "0  2020-06-05 10:30:54-04:00     A  \n",
              "1  2020-06-03 10:45:20-04:00     A  \n",
              "2  2020-05-26 04:30:07-04:00     A  \n",
              "3  2020-05-22 12:45:06-04:00     A  \n",
              "4  2020-05-22 11:38:59-04:00     A  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-20bc0869-6600-49a8-adb0-7d204fb047ba\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>headline</th>\n",
              "      <th>url</th>\n",
              "      <th>publisher</th>\n",
              "      <th>date</th>\n",
              "      <th>stock</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>Stocks That Hit 52-Week Highs On Friday</td>\n",
              "      <td>https://www.benzinga.com/news/20/06/16190091/s...</td>\n",
              "      <td>Benzinga Insights</td>\n",
              "      <td>2020-06-05 10:30:54-04:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>Stocks That Hit 52-Week Highs On Wednesday</td>\n",
              "      <td>https://www.benzinga.com/news/20/06/16170189/s...</td>\n",
              "      <td>Benzinga Insights</td>\n",
              "      <td>2020-06-03 10:45:20-04:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>71 Biggest Movers From Friday</td>\n",
              "      <td>https://www.benzinga.com/news/20/05/16103463/7...</td>\n",
              "      <td>Lisa Levin</td>\n",
              "      <td>2020-05-26 04:30:07-04:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>46 Stocks Moving In Friday's Mid-Day Session</td>\n",
              "      <td>https://www.benzinga.com/news/20/05/16095921/4...</td>\n",
              "      <td>Lisa Levin</td>\n",
              "      <td>2020-05-22 12:45:06-04:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>B of A Securities Maintains Neutral on Agilent...</td>\n",
              "      <td>https://www.benzinga.com/news/20/05/16095304/b...</td>\n",
              "      <td>Vick Meyer</td>\n",
              "      <td>2020-05-22 11:38:59-04:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-20bc0869-6600-49a8-adb0-7d204fb047ba')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-20bc0869-6600-49a8-adb0-7d204fb047ba button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-20bc0869-6600-49a8-adb0-7d204fb047ba');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-601dbaa2-908e-4c99-bbbe-2b056356d96f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-601dbaa2-908e-4c99-bbbe-2b056356d96f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-601dbaa2-908e-4c99-bbbe-2b056356d96f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(raw_analyst_ratings)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "375g0fISbNBl",
        "outputId": "5e63a9b4-ae46-4e52-9e16-b379e8c26aed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1407328"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "raw_partner_headlines.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "a1oX8gWlSk1Z",
        "outputId": "9d987a16-da56-4bda-e945-92b8da2296c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0                                           headline  \\\n",
              "0           2  Agilent Technologies Announces Pricing of $5……...   \n",
              "1           3  Agilent (A) Gears Up for Q2 Earnings: What's i...   \n",
              "2           4  J.P. Morgan Asset Management Announces Liquida...   \n",
              "3           5  Pershing Square Capital Management, L.P. Buys ...   \n",
              "4           6  Agilent Awards Trilogy Sciences with a Golden ...   \n",
              "\n",
              "                                                 url  publisher  \\\n",
              "0  http://www.gurufocus.com/news/1153187/agilent-...  GuruFocus   \n",
              "1  http://www.zacks.com/stock/news/931205/agilent...      Zacks   \n",
              "2  http://www.gurufocus.com/news/1138923/jp-morga...  GuruFocus   \n",
              "3  http://www.gurufocus.com/news/1138704/pershing...  GuruFocus   \n",
              "4  http://www.gurufocus.com/news/1134012/agilent-...  GuruFocus   \n",
              "\n",
              "                  date stock  \n",
              "0  2020-06-01 00:00:00     A  \n",
              "1  2020-05-18 00:00:00     A  \n",
              "2  2020-05-15 00:00:00     A  \n",
              "3  2020-05-15 00:00:00     A  \n",
              "4  2020-05-12 00:00:00     A  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2a488000-f7a0-4ee4-ad2e-39e2f3323156\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>headline</th>\n",
              "      <th>url</th>\n",
              "      <th>publisher</th>\n",
              "      <th>date</th>\n",
              "      <th>stock</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2</td>\n",
              "      <td>Agilent Technologies Announces Pricing of $5……...</td>\n",
              "      <td>http://www.gurufocus.com/news/1153187/agilent-...</td>\n",
              "      <td>GuruFocus</td>\n",
              "      <td>2020-06-01 00:00:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3</td>\n",
              "      <td>Agilent (A) Gears Up for Q2 Earnings: What's i...</td>\n",
              "      <td>http://www.zacks.com/stock/news/931205/agilent...</td>\n",
              "      <td>Zacks</td>\n",
              "      <td>2020-05-18 00:00:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4</td>\n",
              "      <td>J.P. Morgan Asset Management Announces Liquida...</td>\n",
              "      <td>http://www.gurufocus.com/news/1138923/jp-morga...</td>\n",
              "      <td>GuruFocus</td>\n",
              "      <td>2020-05-15 00:00:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>5</td>\n",
              "      <td>Pershing Square Capital Management, L.P. Buys ...</td>\n",
              "      <td>http://www.gurufocus.com/news/1138704/pershing...</td>\n",
              "      <td>GuruFocus</td>\n",
              "      <td>2020-05-15 00:00:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>6</td>\n",
              "      <td>Agilent Awards Trilogy Sciences with a Golden ...</td>\n",
              "      <td>http://www.gurufocus.com/news/1134012/agilent-...</td>\n",
              "      <td>GuruFocus</td>\n",
              "      <td>2020-05-12 00:00:00</td>\n",
              "      <td>A</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2a488000-f7a0-4ee4-ad2e-39e2f3323156')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2a488000-f7a0-4ee4-ad2e-39e2f3323156 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2a488000-f7a0-4ee4-ad2e-39e2f3323156');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-bdec4c41-de48-4448-89fc-6e990a4d735f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-bdec4c41-de48-4448-89fc-6e990a4d735f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-bdec4c41-de48-4448-89fc-6e990a4d735f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(raw_partner_headlines)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q9YQ3-osbQGO",
        "outputId": "a34dad59-7aeb-4b09-94a6-e19bcf13ba38"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1845559"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "stock_tickers = analyst_ratings_processed['stock'].unique()"
      ],
      "metadata": {
        "id": "BU1N7f8VS9AM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "stock_tickers[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p7HDlqt-TImu",
        "outputId": "3f47e842-8c2f-4013-dd86-956b5bb2a2de"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['A', 'AAMC', 'AAME', 'AAN', nan, 'AAOI', 'AAON', 'AAP', 'AAPL',\n",
              "       'AAU'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(stock_tickers)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PlcjHEE7TL3D",
        "outputId": "2a69cf63-53c7-4c9d-96a7-15debc00bc7d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6193"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "partners_stock_tickers = raw_partner_headlines['stock'].unique()"
      ],
      "metadata": {
        "id": "R124QiSVa3br"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "partners_stock_tickers[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uXQIbgOMa86O",
        "outputId": "c3d023d9-75c4-4379-b197-4ac1d696def6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['A', 'AA', 'AAC', 'AADR', 'AAL', 'AAMC', 'AAME', 'AAN', 'AAOI',\n",
              "       'AAON'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(partners_stock_tickers)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0-wvrzvna-nz",
        "outputId": "4e066f24-cd38-464f-91c7-ff8e641e2ee2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6552"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "analyst_ratings_processed[analyst_ratings_processed['stock'] == 'AAPL']['title'][:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7s8qUCIKr3TP",
        "outputId": "2abfe86b-f92f-4812-de53-2080e6d5de77"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3668    Tech Stocks And FAANGS Strong Again To Start D...\n",
              "3669        10 Biggest Price Target Changes For Wednesday\n",
              "3670    Benzinga Pro's Top 5 Stocks To Watch For Wed.,...\n",
              "3671    Deutsche Bank Maintains Buy on Apple, Raises P...\n",
              "3672    Apple To Let Users Trade In Their Mac Computer...\n",
              "3673    Big Tech Reaches New Record Heights At The Sto...\n",
              "3674            Why Apple's Stock Is Trading Higher Today\n",
              "3675    Apple Could Announce In-House Chips For Macs A...\n",
              "3676    Apple shares are trading higher despite market...\n",
              "3677    Sonos Shares Spike To Session High, Now Up 9.5...\n",
              "Name: title, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "raw_analyst_ratings[raw_analyst_ratings['stock'] == 'AAPL']['headline'][:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fOJfJ9ZUsICI",
        "outputId": "d906539e-c082-4dbc-877d-dcdf0ef5f864"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6680    Tech Stocks And FAANGS Strong Again To Start D...\n",
              "6681        10 Biggest Price Target Changes For Wednesday\n",
              "6682    Benzinga Pro's Top 5 Stocks To Watch For Wed.,...\n",
              "6683    Deutsche Bank Maintains Buy on Apple, Raises P...\n",
              "6684    Apple To Let Users Trade In Their Mac Computer...\n",
              "6685    Big Tech Reaches New Record Heights At The Sto...\n",
              "6686            Why Apple's Stock Is Trading Higher Today\n",
              "6687    Apple Could Announce In-House Chips For Macs A...\n",
              "6688    Apple shares are trading higher despite market...\n",
              "6689    Sonos Shares Spike To Session High, Now Up 9.5...\n",
              "Name: headline, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "raw_partner_headlines[raw_partner_headlines['stock'] == 'AAPL']['headline'][:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3Mwf2Bu5sPwt",
        "outputId": "6b068209-3d45-458e-d3e9-9c31ade4a4f9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4067                                         American Pie\n",
              "4068            Tech Giants Dare Antitrust Deal Watchdogs\n",
              "4069    MoneyGram Shares Jump 50% As Western Union Rep...\n",
              "4070                        All Eyes on Market Volatility\n",
              "4071    Warren Buffett's Berkshire Hathaway Turns Up S...\n",
              "4072                                           COVID 2.0?\n",
              "4073    Apple CEO Writes To Employees About George Flo...\n",
              "4074    First the Deflationary Deluge of Assets Crashi...\n",
              "4075                   Market Briefing For Monday, June 1\n",
              "4076    Five Monster Stock Market Predictions For The ...\n",
              "Name: headline, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers"
      ],
      "metadata": {
        "id": "fqKDHoYG_TDN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertTokenizer\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "# Load the BertTokenizer\n",
        "tokenizer = BertTokenizer.from_pretrained(\"ProsusAI/finbert\")\n",
        "\n",
        "# Assuming 'analyst_ratings_processed' is your dataframe and it has a column named 'title'\n",
        "# Replace 'analyst_ratings_processed' with your actual dataframe variable name\n",
        "analyst_ratings_processed_temp = analyst_ratings_processed\n",
        "\n",
        "# Tokenize the titles and calculate the token length for each title\n",
        "token_lengths = analyst_ratings_processed_temp['title'].apply(lambda x: len(tokenizer.encode(x, add_special_tokens=True)))\n",
        "\n",
        "# Calculate the mean, median, max, and top 95% percentile of the token lengths\n",
        "mean_length = token_lengths.mean()\n",
        "median_length = token_lengths.median()\n",
        "max_length = token_lengths.max()\n",
        "percentile_95_length = np.percentile(token_lengths, 95)\n",
        "\n",
        "print(f\"Mean token length: {mean_length}\")\n",
        "print(f\"Median token length: {median_length}\")\n",
        "print(f\"Max token length: {max_length}\")\n",
        "print(f\"Top 95% percentile token length: {percentile_95_length}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301,
          "referenced_widgets": [
            "6f5731be38894a5c9738d3e9aefa4b07",
            "6ea10a8536514ee78d5152f42bbe915a",
            "3596645b05804b90b93e58ed3200fad0",
            "fd50cc1261514ec09f4025fc480e6a93",
            "98cacf4f1f7a4f67a1a3b427b4e0cef9",
            "94d7e0862eef4282a6a24450f768cd53",
            "184cc6a699d54fc8a658553dacc34b61",
            "ef4be62bfcd047f19adadbbc4cfa9409",
            "f553d1fc6ad6448bbb627a6f092c07c5",
            "f841ecd585e74d028472d881ec628358",
            "8d3ad8e0a8514298abd41418033470b4",
            "d86d06f47a7948208da4636192b98d09",
            "1c4a460434d141bdbd382e399b532f4e",
            "ce5f24fb982644cbb557b9577732dc6f",
            "8a8eaf9c894e41dbaf4941fcc99e3512",
            "a06aad81a5b74a1bbebcebe03426eb2c",
            "eea5a581f3d147e0adc84a6c76603b47",
            "ded1874727b643d4be877b108c623b2c",
            "fb9221cc11094d889ec892e9dbe3c2a3",
            "71589bc3e22c4d1f9b80866e4d6b5cec",
            "b803f8139c6645f095ce080fc3245764",
            "301adffc637c42c7a57476b371aa3062",
            "babb14c4355a4f48b19c78effaa1ac18",
            "b2d83a05c84a410b8716d1f1458b2355",
            "28f6de94b497429fb3a3b8b0200ae6c4",
            "ea6b739651fc42ee8011b1b16ed7a70a",
            "4a03c614df474c08991042ed2d3523f9",
            "f07e928d3cf74eeaa925e8f6646d66fe",
            "b9ee6323db8245c482d9f3f288f96320",
            "20a66bccb930424ea58a008f5383ced2",
            "563855d76cd84683bacf4c3429dad1d4",
            "1d14a9a005154750af17573f7b6102d1",
            "8a8f21d7c2c9423790e99141af8181b2",
            "ae306c07391448b5a62f8386804dbb8c",
            "204abcf278c24e73806b706fdf35cec2",
            "b90674bec8e64cd59ed476d7977a0370",
            "61cd6642972745f399be0892fd414788",
            "a0967881414d4efa990427f51950bf81",
            "7eb327d14858488582586c8dd2d22462",
            "6e8c786692b94920998e37f142ba871c",
            "c1098df9c0e640eb88699752ea341e8b",
            "54ed606860b841cab49f2712082a0ffc",
            "cfae771ecf2f4062bde2798c995339f2",
            "19ebcf3c4f7340a99bd4f93e7dba69ae"
          ]
        },
        "id": "1g_f5-OV_JnT",
        "outputId": "1a5bb9c8-6886-4a0e-de79-59edf42ec5a7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Exception ignored in: <function _xla_gc_callback at 0x7e05de823250>\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/jax/_src/lib/__init__.py\", line 101, in _xla_gc_callback\n",
            "    def _xla_gc_callback(*args):\n",
            "KeyboardInterrupt: \n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/252 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6f5731be38894a5c9738d3e9aefa4b07"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d86d06f47a7948208da4636192b98d09"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "babb14c4355a4f48b19c78effaa1ac18"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/758 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ae306c07391448b5a62f8386804dbb8c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mean token length: 20.305590484330608\n",
            "Median token length: 17.0\n",
            "Max token length: 139\n",
            "Top 95% percentile token length: 42.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data Processing\n",
        "\n",
        "- Run this section to get `training_data.jsonl` file.\n",
        "- Skip this section if you already have `training_data.jsonl` file."
      ],
      "metadata": {
        "id": "o9w3BHbhsitx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd"
      ],
      "metadata": {
        "id": "fGjTgsI1w67C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv('analyst_ratings_processed.csv')"
      ],
      "metadata": {
        "id": "e89zjR3_t4OM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert 'date' from string to datetime object\n",
        "data['date'] = pd.to_datetime(data['date'], errors='coerce')\n",
        "\n",
        "# Filter out rows where 'date' is NaT (i.e., where date was invalid)\n",
        "data = data.dropna(subset=['date'])\n",
        "\n",
        "# Extract month and year into a new column\n",
        "data['month_year'] = data['date'].apply(lambda x: x.strftime('%Y-%m'))\n",
        "\n",
        "# Group by 'stock' and 'month_year', and aggregate 'title' and 'date'\n",
        "grouped = data.groupby(['stock', 'month_year']).apply(lambda x: x[['title', 'date']].to_dict('records')).reset_index(name='data')\n",
        "\n",
        "# Convert the grouped data into the desired dictionary structure\n",
        "grouped_data = {}\n",
        "for index, row in grouped.iterrows():\n",
        "    stock = row['stock']\n",
        "    month_year = row['month_year']\n",
        "    if stock not in grouped_data:\n",
        "        grouped_data[stock] = {}\n",
        "    grouped_data[stock][month_year] = row['data']\n",
        "\n",
        "# Now you can access the data as described\n",
        "# For example: grouped_data['A']['2020-06']"
      ],
      "metadata": {
        "id": "EDPxg3-CvV2t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data['date'].min()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p5Rh566DRKy3",
        "outputId": "8483e1ba-f444-492e-aa6a-4d5e8d8f1103"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "datetime.datetime(2009, 2, 14, 14, 2, tzinfo=tzoffset(None, -18000))"
            ]
          },
          "metadata": {},
          "execution_count": 145
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data['date'].max()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VZ0uasN1RuHr",
        "outputId": "bb9e1758-3765-4288-e540-20878a050eca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "datetime.datetime(2020, 6, 11, 17, 12, tzinfo=tzoffset(None, -14400))"
            ]
          },
          "metadata": {},
          "execution_count": 146
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "grouped_data.keys()"
      ],
      "metadata": {
        "id": "U0xQJFSW2BuU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ticker = 'AAPL'\n",
        "\n",
        "for month_year in grouped_data[ticker]:\n",
        "  print(f\"{month_year}: (total of {len(grouped_data[ticker][month_year])} items) {grouped_data[ticker][month_year][0]['title']} ({grouped_data[ticker][month_year][0]['date']}), ...\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sXcLwnwX3BrX",
        "outputId": "c626f525-0796-4a62-cf4c-bd01df710c4f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2020-03: (total of 130 items) Apple Acquires DarkSky, Weather App To Be Pulled From Android (2020-03-31 22:36:00-04:00), ...\n",
            "2020-04: (total of 164 items) Apple CEO Tim Cook: Store Traffic Not Back To Pre-COVID-19 Levels In China (2020-04-30 17:53:00-04:00), ...\n",
            "2020-05: (total of 123 items) Apple Says With Health and Safety of Its Teams in Mind, Decided to Keep Number of Its Stores in the U.S. Closed on Sunday (2020-05-31 18:19:00-04:00), ...\n",
            "2020-06: (total of 52 items) Tech Stocks And FAANGS Strong Again To Start Day As Market Awaits Fed (2020-06-10 11:33:00-04:00), ...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "total_months_count = 0\n",
        "usable_months_count = 0\n",
        "usable_months_chunk_count = 0\n",
        "\n",
        "for ticker in grouped_data.keys():\n",
        "  for month_year in grouped_data[ticker]:\n",
        "    year = int(month_year.split(\"-\")[0])\n",
        "    if year < 2019:\n",
        "        continue\n",
        "    total_months_count += 1\n",
        "    if len(grouped_data[ticker][month_year]) >= 5:\n",
        "        usable_months_count += 1\n",
        "    for i in range (0, len(grouped_data[ticker][month_year]) // 5):\n",
        "        usable_months_chunk_count += 1"
      ],
      "metadata": {
        "id": "aKEZlyJm7SjW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "total_months_count, usable_months_count, usable_months_chunk_count"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kRVHZLZB7o-M",
        "outputId": "a4dedbe7-e045-4570-ca63-78e9fb74887b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(41342, 16382, 35552)"
            ]
          },
          "metadata": {},
          "execution_count": 226
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import copy\n",
        "\n",
        "# Deep copy the original grouped_data to ensure it's not modified\n",
        "training_data = copy.deepcopy(grouped_data)\n",
        "\n",
        "# Iterate over the copied dictionary and filter out month_year with less than 5 items\n",
        "for stock in list(training_data.keys()):\n",
        "    for month_year in list(training_data[stock].keys()):\n",
        "        if len(training_data[stock][month_year]) < 5:\n",
        "            del training_data[stock][month_year]\n",
        "\n",
        "    # If a stock has no remaining month_year, remove the stock from the dictionary\n",
        "    if not training_data[stock]:\n",
        "        del training_data[stock]\n",
        "\n",
        "# 'training_data' now contains only those month_year entries with 5 or more items"
      ],
      "metadata": {
        "id": "EB_f6s_wataL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "import datetime\n",
        "\n",
        "def format_datetime(dt):\n",
        "    # Format the datetime object to a string without the timezone\n",
        "    dt_str = dt.strftime('%Y-%m-%d %H:%M:%S')\n",
        "    # Get the timezone offset in hours and minutes\n",
        "    offset_hour = int(dt.utcoffset().total_seconds() / 3600)\n",
        "    offset_min = int((dt.utcoffset().total_seconds() % 3600) / 60)\n",
        "    # Format the timezone offset\n",
        "    offset_str = f\"{offset_hour:+03d}:{offset_min:02d}\"\n",
        "    # Combine the datetime string with the timezone offset\n",
        "    return f\"{dt_str}{offset_str}\"\n",
        "\n",
        "# File path for the .jsonl file\n",
        "file_path = 'training_data.jsonl'\n",
        "\n",
        "# Open the file in write mode\n",
        "with open(file_path, 'w') as file:\n",
        "    # Iterate over each stock in training_data\n",
        "    for stock, month_years in training_data.items():\n",
        "        # Iterate over each month_year and its corresponding data\n",
        "        for month_year, entries in month_years.items():\n",
        "            # Convert datetime objects to formatted strings in each entry\n",
        "            for entry in entries:\n",
        "                if isinstance(entry['date'], datetime.datetime):\n",
        "                    entry['date'] = format_datetime(entry['date'])\n",
        "            # Prepare the JSON object for each entry\n",
        "            json_object = json.dumps({\n",
        "                \"stock\": stock,\n",
        "                \"month_year\": month_year,\n",
        "                \"data\": entries\n",
        "            })\n",
        "            # Write the JSON object to the file with a newline character\n",
        "            file.write(json_object + '\\n')\n",
        "\n",
        "# The file training_data.jsonl now contains all the training data with formatted dates"
      ],
      "metadata": {
        "id": "N_33gPDDbMjx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "> Specifications\n",
        "\n",
        "- Only months with 5 or more headlines will be used for monthly stock price prediction.\n",
        "- Each input to FinBERT will be grouped into 5 headlines.\n",
        "- The stock price prediction will be based on comparing the percentage change of the aggregated volumn weighted average price between the current month and the next month."
      ],
      "metadata": {
        "id": "Ydi6rhfbXu76"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Training Data Uploading\n",
        "\n",
        "- Run this section if you already have `training_data.jsonl` file."
      ],
      "metadata": {
        "id": "5owp_g5YzVUL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Upload training_data.jsonl file into a variable called \"training_data\", and start using the notebook from this point on\n",
        "# Upload zipped stock_data folder if the stock price history fetching from Polygon.io is still ongoing\n",
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()"
      ],
      "metadata": {
        "id": "4bSH2-3AzeZF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 108
        },
        "outputId": "06d23206-9669-44de-b1a7-bddb98780b5b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-8ced172c-80e7-4864-ba59-fce0a6081707\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-8ced172c-80e7-4864-ba59-fce0a6081707\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving stock_data_0_to_1000.zip to stock_data_0_to_1000.zip\n",
            "Saving training_data.jsonl to training_data.jsonl\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip stock_data_0_to_1000.zip"
      ],
      "metadata": {
        "id": "0F1lWssE8ESV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!rm stock_data_0_to_1000.zip"
      ],
      "metadata": {
        "id": "qoOt20JW8-aT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "\n",
        "# Path to your .jsonl file\n",
        "file_path = 'training_data.jsonl'\n",
        "\n",
        "# Initialize an empty list to store the dictionaries\n",
        "training_data_list = []\n",
        "\n",
        "# Open the file and read line by line\n",
        "with open(file_path, 'r') as file:\n",
        "    for line in file:\n",
        "        # Convert each line into a dictionary and append to the list\n",
        "        training_data_list.append(json.loads(line))"
      ],
      "metadata": {
        "id": "Jznnay769utU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "training_data = {}\n",
        "\n",
        "for entry in training_data_list:\n",
        "    if entry['stock'] not in training_data:\n",
        "        training_data[entry['stock']] = {}\n",
        "    training_data[entry['stock']][entry['month_year']] = entry['data']"
      ],
      "metadata": {
        "id": "cTFxFeWB-pvp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Stock Price History with Polygon (Demo)\n",
        "\n",
        "- Run this section to get `stock_data/*` folder with `{stock}.jsonl` files."
      ],
      "metadata": {
        "id": "onou2KyLSKdw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "polygon_api_key = '<Polygon API Key>'"
      ],
      "metadata": {
        "id": "gx9TzsZuSbOv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "\n",
        "def get_stock_data(ticker: str, start_date: str, end_date: str):\n",
        "  # Polygon.io API URL for AAPL stock data\n",
        "  url = f'https://api.polygon.io/v2/aggs/ticker/{ticker}/range/1/month/{start_date}/{end_date}?apiKey={polygon_api_key}'\n",
        "\n",
        "  # Make the GET request\n",
        "  response = requests.get(url)\n",
        "\n",
        "  # Check if the request was successful\n",
        "  if response.status_code == 200:\n",
        "      # Parse the JSON data\n",
        "      data = response.json()\n",
        "      return data\n",
        "  else:\n",
        "      print(f'Failed to retrieve data: {response.status_code}')"
      ],
      "metadata": {
        "id": "U0D1_RXKSN0C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_jan = get_stock_data('AAPL', '2023-01-01', '2023-01-31')\n",
        "data_feb = get_stock_data('AAPL', '2023-02-01', '2023-02-28')"
      ],
      "metadata": {
        "id": "XaQYl_PYSzw_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_jan, data_feb"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eNJYEiAdTQ_O",
        "outputId": "1ffa002a-e920-42a4-dade-7e913995dd58"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'ticker': 'AAPL',\n",
              "  'queryCount': 20,\n",
              "  'resultsCount': 1,\n",
              "  'adjusted': True,\n",
              "  'results': [{'v': 1443328754.0,\n",
              "    'vw': 134.9167,\n",
              "    'o': 130.28,\n",
              "    'c': 144.29,\n",
              "    'h': 147.23,\n",
              "    'l': 124.17,\n",
              "    't': 1672549200000,\n",
              "    'n': 12126333}],\n",
              "  'status': 'OK',\n",
              "  'request_id': '3f8e55b5b851563aeacca18ea3f0a2a6',\n",
              "  'count': 1},\n",
              " {'ticker': 'AAPL',\n",
              "  'queryCount': 19,\n",
              "  'resultsCount': 1,\n",
              "  'adjusted': True,\n",
              "  'results': [{'v': 1305872484.0,\n",
              "    'vw': 151.0488,\n",
              "    'o': 143.97,\n",
              "    'c': 147.41,\n",
              "    'h': 157.38,\n",
              "    'l': 141.32,\n",
              "    't': 1675227600000,\n",
              "    'n': 10694649}],\n",
              "  'status': 'OK',\n",
              "  'request_id': 'de8b438e1fc22c2ac6f2ae62d65c981c',\n",
              "  'count': 1})"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Stock Price History with Polygon (Data Fetching Demo)\n",
        "\n",
        "- Run this section to get `stock_data/*` folder with `{stock}.jsonl` files."
      ],
      "metadata": {
        "id": "rdoqONX2dkNP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import datetime\n",
        "import calendar\n",
        "\n",
        "def get_first_last_days(year_month_str):\n",
        "    # Parse the year and month from the input string\n",
        "    year, month = map(int, year_month_str.split('-'))\n",
        "\n",
        "    # First day of the month is always 1\n",
        "    first_day = datetime.date(year, month, 1)\n",
        "\n",
        "    # Get the last day of the month\n",
        "    last_day = datetime.date(year, month, calendar.monthrange(year, month)[1])\n",
        "\n",
        "    # Calculate the first day of the next month\n",
        "    if month == 12:\n",
        "        next_month_first_day = datetime.date(year + 1, 1, 1)\n",
        "    else:\n",
        "        next_month_first_day = datetime.date(year, month + 1, 1)\n",
        "\n",
        "    # Get the last day of the next month\n",
        "    next_month_last_day = datetime.date(next_month_first_day.year, next_month_first_day.month, calendar.monthrange(next_month_first_day.year, next_month_first_day.month)[1])\n",
        "\n",
        "    # Format the dates to the desired format\n",
        "    this_month_first_day_str = first_day.strftime('%Y-%m-%d')\n",
        "    this_month_last_day_str = last_day.strftime('%Y-%m-%d')\n",
        "    next_month_first_day_str = next_month_first_day.strftime('%Y-%m-%d')\n",
        "    next_month_last_day_str = next_month_last_day.strftime('%Y-%m-%d')\n",
        "\n",
        "    return this_month_first_day_str, this_month_last_day_str, next_month_first_day_str, next_month_last_day_str\n",
        "\n",
        "# Example usage\n",
        "month_year = \"2023-01\"\n",
        "this_month_first_day_str, this_month_last_day_str, next_month_first_day_str, next_month_last_day_str = get_first_last_days(month_year)\n",
        "print(\"This Month First Day:\", this_month_first_day_str)\n",
        "print(\"This Month Last Day:\", this_month_last_day_str)\n",
        "print(\"Next Month First Day:\", next_month_first_day_str)\n",
        "print(\"Next Month Last Day:\", next_month_last_day_str)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xaTPhXU8dr3b",
        "outputId": "23dadba6-b2ce-4627-a496-74751ec30a45"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "This Month First Day: 2023-01-01\n",
            "This Month Last Day: 2023-01-31\n",
            "Next Month First Day: 2023-02-01\n",
            "Next Month Last Day: 2023-02-28\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_all_monthly_stock_data(ticker: str):\n",
        "    stock_data = []\n",
        "    for month_year in training_data[ticker].keys():\n",
        "        # Extract year and month from month_year\n",
        "        year, month = map(int, month_year.split(\"-\"))\n",
        "\n",
        "        # Skip if month_year is before December 2018 (Polygon.io API has stock history available upto 5 years; modify this number accordingly to the current date; remove this if you have access to Polygon.io premium account)\n",
        "        if year < 2018 or (year == 2018 and month < 12):\n",
        "            continue\n",
        "\n",
        "        this_month_first_day_str, this_month_last_day_str, next_month_first_day_str, next_month_last_day_str = get_first_last_days(month_year)\n",
        "        try:\n",
        "            this_month_data = get_stock_data(ticker, this_month_first_day_str, this_month_last_day_str)\n",
        "            next_month_data = get_stock_data(ticker, next_month_first_day_str, next_month_last_day_str)\n",
        "            monthly_stock_data = {\n",
        "                \"stock\": ticker,\n",
        "                \"month_year\": month_year,\n",
        "                \"this_month_vw\": this_month_data['results'][0]['vw'],\n",
        "                \"next_month_vw\": next_month_data['results'][0]['vw'],\n",
        "            }\n",
        "            monthly_stock_data['percentage_change'] = (monthly_stock_data['next_month_vw'] - monthly_stock_data['this_month_vw']) / monthly_stock_data['this_month_vw']\n",
        "            stock_data.append(monthly_stock_data)\n",
        "            print(f'Retrieved data for {ticker} in {month_year}')\n",
        "        except:\n",
        "            print(f'Failed to retrieve data for {ticker} in {month_year}')\n",
        "            continue\n",
        "    return stock_data"
      ],
      "metadata": {
        "id": "LzhAtxbBfD4D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "aapl_stock_data = get_all_monthly_stock_data('AAPL')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IuEPp7_3h6zP",
        "outputId": "fbcc711c-44b2-40df-c359-098a7eebdcf6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Retrieved data for AAPL in 2020-03\n",
            "Retrieved data for AAPL in 2020-04\n",
            "Retrieved data for AAPL in 2020-05\n",
            "Retrieved data for AAPL in 2020-06\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "aapl_stock_data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rme4BdOqi2Ia",
        "outputId": "a365bee5-4b39-4003-c1eb-1d8a18958684"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'stock': 'AAPL',\n",
              "  'month_year': '2020-03',\n",
              "  'this_month_vw': 65.1759,\n",
              "  'next_month_vw': 67.85,\n",
              "  'percentage_change': 0.04102896929693331},\n",
              " {'stock': 'AAPL',\n",
              "  'month_year': '2020-04',\n",
              "  'this_month_vw': 67.85,\n",
              "  'next_month_vw': 77.255,\n",
              "  'percentage_change': 0.13861459100957998},\n",
              " {'stock': 'AAPL',\n",
              "  'month_year': '2020-05',\n",
              "  'this_month_vw': 77.255,\n",
              "  'next_month_vw': 87.0114,\n",
              "  'percentage_change': 0.12628826613164196},\n",
              " {'stock': 'AAPL',\n",
              "  'month_year': '2020-06',\n",
              "  'this_month_vw': 87.0114,\n",
              "  'next_month_vw': 96.1616,\n",
              "  'percentage_change': 0.10516093293522473}]"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Stock Price History with Polygon (Data Fetching)\n",
        "\n",
        "- Run this section to get `stock_data/*` folder with `{stock}.jsonl` files."
      ],
      "metadata": {
        "id": "Ek4ifCL-j1T6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Adjust the index as needed\n",
        "start_index = 0\n",
        "end_index = min(100, len(list(training_data.keys())))"
      ],
      "metadata": {
        "id": "1EOPqcNCkbQt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import json\n",
        "import time\n",
        "from tqdm import tqdm\n",
        "import concurrent.futures\n",
        "\n",
        "def process_ticker(ticker):\n",
        "    \"\"\"Function to process each ticker.\"\"\"\n",
        "    stock_data = get_all_monthly_stock_data(ticker)\n",
        "\n",
        "    # Only proceed if stock_data has entries\n",
        "    if len(stock_data) > 0:\n",
        "        # Define the file path for the .jsonl file\n",
        "        file_path = f'stock_data/{ticker}.jsonl'\n",
        "\n",
        "        # Open the file in write mode and write the data\n",
        "        with open(file_path, 'w') as file:\n",
        "            # Write each entry in stock_data as a separate line in JSON format\n",
        "            for entry in stock_data:\n",
        "                json_line = json.dumps(entry)\n",
        "                file.write(json_line + '\\n')\n",
        "        print(f'Saved data for {ticker} in {file_path}')\n",
        "    else:\n",
        "        print(f'No data available for {ticker}, skipping file creation.')\n",
        "\n",
        "# Create the 'stock_data' directory if it does not exist\n",
        "if not os.path.exists('stock_data'):\n",
        "    os.makedirs('stock_data')\n",
        "\n",
        "# Start the timer\n",
        "start_time = time.time()\n",
        "\n",
        "# List of tickers to process\n",
        "tickers = list(training_data.keys())[start_index:end_index]\n",
        "\n",
        "# Use ThreadPoolExecutor to process tickers in parallel\n",
        "with concurrent.futures.ThreadPoolExecutor() as executor:\n",
        "    # Use list to consume the results and provide progress with tqdm\n",
        "    list(tqdm(executor.map(process_ticker, tickers), total=len(tickers), desc=\"Processing tickers\"))\n",
        "\n",
        "# End the timer\n",
        "end_time = time.time()\n",
        "\n",
        "# Calculate and print the duration\n",
        "duration = end_time - start_time\n",
        "print(f\"Total run time: {duration:.2f} seconds\")"
      ],
      "metadata": {
        "id": "ausGxROkj5pf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!zip -r stock_data.zip stock_data/*"
      ],
      "metadata": {
        "id": "yF7aqnDFwpp0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Remove All Invalid Stock & Month-Year Data\n",
        "\n",
        "- Run this section after all previous sections to reduce the size of the `training_data.jsonl` further."
      ],
      "metadata": {
        "id": "-9BDmGuK97p5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Upload training_data.jsonl file into a variable called \"training_data\"\n",
        "# Upload zipped stock_data folder\n",
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()"
      ],
      "metadata": {
        "id": "K9lf98KK-BoK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip stock_data.zip\n",
        "!rm stock_data.zip"
      ],
      "metadata": {
        "id": "RtSGJlVy-DTG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Prepare stock price history data\n",
        "import json\n",
        "import os\n",
        "\n",
        "# Path to your folder containing .jsonl files\n",
        "folder_path = 'stock_data/'\n",
        "\n",
        "# Initialize an empty list to store the dictionaries from all files\n",
        "stock_price_history_data_list = []\n",
        "\n",
        "# Iterate over each file in the folder\n",
        "for filename in os.listdir(folder_path):\n",
        "    if filename.endswith('.jsonl'):\n",
        "        file_path = os.path.join(folder_path, filename)\n",
        "\n",
        "        # Open and read each file\n",
        "        with open(file_path, 'r') as file:\n",
        "            for line in file:\n",
        "                # Convert each line into a dictionary and append to the list\n",
        "                stock_price_history_data_list.append(json.loads(line))\n",
        "\n",
        "stock_price_history_data = {}\n",
        "\n",
        "for entry in stock_price_history_data_list:\n",
        "    if entry['stock'] not in stock_price_history_data:\n",
        "        stock_price_history_data[entry['stock']] = {}\n",
        "    stock_price_history_data[entry['stock']][entry['month_year']] = {\n",
        "        'this_month_vw': entry['this_month_vw'],\n",
        "        'next_month_vw': entry['next_month_vw'],\n",
        "        'percentage_change': entry['percentage_change'],\n",
        "    }"
      ],
      "metadata": {
        "id": "rBqTP2am-OKL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Prepare training data\n",
        "import json\n",
        "\n",
        "# Path to your .jsonl file\n",
        "file_path = 'training_data.jsonl'\n",
        "\n",
        "# Initialize an empty list to store the dictionaries\n",
        "training_data_list = []\n",
        "\n",
        "# Open the file and read line by line\n",
        "with open(file_path, 'r') as file:\n",
        "    for line in file:\n",
        "        # Convert each line into a dictionary and append to the list\n",
        "        training_data_list.append(json.loads(line))\n",
        "\n",
        "training_data = {}\n",
        "\n",
        "for entry in training_data_list:\n",
        "    if entry['stock'] not in training_data:\n",
        "        training_data[entry['stock']] = {}\n",
        "    training_data[entry['stock']][entry['month_year']] = entry['data']"
      ],
      "metadata": {
        "id": "fGVvwCMj-Opi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for ticker in stock_price_history_data.keys():\n",
        "    for month_year in stock_price_history_data[ticker].keys():\n",
        "        if month_year in training_data[ticker]:\n",
        "            entry = {\n",
        "                'stock': ticker,\n",
        "                'month_year': month_year,\n",
        "                'data': training_data[ticker][month_year]\n",
        "            }\n",
        "            # Store this to .jsonl file\n",
        "            with open('cleaned_training_data.jsonl', 'a') as file:\n",
        "                file.write(json.dumps(entry) + '\\n')\n",
        "\n",
        "# This reduces file size from 128MB to 32MB"
      ],
      "metadata": {
        "id": "dHCiMo4x--K8"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}