Prune_tutorial.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Prune_tutorial",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "031ae4767cee40caa77d6ccd21650573": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d52e682ec9df4987bcab5bf42beaa5eb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8182274d71444660bb86f4e955f14404",
              "IPY_MODEL_0adcf6bc0bc14c2b844782b71fe4d2c2",
              "IPY_MODEL_5a5ecc49e3134430977fe6d3e8e49b98"
            ]
          }
        },
        "d52e682ec9df4987bcab5bf42beaa5eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8182274d71444660bb86f4e955f14404": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bdfb10098610403ebe17a58d80c25584",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 17%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3658c715a3704052b0cad5d244e274f8"
          }
        },
        "0adcf6bc0bc14c2b844782b71fe4d2c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9b7ed43628e74d31a8a6e32d3c9ce29d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 30,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 5,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_613332ba6a064f799f9f7a93947225e7"
          }
        },
        "5a5ecc49e3134430977fe6d3e8e49b98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_80c0b8f7d51b44719f730b9089e2ad84",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5/30 [00:38&lt;02:59,  7.18s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c1ea27e021304367b3f9dbef9f780d11"
          }
        },
        "bdfb10098610403ebe17a58d80c25584": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3658c715a3704052b0cad5d244e274f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9b7ed43628e74d31a8a6e32d3c9ce29d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "613332ba6a064f799f9f7a93947225e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "80c0b8f7d51b44719f730b9089e2ad84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c1ea27e021304367b3f9dbef9f780d11": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5ea80c450b8f46468d0b14d8eaa3a592": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_462dbd8189884fb9b88516d591ab4af2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_066451ee8ba441f2a56abb274dd6d3a0",
              "IPY_MODEL_031647fa23274528bcfb7d83d6af063b",
              "IPY_MODEL_1500b955e450464fb8ba98262d9af117"
            ]
          }
        },
        "462dbd8189884fb9b88516d591ab4af2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "066451ee8ba441f2a56abb274dd6d3a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7ecdf21c174f4e2abcc01b7b3088ba6f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bbe0026d8b0441158753713505d6a7a3"
          }
        },
        "031647fa23274528bcfb7d83d6af063b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_281558f9df294086ac50515f1bc15808",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 20,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 20,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e205ea6d9393442dabfdf2726c0c79e8"
          }
        },
        "1500b955e450464fb8ba98262d9af117": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f89ba01c5a6b4158a60d6033277c3e5f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 20/20 [00:00&lt;00:00, 21.41it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0daa368c3fd0438eba8cb0ae29399a30"
          }
        },
        "7ecdf21c174f4e2abcc01b7b3088ba6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bbe0026d8b0441158753713505d6a7a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "281558f9df294086ac50515f1bc15808": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e205ea6d9393442dabfdf2726c0c79e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f89ba01c5a6b4158a60d6033277c3e5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0daa368c3fd0438eba8cb0ae29399a30": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d790bf19e0064cbfb375b3a0a0986939": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_203c633edaaf4abbb7d5213585635f0a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cd99c59f160d4559b8efaa2815a6f08f",
              "IPY_MODEL_4d03915467a540d09a903efba47f4d3c",
              "IPY_MODEL_f0673fc2162849149208cda0d8c9e885"
            ]
          }
        },
        "203c633edaaf4abbb7d5213585635f0a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cd99c59f160d4559b8efaa2815a6f08f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a7dfbe9a63e947839496b485afb75a48",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6c99166c906d4b55b394f43a7f244b26"
          }
        },
        "4d03915467a540d09a903efba47f4d3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b89904e79fa34c56a1713208de1b3270",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 20,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 20,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0c3ffa5920cb4167a4f2d4ee57f6eb95"
          }
        },
        "f0673fc2162849149208cda0d8c9e885": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2d0d41eac2634c6d86ace844d8b7832e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 20/20 [00:00&lt;00:00, 22.57it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a3d1ab4501e24b3e9bdc986b0eb9546e"
          }
        },
        "a7dfbe9a63e947839496b485afb75a48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6c99166c906d4b55b394f43a7f244b26": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b89904e79fa34c56a1713208de1b3270": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0c3ffa5920cb4167a4f2d4ee57f6eb95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2d0d41eac2634c6d86ace844d8b7832e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a3d1ab4501e24b3e9bdc986b0eb9546e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b2de4a712ab74abe9624c73fe906ebdc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e0fa42127bbd441f8280e9ca03e9d6f2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_40eaea5c792644048823868a050667c9",
              "IPY_MODEL_ae0af8897b9d41ea80d084f886d5e3b1",
              "IPY_MODEL_c1b01ba1ae2548b1a559e1ea529a5264"
            ]
          }
        },
        "e0fa42127bbd441f8280e9ca03e9d6f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "40eaea5c792644048823868a050667c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_564562f923394ec7843a0a0a1688861f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d3d3aad666014f4fb3e31217a4133297"
          }
        },
        "ae0af8897b9d41ea80d084f886d5e3b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2ee57cf2980848cdbc846442b505354b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 20,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 20,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_91226977cec941cfa367559bceb3acd7"
          }
        },
        "c1b01ba1ae2548b1a559e1ea529a5264": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7a235a982af14628b38ed8576ee94c85",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 20/20 [00:00&lt;00:00, 23.94it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5c061a904331438996789c13232c6a50"
          }
        },
        "564562f923394ec7843a0a0a1688861f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d3d3aad666014f4fb3e31217a4133297": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2ee57cf2980848cdbc846442b505354b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "91226977cec941cfa367559bceb3acd7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7a235a982af14628b38ed8576ee94c85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5c061a904331438996789c13232c6a50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1616b7d3dda24ccf8a5be364635ceed5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_44494507f8a54617b82e948a8440b4f9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5c90097574d14971b74143959eb732d2",
              "IPY_MODEL_6e2b3216a27d4c3aa0ca17bfea6365b1",
              "IPY_MODEL_f28143c51649461bb69d3bcdb683bed2"
            ]
          }
        },
        "44494507f8a54617b82e948a8440b4f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5c90097574d14971b74143959eb732d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1fe5fdae3f954f839738a88f10f76caf",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7cf1406f32ea46378537d1237c3532fb"
          }
        },
        "6e2b3216a27d4c3aa0ca17bfea6365b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8fa039daa7774ce483ca0206d513e789",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 20,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 20,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cee4e0683e55463b9fa4ae93c004bb8e"
          }
        },
        "f28143c51649461bb69d3bcdb683bed2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_54eea763862c45528fdf4ea786f63466",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 20/20 [00:00&lt;00:00, 24.71it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f07969b8e40e47ed8e77861f207290d5"
          }
        },
        "1fe5fdae3f954f839738a88f10f76caf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7cf1406f32ea46378537d1237c3532fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8fa039daa7774ce483ca0206d513e789": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cee4e0683e55463b9fa4ae93c004bb8e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "54eea763862c45528fdf4ea786f63466": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f07969b8e40e47ed8e77861f207290d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7b25f64f0a154084bd9c3504bc3ddf51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_42814286b710410ca0914e097336e846",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f354b00f62594875bd5993336617f1d3",
              "IPY_MODEL_fce80578cafb404b931e8b90d008fe8d",
              "IPY_MODEL_8b34d93514a247dc8cd8384b7d714575"
            ]
          }
        },
        "42814286b710410ca0914e097336e846": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f354b00f62594875bd5993336617f1d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b9f5c4906ab24af084f9072800ae6a67",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a9da1491862049dab95a46c477f82c6d"
          }
        },
        "fce80578cafb404b931e8b90d008fe8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_744447f5ffc5428abb500be1677d80cb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 20,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 20,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_88e6f320ff2842869774f10ccd18876c"
          }
        },
        "8b34d93514a247dc8cd8384b7d714575": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_5cc9167ef6bf472ba51c9d37cd77bd49",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 20/20 [00:00&lt;00:00, 24.07it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c95742d6160449258aa71c78f18eacc8"
          }
        },
        "b9f5c4906ab24af084f9072800ae6a67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a9da1491862049dab95a46c477f82c6d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "744447f5ffc5428abb500be1677d80cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "88e6f320ff2842869774f10ccd18876c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5cc9167ef6bf472ba51c9d37cd77bd49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c95742d6160449258aa71c78f18eacc8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/EherSenaw/Prune_tutorial/blob/main/Prune_tutorial.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Colab notebook (.ipynb) for [Prune_tutorial](https://github.com/EherSenaw/Prune_tutorial)\n",
        "\n",
        "- \\<Imports\\>\n",
        "> Imports needed for this tutorial.\n",
        "- \\<Configs\\>\n",
        "> Configurations used for this tutorial. Change this to customize.\n",
        "- \\<Utils\\>\n",
        "> Utils used for this tutorial. Might be helpful for reader whom searching for starting-point of pruning-related research/implementation.\n",
        "- \\<Model\\>\n",
        "> Sample model used for this tutorial. Any other models could be used...\n"
      ],
      "metadata": {
        "id": "jLwVt1VqKge6"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Imports"
      ],
      "metadata": {
        "id": "5axua8ACLZGl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import math\n",
        "\n",
        "import numpy as np \n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.nn.utils.prune as prune\n",
        "\n",
        "from copy import deepcopy\n",
        "from tqdm.notebook import tqdm\n",
        "from torchvision import transforms\n",
        "from torch.nn.modules.batchnorm import BatchNorm2d\n",
        "from torchvision.datasets import CIFAR10, SVHN, MNIST"
      ],
      "metadata": {
        "id": "Jb17xc34vCGu"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Configs"
      ],
      "metadata": {
        "id": "0nMUt5sZObet"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 512\n",
        "step_size = 10\n",
        "total_epochs = 30\n",
        "learning_rate = 3e-4 \n",
        "lr_decay = 1e-1\n",
        "\n",
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
        "save_file_path = './best.pth'\n",
        "\n",
        "## mask_dict: True -> don't change, False -> change to 0.\n",
        "true_3x3 = [[True for _ in range(3)] for _ in range(3)]\n",
        "false_3x3 = [[False for _ in range(3)] for _ in range(3)]\n",
        "cross_3x3 = [[True, False, True], [False, True, False], [True, False, True]]\n",
        "custom_3x3 = [[False, True, True], [True, True, True], [True, True, True]] \n",
        "## Change these dict to change masking\n",
        "kernel_mask_dict = {\n",
        "    'conv_1_24_3': true_3x3, \n",
        "    'conv_24_32_3': custom_3x3, \n",
        "    'conv_32_32_3': true_3x3, \n",
        "    'conv_32_64_3': true_3x3, \n",
        "    'conv_64_64_3': true_3x3, \n",
        "}\n",
        "IC_mask_dict = {\n",
        "    'conv_1_24_3': [true_3x3 for i in range(1)],\n",
        "    'conv_24_32_3': [true_3x3 for i in range(24)],\n",
        "    'conv_32_32_3': [true_3x3 for i in range(32)],\n",
        "    'conv_32_64_3': [true_3x3 for i in range(32)],\n",
        "    'conv_64_64_3': [true_3x3 for i in range(64)],\n",
        "}\n",
        "OC_mask_dict = {\n",
        "    'conv_1_24_3': [[true_3x3 for _ in range(1)] for i in range(24)],\n",
        "    'conv_24_32_3': [[true_3x3 for _ in range(24)] for i in range(32)],\n",
        "    'conv_32_32_3': [[true_3x3 for _ in range(32)] for i in range(32)],\n",
        "    'conv_32_64_3': [[true_3x3 for _ in range(32)] for i in range(64)],\n",
        "    'conv_64_64_3': [[true_3x3 for _ in range(64)] for i in range(64)],\n",
        "}"
      ],
      "metadata": {
        "id": "a_PjyPJeOc6X"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Utils"
      ],
      "metadata": {
        "id": "WvwOkDPLKdP2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Hierarchical Masking of weight initialization\n",
        "##Use with 'torch.nn.module.apply()'\n",
        "@torch.no_grad()\n",
        "def weight_init_custom(submodule):\n",
        "    if isinstance(submodule, torch.nn.Conv2d):\n",
        "        # Set all element of conv2d layers to zero (both weight & bias)\n",
        "        if isinstance(submodule.kernel_size, tuple) and submodule.kernel_size == (3,3):\n",
        "            if submodule.weight is not None:\n",
        "                dict_key = \"conv_\" + str(submodule.in_channels) +\"_\"+ str(submodule.out_channels) +\"_\"+ str(submodule.kernel_size[0])\n",
        "                # Kernel-mask\n",
        "                mask_tensor = torch.tensor([[kernel_mask_dict[dict_key] for _ in range(submodule.in_channels)] for _ in range(submodule.out_channels)], requires_grad=False).float()\n",
        "                # IC-mask\n",
        "                mask_tensor = mask_tensor * torch.tensor([IC_mask_dict[dict_key] for _ in range(submodule.out_channels)], requires_grad=False).float()\n",
        "                # OC-mask\n",
        "                mask_tensor = mask_tensor * torch.tensor(OC_mask_dict[dict_key], requires_grad=False).float()\n",
        "                submodule.weight.copy_(submodule.weight * mask_tensor.float())\n",
        "    #print(f\"non-zero submodule.weight.data count: {len(submodule.weight.data[submodule.weight.data.nonzero(as_tuple=True)])}\")\n",
        "\n",
        "# Show parameters of given model\n",
        "@torch.no_grad()\n",
        "def profiler_custom(model):\n",
        "    for name, param in model.named_parameters():\n",
        "        if param.requires_grad:\n",
        "            print(name, param.data)\n",
        "# Show modules of given model\n",
        "@torch.no_grad()\n",
        "def profiler_modules(model):\n",
        "    for name, module in model.named_modules():\n",
        "        print(f\"{name}: {module}\")\n",
        "\n",
        "# Hook, for masking gradients (and debugging)\n",
        "class Hook():\n",
        "    ## TODO: Define gradient mask for mask-target layers\n",
        "    #mask_dict = {\n",
        "    #    'conv_1_24_3': [[[[False, True, False] for _ in range(3)] for i in range(1)] for j in range(24)],\n",
        "    #    'conv_24_32_3': [[[[False, True, False] for _ in range(3)] for i in range(24)] for j in range(32)],\n",
        "    #    'conv_32_32_3': [[[[False, True, False] for _ in range(3)] for i in range(32)] for j in range(32)],\n",
        "    #    'conv_32_64_3': [[[[False, True, False] for _ in range(3)] for i in range(32)] for j in range(64)],\n",
        "    #    'conv_64_64_3': [[[[False, True, False] for _ in range(3)] for i in range(64)] for j in range(64)],\n",
        "    #}\n",
        "    def __init__(self, module, backward=False):\n",
        "        if not backward:\n",
        "            if isinstance(module[1], nn.Sequential):\n",
        "                self.hook = []\n",
        "                for submodule in list(module[1]._modules.items()):\n",
        "                    curHook = submodule[1].register_forward_hook(self.hook_fn_fwd)\n",
        "                    self.hook.append(curHook)\n",
        "            else:\n",
        "                self.hook = module[1].register_forward_hook(self.hook_fn_fwd)\n",
        "        else:\n",
        "            self.module_name = module[0]\n",
        "            if isinstance(module[1], nn.Sequential):\n",
        "                self.hook = []\n",
        "                for submodule in list(module[1]._modules.items()):\n",
        "                    curHook = submodule[1].register_full_backward_hook(self.hook_fn_full_bwd)\n",
        "                    self.hook.append(curHook)\n",
        "            else:\n",
        "                self.hook = module[1].register_full_backward_hook(self.hook_fn_full_bwd)\n",
        "    def hook_fn_fwd(self, module, input, output):\n",
        "        # Forward hook\n",
        "        # If you want to check forward pass, use this.\n",
        "        pass\n",
        "    def hook_fn_full_bwd(self, module, grad_in=None, grad_out=None):\n",
        "        # Backward Hook\n",
        "        # TODO: Repair commented parts, which was intended for gradient masking.\n",
        "        with torch.no_grad():\n",
        "            if not (isinstance(module, nn.Linear) or isinstance(module, nn.GELU) \\\n",
        "                    or isinstance(module, nn.BatchNorm2d) or isinstance(module, nn.Dropout)\\\n",
        "                    or isinstance(module, nn.MaxPool2d) or isinstance(module, nn.Conv2d)):\n",
        "                if grad_out is not None:\n",
        "                    if grad_out[0] is not None:\n",
        "                        assert not torch.isnan(grad_out[0]).any()\n",
        "                        #print(f\"\\t\\t<GRAD_OUT shape> {grad_out[0].size()}\")\n",
        "                        #masked_grad = grad_out[0] * torch.tensor(self.mask_dict[module[0]], requires_grad=False).float()\n",
        "                        #return (masked_grad, )\n",
        "            else:\n",
        "                # Unsubscriptable\n",
        "                if grad_out is not None:\n",
        "                    if grad_out[0] is not None:\n",
        "                        assert not torch.isnan(grad_out[0]).any()\n",
        "                        #print(f\"\\t\\t<GRAD_OUT shape> {grad_out[0].size()}\")\n",
        "                        #masked_grad = grad_out[0] * torch.tensor(self.mask_dict[module], requires_grad=False).float()\n",
        "                        #return (masked_grad, )\n",
        "        #return grad_in\n",
        "    def close(self):\n",
        "        if self.hook is not None:\n",
        "            if isinstance(self.hook, list):\n",
        "                for subHook in self.hook:\n",
        "                    subHook.remove()\n",
        "            else:\n",
        "                self.hook.remove()\n"
      ],
      "metadata": {
        "id": "UQm5SvOVKRTR"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Prune-related stuffs"
      ],
      "metadata": {
        "id": "tCRWezz3WMMZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "MPM_dict_key = None\n",
        "\n",
        "class MaskPruningMethod(prune.BasePruningMethod):\n",
        "    \"\"\"Prune tensor\"\"\"\n",
        "    PRUNING_TYPE = 'unstructured'\n",
        "    def compute_mask(self, t, default_mask):\n",
        "        global MPM_dict_key\n",
        "        OC_NUM, IC_NUM, _, _ = torch.tensor(OC_mask_dict[MPM_dict_key], requires_grad=False).size()\n",
        "        mask = default_mask.clone()\n",
        "        # Kernel-mask\n",
        "        mask = mask * torch.tensor([[kernel_mask_dict[MPM_dict_key] for _ in range(IC_NUM)] for _ in range(OC_NUM)], requires_grad=False).float()\n",
        "        # IC-mask\n",
        "        mask = mask * torch.tensor([IC_mask_dict[MPM_dict_key] for _ in range(OC_NUM)], requires_grad=False).float()\n",
        "        # OC-mask\n",
        "        mask = mask * torch.tensor(OC_mask_dict[MPM_dict_key], requires_grad=False).float()\n",
        "        return mask\n",
        "\n",
        "def mask_prune(module):\n",
        "    global MPM_dict_key\n",
        "    for submodule in module.modules():\n",
        "        if not isinstance(submodule, nn.Conv2d):\n",
        "            continue\n",
        "        dict_key = \"conv_\" + str(submodule.in_channels) +\"_\"+ str(submodule.out_channels) +\"_\"+ str(submodule.kernel_size[0])\n",
        "        if dict_key not in kernel_mask_dict:\n",
        "            continue\n",
        "        MPM_dict_key = dict_key\n",
        "        if hasattr(submodule, 'weight'):\n",
        "            # Apply prune (weights saved as 'weight_orig')\n",
        "            MaskPruningMethod.apply(submodule, 'weight')\n",
        "            ## !!!SHOULD CALL 'permenant_prune()' BEFORE saving model!!!\n",
        "# Use this BEFORE saving model.\n",
        "def permanent_prune(module):\n",
        "    for submodule in module.modules():\n",
        "        if not isinstance(submodule, nn.Conv2d):\n",
        "            continue\n",
        "        dict_key = \"conv_\" + str(submodule.in_channels) +\"_\"+ str(submodule.out_channels) +\"_\"+ str(submodule.kernel_size[0])\n",
        "        if dict_key not in kernel_mask_dict:\n",
        "            continue\n",
        "        # Make prune permanent (weights saved as 'weight')\n",
        "        prune.remove(submodule, 'weight')"
      ],
      "metadata": {
        "id": "i-Q1RDDwWOQa"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Model"
      ],
      "metadata": {
        "id": "d8L8-R_XL56P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Sample_Model(nn.Module):\n",
        "    def __init__(self, num_classes=10): \n",
        "        super(Sample_Model, self).__init__()\n",
        "\n",
        "        self.layers = nn.Sequential(\n",
        "            nn.Conv2d(1, 24, 3, padding=1, bias=True), \n",
        "            nn.MaxPool2d(2),\n",
        "            nn.GELU(),\n",
        "\n",
        "            nn.Conv2d(24, 32, 3, padding=1, bias=False),\n",
        "            nn.BatchNorm2d(32, eps=1e-4),\n",
        "            nn.GELU(),\n",
        "\n",
        "            nn.Conv2d(32, 32, 3, padding=1, bias=False),\n",
        "            nn.BatchNorm2d(32, eps=1e-4),\n",
        "            nn.MaxPool2d(2),\n",
        "            nn.GELU(),\n",
        "\n",
        "            nn.Conv2d(32, 32, 3, padding=0, bias=False),\n",
        "            nn.BatchNorm2d(32, eps=1e-4),\n",
        "            nn.GELU(),\n",
        "\n",
        "            nn.Conv2d(32, 64, 3, padding=1, bias=False),\n",
        "            nn.BatchNorm2d(64, eps=1e-4),\n",
        "            nn.GELU(),\n",
        "            \n",
        "            nn.Conv2d(64, 64, 3, padding=0, bias=False),\n",
        "            nn.BatchNorm2d(64, eps=1e-4),\n",
        "            nn.GELU(),\n",
        "\n",
        "            nn.Dropout(0.5),\n",
        "            nn.Conv2d(64, 64, 3, padding=0, bias=False),\n",
        "            nn.GELU(),\n",
        "        )\n",
        "        # Apply Prune\n",
        "        mask_prune(self.layers)\n",
        "        self.fc = nn.Linear(64, 10) \n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.layers(x)\n",
        "        x = x.view([x.shape[0], -1])\n",
        "        x = self.fc(x)\n",
        "        return x"
      ],
      "metadata": {
        "id": "fKnSCEMclQgE"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_dataloader():\n",
        "    # MNIST\n",
        "    train_loader = torch.utils.data.DataLoader(\n",
        "        MNIST('./Data',\n",
        "            transform=transforms.Compose([\n",
        "                transforms.ToTensor(),\n",
        "            ]),\n",
        "            download=True,\n",
        "            train=True\n",
        "        ),\n",
        "        shuffle=True,\n",
        "        batch_size=batch_size,\n",
        "        num_workers=0\n",
        "    )\n",
        "    test_loader = torch.utils.data.DataLoader(\n",
        "        MNIST('./Data',\n",
        "            transform=transforms.Compose([\n",
        "                transforms.ToTensor(),\n",
        "            ]),\n",
        "            download=True,\n",
        "            train=False\n",
        "        ), \n",
        "        shuffle=False, \n",
        "        batch_size=batch_size, \n",
        "        num_workers=0\n",
        "    )\n",
        "    return train_loader, test_loader\n",
        "\n",
        "@torch.no_grad()\n",
        "def eval(model, test_loader):\n",
        "    correct = 0\n",
        "    total = 0\n",
        "\n",
        "    model.to(device)\n",
        "    model.eval()\n",
        "\n",
        "    for i, (img, target) in enumerate(tqdm(test_loader)):\n",
        "        img = img.to(device)\n",
        "        out = model(img)\n",
        "        pred = out.max(1)[1].detach().cpu().numpy()\n",
        "        target = target.cpu().numpy()\n",
        "        correct += (pred==target).sum()\n",
        "        total += len(target)\n",
        "\n",
        "    return correct / total\n",
        "\n",
        "\n",
        "def train_model(model, train_loader, test_loader):\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
        "    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size, lr_decay) \n",
        "\n",
        "    best_acc = -1\n",
        "    # For debugging, uncomment following line \n",
        "    #torch.autograd.set_detect_anomaly(True)\n",
        "    for epoch in tqdm(range(total_epochs)):\n",
        "        # Train\n",
        "        model.train()\n",
        "        cnt = 0\n",
        "        loss_sum = 0\n",
        "        loss_ = 0\n",
        "        for i, (img, target) in enumerate(train_loader):\n",
        "            cnt += 1\n",
        "            img, target = img.to(device), target.to(device)\n",
        "            optimizer.zero_grad()\n",
        "            out = model(img)\n",
        "            loss = F.cross_entropy(out, target)\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "            loss_sum += loss.item()\n",
        "        \n",
        "        loss_sum = loss_sum / cnt\n",
        "        loss_= round(loss_sum, 4)\n",
        "\n",
        "        # Evaluate\n",
        "        model.eval()\n",
        "        acc = eval(model, test_loader)\n",
        "        if acc > best_acc :\n",
        "          best_acc = acc\n",
        "          print('Best accuracy is updated: %.4f '%(best_acc))\n",
        "          # Save model\n",
        "          copied_model = deepcopy(model)\n",
        "          permanent_prune(copied_model.layers)\n",
        "          torch.save(copied_model.state_dict(), save_file_path)\n",
        "          del copied_model\n",
        "\n",
        "        scheduler.step() \n",
        "\n",
        "    print(\"Best Accuracy: %.4f\"%(best_acc))"
      ],
      "metadata": {
        "id": "hHzkqPjGkF_n"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Train, Evaluate, Save"
      ],
      "metadata": {
        "id": "XnAjQdkrT0ir"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def main():\n",
        "    train_loader, test_loader = get_dataloader()\n",
        "    model = Sample_Model(num_classes=10).cuda()\n",
        "\n",
        "    # Register hook, for masking (and so on)\n",
        "    layers = list(model._modules.items())\n",
        "    #HookF = [Hook(layer, backward=False) for layer in layers]\n",
        "    HookB = [Hook(layer, backward=True) for layer in layers]\n",
        "\n",
        "    # Train, evaluate, save model\n",
        "    train_model(model, train_loader, test_loader)\n",
        "\n",
        "    # Unregister hook\n",
        "    for hook in HookB:\n",
        "        hook.close()\n",
        "    #for hook in HookF:\n",
        "    #    hook.close()\n",
        "    del HookB\n",
        "    #del HookF\n",
        "\n",
        "if __name__=='__main__':\n",
        "  main()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 625,
          "referenced_widgets": [
            "031ae4767cee40caa77d6ccd21650573",
            "d52e682ec9df4987bcab5bf42beaa5eb",
            "8182274d71444660bb86f4e955f14404",
            "0adcf6bc0bc14c2b844782b71fe4d2c2",
            "5a5ecc49e3134430977fe6d3e8e49b98",
            "bdfb10098610403ebe17a58d80c25584",
            "3658c715a3704052b0cad5d244e274f8",
            "9b7ed43628e74d31a8a6e32d3c9ce29d",
            "613332ba6a064f799f9f7a93947225e7",
            "80c0b8f7d51b44719f730b9089e2ad84",
            "c1ea27e021304367b3f9dbef9f780d11",
            "5ea80c450b8f46468d0b14d8eaa3a592",
            "462dbd8189884fb9b88516d591ab4af2",
            "066451ee8ba441f2a56abb274dd6d3a0",
            "031647fa23274528bcfb7d83d6af063b",
            "1500b955e450464fb8ba98262d9af117",
            "7ecdf21c174f4e2abcc01b7b3088ba6f",
            "bbe0026d8b0441158753713505d6a7a3",
            "281558f9df294086ac50515f1bc15808",
            "e205ea6d9393442dabfdf2726c0c79e8",
            "f89ba01c5a6b4158a60d6033277c3e5f",
            "0daa368c3fd0438eba8cb0ae29399a30",
            "d790bf19e0064cbfb375b3a0a0986939",
            "203c633edaaf4abbb7d5213585635f0a",
            "cd99c59f160d4559b8efaa2815a6f08f",
            "4d03915467a540d09a903efba47f4d3c",
            "f0673fc2162849149208cda0d8c9e885",
            "a7dfbe9a63e947839496b485afb75a48",
            "6c99166c906d4b55b394f43a7f244b26",
            "b89904e79fa34c56a1713208de1b3270",
            "0c3ffa5920cb4167a4f2d4ee57f6eb95",
            "2d0d41eac2634c6d86ace844d8b7832e",
            "a3d1ab4501e24b3e9bdc986b0eb9546e",
            "b2de4a712ab74abe9624c73fe906ebdc",
            "e0fa42127bbd441f8280e9ca03e9d6f2",
            "40eaea5c792644048823868a050667c9",
            "ae0af8897b9d41ea80d084f886d5e3b1",
            "c1b01ba1ae2548b1a559e1ea529a5264",
            "564562f923394ec7843a0a0a1688861f",
            "d3d3aad666014f4fb3e31217a4133297",
            "2ee57cf2980848cdbc846442b505354b",
            "91226977cec941cfa367559bceb3acd7",
            "7a235a982af14628b38ed8576ee94c85",
            "5c061a904331438996789c13232c6a50",
            "1616b7d3dda24ccf8a5be364635ceed5",
            "44494507f8a54617b82e948a8440b4f9",
            "5c90097574d14971b74143959eb732d2",
            "6e2b3216a27d4c3aa0ca17bfea6365b1",
            "f28143c51649461bb69d3bcdb683bed2",
            "1fe5fdae3f954f839738a88f10f76caf",
            "7cf1406f32ea46378537d1237c3532fb",
            "8fa039daa7774ce483ca0206d513e789",
            "cee4e0683e55463b9fa4ae93c004bb8e",
            "54eea763862c45528fdf4ea786f63466",
            "f07969b8e40e47ed8e77861f207290d5",
            "7b25f64f0a154084bd9c3504bc3ddf51",
            "42814286b710410ca0914e097336e846",
            "f354b00f62594875bd5993336617f1d3",
            "fce80578cafb404b931e8b90d008fe8d",
            "8b34d93514a247dc8cd8384b7d714575",
            "b9f5c4906ab24af084f9072800ae6a67",
            "a9da1491862049dab95a46c477f82c6d",
            "744447f5ffc5428abb500be1677d80cb",
            "88e6f320ff2842869774f10ccd18876c",
            "5cc9167ef6bf472ba51c9d37cd77bd49",
            "c95742d6160449258aa71c78f18eacc8"
          ]
        },
        "id": "r2NIplUaT7_f",
        "outputId": "20148c14-0f73-4e60-91d2-7ff8658e3df1"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "031ae4767cee40caa77d6ccd21650573",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/30 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5ea80c450b8f46468d0b14d8eaa3a592",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/20 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best accuracy is updated: 0.9791 \n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d790bf19e0064cbfb375b3a0a0986939",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/20 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best accuracy is updated: 0.9881 \n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b2de4a712ab74abe9624c73fe906ebdc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/20 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best accuracy is updated: 0.9889 \n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1616b7d3dda24ccf8a5be364635ceed5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/20 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best accuracy is updated: 0.9902 \n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7b25f64f0a154084bd9c3504bc3ddf51",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/20 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best accuracy is updated: 0.9908 \n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-11-c20f94ce9279>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m   \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-11-c20f94ce9279>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0;31m# Train, evaluate, save model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m     \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0;31m# Unregister hook\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-10-67d56012b55f>\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(model, train_loader, test_loader)\u001b[0m\n\u001b[1;32m     65\u001b[0m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcross_entropy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 67\u001b[0;31m             \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     68\u001b[0m             \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m             \u001b[0mloss_sum\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/_tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    305\u001b[0m                 \u001b[0mcreate_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m                 inputs=inputs)\n\u001b[0;32m--> 307\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    309\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    154\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m    155\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 156\u001b[0;31m         allow_unreachable=True, accumulate_grad=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    157\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load best model and check pruning"
      ],
      "metadata": {
        "id": "GEL5X960S-dR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = Sample_Model(num_classes=10)\n",
        "# To match shape of model's parameters, use 'permanent_prune'.\n",
        "permanent_prune(model.layers)\n",
        "model.load_state_dict(torch.load(save_file_path))\n",
        "model = model.cuda()\n",
        "for name, p in model.named_parameters():\n",
        "\tprint(name)\n",
        " "
      ],
      "metadata": {
        "id": "Mm7IbYgP-NLj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a058be38-e12c-4e75-b3db-ac06888d36b5"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "layers.0.bias\n",
            "layers.0.weight\n",
            "layers.3.weight\n",
            "layers.4.weight\n",
            "layers.4.bias\n",
            "layers.6.weight\n",
            "layers.7.weight\n",
            "layers.7.bias\n",
            "layers.10.weight\n",
            "layers.11.weight\n",
            "layers.11.bias\n",
            "layers.13.weight\n",
            "layers.14.weight\n",
            "layers.14.bias\n",
            "layers.16.weight\n",
            "layers.17.weight\n",
            "layers.17.bias\n",
            "layers.20.weight\n",
            "fc.weight\n",
            "fc.bias\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.get_parameter('layers.3.weight').shape"
      ],
      "metadata": {
        "id": "6xyEQTDcHFhO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "97f76e77-4d3d-44d5-920f-aa99985dbc91"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([32, 24, 3, 3])"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(model.get_parameter('layers.3.weight'))"
      ],
      "metadata": {
        "id": "4c8LYBdXpplz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7f89acd6-46b3-4195-dcd0-369dcd8a4104"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Parameter containing:\n",
            "tensor([[[[ 0.0000,  0.0347, -0.0546],\n",
            "          [-0.0356, -0.0218, -0.0172],\n",
            "          [-0.0383,  0.0507, -0.0138]],\n",
            "\n",
            "         [[-0.0000,  0.0390,  0.0070],\n",
            "          [ 0.0354, -0.0549,  0.0540],\n",
            "          [-0.0450,  0.0202, -0.0072]],\n",
            "\n",
            "         [[-0.0000, -0.0391,  0.0122],\n",
            "          [-0.0549,  0.0374,  0.0332],\n",
            "          [ 0.0283, -0.0039, -0.0281]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-0.0000, -0.0011,  0.0185],\n",
            "          [ 0.0334, -0.0079,  0.0712],\n",
            "          [ 0.0406,  0.0461,  0.0166]],\n",
            "\n",
            "         [[ 0.0000,  0.0709,  0.0272],\n",
            "          [ 0.0035,  0.0010, -0.0408],\n",
            "          [ 0.0343, -0.0102, -0.0494]],\n",
            "\n",
            "         [[-0.0000,  0.0581,  0.0133],\n",
            "          [ 0.0163, -0.0444,  0.0078],\n",
            "          [-0.0094,  0.0470,  0.0024]]],\n",
            "\n",
            "\n",
            "        [[[-0.0000, -0.0185, -0.0652],\n",
            "          [-0.0339, -0.0225,  0.0094],\n",
            "          [-0.0115, -0.0453, -0.0186]],\n",
            "\n",
            "         [[-0.0000,  0.0058, -0.0376],\n",
            "          [ 0.0393,  0.0737, -0.0200],\n",
            "          [-0.0516, -0.0035,  0.0219]],\n",
            "\n",
            "         [[-0.0000,  0.0325,  0.0278],\n",
            "          [-0.0452,  0.0201, -0.0001],\n",
            "          [-0.0566,  0.0218,  0.0154]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[ 0.0000, -0.0149, -0.0435],\n",
            "          [ 0.0193,  0.0710, -0.0605],\n",
            "          [ 0.0845,  0.0703,  0.0632]],\n",
            "\n",
            "         [[ 0.0000, -0.0640, -0.0245],\n",
            "          [ 0.0063, -0.0263,  0.0037],\n",
            "          [-0.0226, -0.0577, -0.0492]],\n",
            "\n",
            "         [[ 0.0000,  0.0231, -0.0125],\n",
            "          [ 0.0365, -0.0664, -0.0400],\n",
            "          [ 0.0233,  0.0023,  0.0009]]],\n",
            "\n",
            "\n",
            "        [[[ 0.0000, -0.0334, -0.0364],\n",
            "          [-0.0635, -0.0531, -0.0031],\n",
            "          [-0.0223,  0.0380,  0.0452]],\n",
            "\n",
            "         [[-0.0000,  0.0495, -0.0366],\n",
            "          [ 0.0277, -0.0645, -0.0199],\n",
            "          [-0.0010,  0.0569, -0.0456]],\n",
            "\n",
            "         [[ 0.0000, -0.0313, -0.0628],\n",
            "          [-0.0206,  0.0411,  0.0062],\n",
            "          [-0.0549,  0.0577,  0.0204]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[ 0.0000,  0.0702,  0.0005],\n",
            "          [ 0.0207, -0.0103, -0.0197],\n",
            "          [-0.0053,  0.0134,  0.0014]],\n",
            "\n",
            "         [[-0.0000, -0.0686, -0.0015],\n",
            "          [-0.0193, -0.0799,  0.0399],\n",
            "          [ 0.0277, -0.0292, -0.0687]],\n",
            "\n",
            "         [[ 0.0000,  0.0144, -0.0276],\n",
            "          [-0.0416,  0.0285, -0.0637],\n",
            "          [-0.0616, -0.0266,  0.0333]]],\n",
            "\n",
            "\n",
            "        ...,\n",
            "\n",
            "\n",
            "        [[[ 0.0000,  0.0278,  0.0596],\n",
            "          [-0.0131,  0.0199,  0.0374],\n",
            "          [ 0.0482, -0.0109, -0.0392]],\n",
            "\n",
            "         [[-0.0000,  0.0719, -0.0074],\n",
            "          [-0.0315, -0.0617, -0.0548],\n",
            "          [ 0.0557,  0.0616, -0.0187]],\n",
            "\n",
            "         [[-0.0000, -0.0131, -0.0264],\n",
            "          [ 0.0351, -0.0181,  0.0005],\n",
            "          [ 0.0553, -0.0352,  0.0135]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[ 0.0000,  0.0282,  0.0426],\n",
            "          [-0.0506, -0.0603,  0.0162],\n",
            "          [-0.0126, -0.0825,  0.0597]],\n",
            "\n",
            "         [[ 0.0000, -0.0391,  0.0003],\n",
            "          [ 0.0662, -0.0262,  0.0032],\n",
            "          [-0.0114,  0.0550,  0.0358]],\n",
            "\n",
            "         [[-0.0000,  0.0226,  0.0227],\n",
            "          [ 0.0710,  0.0572,  0.0087],\n",
            "          [-0.0366, -0.0440, -0.0163]]],\n",
            "\n",
            "\n",
            "        [[[ 0.0000, -0.0226,  0.0402],\n",
            "          [ 0.0023, -0.0010, -0.0095],\n",
            "          [ 0.0477, -0.0194,  0.0328]],\n",
            "\n",
            "         [[ 0.0000,  0.0286,  0.0272],\n",
            "          [-0.0028,  0.0215,  0.0156],\n",
            "          [-0.0549, -0.0201,  0.0657]],\n",
            "\n",
            "         [[ 0.0000, -0.0232, -0.0478],\n",
            "          [-0.0047, -0.0559, -0.0336],\n",
            "          [ 0.0247, -0.0350,  0.0186]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[ 0.0000, -0.0318, -0.0507],\n",
            "          [-0.0237,  0.0531,  0.0685],\n",
            "          [ 0.0529, -0.0137, -0.0363]],\n",
            "\n",
            "         [[-0.0000, -0.0230,  0.0576],\n",
            "          [-0.0404, -0.0441, -0.0078],\n",
            "          [-0.0717, -0.0368,  0.0070]],\n",
            "\n",
            "         [[ 0.0000,  0.0263, -0.0623],\n",
            "          [-0.0326, -0.0251, -0.0531],\n",
            "          [ 0.0190,  0.0515,  0.0111]]],\n",
            "\n",
            "\n",
            "        [[[ 0.0000, -0.0522, -0.0083],\n",
            "          [ 0.0198,  0.0352,  0.0255],\n",
            "          [ 0.0498,  0.0010,  0.0496]],\n",
            "\n",
            "         [[ 0.0000, -0.0487, -0.0098],\n",
            "          [-0.0367, -0.0292,  0.0186],\n",
            "          [ 0.0242,  0.0040,  0.0324]],\n",
            "\n",
            "         [[-0.0000,  0.0534,  0.0557],\n",
            "          [ 0.0222,  0.0457,  0.0577],\n",
            "          [-0.0163,  0.0347, -0.0332]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[ 0.0000, -0.0085,  0.0258],\n",
            "          [ 0.0027, -0.0501, -0.0395],\n",
            "          [-0.0110,  0.0067,  0.0698]],\n",
            "\n",
            "         [[-0.0000, -0.0509,  0.0645],\n",
            "          [ 0.0129, -0.0324,  0.0353],\n",
            "          [ 0.0271,  0.0628, -0.0262]],\n",
            "\n",
            "         [[ 0.0000,  0.0357,  0.0072],\n",
            "          [ 0.0354, -0.0325,  0.0403],\n",
            "          [-0.0211,  0.0842,  0.0476]]]], device='cuda:0', requires_grad=True)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "PRxJ4rqtqu72"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}