Run_Democratized_build_in_models_of_AIDK.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Run Democratized build-in models of AIDK.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPDfgsx0RQQbP7mhFvhD/r4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/XinyaoWa/hydroai-colab/blob/main/Run_Democratized_build_in_models_of_AIDK.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **To be deleted**"
      ],
      "metadata": {
        "id": "GLLjXfsmUcJM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The below will be deleted when hydro.ai can be installed with pip"
      ],
      "metadata": {
        "id": "h9tasNqTmNyc"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "Z3vJSyZV7EJZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a3885346-6a79-48e9-c610-a0d0f20785fb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount(\"/content/drive/\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "import os\n",
        "drive.mount('/content/drive/')\n",
        "os.chdir('/content/drive/My Drive/integration/frameworks.bigdata.bluewhale-main')\n",
        "aidk_path = '/content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main'"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -r requirements.txt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VaqolOXCUSyM",
        "outputId": "2e2be948-d982-457a-db3a-7b78bfb75a47"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: sigopt==7.5.0 in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 1)) (7.5.0)\n",
            "Requirement already satisfied: ipython>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from sigopt==7.5.0->-r requirements.txt (line 1)) (5.5.0)\n",
            "Requirement already satisfied: PyYAML>=5.4.1 in /usr/local/lib/python3.7/dist-packages (from sigopt==7.5.0->-r requirements.txt (line 1)) (6.0)\n",
            "Requirement already satisfied: click>=7.0.0 in /usr/local/lib/python3.7/dist-packages (from sigopt==7.5.0->-r requirements.txt (line 1)) (7.1.2)\n",
            "Requirement already satisfied: requests>=2.11.1 in /usr/local/lib/python3.7/dist-packages (from sigopt==7.5.0->-r requirements.txt (line 1)) (2.23.0)\n",
            "Requirement already satisfied: pypng>=0.0.20 in /usr/local/lib/python3.7/dist-packages (from sigopt==7.5.0->-r requirements.txt (line 1)) (0.0.21)\n",
            "Requirement already satisfied: GitPython>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from sigopt==7.5.0->-r requirements.txt (line 1)) (3.1.26)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.7/dist-packages (from GitPython>=2.0.0->sigopt==7.5.0->-r requirements.txt (line 1)) (4.0.9)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from GitPython>=2.0.0->sigopt==7.5.0->-r requirements.txt (line 1)) (3.10.0.2)\n",
            "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.7/dist-packages (from gitdb<5,>=4.0.1->GitPython>=2.0.0->sigopt==7.5.0->-r requirements.txt (line 1)) (5.0.0)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->sigopt==7.5.0->-r requirements.txt (line 1)) (0.8.1)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->sigopt==7.5.0->-r requirements.txt (line 1)) (5.1.1)\n",
            "Requirement already satisfied: pexpect in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->sigopt==7.5.0->-r requirements.txt (line 1)) (4.8.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->sigopt==7.5.0->-r requirements.txt (line 1)) (0.7.5)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->sigopt==7.5.0->-r requirements.txt (line 1)) (57.4.0)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->sigopt==7.5.0->-r requirements.txt (line 1)) (1.0.18)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->sigopt==7.5.0->-r requirements.txt (line 1)) (2.6.1)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->sigopt==7.5.0->-r requirements.txt (line 1)) (4.4.2)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=5.0.0->sigopt==7.5.0->-r requirements.txt (line 1)) (0.2.5)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=5.0.0->sigopt==7.5.0->-r requirements.txt (line 1)) (1.15.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.11.1->sigopt==7.5.0->-r requirements.txt (line 1)) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.11.1->sigopt==7.5.0->-r requirements.txt (line 1)) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.11.1->sigopt==7.5.0->-r requirements.txt (line 1)) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.11.1->sigopt==7.5.0->-r requirements.txt (line 1)) (2021.10.8)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.7/dist-packages (from pexpect->ipython>=5.0.0->sigopt==7.5.0->-r requirements.txt (line 1)) (0.7.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from run_hydroai import run_aidk"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sEWkmA1mXUkt",
        "outputId": "4b115372-dfae-4a85-a709-458e0ae3685f"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/My Drive/integration/frameworks.bigdata.bluewhale-main\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Overview**\n",
        "\n",
        "### **AIDK Introduction**\n",
        "\n",
        "AIDK(AI Democratization Toolkit) is an end-to-end AI toolkit which can deliver high performance lightweight models efficiently on commodity hardware.\n",
        "\n",
        "##### **AIDK Key Features**\n",
        "\n",
        "With AIDK, you can:\n",
        "\n",
        "- Get direct access to democratized models for a wide range of areas( recommendation systems, computer vision, speech recognition, nature language processing.....), which can bring 100x speedup over stock models.\n",
        "- Automatically optimize and accelerate the candidate models with democratized tools in AIDK(SDA, SDNN...) while keeping comparable score.\n",
        "- Plug the democratized module(RecDP, SDA, SDNN...) into your own AI pipeline for specific usage.\n",
        "\n",
        "###**Notebook Content**\n",
        "\n",
        "This notebook gives a quick start for AIDK, which mainly shows how to deploy a build-in model to solve your problems.\n",
        "\n",
        "- **Quick Start**  \n",
        "\n",
        "  Quick start for AIDK usage.\n",
        "\n",
        "- **Case1: Directly run build-in democratized models**  \n",
        "\n",
        "  If you start an AI project but with no candidate model or training dataset, you can just pick the democratized model in AIDK modelzoo for a quick deployment.\n",
        "\n",
        "- **Case2: Optimize build-in models for specific dataset**\n",
        "\n",
        "  If you want a better performance for a specific dataset, you can optimize the model with the build-in democratized algorithms, which will output a new model with the best performance."
      ],
      "metadata": {
        "id": "dNXznmsPVJF2"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **AIDK Install**\n",
        "\n",
        "Install AIDK with pip."
      ],
      "metadata": {
        "id": "8jEEcwBbmZu6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# pip install aidk"
      ],
      "metadata": {
        "id": "2-4POVvs8ss4"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# from aidk import *\n",
        "# aidk_path = aidk.__path__[0]\n",
        "aidk_path"
      ],
      "metadata": {
        "id": "83MWtcZ0XQIc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "41afa875-a3f4-4e58-bc8b-c45747899395"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main'"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##**Quick start**\n",
        "\n",
        "A quick start for pipeline test, which do a classification job on Sklearn Wine dataset."
      ],
      "metadata": {
        "id": "1lQ_4SQxdKTl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "parameters = {'model_name': 'pipeline_test'}\n",
        "run_aidk(parameters)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JBLsslbDi-fh",
        "outputId": "2ebb8a49-c0ab-480e-f7d3-cd80c811cde9"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2022-02-17 01:46:11,721 - HYDRO.AI.SDA - INFO - Model Advisor created\n",
            "2022-02-17 01:46:11,724 - HYDRO.AI.SDA - INFO - model parameter initialized\n",
            "2022-02-17 01:46:11,726 - HYDRO.AI.SDA - INFO - start to launch training\n",
            "2022-02-17 01:46:11,732 - sigopt - INFO - training launch command: ['python', '/content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/example/sklearn_train.py', '--max_depth', '11', '--learning_rate', '0.9294458527831317', '--min_split_loss', '6.88375281543753', '--saved_path', '/content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/result/pipeline_test/20220217_014611/1768ea50c3ead24d0a19ff305fe537ca']\n",
            "2022-02-17 01:46:13,070 - sigopt - INFO - Training completed based in sigopt suggestion, took 1.3386046886444092 secs\n",
            "2022-02-17 01:46:13,073 - HYDRO.AI.SDA - INFO - training script completed\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "We found the best model! Here is the model explaination\n",
            "\n",
            "===============================================\n",
            "***    Best Trained Model    ***\n",
            "===============================================\n",
            "  Model Type: pipeline_test\n",
            "  Model Saved Path: /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/result/pipeline_test/20220217_014611/1768ea50c3ead24d0a19ff305fe537ca\n",
            "  Sigopt Experiment id is None\n",
            "  === Result Metrics ===\n",
            "    accuracy: 0.7241269841269842\n",
            "    training_time: 1.3386046886444092\n",
            "===============================================\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Case1: Directly run build-in democratized models**\n",
        "\n",
        "You can pick the build-in democratized models in AIDK modelzoo and directly run it, especially in two cases:\n",
        "\n",
        "- When you start an AI project, but don't want to carefully select the candidate model or have no dataset for model training.\n",
        "\n",
        "- You want an optimized model with better performance on commodity hardware.\n",
        "\n",
        "**AIDK modelzoo**\n",
        "\n",
        "AIDK modelzoo contains some democratized models for a wide range of areas:\n",
        "\n",
        "- Reconmmendation System: DLRM, DIEN, WnD, TwitterRecSys2021\n",
        "\n",
        "- Computer Vision: ResNet\n",
        "\n",
        "- Nature Language Processing: Bert\n",
        "\n",
        "- Speech Recognition: RNN-T\n",
        "\n",
        "All the democratized models have been specifically optimized on commodity hardware and can bring up to 100x perfrmance speedup."
      ],
      "metadata": {
        "id": "afT68xxindDb"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "###**Example: Reconmmendation System**\n",
        "\n",
        "For reconmmendation system, AIDK contains four democratized models. \n",
        "\n",
        "Below we select TwitterRecSys2021 as an example, which gives an optimized solution on Twitter RecSys 2021 Dataset."
      ],
      "metadata": {
        "id": "g0QAMknspKfi"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Download sample dataset**"
      ],
      "metadata": {
        "id": "fqqpZdUjn2bR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "! wget --no-check-certificate 'https://docs.google.com/uc?export=download&id=14Lkvye3Wat3bjZelGuQJ4y_vXbHBUAFF' -O TwitterRecSys2021Dataset_sampledata.zip\n",
        "! unzip TwitterRecSys2021Dataset_sampledata.zip"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WEB0lKm-SnKT",
        "outputId": "4022979d-94f8-469c-a4c9-3a0b1ed2b3bd"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2022-02-17 01:44:36--  https://docs.google.com/uc?export=download&id=14Lkvye3Wat3bjZelGuQJ4y_vXbHBUAFF\n",
            "Resolving docs.google.com (docs.google.com)... 142.251.8.101, 142.251.8.138, 142.251.8.102, ...\n",
            "Connecting to docs.google.com (docs.google.com)|142.251.8.101|:443... connected.\n",
            "HTTP request sent, awaiting response... 303 See Other\n",
            "Location: https://doc-14-1s-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/dn3mhvhl97se4pbkd56st61a7vav6vjc/1645062225000/18016102022855825607/*/14Lkvye3Wat3bjZelGuQJ4y_vXbHBUAFF?e=download [following]\n",
            "Warning: wildcards not supported in HTTP.\n",
            "--2022-02-17 01:44:38--  https://doc-14-1s-docs.googleusercontent.com/docs/securesc/ha0ro937gcuc7l7deffksulhg5h7mbp1/dn3mhvhl97se4pbkd56st61a7vav6vjc/1645062225000/18016102022855825607/*/14Lkvye3Wat3bjZelGuQJ4y_vXbHBUAFF?e=download\n",
            "Resolving doc-14-1s-docs.googleusercontent.com (doc-14-1s-docs.googleusercontent.com)... 64.233.189.132, 2404:6800:4008:c07::84\n",
            "Connecting to doc-14-1s-docs.googleusercontent.com (doc-14-1s-docs.googleusercontent.com)|64.233.189.132|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 5439940 (5.2M) [application/x-zip-compressed]\n",
            "Saving to: ‘TwitterRecSys2021Dataset_sampledata.zip’\n",
            "\n",
            "TwitterRecSys2021Da 100%[===================>]   5.19M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2022-02-17 01:44:39 (35.8 MB/s) - ‘TwitterRecSys2021Dataset_sampledata.zip’ saved [5439940/5439940]\n",
            "\n",
            "Archive:  TwitterRecSys2021Dataset_sampledata.zip\n",
            "   creating: TwitterRecSys2021Dataset_sample/\n",
            "   creating: TwitterRecSys2021Dataset_sample/valid/\n",
            "  inflating: TwitterRecSys2021Dataset_sample/valid/valid.parquet  \n",
            " extracting: TwitterRecSys2021Dataset_sample/metadata.yaml  \n",
            "   creating: TwitterRecSys2021Dataset_sample/train/\n",
            "  inflating: TwitterRecSys2021Dataset_sample/train/train.parquet  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Parameter setting**\n",
        "\n",
        "To run a build-in model, specify some basic parameters for AIDK running:\n",
        "\n",
        "- **model_name** The selected democratized model name, can be: \n",
        "\n",
        "   *'pipeline_test', 'dlrm', 'dien', 'wnd', 'twitter_recsys', 'rnnt', 'tpcxai09', 'upm'*\n",
        "\n",
        "- **data_path** The path to training dataset.\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "h4JFnDmbn__E"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "parameters = {\n",
        "    'model_name': 'twitter_recsys',\n",
        "    'data_path': './TwitterRecSys2021Dataset_sample'\n",
        "}"
      ],
      "metadata": {
        "id": "NHtvblhjtz7k"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Run AIDK**\n",
        "\n",
        "Use *run_aidk* can quickly train the democratized model on provided dataset and present the performance."
      ],
      "metadata": {
        "id": "ruU8WkOjvyYS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "## Run AIDK to train the democratized Twitter RecSys model and return the performance on sample dataset\n",
        "run_aidk(parameters)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gaE3j4ArvO3r",
        "outputId": "f6106350-14dc-4b9e-ecc2-8da119d241fa"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2022-02-17 01:44:48,517 - HYDRO.AI.SDA - INFO - Model Advisor created\n",
            "2022-02-17 01:44:48,518 - HYDRO.AI.SDA - INFO - model parameter initialized\n",
            "2022-02-17 01:44:48,520 - HYDRO.AI.SDA - INFO - start to launch training\n",
            "2022-02-17 01:44:48,531 - sigopt - INFO - training launch command: python -u /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/modelzoo/TwitterRecSys/TwitterRecSys2021-intel-opt/model_hydroai/xgboost/train.py --stage stage1 --target reply --train_data_path ./TwitterRecSys2021Dataset_sample/train/train.parquet --valid_data_path ./TwitterRecSys2021Dataset_sample/valid/valid.parquet --model_save_path /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/result/twitter_recsys/20220217_014448/145ebb148387d95b7ff12d81001208b3 --max_depth 8 --learning_rate 0.1 --subsample 0.8 --colsample_bytree 0.8 --num_boost_round 250 \n",
            "2022-02-17 01:44:53,314 - sigopt - INFO - Training completed based in sigopt suggestion, took 4.778974771499634 secs\n",
            "2022-02-17 01:44:53,316 - HYDRO.AI.SDA - INFO - training script completed\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "We found the best model! Here is the model explaination\n",
            "\n",
            "===============================================\n",
            "***    Best Trained Model    ***\n",
            "===============================================\n",
            "  Model Type: twitter_recsys\n",
            "  Model Saved Path: /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/result/twitter_recsys/20220217_014448/145ebb148387d95b7ff12d81001208b3\n",
            "  Sigopt Experiment id is None\n",
            "  === Result Metrics ===\n",
            "    AP: 0.09273384406261408\n",
            "    RCE: 9.148431114577527\n",
            "===============================================\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Case2: Optimize build-in models for specific dataset**\n",
        "\n",
        "If you have your own training dataset, and want to optimize the build-in model for better performance on the specific data, you can enable *demoratize* flag, which will start democratize modules in AIDK to search a better model.\n",
        "\n",
        "In this case, we still take Twitter RecSys as an example."
      ],
      "metadata": {
        "id": "vt79cDOjdOvV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Prepare dataset**\n",
        "\n",
        "Originize you dataset as listed structure:\n",
        "\n",
        "- **train** - *fold*, contains all the training data\n",
        "\n",
        "- **validate** - *fold*, contains all the validation data\n",
        "\n",
        "- **metadata.yaml** - *file*, contains the description of the data, can be empty \n",
        "\n",
        "Below we use the downloaded sample dataset for simplicity."
      ],
      "metadata": {
        "id": "_3grWtKy2wXC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!ls TwitterRecSys2021Dataset_sample"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FoBrLclk3vZL",
        "outputId": "624f4915-cb02-4012-ecaa-521e985a2830"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "metadata.yaml  train  valid\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Parameter setting**\n",
        "\n",
        "Besides the above parameters, we need to enable *demoratize* flag and provide sigopt API token:\n",
        "\n",
        "- **democratize**  Turn on the flag if you want to optimize the model, *default=False*\n",
        "\n",
        "- **sigopt_api_token** Give your sigopt API token for Sigopt parameter tuning, *default=None*.\n",
        "\n",
        "- **observation_budget** *optinal*, Experiment budget for Sigopt optimization, *default=1*"
      ],
      "metadata": {
        "id": "oZawALbQ35Zo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "parameters = {\n",
        "    'model_name': 'twitter_recsys',\n",
        "    'data_path': './TwitterRecSys2021Dataset_sample',\n",
        "    'democratize': True,\n",
        "    'sigopt_api_token': '',\n",
        "    'observation_budget': 5\n",
        "} "
      ],
      "metadata": {
        "id": "mmYUkpgOVrKr"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Run AIDK**\n",
        "\n",
        "Use *run_aidk* to start the democratized process for the Twitter RecSys model."
      ],
      "metadata": {
        "id": "_5Em2KaiG22k"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "run_aidk(parameters)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CF-Gj18GGGFp",
        "outputId": "2747b224-27c9-4277-b7b6-69bc44642999"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2022-02-17 01:45:19,889 - HYDRO.AI.SDA - INFO - Model Advisor created\n",
            "2022-02-17 01:45:19,891 - HYDRO.AI.SDA - INFO - Start to init sigopt\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "api_token: \n",
            "experiment: RecSys\n",
            "metrics:\n",
            "- name: AP\n",
            "  objective: maximize\n",
            "  strategy: optimize\n",
            "- name: RCE\n",
            "  objective: maximize\n",
            "observation_budget: 5\n",
            "parameters:\n",
            "- bounds:\n",
            "    max: 20\n",
            "    min: 5\n",
            "  name: max_depth\n",
            "  type: int\n",
            "- bounds:\n",
            "    max: 1.0\n",
            "    min: 0.0\n",
            "  name: learning_rate\n",
            "  type: double\n",
            "- bounds:\n",
            "    max: 1\n",
            "    min: 0.5\n",
            "  name: subsample\n",
            "  type: double\n",
            "- bounds:\n",
            "    max: 1.0\n",
            "    min: 0.5\n",
            "  name: colsample_bytree\n",
            "  type: double\n",
            "- bounds:\n",
            "    max: 1000\n",
            "    min: 100\n",
            "  name: num_boost_round\n",
            "  type: int\n",
            "project: hydro.ai\n",
            "Please confirm with sigopt parameters?(n for exit)y\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2022-02-17 01:45:24,554 - HYDRO.AI.SDA - INFO - model parameter initialized\n",
            "2022-02-17 01:45:24,555 - HYDRO.AI.SDA - INFO - start to launch training\n",
            "2022-02-17 01:45:24,557 - sigopt - INFO - Experiment budget: 5\n",
            "2022-02-17 01:45:25,707 - sigopt - INFO - training launch command: python -u /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/modelzoo/TwitterRecSys/TwitterRecSys2021-intel-opt/model_hydroai/xgboost/train.py --stage stage1 --target reply --train_data_path ./TwitterRecSys2021Dataset_sample/train/train.parquet --valid_data_path ./TwitterRecSys2021Dataset_sample/valid/valid.parquet --model_save_path /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/result/twitter_recsys/20220217_014519/b47472901e63ef571ed116f7bafd2c15 --max_depth 5 --learning_rate 0.40669609955126335 --subsample 0.8355585083313469 --colsample_bytree 0.5946987130986952 --num_boost_round 440 \n",
            "2022-02-17 01:45:28,164 - sigopt - INFO - Received sigopt suggestion, assignment is                     OrderedDict([('colsample_bytree', 0.5946987130986952), ('learning_rate', 0.40669609955126335), ('max_depth', 5), ('num_boost_round', 440), ('subsample', 0.8355585083313469)])\n",
            "2022-02-17 01:45:28,166 - sigopt - INFO - Training completed with sigopt suggestion,                     metrics is [{'name': 'AP', 'value': 0.08252897857048286}, {'name': 'RCE', 'value': -2.3449023882579523}]\n",
            "2022-02-17 01:45:30,815 - sigopt - INFO - training launch command: python -u /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/modelzoo/TwitterRecSys/TwitterRecSys2021-intel-opt/model_hydroai/xgboost/train.py --stage stage1 --target reply --train_data_path ./TwitterRecSys2021Dataset_sample/train/train.parquet --valid_data_path ./TwitterRecSys2021Dataset_sample/valid/valid.parquet --model_save_path /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/result/twitter_recsys/20220217_014519/ffcb360493ab63eb15df41c6335a1908 --max_depth 15 --learning_rate 0.9108026436301443 --subsample 0.6784704624079242 --colsample_bytree 0.8665967344932877 --num_boost_round 541 \n",
            "2022-02-17 01:45:33,350 - sigopt - INFO - Received sigopt suggestion, assignment is                     OrderedDict([('colsample_bytree', 0.8665967344932877), ('learning_rate', 0.9108026436301443), ('max_depth', 15), ('num_boost_round', 541), ('subsample', 0.6784704624079242)])\n",
            "2022-02-17 01:45:33,353 - sigopt - INFO - Training completed with sigopt suggestion,                     metrics is [{'name': 'AP', 'value': 0.06585947351752164}, {'name': 'RCE', 'value': -62.724352606523894}]\n",
            "2022-02-17 01:45:35,283 - sigopt - INFO - training launch command: python -u /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/modelzoo/TwitterRecSys/TwitterRecSys2021-intel-opt/model_hydroai/xgboost/train.py --stage stage1 --target reply --train_data_path ./TwitterRecSys2021Dataset_sample/train/train.parquet --valid_data_path ./TwitterRecSys2021Dataset_sample/valid/valid.parquet --model_save_path /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/result/twitter_recsys/20220217_014519/50c6e5bb694df19527fffc31e65c6a81 --max_depth 12 --learning_rate 0.15031889881488866 --subsample 0.949747528626315 --colsample_bytree 0.9153005880914984 --num_boost_round 875 \n",
            "2022-02-17 01:45:40,190 - sigopt - INFO - Received sigopt suggestion, assignment is                     OrderedDict([('colsample_bytree', 0.9153005880914984), ('learning_rate', 0.15031889881488866), ('max_depth', 12), ('num_boost_round', 875), ('subsample', 0.949747528626315)])\n",
            "2022-02-17 01:45:40,192 - sigopt - INFO - Training completed with sigopt suggestion,                     metrics is [{'name': 'AP', 'value': 0.09593859897258228}, {'name': 'RCE', 'value': 0.8585916358498369}]\n",
            "2022-02-17 01:45:41,149 - sigopt - INFO - training launch command: python -u /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/modelzoo/TwitterRecSys/TwitterRecSys2021-intel-opt/model_hydroai/xgboost/train.py --stage stage1 --target reply --train_data_path ./TwitterRecSys2021Dataset_sample/train/train.parquet --valid_data_path ./TwitterRecSys2021Dataset_sample/valid/valid.parquet --model_save_path /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/result/twitter_recsys/20220217_014519/fcfa6fec267517a7d553ad4aa52433b4 --max_depth 8 --learning_rate 0.34058476118647685 --subsample 0.7254466201419041 --colsample_bytree 0.6718400162655178 --num_boost_round 690 \n",
            "2022-02-17 01:45:44,080 - sigopt - INFO - Received sigopt suggestion, assignment is                     OrderedDict([('colsample_bytree', 0.6718400162655178), ('learning_rate', 0.34058476118647685), ('max_depth', 8), ('num_boost_round', 690), ('subsample', 0.7254466201419041)])\n",
            "2022-02-17 01:45:44,082 - sigopt - INFO - Training completed with sigopt suggestion,                     metrics is [{'name': 'AP', 'value': 0.08372948508478788}, {'name': 'RCE', 'value': -8.674592930971947}]\n",
            "2022-02-17 01:45:45,469 - sigopt - INFO - training launch command: python -u /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/modelzoo/TwitterRecSys/TwitterRecSys2021-intel-opt/model_hydroai/xgboost/train.py --stage stage1 --target reply --train_data_path ./TwitterRecSys2021Dataset_sample/train/train.parquet --valid_data_path ./TwitterRecSys2021Dataset_sample/valid/valid.parquet --model_save_path /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/result/twitter_recsys/20220217_014519/2de2e7dd8b38b4a5db4c40d784be5402 --max_depth 19 --learning_rate 0.674739803351327 --subsample 0.5031079990190965 --colsample_bytree 0.7221693673540982 --num_boost_round 211 \n",
            "2022-02-17 01:45:47,882 - sigopt - INFO - Received sigopt suggestion, assignment is                     OrderedDict([('colsample_bytree', 0.7221693673540982), ('learning_rate', 0.674739803351327), ('max_depth', 19), ('num_boost_round', 211), ('subsample', 0.5031079990190965)])\n",
            "2022-02-17 01:45:47,885 - sigopt - INFO - Training completed with sigopt suggestion,                     metrics is [{'name': 'AP', 'value': 0.05784171880487565}, {'name': 'RCE', 'value': -48.470825914799256}]\n",
            "2022-02-17 01:45:49,312 - sigopt - INFO - Training with sigopt is completed!                 https://app.sigopt.com/experiment/483510 \n",
            "/usr/local/lib/python3.7/dist-packages/sigopt/objects.py:327: RuntimeWarning: The .data field only contains a single page of results, which may be incomplete for large responses. Prefer to use the `.iterate_pages() to ensure that you iterate through all elements in the response.\n",
            "  RuntimeWarning,\n",
            "2022-02-17 01:45:49,583 - HYDRO.AI.SDA - INFO - training script completed\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "We found the best model! Here is the model explaination\n",
            "\n",
            "===============================================\n",
            "***    Best Trained Model    ***\n",
            "===============================================\n",
            "  Model Type: twitter_recsys\n",
            "  Model Saved Path: /content/drive/MyDrive/integration/frameworks.bigdata.bluewhale-main/result/twitter_recsys/20220217_014519/50c6e5bb694df19527fffc31e65c6a81\n",
            "  Sigopt Experiment id is 483510\n",
            "  === Result Metrics ===\n",
            "    AP: 0.09593859897258228\n",
            "    RCE: 0.8585916358498369\n",
            "===============================================\n"
          ]
        }
      ]
    }
  ]
}