CoTexT - Bug Repair.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"CoTexT - Bug Repair.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"TPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"OxPd3O1Xl7CA"},"source":["# All"]},{"cell_type":"markdown","metadata":{"id":"U1Ar53sBl7CB"},"source":["## Set up"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"pDVvzNA7l7CB","outputId":"137a7d8c-84cf-446a-87b5-5a1004965c37","executionInfo":{"status":"ok","timestamp":1660156880872,"user_tz":-300,"elapsed":109663,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["print(\"Installing dependencies...\")\n","%tensorflow_version 2.x\n","!pip install -q t5\n","\n","import functools\n","import os\n","import time\n","import warnings\n","warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n","\n","import tensorflow.compat.v1 as tf\n","import tensorflow_datasets as tfds\n","\n","import t5"],"execution_count":1,"outputs":[{"output_type":"stream","name":"stdout","text":["Installing dependencies...\n","Colab only includes TensorFlow 2.x; %tensorflow_version has no effect.\n","\u001b[K     |████████████████████████████████| 153 kB 4.8 MB/s \n","\u001b[K     |████████████████████████████████| 4.5 MB 48.1 MB/s \n","\u001b[K     |████████████████████████████████| 4.7 MB 34.0 MB/s \n","\u001b[K     |████████████████████████████████| 4.6 MB 45.1 MB/s \n","\u001b[K     |████████████████████████████████| 306 kB 52.7 MB/s \n","\u001b[K     |████████████████████████████████| 385 kB 54.6 MB/s \n","\u001b[K     |████████████████████████████████| 1.3 MB 47.6 MB/s \n","\u001b[K     |████████████████████████████████| 116 kB 57.5 MB/s \n","\u001b[K     |████████████████████████████████| 596 kB 47.6 MB/s \n","\u001b[K     |████████████████████████████████| 6.6 MB 43.4 MB/s \n","\u001b[K     |████████████████████████████████| 101 kB 9.6 MB/s \n","\u001b[K     |████████████████████████████████| 511.7 MB 5.8 kB/s \n","\u001b[K     |████████████████████████████████| 5.8 MB 35.5 MB/s \n","\u001b[K     |████████████████████████████████| 438 kB 41.6 MB/s \n","\u001b[K     |████████████████████████████████| 1.6 MB 46.6 MB/s \n","\u001b[?25h  Building wheel for rouge-score (setup.py) ... \u001b[?25l\u001b[?25hdone\n"]}]},{"cell_type":"code","source":["!pip install -U tensorflow-gcs-config==2.9.1"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vQcnfuLnxpMi","executionInfo":{"status":"ok","timestamp":1660156885757,"user_tz":-300,"elapsed":4895,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}},"outputId":"79c1312e-1d0b-499f-a6e5-ddd6fd96c310"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting tensorflow-gcs-config==2.9.1\n","  Downloading tensorflow_gcs_config-2.9.1-py3-none-any.whl (346 kB)\n","\u001b[K     |████████████████████████████████| 346 kB 5.4 MB/s \n","\u001b[?25hInstalling collected packages: tensorflow-gcs-config\n","  Attempting uninstall: tensorflow-gcs-config\n","    Found existing installation: tensorflow-gcs-config 2.8.0\n","    Uninstalling tensorflow-gcs-config-2.8.0:\n","      Successfully uninstalled tensorflow-gcs-config-2.8.0\n","Successfully installed tensorflow-gcs-config-2.9.1\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":141,"resources":{"http://localhost:8080/nbextensions/google.colab/files.js":{"data":"Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgZG8gewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwoKICAgICAgbGV0IHBlcmNlbnREb25lID0gZmlsZURhdGEuYnl0ZUxlbmd0aCA9PT0gMCA/CiAgICAgICAgICAxMDAgOgogICAgICAgICAgTWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCk7CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPSBgJHtwZXJjZW50RG9uZX0lIGRvbmVgOwoKICAgIH0gd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCk7CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK","ok":true,"headers":[["content-type","application/javascript"]],"status":200,"status_text":""}}},"id":"g1CKuTdVl7CE","outputId":"229d9198-1130-46aa-cf1b-c36abac41e3a","executionInfo":{"status":"ok","timestamp":1660156966454,"user_tz":-300,"elapsed":80712,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["ON_CLOUD = True\n","\n","\n","if ON_CLOUD:\n","  print(\"Setting up GCS access...\")\n","  import tensorflow_gcs_config\n","  from google.colab import auth\n","  # Set credentials for GCS reading/writing from Colab and TPU.\n","  TPU_TOPOLOGY = \"v3-8\"\n","  try:\n","    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU zdetection\n","    TPU_ADDRESS = tpu.get_master()\n","    print('Running on TPU:', TPU_ADDRESS)\n","  except ValueError:\n","    raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')\n","  auth.authenticate_service_account()\n","  tf.config.experimental_connect_to_host(TPU_ADDRESS)\n","  tensorflow_gcs_config.configure_gcs_from_colab_auth()\n","\n","tf.disable_v2_behavior()\n","\n","# Improve logging.\n","from contextlib import contextmanager\n","import logging as py_logging\n","\n","if ON_CLOUD:\n","  tf.get_logger().propagate = False\n","  py_logging.root.setLevel('INFO')\n","\n","@contextmanager\n","def tf_verbosity_level(level):\n","  og_level = tf.logging.get_verbosity()\n","  tf.logging.set_verbosity(level)\n","  yield\n","  tf.logging.set_verbosity(og_level)\n"],"execution_count":3,"outputs":[{"output_type":"stream","name":"stdout","text":["Setting up GCS access...\n","Running on TPU: grpc://10.43.157.194:8470\n"]},{"output_type":"stream","name":"stderr","text":["WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/compat/v2_compat.py:107: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","non-resource variables are not supported in the long term\n"]},{"output_type":"stream","name":"stdout","text":["Successfully saved credentials for daim-938@local-shoreline-357513.iam.gserviceaccount.com\n"]}]},{"cell_type":"code","metadata":{"id":"b3EkFkJ21G6p","executionInfo":{"status":"ok","timestamp":1660156966455,"user_tz":-300,"elapsed":11,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["# print(mesh_tensorflow.__version__)"],"execution_count":4,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"A969X_jAr2qE","outputId":"2561e242-6945-4a3f-b5a5-ac9d0f2d1fba","executionInfo":{"status":"ok","timestamp":1660156966456,"user_tz":-300,"elapsed":11,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["print(t5.__version__)"],"execution_count":5,"outputs":[{"output_type":"stream","name":"stdout","text":["0.9.3\n"]}]},{"cell_type":"code","metadata":{"id":"jaPI1TmAeh0_","executionInfo":{"status":"ok","timestamp":1660156966456,"user_tz":-300,"elapsed":9,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["# import gin\n","# import subprocess\n","# gin.parse_config_file(\n","#         'gs://t5-data/pretrained_models/base/operative_config.gin'\n","#     )\n"],"execution_count":6,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"9xtIvOasl7CI"},"source":["## Register Tasks Medium\n"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"qnIhn9K_MBM-","outputId":"2b2ac638-2a1d-4df5-9daf-856d3d08867c","executionInfo":{"status":"ok","timestamp":1660156968229,"user_tz":-300,"elapsed":1782,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["def dumping_dataset(split, shuffle_files = False):\n","    del shuffle_files\n","    if split == 'train':\n","      ds = tf.data.TextLineDataset(\n","            [\n","            'gs://cotext/data/bug2fix/medium/train.tsv',\n","            'gs://cotext/data/bug2fix/medium/valid.tsv',\n","\n","            ]\n","          )\n","    else:\n","      ds = tf.data.TextLineDataset(\n","            [\n","            ]\n","          )\n","\n","    ds = ds.map(\n","        functools.partial(tf.io.decode_csv, record_defaults=[\"\", \"\"],\n","                          field_delim=\"\\t\", use_quote_delim=False),\n","        num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","\n","    ds = ds.map(lambda *ex: dict(zip([\"input\", \"target\"], ex)))\n","    return ds\n","\n","def ner_preprocessor(ds):\n","  def normalize_text(text):\n","    return text\n","\n","  def to_inputs_and_targets(ex):\n","    \"\"\"Map {\"inputs\": ..., \"targets\": ...}->{\"inputs\": ner..., \"targets\": ...}.\"\"\"\n","    return {\n","        \"inputs\":\n","             tf.strings.join(\n","                 [\"medium: \", normalize_text(ex[\"input\"])]),\n","        \"targets\": normalize_text(ex[\"target\"])\n","    }\n","  return ds.map(to_inputs_and_targets, \n","                num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","\n","print(\"A few raw validation examples...\")\n","for ex in tfds.as_numpy(dumping_dataset(\"train\").take(5)):\n","  print(ex)"],"execution_count":7,"outputs":[{"output_type":"stream","name":"stdout","text":["A few raw validation examples...\n","{'input': b'public static TYPE_1 init ( java.lang.String name , java.util.Date date )  OPEN_CURLY_TOKEN  TYPE_1 VAR_1 = new TYPE_1 ( ) ; VAR_1 . METHOD_1 ( name ) ; java.util.Calendar VAR_2 = java.util.Calendar.getInstance ( ) ; VAR_2 . METHOD_2 ( date ) ; VAR_1 . METHOD_3 ( VAR_2 ) ; return VAR_1 ;  CLOSE_CURLY_TOKEN ', 'target': b'public static TYPE_1 init ( java.lang.String name , java.util.Date date )  OPEN_CURLY_TOKEN  TYPE_1 VAR_1 = new TYPE_1 ( ) ; VAR_1 . METHOD_1 ( name ) ; java.util.Calendar VAR_2 = null ; if ( date != null )  OPEN_CURLY_TOKEN  VAR_2 = java.util.Calendar.getInstance ( ) ; VAR_2 . METHOD_2 ( date ) ;  CLOSE_CURLY_TOKEN  VAR_1 . METHOD_3 ( VAR_2 ) ; return VAR_1 ;  CLOSE_CURLY_TOKEN '}\n","{'input': b'public TYPE_1 METHOD_1 ( java.lang.String name )  OPEN_CURLY_TOKEN  if ( name . equals ( STRING_1 ) ) return new TYPE_2 ( STRING_2 , true ) ; if ( name . equals ( STRING_3 ) ) return new TYPE_3 ( STRING_4 , true ) ; if ( name . equals ( STRING_5 ) ) return new TYPE_4 ( ) ; return super . METHOD_1 ( name ) ;  CLOSE_CURLY_TOKEN ', 'target': b'public TYPE_1 METHOD_1 ( java.lang.String name )  OPEN_CURLY_TOKEN  if ( name . equals ( STRING_3 ) ) return new TYPE_3 ( STRING_4 , true ) ; if ( name . equals ( STRING_5 ) ) return new TYPE_4 ( ) ; return super . METHOD_1 ( name ) ;  CLOSE_CURLY_TOKEN '}\n","{'input': b'private boolean METHOD_1 ( TYPE_1 VAR_1 )  OPEN_CURLY_TOKEN  boolean VAR_2 = false ; VAR_2 = VAR_2 || ( ( VAR_3 . compareTo ( VAR_1 . METHOD_2 ( ) ) )  SMALLER_TOKEN  0 ) ; VAR_2 = VAR_2 || ( ! ( VAR_1 . METHOD_3 ( ) . METHOD_4 ( ) . equals ( VAR_4 ) ) ) ; return VAR_2 ;  CLOSE_CURLY_TOKEN ', 'target': b'private boolean METHOD_1 ( TYPE_1 VAR_1 )  OPEN_CURLY_TOKEN  boolean VAR_2 = ( VAR_3 . compareTo ( VAR_1 . METHOD_2 ( ) ) )  SMALLER_TOKEN  0 ; VAR_2 = VAR_2 || ( ! ( VAR_1 . METHOD_3 ( ) . METHOD_4 ( ) . equals ( VAR_4 ) ) ) ; return VAR_2 ;  CLOSE_CURLY_TOKEN '}\n","{'input': b'public void METHOD_1 ( TYPE_1 VAR_1 , boolean VAR_2 )  OPEN_CURLY_TOKEN  if ( VAR_2 )  OPEN_CURLY_TOKEN  VAR_3 . METHOD_2 ( 1 , CHAR_1 ) ; VAR_4 . METHOD_3 ( VAR_3 . toString ( ) ) ;  CLOSE_CURLY_TOKEN  else  OPEN_CURLY_TOKEN  VAR_3 . METHOD_2 ( 1 , CHAR_2 ) ; VAR_4 . METHOD_3 ( VAR_3 . toString ( ) ) ;  CLOSE_CURLY_TOKEN  TYPE_2 VAR_5 = TYPE_2 . METHOD_4 ( getActivity ( ) , VAR_4 . METHOD_5 ( ) , VAR_6 ) ; VAR_5 . show ( ) ;  CLOSE_CURLY_TOKEN ', 'target': b'public void METHOD_1 ( TYPE_1 VAR_1 , boolean VAR_2 )  OPEN_CURLY_TOKEN  if ( VAR_2 )  OPEN_CURLY_TOKEN  VAR_3 . METHOD_2 ( 1 , CHAR_1 ) ; VAR_4 . METHOD_3 ( VAR_3 . toString ( ) ) ;  CLOSE_CURLY_TOKEN  else  OPEN_CURLY_TOKEN  VAR_3 . METHOD_2 ( 1 , CHAR_2 ) ; VAR_4 . METHOD_3 ( VAR_3 . toString ( ) ) ;  CLOSE_CURLY_TOKEN   CLOSE_CURLY_TOKEN '}\n","{'input': b'public boolean METHOD_1 ( )  OPEN_CURLY_TOKEN  if ( ( VAR_1 ) == ( ( VAR_2 . METHOD_2 ( VAR_1 ) ) - 1 ) )  OPEN_CURLY_TOKEN  return false ;  CLOSE_CURLY_TOKEN  if ( ( METHOD_3 ( ) . getValue ( ) )  SMALLER_TOKEN = ( METHOD_4 ( ( ( VAR_1 ) + 1 ) , VAR_3 ) . getValue ( ) ) )  OPEN_CURLY_TOKEN  return false ;  CLOSE_CURLY_TOKEN  ( VAR_1 ) ++ ; return true ;  CLOSE_CURLY_TOKEN ', 'target': b'public boolean METHOD_1 ( )  OPEN_CURLY_TOKEN  if ( ( VAR_1 )  GREATER_TOKEN = ( ( VAR_2 . METHOD_2 ( VAR_3 ) ) - 1 ) )  OPEN_CURLY_TOKEN  return false ;  CLOSE_CURLY_TOKEN  if ( ( METHOD_3 ( ) . getValue ( ) )  SMALLER_TOKEN = ( METHOD_4 ( ( ( VAR_1 ) + 1 ) , VAR_3 ) . getValue ( ) ) )  OPEN_CURLY_TOKEN  return false ;  CLOSE_CURLY_TOKEN  ( VAR_1 ) ++ ; return true ;  CLOSE_CURLY_TOKEN '}\n"]}]},{"cell_type":"code","metadata":{"id":"spun7-5XMBM_","colab":{"base_uri":"https://localhost:8080/"},"outputId":"3916f52f-595f-485e-f2e0-8124040eda7b","executionInfo":{"status":"ok","timestamp":1660156968229,"user_tz":-300,"elapsed":5,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["t5.data.TaskRegistry.remove('medium')\n","t5.data.TaskRegistry.add(\n","    \"medium\",\n","    # Supply a function which returns a tf.data.Dataset.\n","    dataset_fn=dumping_dataset,\n","    splits=[\"train\", \"validation\"],\n","    # Supply a function which preprocesses text from the tf.data.Dataset.\n","    text_preprocessor=[ner_preprocessor],\n","    # Lowercase targets before computing metrics.\n","    # We'll use accuracy as our evaluation metric.\n","    # output_features=t5.data.Feature(vocabulary=t5.data.SentencePieceVocabulary(vocab)),\n","\n","    # metric_fns=[t5.evaluation.metrics.accuracy, \n","    #            t5.evaluation.metrics.sequence_accuracy, \n","    #             ],\n","    # output_features=t5.data.Feature(vocabulary=t5.data.SentencePieceVocabulary(vocab))\n",")"],"execution_count":8,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<t5.data.dataset_providers.FunctionTask at 0x7ff660b9ed90>"]},"metadata":{},"execution_count":8}]},{"cell_type":"markdown","metadata":{"id":"fGQirLNeX0dr"},"source":["## Register Tasks small\n"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"q63OQCTnX0dt","outputId":"5d01e5e1-5745-40f4-c03f-10cf0c8f2dcb","executionInfo":{"status":"ok","timestamp":1660156968614,"user_tz":-300,"elapsed":389,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["def dumping_dataset(split, shuffle_files = False):\n","    del shuffle_files\n","    if split == 'train':\n","      ds = tf.data.TextLineDataset(\n","            [\n","            'gs://cotext/data/bug2fix/small/train.tsv',\n","            'gs://cotext/data/bug2fix/small/valid.tsv',\n","\n","            ]\n","          )\n","    else:\n","      ds = tf.data.TextLineDataset(\n","            [\n","            ]\n","          )\n","    # Split each \"<t1>\\t<t2>\" example into (input), target) tuple.\n","    ds = ds.map(\n","        functools.partial(tf.io.decode_csv, record_defaults=[\"\", \"\"],\n","                          field_delim=\"\\t\", use_quote_delim=False),\n","        num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","    # Map each tuple to a {\"input\": ... \"target\": ...} dict.\n","    ds = ds.map(lambda *ex: dict(zip([\"input\", \"target\"], ex)))\n","    return ds\n","\n","def ner_preprocessor(ds):\n","  def normalize_text(text):\n","    return text\n","\n","  def to_inputs_and_targets(ex):\n","    \"\"\"Map {\"inputs\": ..., \"targets\": ...}->{\"inputs\": ner..., \"targets\": ...}.\"\"\"\n","    return {\n","        \"inputs\":\n","             tf.strings.join(\n","                 [\"small: \", normalize_text(ex[\"input\"])]),\n","        \"targets\": normalize_text(ex[\"target\"])\n","    }\n","  return ds.map(to_inputs_and_targets, \n","                num_parallel_calls=tf.data.experimental.AUTOTUNE)\n","\n","print(\"A few raw validation examples...\")\n","for ex in tfds.as_numpy(dumping_dataset(\"train\").take(5)):\n","  print(ex)"],"execution_count":9,"outputs":[{"output_type":"stream","name":"stdout","text":["A few raw validation examples...\n","{'input': b'public java.lang.String METHOD_1 ( )  OPEN_CURLY_TOKEN  return new TYPE_1 ( STRING_1 ) . format ( VAR_1  OPEN_SQUARE_TOKEN  ( ( VAR_1 . length ) - 1 )  CLOSE_SQUARE_TOKEN  . getTime ( ) ) ;  CLOSE_CURLY_TOKEN ', 'target': b'public java.lang.String METHOD_1 ( )  OPEN_CURLY_TOKEN  return new TYPE_1 ( STRING_1 ) . format ( VAR_1  OPEN_SQUARE_TOKEN  ( ( type ) - 1 )  CLOSE_SQUARE_TOKEN  . getTime ( ) ) ;  CLOSE_CURLY_TOKEN '}\n","{'input': b'public boolean METHOD_1 ( java.lang.String name )  OPEN_CURLY_TOKEN  TYPE_1 VAR_1 = TYPE_1 . METHOD_2 ( VAR_2 ) ; return ( ! ( METHOD_3 ( name ) ) ) && ( VAR_1 . contains ( name ) ) ;  CLOSE_CURLY_TOKEN ', 'target': b'public boolean METHOD_1 ( java.lang.String name )  OPEN_CURLY_TOKEN  return ( ! ( METHOD_3 ( name ) ) ) && ( TYPE_1 . METHOD_2 ( VAR_2 ) . contains ( name ) ) ;  CLOSE_CURLY_TOKEN '}\n","{'input': b'public char METHOD_1 ( java.lang.String VAR_1 , java.lang.String name )  OPEN_CURLY_TOKEN  return null ;  CLOSE_CURLY_TOKEN ', 'target': b'public char METHOD_1 ( java.lang.String VAR_1 , java.lang.String name )  OPEN_CURLY_TOKEN  return 0 ;  CLOSE_CURLY_TOKEN '}\n","{'input': b'private void METHOD_1 ( )  OPEN_CURLY_TOKEN  VAR_1 . METHOD_2 ( STRING_1 ) ; VAR_2 = false ; ( VAR_3 ) ++ ; if ( ( VAR_3 ) == ( VAR_4 . size ( ) ) )  OPEN_CURLY_TOKEN  METHOD_3 ( ) ;  CLOSE_CURLY_TOKEN  METHOD_4 ( ) ;  CLOSE_CURLY_TOKEN ', 'target': b'private void METHOD_1 ( )  OPEN_CURLY_TOKEN  VAR_1 . METHOD_2 ( STRING_1 ) ; VAR_2 = false ; ( VAR_3 ) ++ ; if ( ( VAR_3 ) == ( VAR_4 . size ( ) ) )  OPEN_CURLY_TOKEN  METHOD_3 ( ) ; return ;  CLOSE_CURLY_TOKEN  METHOD_4 ( ) ;  CLOSE_CURLY_TOKEN '}\n","{'input': b'public TYPE_1 METHOD_1 ( )  OPEN_CURLY_TOKEN  java.lang.System.out.println ( STRING_1 ) ; return this . VAR_1 . METHOD_1 ( ) ;  CLOSE_CURLY_TOKEN ', 'target': b'public TYPE_1 METHOD_1 ( )  OPEN_CURLY_TOKEN  return this . VAR_1 . METHOD_1 ( ) ;  CLOSE_CURLY_TOKEN '}\n"]}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"3QKOr2cPX0du","outputId":"ad9e19f9-9616-46b4-dc40-5daa3bf60aa5","executionInfo":{"status":"ok","timestamp":1660156968615,"user_tz":-300,"elapsed":11,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["t5.data.TaskRegistry.remove('small')\n","t5.data.TaskRegistry.add(\n","    \"small\",\n","    # Supply a function which returns a tf.data.Dataset.\n","    dataset_fn=dumping_dataset,\n","    splits=[\"train\", \"validation\"],\n","    # Supply a function which preprocesses text from the tf.data.Dataset.\n","    text_preprocessor=[ner_preprocessor],\n",")"],"execution_count":10,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<t5.data.dataset_providers.FunctionTask at 0x7ff62cdffa90>"]},"metadata":{},"execution_count":10}]},{"cell_type":"markdown","metadata":{"id":"gAqbaQ7Ol7EK"},"source":["## Mixtures"]},{"cell_type":"code","metadata":{"id":"ceK0-uXzl7EO","colab":{"base_uri":"https://localhost:8080/"},"outputId":"e500e60d-9314-4221-c73b-a1583dce99e8","executionInfo":{"status":"ok","timestamp":1660156969284,"user_tz":-300,"elapsed":674,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["t5.data.MixtureRegistry.remove(\"all_mix\")\n","t5.data.MixtureRegistry.add(\n","    \"all_mix\",\n","    [\n","     'medium',\n","     'small'\n","     ],\n","     default_rate=1.0\n",")"],"execution_count":11,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<seqio.dataset_providers.Mixture at 0x7ff628c0e0d0>"]},"metadata":{},"execution_count":11}]},{"cell_type":"markdown","metadata":{"id":"3y7-5c2Gl7EO"},"source":["## Define Model"]},{"cell_type":"code","metadata":{"id":"fwvZLQy4wv0I","executionInfo":{"status":"ok","timestamp":1660156969284,"user_tz":-300,"elapsed":80,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["# !gsutil -m rm -r {MODEL_DIR}"],"execution_count":12,"outputs":[]},{"cell_type":"code","metadata":{"id":"xmSzYqw_l7EP","colab":{"base_uri":"https://localhost:8080/","height":634},"executionInfo":{"status":"error","timestamp":1660156969314,"user_tz":-300,"elapsed":110,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}},"outputId":"c3c65764-96be-4a78-8fa0-fc084a8cd5c6"},"source":["# Using pretrained_models from wiki + books\n","MODEL_SIZE = \"base\"\n","BASE_PRETRAINED_DIR = \"gs://cotext/cc/\"\n","\n","PRETRAINED_DIR = BASE_PRETRAINED_DIR\n","\n","MODEL_DIR = \"gs://t5_training/models/code/bug2fix_base_v2/\"\n","MODEL_DIR = os.path.join(MODEL_DIR, MODEL_SIZE)\n","\n","\n","# Set parallelism and batch size to fit on v2-8 TPU (if possible).\n","# Limit number of checkpoints to fit within 5GB (if possible).\n","model_parallelism, train_batch_size, keep_checkpoint_max = {\n","    \"small\": (1, 256, 16),\n","    \"base\": (2, 128, 8),\n","    \"large\": (8, 64, 4),\n","    \"3B\": (8, 16, 1),\n","    \"11B\": (8, 16, 1)}[MODEL_SIZE]\n","\n","tf.io.gfile.makedirs(MODEL_DIR)\n","# The models from our paper are based on the Mesh Tensorflow Transformer.\n","model = t5.models.MtfModel(\n","    model_dir=MODEL_DIR,\n","    tpu=TPU_ADDRESS,\n","    tpu_topology=TPU_TOPOLOGY,\n","    model_parallelism=model_parallelism,\n","    batch_size=train_batch_size,\n","    sequence_length={\"inputs\": 512, \"targets\": 512},\n","    learning_rate_schedule=0.001,\n","    save_checkpoints_steps=1000,\n","    keep_checkpoint_max=keep_checkpoint_max if ON_CLOUD else None,\n","    iterations_per_loop=100,\n",")\n"],"execution_count":13,"outputs":[{"output_type":"error","ename":"PermissionDeniedError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mPermissionDeniedError\u001b[0m                     Traceback (most recent call last)","\u001b[0;32m<ipython-input-13-2be5d9eb481b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m     \"11B\": (8, 16, 1)}[MODEL_SIZE]\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mMODEL_DIR\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m \u001b[0;31m# The models from our paper are based on the Mesh Tensorflow Transformer.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m model = t5.models.MtfModel(\n","\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/lib/io/file_io.py\u001b[0m in \u001b[0;36mrecursive_create_dir_v2\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m    509\u001b[0m     \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mIf\u001b[0m \u001b[0mthe\u001b[0m \u001b[0moperation\u001b[0m \u001b[0mfails\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    510\u001b[0m   \"\"\"\n\u001b[0;32m--> 511\u001b[0;31m   \u001b[0m_pywrap_file_io\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRecursivelyCreateDir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath_to_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    512\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    513\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mPermissionDeniedError\u001b[0m: Error executing an HTTP request: HTTP response code 403 with body '{\n  \"error\": {\n    \"code\": 403,\n    \"message\": \"daim-938@local-shoreline-357513.iam.gserviceaccount.com does not have storage.objects.get access to the Google Cloud Storage object.\",\n    \"errors\": [\n      {\n        \"message\": \"daim-938@local-shoreline-357513.iam.gserviceaccount.com does not have storage.objects.get access to the Google Cloud Storage object.\",\n        \"domain\": \"global\",\n        \"reason\": \"forbidden\"\n      }\n    ]\n  }\n}\n'\n\t when reading metadata of gs://t5_training/models/code/bug2fix_base_v2/base"]}]},{"cell_type":"markdown","metadata":{"id":"z-vSwM9Tl7EQ"},"source":["## Finetune"]},{"cell_type":"code","metadata":{"id":"YBkygCUDl7EQ","executionInfo":{"status":"aborted","timestamp":1660156969309,"user_tz":-300,"elapsed":102,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["FINETUNE_STEPS = 45000\n","\n","model.finetune(\n","    mixture_or_task_name=\"all_mix\",\n","    pretrained_model_dir=PRETRAINED_DIR,\n","    finetune_steps=FINETUNE_STEPS\n",")"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"RYVwWtQLIl3X"},"source":["## Predict"]},{"cell_type":"code","metadata":{"id":"pU6Qq9igIlIW","executionInfo":{"status":"aborted","timestamp":1660156969310,"user_tz":-300,"elapsed":102,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["tasks = [\n","         ['bug2fix', 'small'],\n","         ['bug2fix', 'medium']\n","         ]\n","output_dir = \"bug2fix_base_v2\"\n","test_file = 'test'\n","%cd /content/"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"q6VR3fJS7y2i","executionInfo":{"status":"aborted","timestamp":1660156969310,"user_tz":-300,"elapsed":102,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["for task in tasks:\n","  !mkdir {task[1]}\n","  !gsutil cp gs://cotext/data/{task[0]}/{task[1]}/{test_file}.tsv {task[1]}/\n","  with open(f'{task[1]}/{test_file}.tsv', 'r') as file:\n","    with open(f'{task[1]}/predict_input.tsv', 'w') as predict_input:\n","      with open(f'{task[1]}/actual_output.tsv', 'w') as actual_output:\n","        for line in file:\n","          line = line.strip().split('\\t')\n","          input = line[0].strip()\n","          \n","          actual = line[1].strip()\n","          actual = ' '.join(actual.split())\n","          actual = actual.strip() \\\n","                .replace(' SMALLER_TOKEN ', ' < ') \\\n","                .replace(' GREATER_TOKEN ', ' > ')\\\n","                .replace(' OPEN_SQUARE_TOKEN ', ' [ ')\\\n","                .replace(' CLOSE_SQUARE_TOKEN ', ' ] ')\\\n","                .replace(' OPEN_CURLY_TOKEN ', ' { ')\\\n","                .replace(' CLOSE_CURLY_TOKEN', ' } ')\\\n","                .replace(' CLOSE_CURLY_TOKEN ', ' } ')\\\n","                .replace(' EXPONENTIAL_TOKEN ', ' ^ ')\\\n","                .replace(' SHARP_TOKEN ', ' # ')\\\n","                .replace(' DOLLAR_TOKEN ', ' $ ')\\\n","                .replace(' UNK_TOKEN ', ' ` ') \\\n","                .replace(' NEW_LINE ', ' \\\\n ') \\\n","                .replace(' INDENT ', ' \\\\t ')\n","\n","          predict_input.write(f'{task[1]}: {input}\\n')\n","          actual_output.write(f'{actual}\\n')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"tAzDXwIlI16q","executionInfo":{"status":"aborted","timestamp":1660156969311,"user_tz":-300,"elapsed":103,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["import tensorflow.compat.v1 as tf\n","\n","for task in tasks:\n","  dir = task[0]\n","  input_file = f'{task[1]}/predict_input.tsv'\n","  output_file = f'{task[1]}/predict_output.tsv'\n","\n","  predict_inputs_path = input_file\n","  predict_outputs_path = output_file\n","\n","  # Manually apply preprocessing by prepending \"triviaqa question:\".\n","  print(predict_inputs_path)\n","  print(predict_outputs_path)\n","  # Ignore any logging so that we only see the model's answers to the questions.\n","  with tf_verbosity_level('ERROR'):\n","    model.batch_size = 8  # Min size for small model on v2-8 with parallelism 1.\n","    model.predict(\n","        input_file=predict_inputs_path,\n","        output_file=predict_outputs_path,\n","        checkpoint_steps=-1,\n","        temperature=0,\n","    )\n","\n","  # The output filename will have the checkpoint appended so we glob to get \n","  # the latest.\n","  prediction_files = sorted(tf.io.gfile.glob(predict_outputs_path + \"*\"))\n","  print(\"Predicted task : \" + dir)\n","  print(\"\\nPredictions using checkpoint %s:\\n\" % prediction_files[-1].split(\"-\")[-1])"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"HBwOCWUTG-J9"},"source":["## Scoring"]},{"cell_type":"code","metadata":{"id":"av7odWGzG7p6","executionInfo":{"status":"aborted","timestamp":1660156969312,"user_tz":-300,"elapsed":104,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["!git clone https://github.com/microsoft/CodeXGLUE.git\n","%cd /content/CodeXGLUE/Code-Code/code-refinement"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"325835ASQhAH","executionInfo":{"status":"aborted","timestamp":1660156969312,"user_tz":-300,"elapsed":104,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["tasks = [\n","         ['bug2fix', 'small'],\n","         ['bug2fix', 'medium']\n","         ]\n","# output_dir = \"defect_detection_code_all_codesearchnet_v1\"\n","test_file = 'test'\n","checkpoint = '1044900'"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"-EMm0YAoMLS6","executionInfo":{"status":"aborted","timestamp":1660156969313,"user_tz":-300,"elapsed":104,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["for task in tasks:\n","  print(task[0], task[1])\n","  \n","  with open(f'/content/{task[1]}/predict_output.tsv-{checkpoint}') as file:\n","    with open(f'/content/{task[1]}/predict_output.tsv', 'w') as out_file:\n","      for line in file:\n","        line = line.strip() \\\n","                .replace('SMALLER_TOKEN', '<') \\\n","                .replace('GREATER_TOKEN', '>')\\\n","                .replace('OPEN_SQUARE_TOKEN', '[')\\\n","                .replace('CLOSE_SQUARE_TOKEN', ']')\\\n","                .replace('OPEN_CURLY_TOKEN', '{')\\\n","                .replace('CLOSE_CURLY_TOKEN', '}')\\\n","                .replace('EXPONENTIAL_TOKEN', '^')\\\n","                .replace('SHARP_TOKEN', '#')\\\n","                .replace('DOLLAR_TOKEN', '$')\\\n","                .replace('UNK_TOKEN', '`') \\\n","                .replace('NEW_LINE', '\\\\n') \\\n","                .replace('INDENT', '\\\\t')\n","        out_file.write(f'{line}\\n')\n","  \n","  !python evaluator/evaluator.py -ref /content/{task[1]}/actual_output.tsv -pre /content/{task[1]}/predict_output.tsv\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"vwujo5nonxZF","executionInfo":{"status":"aborted","timestamp":1660156969313,"user_tz":-300,"elapsed":104,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":["!pip install tree_sitter==0.2.0\n","%cd /content/CodeXGLUE/Code-Code/code-to-code-trans/evaluator/CodeBLEU\n","\n","for task in tasks:\n","  print(task[0], task[1])\n","  !python calc_code_bleu.py --refs /content/{task[1]}/actual_output.tsv --hyp /content/{task[1]}/predict_output.tsv --lang java"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"abXufzHhn9gO","executionInfo":{"status":"aborted","timestamp":1660156969314,"user_tz":-300,"elapsed":104,"user":{"displayName":"Daim Armaghan","userId":"18187142118366403441"}}},"source":[""],"execution_count":null,"outputs":[]}]}