scripts/03_tally_imports.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "48f05ea6-3de7-4d4c-a85d-8f71ac6829ad",
   "metadata": {},
   "source": [
    "### Imports Per File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d4f5b252-c390-4131-845d-baa88f197545",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import ast\n",
    "import pandas as pd\n",
    "import logging\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0572d044-9120-41cb-ae0e-4d201a22c7b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up logging configuration\n",
    "logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s')\n",
    "\n",
    "# Create a logger\n",
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "# Create a file handler and set the logging level to ERROR\n",
    "file_handler = logging.FileHandler('error.log')\n",
    "file_handler.setLevel(logging.ERROR)\n",
    "\n",
    "# Create a console handler and set the logging level to INFO\n",
    "console_handler = logging.StreamHandler()\n",
    "console_handler.setLevel(logging.INFO)\n",
    "\n",
    "# Create a formatter and add it to both handlers\n",
    "formatter = logging.Formatter('%(levelname)s: %(message)s')\n",
    "file_handler.setFormatter(formatter)\n",
    "console_handler.setFormatter(formatter)\n",
    "\n",
    "# Add the handlers to the logger\n",
    "logger.addHandler(file_handler)\n",
    "logger.addHandler(console_handler)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02b0a972-0459-4569-8d75-afb60b8f3a0e",
   "metadata": {},
   "source": [
    "### Extract R and Python Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f3077807-1352-44fd-a76f-f58af30cd229",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_imported_r_libs(text):\n",
    "    pattern = r'\\b(?:library|require)\\s*\\(\\s*[\"\\']?([^\"\\']+?)[\"\\']?\\s*\\)'\n",
    "    libs = re.findall(pattern, text, flags=re.IGNORECASE)\n",
    "    return list(set(libs))\n",
    "\n",
    "def extract_imported_py_libs(text):\n",
    "    logger = logging.getLogger(__name__)\n",
    "\n",
    "    libs = []\n",
    "    \n",
    "    try:\n",
    "        tree = ast.parse(text)\n",
    "    except SyntaxError as e:\n",
    "        logger.warning(f\"{e}\")\n",
    "        \n",
    "    for node in ast.walk(tree):\n",
    "        if isinstance(node, ast.Import):\n",
    "            for alias in node.names:\n",
    "                libs.append(alias.name)\n",
    "        elif isinstance(node, ast.ImportFrom):  \n",
    "            libs.append(node.module)\n",
    "\n",
    "    return list(set(libs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fb42cafd-5f09-49ba-b716-7d6d5de5ea1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_imports(file_path):\n",
    "\n",
    "  with open(file_path, 'r', encoding='latin1') as f:\n",
    "    text = f.read()\n",
    "\n",
    "  if file_path.endswith(('.R', '.Rscript')):\n",
    "    imports = extract_imported_r_libs(text)\n",
    "    language = 'R'\n",
    "\n",
    "  elif file_path.endswith(('.py', '.ipynb')):\n",
    "    imports = extract_imported_py_libs(text)\n",
    "    language = 'Python'\n",
    "\n",
    "  else:\n",
    "    return None\n",
    "\n",
    "  df = pd.DataFrame({'package': imports, \n",
    "                     'language': [language]*len(imports),\n",
    "                     'file_path': [file_path]*len(imports)})\n",
    "  return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dced79a0-dbe4-4b47-9527-1c735fd2a53a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_imports_from_folder(folder_path):\n",
    "    dfs = []\n",
    "    for root, dirs, files in os.walk(folder_path):\n",
    "        for file in files:\n",
    "            file_path = os.path.join(root, file)\n",
    "            \n",
    "            try:\n",
    "                df = get_imports(file_path)\n",
    "            except Exception as e:\n",
    "                logger.error(f\"Error processing {file_path}: {e}\")\n",
    "                continue\n",
    "    \n",
    "            if df is not None:\n",
    "                dfs.append(df)\n",
    "\n",
    "    imports_df = pd.concat(dfs, ignore_index=True)\n",
    "  \n",
    "    return imports_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3fa51f9d-c0cc-493c-89f7-b3dfd55d8192",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: invalid character in identifier (<unknown>, line 4)\n",
      "WARNING: invalid character in identifier (<unknown>, line 4)\n",
      "ERROR: Error processing scripts/JPE_Macroeconomics_datasets_files/CMBSPF/procedures.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JPE_Macroeconomics_datasets_files/CMBSPF/procedures.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JPE_Macroeconomics_datasets_files/VQV9AB/DS_Agg.py: source code string cannot contain null bytes\n",
      "ERROR: Error processing scripts/JPE_Macroeconomics_datasets_files/VQV9AB/DS_Agg.py: source code string cannot contain null bytes\n",
      "ERROR: Error processing scripts/JPE_Macroeconomics_datasets_files/VQV9AB/MS.py: source code string cannot contain null bytes\n",
      "ERROR: Error processing scripts/JPE_Macroeconomics_datasets_files/VQV9AB/MS.py: source code string cannot contain null bytes\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"starting search\")? (<unknown>, line 75)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"starting search\")? (<unknown>, line 75)\n",
      "ERROR: Error processing scripts/isq_datasets_files/PSOFAU/searchAllCountryMentionsUS.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/isq_datasets_files/PSOFAU/searchAllCountryMentionsUS.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('Beginning year ' + yr + ': PESC')? (<unknown>, line 43)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('Beginning year ' + yr + ': PESC')? (<unknown>, line 43)\n",
      "ERROR: Error processing scripts/isq_datasets_files/PSOFAU/EUscrape14jul2010.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/isq_datasets_files/PSOFAU/EUscrape14jul2010.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Starting document \",count)? (<unknown>, line 29)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Starting document \",count)? (<unknown>, line 29)\n",
      "ERROR: Error processing scripts/isq_datasets_files/PSOFAU/EU-finalize-briefs14jul2010.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/isq_datasets_files/PSOFAU/EU-finalize-briefs14jul2010.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"problem with brief or date in\", path)? (<unknown>, line 141)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"problem with brief or date in\", path)? (<unknown>, line 141)\n",
      "ERROR: Error processing scripts/isq_datasets_files/PSOFAU/EU-NameAndSlice14jul2010.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/isq_datasets_files/PSOFAU/EU-NameAndSlice14jul2010.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"starting search\")? (<unknown>, line 76)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"starting search\")? (<unknown>, line 76)\n",
      "ERROR: Error processing scripts/isq_datasets_files/PSOFAU/searchAllCountryMentions.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/isq_datasets_files/PSOFAU/searchAllCountryMentions.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 63)\n",
      "WARNING: invalid syntax (<unknown>, line 63)\n",
      "ERROR: Error processing scripts/ipsr-risp_datasets_files/E05GUN/nearest_centroid_eval.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/ipsr-risp_datasets_files/E05GUN/nearest_centroid_eval.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(row['candidateid'])? (<unknown>, line 11)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(row['candidateid'])? (<unknown>, line 11)\n",
      "ERROR: Error processing scripts/BJPolS_datasets_files/CGMMHG/test_all_cands_work.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/BJPolS_datasets_files/CGMMHG/test_all_cands_work.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 115)\n",
      "WARNING: invalid syntax (<unknown>, line 115)\n",
      "ERROR: Error processing scripts/BJPolS_datasets_files/CGMMHG/ConstructAPIResponse.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/BJPolS_datasets_files/CGMMHG/ConstructAPIResponse.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('Now on vote id %s' % vote_id)? (<unknown>, line 35)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('Now on vote id %s' % vote_id)? (<unknown>, line 35)\n",
      "ERROR: Error processing scripts/BJPolS_datasets_files/CGMMHG/1b_scrape_votes.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/BJPolS_datasets_files/CGMMHG/1b_scrape_votes.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 32)\n",
      "WARNING: invalid syntax (<unknown>, line 32)\n",
      "ERROR: Error processing scripts/BJPolS_datasets_files/CGMMHG/1a_scrape_MCs.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/BJPolS_datasets_files/CGMMHG/1a_scrape_MCs.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(r.url)? (<unknown>, line 11)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(r.url)? (<unknown>, line 11)\n",
      "ERROR: Error processing scripts/BJPolS_datasets_files/CGMMHG/1c_scrape_sig_ratings.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/BJPolS_datasets_files/CGMMHG/1c_scrape_sig_ratings.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(it)? (<unknown>, line 49)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(it)? (<unknown>, line 49)\n",
      "ERROR: Error processing scripts/JPublicPolicy_datasets_files/BG1FEO/createDocByTopMatrix_modified.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JPublicPolicy_datasets_files/BG1FEO/createDocByTopMatrix_modified.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 17)\n",
      "WARNING: invalid syntax (<unknown>, line 17)\n",
      "ERROR: Error processing scripts/JPublicPolicy_datasets_files/BG1FEO/get_year_mallet_data.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JPublicPolicy_datasets_files/BG1FEO/get_year_mallet_data.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(dirList[d])? (<unknown>, line 48)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(dirList[d])? (<unknown>, line 48)\n",
      "ERROR: Error processing scripts/JPublicPolicy_datasets_files/BG1FEO/article_subformat2.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JPublicPolicy_datasets_files/BG1FEO/article_subformat2.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Estimates\")? (<unknown>, line 195)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Estimates\")? (<unknown>, line 195)\n",
      "ERROR: Error processing scripts/PSRM_datasets_files/1JBNDC/CMLE_sanctions_1yearsT12.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/PSRM_datasets_files/1JBNDC/CMLE_sanctions_1yearsT12.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Estimates\")? (<unknown>, line 195)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Estimates\")? (<unknown>, line 195)\n",
      "ERROR: Error processing scripts/PSRM_datasets_files/1JBNDC/CMLE_estimation.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/PSRM_datasets_files/1JBNDC/CMLE_estimation.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Estimates\")? (<unknown>, line 195)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Estimates\")? (<unknown>, line 195)\n",
      "ERROR: Error processing scripts/PSRM_datasets_files/1JBNDC/CMLE_sanctions_5years.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/PSRM_datasets_files/1JBNDC/CMLE_sanctions_5years.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 17)\n",
      "WARNING: invalid syntax (<unknown>, line 17)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/SEOCQ0/get_and_format_tweet_geo_args.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/SEOCQ0/get_and_format_tweet_geo_args.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(name, year)? (<unknown>, line 41)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(name, year)? (<unknown>, line 41)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/LHTRWM/fl.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/LHTRWM/fl.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(session , bill, sponsor)? (<unknown>, line 27)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(session , bill, sponsor)? (<unknown>, line 27)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/LHTRWM/ca.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/LHTRWM/ca.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(fname)? (<unknown>, line 41)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(fname)? (<unknown>, line 41)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/QM15HX/extract_from_voterfile_1.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/QM15HX/extract_from_voterfile_1.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Done with letter %s\" % data_tuple['letter'])? (<unknown>, line 115)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Done with letter %s\" % data_tuple['letter'])? (<unknown>, line 115)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/QM15HX/synthetic_data_generation_3.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/QM15HX/synthetic_data_generation_3.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(fname)? (<unknown>, line 37)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(fname)? (<unknown>, line 37)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/QM15HX/correct_spellings_3.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/QM15HX/correct_spellings_3.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Done with letter %s\" % data_tuple['letter'])? (<unknown>, line 115)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Done with letter %s\" % data_tuple['letter'])? (<unknown>, line 115)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/QM15HX/synthetic_data_generation_3_anon.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/QM15HX/synthetic_data_generation_3_anon.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"running!\")? (<unknown>, line 6)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"running!\")? (<unknown>, line 6)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/TWVXKZ/parsedefendantnames.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/TWVXKZ/parsedefendantnames.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 182)\n",
      "WARNING: invalid syntax (<unknown>, line 182)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/XI7ENB/collectData.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/XI7ENB/collectData.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(file)? (<unknown>, line 21)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(file)? (<unknown>, line 21)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/XI7ENB/Guatemala_VoteCount.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/XI7ENB/Guatemala_VoteCount.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(name)? (<unknown>, line 38)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(name)? (<unknown>, line 38)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/XI7ENB/France_Lower_VoteCount.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/XI7ENB/France_Lower_VoteCount.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(name)? (<unknown>, line 33)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(name)? (<unknown>, line 33)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/XI7ENB/France_Upper_VoteCount.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/XI7ENB/France_Upper_VoteCount.py: local variable 'tree' referenced before assignment\n",
      "WARNING: inconsistent use of tabs and spaces in indentation (<unknown>, line 131)\n",
      "WARNING: inconsistent use of tabs and spaces in indentation (<unknown>, line 131)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/XI7ENB/Belgium_VoteCount.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/XI7ENB/Belgium_VoteCount.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(votelist_date)? (<unknown>, line 23)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(votelist_date)? (<unknown>, line 23)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/XI7ENB/Netherlands_VoteCount.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/XI7ENB/Netherlands_VoteCount.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(file_name)? (<unknown>, line 14)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(file_name)? (<unknown>, line 14)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/XI7ENB/Uruguay_VoteCount.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/XI7ENB/Uruguay_VoteCount.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('hello world')? (<unknown>, line 10)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('hello world')? (<unknown>, line 10)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/NFZLI3/Transcripts_DTM_v1.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/NFZLI3/Transcripts_DTM_v1.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 62)\n",
      "WARNING: invalid syntax (<unknown>, line 62)\n",
      "ERROR: Error processing scripts/the_review_datasets_files/NFZLI3/flatten_dict.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/the_review_datasets_files/NFZLI3/flatten_dict.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Finding PID duplicates and high HID occupancies:\")? (<unknown>, line 67)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Finding PID duplicates and high HID occupancies:\")? (<unknown>, line 67)\n",
      "ERROR: Error processing scripts/polbehavior_datasets_files/TKORD8/format_households1.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/polbehavior_datasets_files/TKORD8/format_households1.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Filtering households and creating ego-alter associations:\")? (<unknown>, line 62)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Filtering households and creating ego-alter associations:\")? (<unknown>, line 62)\n",
      "ERROR: Error processing scripts/polbehavior_datasets_files/TKORD8/format_households2.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/polbehavior_datasets_files/TKORD8/format_households2.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 39)\n",
      "WARNING: invalid syntax (<unknown>, line 39)\n",
      "ERROR: Error processing scripts/qje_datasets_files/NAGMJQ/final_bert_class.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/qje_datasets_files/NAGMJQ/final_bert_class.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"I am importing the libraries.\")? (<unknown>, line 36)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"I am importing the libraries.\")? (<unknown>, line 36)\n",
      "ERROR: Error processing scripts/qje_datasets_files/3A8CBI/02_Geoprocessing.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/qje_datasets_files/3A8CBI/02_Geoprocessing.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 36)\n",
      "WARNING: invalid syntax (<unknown>, line 36)\n",
      "ERROR: Error processing scripts/economics_datasets_files/PIGIJ8/sweepover.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/economics_datasets_files/PIGIJ8/sweepover.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"NO module found\")? (<unknown>, line 104)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"NO module found\")? (<unknown>, line 104)\n",
      "ERROR: Error processing scripts/tpmnewsletter_datasets_files/YINHYL/13MayUDFs.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/tpmnewsletter_datasets_files/YINHYL/13MayUDFs.py: local variable 'tree' referenced before assignment\n",
      "WARNING: (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (<unknown>, line 12)\n",
      "WARNING: (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (<unknown>, line 12)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinYearlyBufferGrid.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinYearlyBufferGrid.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Joining attributes from \" + JoinClass + \" to grid.\")? (<unknown>, line 31)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Joining attributes from \" + JoinClass + \" to grid.\")? (<unknown>, line 31)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinLakeGrid.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinLakeGrid.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\" Writing table ... \")? (<unknown>, line 40)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\" Writing table ... \")? (<unknown>, line 40)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinSugarDistanceGrid.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinSugarDistanceGrid.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Joining attributes from \" + JoinClass + \" to grid.\")? (<unknown>, line 46)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Joining attributes from \" + JoinClass + \" to grid.\")? (<unknown>, line 46)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinLastOfWildGrid.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinLastOfWildGrid.py: local variable 'tree' referenced before assignment\n",
      "WARNING: (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (<unknown>, line 12)\n",
      "WARNING: (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (<unknown>, line 12)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinXY.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinXY.py: local variable 'tree' referenced before assignment\n",
      "WARNING: (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (<unknown>, line 12)\n",
      "WARNING: (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (<unknown>, line 12)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/TransCostEsalqLogDestinations.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/TransCostEsalqLogDestinations.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\" \")? (<unknown>, line 31)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\" \")? (<unknown>, line 31)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinRoadCostDistanceGrid.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinRoadCostDistanceGrid.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\" Creating match between coarse and thin grids ...\")? (<unknown>, line 34)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\" Creating match between coarse and thin grids ...\")? (<unknown>, line 34)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinGaezGrid.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinGaezGrid.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\" Creating match between coarse and thin grids ...\")? (<unknown>, line 38)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\" Creating match between coarse and thin grids ...\")? (<unknown>, line 38)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinClimateGrid.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinClimateGrid.py: local variable 'tree' referenced before assignment\n",
      "WARNING: (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (<unknown>, line 12)\n",
      "WARNING: (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (<unknown>, line 12)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/SugarcaneDistances.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/SugarcaneDistances.py: local variable 'tree' referenced before assignment\n",
      "WARNING: (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (<unknown>, line 12)\n",
      "WARNING: (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (<unknown>, line 12)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/RoadTransCostRaster.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/RoadTransCostRaster.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Joining attributes from CSBuffer40km to grid.\")? (<unknown>, line 31)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Joining attributes from CSBuffer40km to grid.\")? (<unknown>, line 31)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinBufferGrid.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinBufferGrid.py: local variable 'tree' referenced before assignment\n",
      "WARNING: (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (<unknown>, line 20)\n",
      "WARNING: (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (<unknown>, line 20)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/YearlyBuffer.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/YearlyBuffer.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Joining attributes from \" + JoinClass + \" to grid.\")? (<unknown>, line 42)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Joining attributes from \" + JoinClass + \" to grid.\")? (<unknown>, line 42)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinMunicGrid.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinMunicGrid.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Mosaicking \" + NewClimateRaster)? (<unknown>, line 29)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Mosaicking \" + NewClimateRaster)? (<unknown>, line 29)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/WorkClimate.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/WorkClimate.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\" \")? (<unknown>, line 30)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\" \")? (<unknown>, line 30)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinElevationSlopeGrid.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinElevationSlopeGrid.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Joining attributes from \" + JoinClass + \" to grid.\")? (<unknown>, line 52)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Joining attributes from \" + JoinClass + \" to grid.\")? (<unknown>, line 52)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinCanaGrid.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinCanaGrid.py: local variable 'tree' referenced before assignment\n",
      "WARNING: (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (<unknown>, line 12)\n",
      "WARNING: (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (<unknown>, line 12)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/CreateFishnets.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/CreateFishnets.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Joining attributes from \" + JoinClass + \" to grid.\")? (<unknown>, line 31)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Joining attributes from \" + JoinClass + \" to grid.\")? (<unknown>, line 31)\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinUrbanGrid.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/1IY2KS/JoinUrbanGrid.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('writeResult: Result Status is not OK!')? (<unknown>, line 71)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('writeResult: Result Status is not OK!')? (<unknown>, line 71)\n",
      "ERROR: Error processing scripts/restat_datasets_files/27228/distmat.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/27228/distmat.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 31)\n",
      "WARNING: invalid syntax (<unknown>, line 31)\n",
      "ERROR: Error processing scripts/restat_datasets_files/BLEBHI/prep_for_analysis.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/BLEBHI/prep_for_analysis.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('*********************************************************************************')? (<unknown>, line 34)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('*********************************************************************************')? (<unknown>, line 34)\n",
      "ERROR: Error processing scripts/restat_datasets_files/XE7YVI/food_predict.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/XE7YVI/food_predict.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 14)\n",
      "WARNING: invalid syntax (<unknown>, line 14)\n",
      "ERROR: Error processing scripts/restat_datasets_files/WHOPIM/00A1_FGW_fig.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/WHOPIM/00A1_FGW_fig.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 40)\n",
      "WARNING: invalid syntax (<unknown>, line 40)\n",
      "ERROR: Error processing scripts/restat_datasets_files/MOTPKF/RES_FindWithin1Mile_newBG.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/MOTPKF/RES_FindWithin1Mile_newBG.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 40)\n",
      "WARNING: invalid syntax (<unknown>, line 40)\n",
      "ERROR: Error processing scripts/restat_datasets_files/MOTPKF/RES_FindWithin1Mile_new.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/MOTPKF/RES_FindWithin1Mile_new.py: local variable 'tree' referenced before assignment\n",
      "WARNING: inconsistent use of tabs and spaces in indentation (<unknown>, line 89)\n",
      "WARNING: inconsistent use of tabs and spaces in indentation (<unknown>, line 89)\n",
      "ERROR: Error processing scripts/restat_datasets_files/UO368B/DOJArchiveScraper.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/UO368B/DOJArchiveScraper.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 35)\n",
      "WARNING: invalid syntax (<unknown>, line 35)\n",
      "ERROR: Error processing scripts/restat_datasets_files/WBH8KW/estimation.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/WBH8KW/estimation.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('n = %d' % n)? (<unknown>, line 60)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('n = %d' % n)? (<unknown>, line 60)\n",
      "ERROR: Error processing scripts/restat_datasets_files/WBH8KW/monte_carlo.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/WBH8KW/monte_carlo.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 34)\n",
      "WARNING: invalid syntax (<unknown>, line 34)\n",
      "ERROR: Error processing scripts/restat_datasets_files/WBH8KW/construct_dataset.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/WBH8KW/construct_dataset.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 138)\n",
      "WARNING: invalid syntax (<unknown>, line 138)\n",
      "ERROR: Error processing scripts/restat_datasets_files/6G6KJI/data_extract.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/restat_datasets_files/6G6KJI/data_extract.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"CONVERTING DATA FROM R TO PYTHON\")? (<unknown>, line 11)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"CONVERTING DATA FROM R TO PYTHON\")? (<unknown>, line 11)\n",
      "ERROR: Error processing scripts/ajps_datasets_files/XBNJD9/convertData_R2py.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/ajps_datasets_files/XBNJD9/convertData_R2py.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(results[0][:given['nReal']])? (<unknown>, line 245)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(results[0][:given['nReal']])? (<unknown>, line 245)\n",
      "ERROR: Error processing scripts/ajps_datasets_files/XBNJD9/mainModelEstimationCode.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/ajps_datasets_files/XBNJD9/mainModelEstimationCode.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Timed out. Waiting to restart...\")? (<unknown>, line 140)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Timed out. Waiting to restart...\")? (<unknown>, line 140)\n",
      "ERROR: Error processing scripts/ajps_datasets_files/NYONTD/scrape_newsbank_sourcelist.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/ajps_datasets_files/NYONTD/scrape_newsbank_sourcelist.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Timed out. Waiting to restart...\")? (<unknown>, line 167)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Timed out. Waiting to restart...\")? (<unknown>, line 167)\n",
      "ERROR: Error processing scripts/ajps_datasets_files/NYONTD/scrape_newsbank.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/ajps_datasets_files/NYONTD/scrape_newsbank.py: local variable 'tree' referenced before assignment\n",
      "WARNING: inconsistent use of tabs and spaces in indentation (<unknown>, line 34)\n",
      "WARNING: inconsistent use of tabs and spaces in indentation (<unknown>, line 34)\n",
      "ERROR: Error processing scripts/jop_datasets_files/CV9AYE/PreProcess.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/jop_datasets_files/CV9AYE/PreProcess.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid character in identifier (<unknown>, line 21)\n",
      "WARNING: invalid character in identifier (<unknown>, line 21)\n",
      "ERROR: Error processing scripts/culturalanalytics_datasets_files/TKOEA1/inference.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/culturalanalytics_datasets_files/TKOEA1/inference.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid character in identifier (<unknown>, line 109)\n",
      "WARNING: invalid character in identifier (<unknown>, line 109)\n",
      "ERROR: Error processing scripts/culturalanalytics_datasets_files/TKOEA1/datasets.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/culturalanalytics_datasets_files/TKOEA1/datasets.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid character in identifier (<unknown>, line 20)\n",
      "WARNING: invalid character in identifier (<unknown>, line 20)\n",
      "ERROR: Error processing scripts/culturalanalytics_datasets_files/TKOEA1/make_table_novels_by_year.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/culturalanalytics_datasets_files/TKOEA1/make_table_novels_by_year.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid character in identifier (<unknown>, line 14)\n",
      "WARNING: invalid character in identifier (<unknown>, line 14)\n",
      "ERROR: Error processing scripts/culturalanalytics_datasets_files/TKOEA1/posterior_predictive_checks.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/culturalanalytics_datasets_files/TKOEA1/posterior_predictive_checks.py: local variable 'tree' referenced before assignment\n",
      "WARNING: unexpected indent (<unknown>, line 28)\n",
      "WARNING: unexpected indent (<unknown>, line 28)\n",
      "ERROR: Error processing scripts/JPE_datasets_files/X4MSWW/Data_10_PC_Response.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JPE_datasets_files/X4MSWW/Data_10_PC_Response.py: local variable 'tree' referenced before assignment\n",
      "WARNING: unexpected indent (<unknown>, line 279)\n",
      "WARNING: unexpected indent (<unknown>, line 279)\n",
      "ERROR: Error processing scripts/JPE_datasets_files/X4MSWW/Data_15_Market_Definition.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JPE_datasets_files/X4MSWW/Data_15_Market_Definition.py: local variable 'tree' referenced before assignment\n",
      "WARNING: unexpected indent (<unknown>, line 56)\n",
      "WARNING: unexpected indent (<unknown>, line 56)\n",
      "ERROR: Error processing scripts/JPE_datasets_files/X4MSWW/Data_9_Price_Change_Count.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JPE_datasets_files/X4MSWW/Data_9_Price_Change_Count.py: local variable 'tree' referenced before assignment\n",
      "WARNING: EOL while scanning string literal (<unknown>, line 305)\n",
      "WARNING: EOL while scanning string literal (<unknown>, line 305)\n",
      "ERROR: Error processing scripts/jlc_datasets_files/YYU9PG/Sentiment Analysis of Supreme Court Confirmation Hearings.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/jlc_datasets_files/YYU9PG/Sentiment Analysis of Supreme Court Confirmation Hearings.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 22)\n",
      "WARNING: invalid syntax (<unknown>, line 22)\n",
      "ERROR: Error processing scripts/pan_datasets_files/4PYOXP/placebo_generation_using_gpt_2.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/pan_datasets_files/4PYOXP/placebo_generation_using_gpt_2.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('  LSM Executing:', model_spec['name'])? (<unknown>, line 392)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('  LSM Executing:', model_spec['name'])? (<unknown>, line 392)\n",
      "ERROR: Error processing scripts/pan_datasets_files/CSAJQY/lsm.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/pan_datasets_files/CSAJQY/lsm.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('  biLCM Executing:', model_spec['name'])? (<unknown>, line 93)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('  biLCM Executing:', model_spec['name'])? (<unknown>, line 93)\n",
      "ERROR: Error processing scripts/pan_datasets_files/CSAJQY/bilcm.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/pan_datasets_files/CSAJQY/bilcm.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('Processing text in training data...')? (<unknown>, line 48)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('Processing text in training data...')? (<unknown>, line 48)\n",
      "ERROR: Error processing scripts/pan_datasets_files/MXKRDE/classifier.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/pan_datasets_files/MXKRDE/classifier.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 23)\n",
      "WARNING: invalid syntax (<unknown>, line 23)\n",
      "ERROR: Error processing scripts/pan_datasets_files/MXKRDE/sentence_article_feature_space.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/pan_datasets_files/MXKRDE/sentence_article_feature_space.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('Processing text in training data...')? (<unknown>, line 49)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print('Processing text in training data...')? (<unknown>, line 49)\n",
      "ERROR: Error processing scripts/pan_datasets_files/MXKRDE/predict-binning.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/pan_datasets_files/MXKRDE/predict-binning.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(bad_list)? (<unknown>, line 38)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(bad_list)? (<unknown>, line 38)\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/ELKQNR/QualityCL.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/ELKQNR/QualityCL.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(dbf_file)? (<unknown>, line 36)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(dbf_file)? (<unknown>, line 36)\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/DBVDU6/dbf_to_csv_EIADATA.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/DBVDU6/dbf_to_csv_EIADATA.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 175)\n",
      "WARNING: invalid syntax (<unknown>, line 175)\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/make_voyage_maps.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/make_voyage_maps.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Create Output GDB\")? (<unknown>, line 67)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Create Output GDB\")? (<unknown>, line 67)\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/process_lines_to_voyages.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/process_lines_to_voyages.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Creating output directories and gdbs\")? (<unknown>, line 69)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Creating output directories and gdbs\")? (<unknown>, line 69)\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/createTracks_month.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/createTracks_month.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 126)\n",
      "WARNING: invalid syntax (<unknown>, line 126)\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/TrackBuilderFunction_M.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/TrackBuilderFunction_M.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"\\n  Reading input point data...\")? (<unknown>, line 141)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"\\n  Reading input point data...\")? (<unknown>, line 141)\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/AIS_TrackBuilder_v3.1_64bit_modified.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/AIS_TrackBuilder_v3.1_64bit_modified.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Testing\")? (<unknown>, line 47)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Testing\")? (<unknown>, line 47)\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/mergeAIScsv_points.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/mergeAIScsv_points.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Create Output GDB\")? (<unknown>, line 88)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Create Output GDB\")? (<unknown>, line 88)\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/merge_tracks_AK_HI_add_interp.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/merge_tracks_AK_HI_add_interp.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Creating output directories and gdbs\")? (<unknown>, line 65)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Creating output directories and gdbs\")? (<unknown>, line 65)\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/cleanAIS_1516.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/cleanAIS_1516.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Create Output GDB\")? (<unknown>, line 125)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Create Output GDB\")? (<unknown>, line 125)\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/add_fuel_damages_to_voyages.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/JAERE_datasets_files/UPBNUI/add_fuel_damages_to_voyages.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 12)\n",
      "WARNING: invalid syntax (<unknown>, line 12)\n",
      "ERROR: Error processing scripts/lsq_datasets_files/OD5MBX/SPAP_2021_LSQ_TableS2_BERT.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/lsq_datasets_files/OD5MBX/SPAP_2021_LSQ_TableS2_BERT.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 40)\n",
      "WARNING: invalid syntax (<unknown>, line 40)\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/featurePickler.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/featurePickler.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"TTest\")? (<unknown>, line 7)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"TTest\")? (<unknown>, line 7)\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/batch_pickle.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/batch_pickle.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Call this as articlePickler start_at_article number_of_articles\")? (<unknown>, line 25)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Call this as articlePickler start_at_article number_of_articles\")? (<unknown>, line 25)\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/articlePickler.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/articlePickler.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(countries)? (<unknown>, line 202)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(countries)? (<unknown>, line 202)\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/country.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/country.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 92)\n",
      "WARNING: invalid syntax (<unknown>, line 92)\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/classifierClass2.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/classifierClass2.py: local variable 'tree' referenced before assignment\n",
      "WARNING: invalid syntax (<unknown>, line 25)\n",
      "WARNING: invalid syntax (<unknown>, line 25)\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/HTMLStripper.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/HTMLStripper.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Call this as articlePickler iterations bigrams unigrams. Working in demo mode!\")? (<unknown>, line 9)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Call this as articlePickler iterations bigrams unigrams. Working in demo mode!\")? (<unknown>, line 9)\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/train-eval.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/train-eval.py: local variable 'tree' referenced before assignment\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Classifying...  \\n\" \\)? (<unknown>, line 46)\n",
      "WARNING: Missing parentheses in call to 'print'. Did you mean print(\"Classifying...  \\n\" \\)? (<unknown>, line 46)\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/classify.py: local variable 'tree' referenced before assignment\n",
      "ERROR: Error processing scripts/researchandpolitics_datasets_files/2XSVWA/classify.py: local variable 'tree' referenced before assignment\n"
     ]
    }
   ],
   "source": [
    "# Specify the root folder where to start the search\n",
    "root_folder = 'scripts/'\n",
    "\n",
    "# Process the files and store the results in a DataFrame\n",
    "data = extract_imports_from_folder(root_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0d9c584c-5dd2-4cad-9e33-e7b93576b0d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "language\n",
       "R         61336\n",
       "Python     7378\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['language'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4e69acaa-9e84-45b3-bd71-cffa118061a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>package</th>\n",
       "      <th>language</th>\n",
       "      <th>file_path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ggplot2</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/FPA_datasets_files/CUUC5A/Mediation Fi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>memisc</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/FPA_datasets_files/6S3SHE/Chung FPA Re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>foreign</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/FPA_datasets_files/6S3SHE/Chung FPA Re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mediation</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/FPA_datasets_files/6S3SHE/Chung FPA Re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tidyverse</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/FPA_datasets_files/LO7A8I/Main Documen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68709</th>\n",
       "      <td>gridExtra</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/uidergisi_datasets_files/RHSDV3/script1.R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68710</th>\n",
       "      <td>tidyverse</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/uidergisi_datasets_files/RHSDV3/script1.R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68711</th>\n",
       "      <td>tm</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/uidergisi_datasets_files/RHSDV3/script1.R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68712</th>\n",
       "      <td>knitr</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/uidergisi_datasets_files/RHSDV3/script1.R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68713</th>\n",
       "      <td>ggwordcloud</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/uidergisi_datasets_files/RHSDV3/script1.R</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>68714 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           package language                                          file_path\n",
       "0          ggplot2        R  scripts/FPA_datasets_files/CUUC5A/Mediation Fi...\n",
       "1           memisc        R  scripts/FPA_datasets_files/6S3SHE/Chung FPA Re...\n",
       "2          foreign        R  scripts/FPA_datasets_files/6S3SHE/Chung FPA Re...\n",
       "3        mediation        R  scripts/FPA_datasets_files/6S3SHE/Chung FPA Re...\n",
       "4        tidyverse        R  scripts/FPA_datasets_files/LO7A8I/Main Documen...\n",
       "...            ...      ...                                                ...\n",
       "68709    gridExtra        R  scripts/uidergisi_datasets_files/RHSDV3/script1.R\n",
       "68710    tidyverse        R  scripts/uidergisi_datasets_files/RHSDV3/script1.R\n",
       "68711           tm        R  scripts/uidergisi_datasets_files/RHSDV3/script1.R\n",
       "68712        knitr        R  scripts/uidergisi_datasets_files/RHSDV3/script1.R\n",
       "68713  ggwordcloud        R  scripts/uidergisi_datasets_files/RHSDV3/script1.R\n",
       "\n",
       "[68714 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b0335722-f979-435d-8b76-60d6fef710b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['dataverse'] = data['file_path'].str.extract(r'/([^/]+)_datasets_files/')\n",
    "data['repo_id'] = data['file_path'].str.extract(r'_datasets_files/([^/]+)/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f046e137-eff2-46ce-a0c8-c8eb0b00a0c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(\"../data/file_imports.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "21169608-1704-44f2-98a7-bc5a0909a625",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>package</th>\n",
       "      <th>language</th>\n",
       "      <th>file_path</th>\n",
       "      <th>dataverse</th>\n",
       "      <th>repo_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ggplot2</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/FPA_datasets_files/CUUC5A/Mediation Fi...</td>\n",
       "      <td>FPA</td>\n",
       "      <td>CUUC5A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>memisc</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/FPA_datasets_files/6S3SHE/Chung FPA Re...</td>\n",
       "      <td>FPA</td>\n",
       "      <td>6S3SHE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>foreign</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/FPA_datasets_files/6S3SHE/Chung FPA Re...</td>\n",
       "      <td>FPA</td>\n",
       "      <td>6S3SHE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mediation</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/FPA_datasets_files/6S3SHE/Chung FPA Re...</td>\n",
       "      <td>FPA</td>\n",
       "      <td>6S3SHE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tidyverse</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/FPA_datasets_files/LO7A8I/Main Documen...</td>\n",
       "      <td>FPA</td>\n",
       "      <td>LO7A8I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68709</th>\n",
       "      <td>gridExtra</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/uidergisi_datasets_files/RHSDV3/script1.R</td>\n",
       "      <td>uidergisi</td>\n",
       "      <td>RHSDV3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68710</th>\n",
       "      <td>tidyverse</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/uidergisi_datasets_files/RHSDV3/script1.R</td>\n",
       "      <td>uidergisi</td>\n",
       "      <td>RHSDV3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68711</th>\n",
       "      <td>tm</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/uidergisi_datasets_files/RHSDV3/script1.R</td>\n",
       "      <td>uidergisi</td>\n",
       "      <td>RHSDV3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68712</th>\n",
       "      <td>knitr</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/uidergisi_datasets_files/RHSDV3/script1.R</td>\n",
       "      <td>uidergisi</td>\n",
       "      <td>RHSDV3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68713</th>\n",
       "      <td>ggwordcloud</td>\n",
       "      <td>R</td>\n",
       "      <td>scripts/uidergisi_datasets_files/RHSDV3/script1.R</td>\n",
       "      <td>uidergisi</td>\n",
       "      <td>RHSDV3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>68714 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           package language  \\\n",
       "0          ggplot2        R   \n",
       "1           memisc        R   \n",
       "2          foreign        R   \n",
       "3        mediation        R   \n",
       "4        tidyverse        R   \n",
       "...            ...      ...   \n",
       "68709    gridExtra        R   \n",
       "68710    tidyverse        R   \n",
       "68711           tm        R   \n",
       "68712        knitr        R   \n",
       "68713  ggwordcloud        R   \n",
       "\n",
       "                                               file_path  dataverse repo_id  \n",
       "0      scripts/FPA_datasets_files/CUUC5A/Mediation Fi...        FPA  CUUC5A  \n",
       "1      scripts/FPA_datasets_files/6S3SHE/Chung FPA Re...        FPA  6S3SHE  \n",
       "2      scripts/FPA_datasets_files/6S3SHE/Chung FPA Re...        FPA  6S3SHE  \n",
       "3      scripts/FPA_datasets_files/6S3SHE/Chung FPA Re...        FPA  6S3SHE  \n",
       "4      scripts/FPA_datasets_files/LO7A8I/Main Documen...        FPA  LO7A8I  \n",
       "...                                                  ...        ...     ...  \n",
       "68709  scripts/uidergisi_datasets_files/RHSDV3/script1.R  uidergisi  RHSDV3  \n",
       "68710  scripts/uidergisi_datasets_files/RHSDV3/script1.R  uidergisi  RHSDV3  \n",
       "68711  scripts/uidergisi_datasets_files/RHSDV3/script1.R  uidergisi  RHSDV3  \n",
       "68712  scripts/uidergisi_datasets_files/RHSDV3/script1.R  uidergisi  RHSDV3  \n",
       "68713  scripts/uidergisi_datasets_files/RHSDV3/script1.R  uidergisi  RHSDV3  \n",
       "\n",
       "[68714 rows x 5 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1137342d-9987-4eb0-ab32-b0ae382d924b",
   "metadata": {},
   "source": [
    "### Let's count an import per repo. only once"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "18c44230-446d-442c-9357-883b37cf55a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataverse</th>\n",
       "      <th>repo_id</th>\n",
       "      <th>package</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BJPolS</td>\n",
       "      <td>0BFF0K</td>\n",
       "      <td>{Matching, lfe, stargazer, ggplot2}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BJPolS</td>\n",
       "      <td>11V2P6</td>\n",
       "      <td>{plotrix, stargazer, lmtest, survival, sensema...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BJPolS</td>\n",
       "      <td>1B1MXY</td>\n",
       "      <td>{reshape, ggplot2, sys, csv, plyr, scales, kni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BJPolS</td>\n",
       "      <td>1QFIA1</td>\n",
       "      <td>{regweight, countrycode, stargazer, corrr, tex...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>BJPolS</td>\n",
       "      <td>1WZRY2</td>\n",
       "      <td>{reshape, grid, ggplot2, erer, spatialreg, plm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3037</th>\n",
       "      <td>xps</td>\n",
       "      <td>YIZEA7</td>\n",
       "      <td>{data.table}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3038</th>\n",
       "      <td>xps</td>\n",
       "      <td>YNXJZO</td>\n",
       "      <td>{emmeans, AER, cobalt, esc, ggeffects, mediati...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3039</th>\n",
       "      <td>xps</td>\n",
       "      <td>YTZZIT</td>\n",
       "      <td>{ggplot2, foreign, plyr, rdd, rdrobust}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3040</th>\n",
       "      <td>xps</td>\n",
       "      <td>YZPGSV</td>\n",
       "      <td>{stargazer, jtools, ggplot2, psych, plm, broom...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3041</th>\n",
       "      <td>xps</td>\n",
       "      <td>ZHWIWG</td>\n",
       "      <td>{ggplot2, rio, randomizr, here, tidyverse, dpl...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3042 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     dataverse repo_id                                            package\n",
       "0       BJPolS  0BFF0K                {Matching, lfe, stargazer, ggplot2}\n",
       "1       BJPolS  11V2P6  {plotrix, stargazer, lmtest, survival, sensema...\n",
       "2       BJPolS  1B1MXY  {reshape, ggplot2, sys, csv, plyr, scales, kni...\n",
       "3       BJPolS  1QFIA1  {regweight, countrycode, stargazer, corrr, tex...\n",
       "4       BJPolS  1WZRY2  {reshape, grid, ggplot2, erer, spatialreg, plm...\n",
       "...        ...     ...                                                ...\n",
       "3037       xps  YIZEA7                                       {data.table}\n",
       "3038       xps  YNXJZO  {emmeans, AER, cobalt, esc, ggeffects, mediati...\n",
       "3039       xps  YTZZIT            {ggplot2, foreign, plyr, rdd, rdrobust}\n",
       "3040       xps  YZPGSV  {stargazer, jtools, ggplot2, psych, plm, broom...\n",
       "3041       xps  ZHWIWG  {ggplot2, rio, randomizr, here, tidyverse, dpl...\n",
       "\n",
       "[3042 rows x 3 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repo_level = data.groupby(['dataverse', 'repo_id'])['package'].agg(set).reset_index()\n",
    "repo_level.to_csv(\"../data/repo_level.csv\", index = False)\n",
    "repo_level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9c14c4d2-6f74-4de0-aa93-2b758497f93d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>package</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ggplot2</td>\n",
       "      <td>1426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>foreign</td>\n",
       "      <td>1068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>stargazer</td>\n",
       "      <td>963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dplyr</td>\n",
       "      <td>864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tidyverse</td>\n",
       "      <td>788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>xtable</td>\n",
       "      <td>634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>plyr</td>\n",
       "      <td>515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>lmtest</td>\n",
       "      <td>480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>MASS</td>\n",
       "      <td>459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>gridExtra</td>\n",
       "      <td>442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>sandwich</td>\n",
       "      <td>421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>haven</td>\n",
       "      <td>393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>car</td>\n",
       "      <td>358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>readstata13</td>\n",
       "      <td>357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>reshape2</td>\n",
       "      <td>349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>stringr</td>\n",
       "      <td>348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>data.table</td>\n",
       "      <td>324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>scales</td>\n",
       "      <td>285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>texreg</td>\n",
       "      <td>280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>tidyr</td>\n",
       "      <td>278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>grid</td>\n",
       "      <td>268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>lubridate</td>\n",
       "      <td>265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Hmisc</td>\n",
       "      <td>253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>readxl</td>\n",
       "      <td>250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>lme4</td>\n",
       "      <td>250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>lfe</td>\n",
       "      <td>222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>broom</td>\n",
       "      <td>210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>RColorBrewer</td>\n",
       "      <td>210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>ggpubr</td>\n",
       "      <td>198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>plm</td>\n",
       "      <td>189</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         package  count\n",
       "0        ggplot2   1426\n",
       "1        foreign   1068\n",
       "2      stargazer    963\n",
       "3          dplyr    864\n",
       "4      tidyverse    788\n",
       "5         xtable    634\n",
       "6           plyr    515\n",
       "7         lmtest    480\n",
       "8           MASS    459\n",
       "9      gridExtra    442\n",
       "10      sandwich    421\n",
       "11         haven    393\n",
       "12           car    358\n",
       "13   readstata13    357\n",
       "14      reshape2    349\n",
       "15       stringr    348\n",
       "16    data.table    324\n",
       "17        scales    285\n",
       "18        texreg    280\n",
       "19         tidyr    278\n",
       "20          grid    268\n",
       "21     lubridate    265\n",
       "22         Hmisc    253\n",
       "23        readxl    250\n",
       "24          lme4    250\n",
       "25           lfe    222\n",
       "26         broom    210\n",
       "27  RColorBrewer    210\n",
       "28        ggpubr    198\n",
       "29           plm    189"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Assuming for now that python and R imports don't clash\n",
    "cites_df = repo_level.explode('package').groupby('package')\\\n",
    "           .size().reset_index(name='count').sort_values(by='count', ascending=False)\\\n",
    "           .reset_index(drop=True)\n",
    "cites_df.head(n = 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "75d3b6af-7f0e-4802-9cf5-97de854d253b",
   "metadata": {},
   "outputs": [],
   "source": [
    "cites_df.to_csv(\"../data/imports_per_package.csv\", index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}