graph_gen_multi.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1fea88b2-e85f-4ae4-8d7f-abe2978b1bfa",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import platform\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm\n",
    "from src.datahandler import DataHandler\n",
    "from src.graphhandler import GraphHandler\n",
    "from src.helper_functions import *\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pickle\n",
    "from concurrent.futures import ProcessPoolExecutor, as_completed\n",
    "import fcntl\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d8ffe788",
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "def process_combination(board_size, hv_size, mb, double_hash, n_samples, op, mbf, paths):\n",
    "    seen_files = set()\n",
    "    dh_string = \"dh2\" if double_hash else \"dh1\"\n",
    "    dataset = f\"{board_size}x{board_size}_{n_samples}_{op}_{mbf}_{hv_size}_{mb}_{dh_string}\"\n",
    "    file_path = paths['graphs'] / f\"{dataset}.pkl\"\n",
    "\n",
    "    # Check if file has been processed\n",
    "    if file_path in seen_files or os.path.exists(file_path):\n",
    "        return str(f\"{board_size}x{board_size} - skipping...\")\n",
    "\n",
    "    seen_files.add(file_path)\n",
    "    \n",
    "    # Load data\n",
    "    dataset_label = f\"{board_size}x{board_size}_{n_samples*2}_{op}_{mbf}\"\n",
    "    data_handler = DataHandler(paths=paths, files={'data': dataset_label}, dataloader='np.genfromtxt', n_samples=n_samples)\n",
    "    data = data_handler.data[:n_samples]\n",
    "\n",
    "    # Prepare training and testing sets\n",
    "    X_data, Y_data = data[:, :-1], data[:, -1]\n",
    "    X_train, X_test, Y_train, Y_test = train_test_split(X_data, Y_data, test_size=0.2)\n",
    "    \n",
    "    # Prepare graph configuration\n",
    "    graphs_train = GraphHandler(paths=paths,\n",
    "                                board_size=board_size,\n",
    "                                data_array=X_train,\n",
    "                                symbols=['RED', 'BLUE','UP', 'DOWN', 'RIGHT','LEFT'],\n",
    "                                hypervector_size=hv_size,\n",
    "                                hypervector_bits=mb,\n",
    "                                double_hashing=double_hash)\n",
    "    \n",
    "    graphs_train.build_complete_graphs()\n",
    "\n",
    "    graphs_test = GraphHandler(paths=paths,\n",
    "                                board_size=board_size,\n",
    "                                data_array=X_test,\n",
    "                                init_with=graphs_train)\n",
    "    \n",
    "    graphs_test.build_complete_graphs()\n",
    "                                \n",
    "    with open(file_path, 'wb') as f:\n",
    "        fcntl.flock(f, fcntl.LOCK_EX) \n",
    "        pickle.dump((graphs_train.graphs, graphs_test.graphs, X_train, Y_train, X_test, Y_test), f)\n",
    "        fcntl.flock(f, fcntl.LOCK_UN)\n",
    "\n",
    "    return str(f\"{board_size}x{board_size} - building...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7b33aaad",
   "metadata": {},
   "outputs": [],
   "source": [
    "machine_name, os_name, user = get_machine_info()\n",
    "paths = get_paths(machine_name, os_name, user)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5995053-0c74-4d0d-a41a-4bcbcf65b765",
   "metadata": {},
   "outputs": [],
   "source": [
    "max_workers = min(20, os.cpu_count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "209e65f3-ee36-4006-9b4d-9effa8af6c02",
   "metadata": {},
   "outputs": [],
   "source": [
    "board_sizes = [4,5,6,7,8,9,10,11,12,13,14,15] #4,,6,7,8,9,10,11,12,13,14,15\n",
    "hv_sizes = [128] # (256 kills the kernel) 512,1024,2048,4096,8192,16384\n",
    "double_hashing = [False] #,True ,False\n",
    "hv_mbs = [2] #,4,8,16,32\n",
    "open_pos = [40] #5,15,25,35,45,0,10,20,,,50 40\n",
    "samples = [1000, 10000, 100000]\n",
    "moves_before = [0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "04edfea5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15x15 - building...: 100%|██████████| 36/36 [43:39<00:00, 72.77s/it]  \n"
     ]
    }
   ],
   "source": [
    "with ProcessPoolExecutor(max_workers=max_workers) as executor: \n",
    "    futures = []\n",
    "    for board_size in board_sizes:\n",
    "        for hv_size in hv_sizes:\n",
    "            for hv_mb in hv_mbs:\n",
    "                for double_hash in double_hashing:\n",
    "                    for n_samples in samples:\n",
    "                        for op in open_pos:\n",
    "                            for mbf in moves_before:\n",
    "                                futures.append(executor.submit(\n",
    "                                    process_combination, board_size, hv_size, hv_mb, double_hash, n_samples, op, mbf, paths\n",
    "                                ))\n",
    "                                time.sleep(2)\n",
    "\n",
    "    with tqdm(total=len(futures)) as pbar:\n",
    "        for future in as_completed(futures):\n",
    "            try:\n",
    "                result = future.result()\n",
    "                pbar.set_description(f\"{result}\")\n",
    "                pbar.update(1)\n",
    "            except Exception as e:\n",
    "                print(f\"Error processing a combination: {e}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}