0_Test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f0967f79",
   "metadata": {},
   "outputs": [],
   "source": [
    "for name in dir():\n",
    "    if not name.startswith('_'):\n",
    "        del globals()[name]\n",
    "        \n",
    "        \n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "#magic commands "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8f089af9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import unittest\n",
    "import warnings\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from QRF.utils.uncertainty_utils import Result\n",
    "from QRF.wrapper.qrf_wrapper import QuantileRegressionForest\n",
    "from sklearn.utils.validation import check_is_fitted\n",
    "\n",
    "from QRF.utils.uncertainty_utils import entropy, entropy_based_uncertainty_decomposition, calculate_quantiles, Uncertainty, Result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0e32470c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# several test to assure computations and error free wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "66049cd4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Aggregating values: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 5304.34it/s]\n",
      "Aggregating counts: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 4144.53it/s]\n",
      "Estimating for samples: 100%|██████████████████████████████████████████████████████| 100/100 [00:00<00:00, 4219.11it/s]\n",
      "2024-08-14 23:10:25,447 - INFO - Calculating Quantiles has been terminated\n",
      "2024-08-14 23:10:25,452 - INFO - Calculating uncertainties has been terminated - Elapsed time 0.00 s\n",
      "Aggregating values: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 2581.27it/s]\n",
      "Estimating for samples: 100%|██████████████████████████████████████████████████████| 100/100 [00:00<00:00, 4958.69it/s]\n",
      "2024-08-14 23:10:35,914 - INFO - Calculating Quantiles has been terminated - Elapsed time 10.46 s\n",
      "Aggregating values: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 4694.61it/s]\n",
      "Aggregating counts: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 4556.95it/s]\n",
      "Estimating for samples: 100%|██████████████████████████████████████████████████████| 100/100 [00:00<00:00, 6633.51it/s]\n",
      "2024-08-14 23:10:37,384 - INFO - Calculating Quantiles has been terminated - Elapsed time 1.47 s\n",
      "2024-08-14 23:10:37,388 - INFO - Calculating uncertainties has been terminated - Elapsed time 0.00 s\n",
      "Aggregating values: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 3282.78it/s]\n",
      "Estimating for samples: 100%|████████████████████████████████████████████████████| 1000/1000 [00:00<00:00, 8569.98it/s]\n",
      "2024-08-14 23:10:40,281 - INFO - Calculating Quantiles has been terminated - Elapsed time 2.89 s\n",
      "Aggregating values: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 3260.22it/s]\n",
      "Estimating for samples: 100%|████████████████████████████████████████████████████| 1000/1000 [00:00<00:00, 9550.72it/s]\n",
      "2024-08-14 23:10:41,681 - INFO - Calculating Quantiles has been terminated - Elapsed time 1.40 s\n",
      "Aggregating counts: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 5953.76it/s]\n",
      "Aggregating values: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 5266.12it/s]\n",
      "..\n",
      "----------------------------------------------------------------------\n",
      "Ran 12 tests in 21.280s\n",
      "\n",
      "OK\n"
     ]
    }
   ],
   "source": [
    "\n",
    "class TestQuantileRegressionForest(unittest.TestCase):\n",
    "\n",
    "    def setUp(self):\n",
    "        # Suppress specific warnings\n",
    "        warnings.filterwarnings(\"ignore\", category=DeprecationWarning, message=\"np.find_common_type is deprecated\")\n",
    "        \n",
    "        self.X_train = np.random.rand(1000, 10)\n",
    "        self.y_train = np.random.rand(1000)\n",
    "        self.sample_weight = np.random.rand(1000)\n",
    "        self.X_test = np.random.rand(100, 10)\n",
    "        self.regressor = QuantileRegressionForest(n_estimators=100)\n",
    "        self.regressor.fit(self.X_train, self.y_train)\n",
    "\n",
    "    def test_fit(self):\n",
    "        self.assertIsNotNone(self.regressor.model)\n",
    "\n",
    "    def test_predict_original(self):\n",
    "        predictions = self.regressor.predict(self.X_test, prediction_type='RF')\n",
    "        self.assertEqual(len(predictions), len(self.X_test))\n",
    "        \n",
    "    def test_predict_rf_vs_qrf(self):\n",
    "        predictions_rf = self.regressor.predict(self.X_test, prediction_type='RF')        \n",
    "        results_df = pd.DataFrame(index=range(len(self.X_test)))\n",
    "        updated_df = self.regressor.calculate_metrics_and_uncertainties(self.X_test, results_df, results_df.index)\n",
    "        predictions_avg = updated_df['AVG']\n",
    "        self.assertTrue((predictions_rf == predictions_avg.values).all())\n",
    "\n",
    "    def test_predict_quantiles(self):\n",
    "        quantiles = [50, 25, 75]\n",
    "        prediction = self.regressor.predict(self.X_test, quantile=quantiles)\n",
    "        self.assertIsInstance(prediction, Result)\n",
    "        self.assertTrue(hasattr(prediction, 'median'))\n",
    "        self.assertTrue(hasattr(prediction, 'lower'))\n",
    "        self.assertTrue(hasattr(prediction, 'upper'))\n",
    "\n",
    "    def test_get_params(self):\n",
    "        params = self.regressor.get_params()\n",
    "        self.assertIn('n_estimators', params)\n",
    "        self.assertEqual(params['class_min'], 0)\n",
    "        self.assertEqual(params['class_max'], 100)\n",
    "\n",
    "    def test_set_params(self):\n",
    "        self.regressor.set_params(class_min=10, class_max=90, n_estimators=100)\n",
    "        params = self.regressor.get_params()\n",
    "        self.assertEqual(params['class_min'], 10)\n",
    "        self.assertEqual(params['class_max'], 90)\n",
    "        self.assertEqual(params['n_estimators'], 100)\n",
    "\n",
    "    def test_retrieve_sample_values(self):\n",
    "        values = self.regressor.retrieve_values(self.X_test)\n",
    "        self.assertEqual(values.shape[0], len(self.X_test))\n",
    "\n",
    "    def test_retrieve_sample_counts(self):\n",
    "        counts = self.regressor.retrieve_counts(self.X_test)\n",
    "        self.assertEqual(counts.shape[0], len(self.X_test))\n",
    "\n",
    "    def test_calculate_metrics_and_uncertainties(self):\n",
    "        results_df = pd.DataFrame(index=range(len(self.X_test)))\n",
    "        sample_indices = np.arange(len(self.X_test))\n",
    "        updated_df = self.regressor.calculate_metrics_and_uncertainties(self.X_test, results_df, sample_indices)\n",
    "        self.assertIn('AVG', updated_df.columns)\n",
    "        self.assertIn('UT', updated_df.columns)\n",
    "        \n",
    "    def test_fit_without_sample_weight(self):\n",
    "        self.regressor.fit(self.X_train, self.y_train)\n",
    "        check_is_fitted(self.regressor.model)\n",
    "\n",
    "    def test_fit_with_sample_weight(self):\n",
    "        self.regressor.fit(self.X_train, self.y_train, sample_weight=self.sample_weight)\n",
    "        check_is_fitted(self.regressor.model)\n",
    "\n",
    "    def test_prediction_with_sample_weight_vs_no_sampe_weight(self):\n",
    "        self.regressor.fit(self.X_train, self.y_train)\n",
    "        predictions_nw = self.regressor.predict(self.X_train)[0]\n",
    "        self.regressor.fit(self.X_train, self.y_train, sample_weight=self.sample_weight)\n",
    "        predictions_w = self.regressor.predict(self.X_train)[0]\n",
    "        self.assertFalse(np.array_equal(predictions_nw, predictions_w))\n",
    "        \n",
    "if __name__ == '__main__':\n",
    "    unittest.main(argv=['first-arg-is-ignored'], exit=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a39279ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Aggregating values: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 4391.94it/s]\n",
      "Aggregating counts: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 4963.09it/s]\n",
      "Estimating for samples: 100%|██████████████████████████████████████████████████████| 100/100 [00:00<00:00, 7925.59it/s]\n",
      "2024-08-14 23:10:46,775 - INFO - Calculating Quantiles has been terminated - Elapsed time 5.09 s\n",
      "2024-08-14 23:10:46,780 - INFO - Calculating uncertainties has been terminated - Elapsed time 0.01 s\n",
      "Aggregating values: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 4293.79it/s]\n",
      "Estimating for samples: 100%|██████████████████████████████████████████████████████| 100/100 [00:00<00:00, 4818.77it/s]\n",
      "2024-08-14 23:10:56,687 - INFO - Calculating Quantiles has been terminated - Elapsed time 9.91 s\n",
      "Aggregating values: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 7799.73it/s]\n",
      "Aggregating counts: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 3028.62it/s]\n",
      "Estimating for samples: 100%|██████████████████████████████████████████████████████| 100/100 [00:00<00:00, 6042.01it/s]\n",
      "2024-08-14 23:10:57,958 - INFO - Calculating Quantiles has been terminated - Elapsed time 1.27 s\n",
      "2024-08-14 23:10:57,958 - INFO - Calculating uncertainties has been terminated - Elapsed time 0.00 s\n",
      "Aggregating values: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 3217.06it/s]\n",
      "Estimating for samples: 100%|███████████████████████████████████████████████████| 1000/1000 [00:00<00:00, 10375.01it/s]\n",
      "2024-08-14 23:11:00,525 - INFO - Calculating Quantiles has been terminated - Elapsed time 2.57 s\n",
      "Aggregating values: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 3921.74it/s]\n",
      "Estimating for samples: 100%|████████████████████████████████████████████████████| 1000/1000 [00:00<00:00, 8814.51it/s]\n",
      "2024-08-14 23:11:01,857 - INFO - Calculating Quantiles has been terminated - Elapsed time 1.33 s\n",
      "Aggregating counts: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 6171.00it/s]\n",
      "Aggregating values: 100%|██████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 6346.45it/s]\n",
      "Estimating for samples: 100%|██████████████████████████████████████████████████████| 100/100 [00:00<00:00, 5979.39it/s]\n",
      "2024-08-14 23:11:05,506 - INFO - Calculating Quantiles has been terminated - Elapsed time 3.65 s\n",
      "..2024-08-14 23:11:05,522 - INFO - Calculating uncertainties has been terminated - Elapsed time 0.02 s\n",
      "..\n",
      "----------------------------------------------------------------------\n",
      "Ran 16 tests in 20.019s\n",
      "\n",
      "OK\n"
     ]
    }
   ],
   "source": [
    "class TestUtilityFunctions(unittest.TestCase):\n",
    "\n",
    "    def setUp(self):\n",
    "        self.sample_counts_matrix = np.array([\n",
    "            [10, 20, 30],\n",
    "            [5, 15, 25],\n",
    "            [20, 20, 20]\n",
    "        ])\n",
    "        self.aggregated_leaf_values = np.random.rand(100, 50)  # Increased size for more robust testing\n",
    "\n",
    "    def test_entropy(self):\n",
    "        probabilities = np.array([\n",
    "            [0.1, 0.2, 0.7],\n",
    "            [0.3, 0.3, 0.4]\n",
    "        ])\n",
    "        expected_entropies = -np.sum(probabilities * np.log(probabilities), axis=-1)\n",
    "        calculated_entropies = entropy(probabilities)\n",
    "        np.testing.assert_almost_equal(calculated_entropies, expected_entropies, decimal=6)\n",
    "\n",
    "    def test_entropy_based_uncertainty_decomposition(self):\n",
    "        uncertainty = entropy_based_uncertainty_decomposition(self.sample_counts_matrix)\n",
    "        self.assertIsInstance(uncertainty, Uncertainty)\n",
    "        self.assertEqual(uncertainty.total.shape[0], self.sample_counts_matrix.shape[0])\n",
    "\n",
    "    def test_calculate_quantiles(self):\n",
    "        quantiles = calculate_quantiles(self.aggregated_leaf_values, quantile=[50, 25, 75])\n",
    "        self.assertIsInstance(quantiles, Result)\n",
    "        self.assertEqual(quantiles.median.shape[0], self.aggregated_leaf_values.shape[0])\n",
    "        self.assertEqual(quantiles.lower.shape[0], self.aggregated_leaf_values.shape[0])\n",
    "        self.assertEqual(quantiles.upper.shape[0], self.aggregated_leaf_values.shape[0])\n",
    "\n",
    "    def test_entropy_calculation_correctness(self):\n",
    "        # Test case with known entropy values\n",
    "        probabilities = np.array([\n",
    "            [0.25, 0.25, 0.25, 0.25],  # Uniform distribution\n",
    "            [0.1, 0.1, 0.1, 0.7]       # Non-uniform distribution\n",
    "        ])\n",
    "        expected_entropies = np.array([\n",
    "            1.386294361,  # Entropy of uniform distribution\n",
    "            0.940246      # Corrected entropy for the given non-uniform distribution\n",
    "        ])\n",
    "        calculated_entropies = entropy(probabilities)\n",
    "        np.testing.assert_almost_equal(calculated_entropies, expected_entropies, decimal=3)\n",
    "\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    unittest.main(argv=['first-arg-is-ignored'], exit=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40144e52",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}