98-数据处理（比pandas快几倍）-pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "54f883cc-4aec-4f69-b602-7956a59dffba",
   "metadata": {},
   "source": [
    "# Polars（据说比Pandas快不少）\n",
    "* 安装方法：pip3 install -U polars[pyarrow]\n",
    "* 用户指南: https://pola-rs.github.io/polars-book/\n",
    "* h2oai's db基准测试结果: https://h2oai.github.io/db-benchmark/\n",
    "* Python文档: https://pola-rs.github.io/polars/py-polars/html/reference/index.html\n",
    "* 用户指南: https://pola-rs.github.io/polars-book/user-guide/index.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "91bdf7a2-1aeb-468f-808b-c9f07bda3322",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import polars as pl\n",
    "import timeit\n",
    "df = pl.DataFrame(\n",
    "    {\n",
    "        \"A\": [1, 2, 3, 4, 5],\n",
    "        \"fruits\": [\"banana\", \"banana\", \"apple\", \"apple\", \"banana\"],\n",
    "        \"B\": [5, 4, 3, 2, 1],\n",
    "         \"cars\": [\"beetle\", \"audi\", \"beetle\", \"beetle\", \"beetle\"],\n",
    "     }\n",
    " )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fc239aaa-b7be-4a12-864e-a88950a91b44",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1 \"class=\"dataframe \">\n",
       "<thead>\n",
       "<tr>\n",
       "<th>\n",
       "A\n",
       "</th>\n",
       "<th>\n",
       "fruits\n",
       "</th>\n",
       "<th>\n",
       "B\n",
       "</th>\n",
       "<th>\n",
       "cars\n",
       "</th>\n",
       "</tr>\n",
       "<tr>\n",
       "<td>\n",
       "i64\n",
       "</td>\n",
       "<td>\n",
       "str\n",
       "</td>\n",
       "<td>\n",
       "i64\n",
       "</td>\n",
       "<td>\n",
       "str\n",
       "</td>\n",
       "</tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr>\n",
       "<td>\n",
       "1\n",
       "</td>\n",
       "<td>\n",
       "\"banana\"\n",
       "</td>\n",
       "<td>\n",
       "5\n",
       "</td>\n",
       "<td>\n",
       "\"beetle\"\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td>\n",
       "2\n",
       "</td>\n",
       "<td>\n",
       "\"banana\"\n",
       "</td>\n",
       "<td>\n",
       "4\n",
       "</td>\n",
       "<td>\n",
       "\"audi\"\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td>\n",
       "3\n",
       "</td>\n",
       "<td>\n",
       "\"apple\"\n",
       "</td>\n",
       "<td>\n",
       "3\n",
       "</td>\n",
       "<td>\n",
       "\"beetle\"\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td>\n",
       "4\n",
       "</td>\n",
       "<td>\n",
       "\"apple\"\n",
       "</td>\n",
       "<td>\n",
       "2\n",
       "</td>\n",
       "<td>\n",
       "\"beetle\"\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td>\n",
       "5\n",
       "</td>\n",
       "<td>\n",
       "\"banana\"\n",
       "</td>\n",
       "<td>\n",
       "1\n",
       "</td>\n",
       "<td>\n",
       "\"beetle\"\n",
       "</td>\n",
       "</tr>\n",
       "</tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "shape: (5, 4)\n",
       "┌─────┬────────┬─────┬────────┐\n",
       "│ A   ┆ fruits ┆ B   ┆ cars   │\n",
       "│ --- ┆ ---    ┆ --- ┆ ---    │\n",
       "│ i64 ┆ str    ┆ i64 ┆ str    │\n",
       "╞═════╪════════╪═════╪════════╡\n",
       "│ 1   ┆ banana ┆ 5   ┆ beetle │\n",
       "├╌╌╌╌╌┼╌╌╌╌╌╌╌╌┼╌╌╌╌╌┼╌╌╌╌╌╌╌╌┤\n",
       "│ 2   ┆ banana ┆ 4   ┆ audi   │\n",
       "├╌╌╌╌╌┼╌╌╌╌╌╌╌╌┼╌╌╌╌╌┼╌╌╌╌╌╌╌╌┤\n",
       "│ 3   ┆ apple  ┆ 3   ┆ beetle │\n",
       "├╌╌╌╌╌┼╌╌╌╌╌╌╌╌┼╌╌╌╌╌┼╌╌╌╌╌╌╌╌┤\n",
       "│ 4   ┆ apple  ┆ 2   ┆ beetle │\n",
       "├╌╌╌╌╌┼╌╌╌╌╌╌╌╌┼╌╌╌╌╌┼╌╌╌╌╌╌╌╌┤\n",
       "│ 5   ┆ banana ┆ 1   ┆ beetle │\n",
       "└─────┴────────┴─────┴────────┘"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a397971b-cad3-4c81-9388-36f8d6c020bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1 \"class=\"dataframe \">\n",
       "<thead>\n",
       "<tr>\n",
       "<th>\n",
       "fruits\n",
       "</th>\n",
       "<th>\n",
       "cars\n",
       "</th>\n",
       "<th>\n",
       "literal_string_fruits\n",
       "</th>\n",
       "<th>\n",
       "B\n",
       "</th>\n",
       "<th>\n",
       "sum_A_by_cars\n",
       "</th>\n",
       "<th>\n",
       "sum_A_by_fruits\n",
       "</th>\n",
       "<th>\n",
       "rev_A_by_fruits\n",
       "</th>\n",
       "<th>\n",
       "sort_A_by_B_by_fruits\n",
       "</th>\n",
       "</tr>\n",
       "<tr>\n",
       "<td>\n",
       "str\n",
       "</td>\n",
       "<td>\n",
       "str\n",
       "</td>\n",
       "<td>\n",
       "str\n",
       "</td>\n",
       "<td>\n",
       "i64\n",
       "</td>\n",
       "<td>\n",
       "i64\n",
       "</td>\n",
       "<td>\n",
       "i64\n",
       "</td>\n",
       "<td>\n",
       "list\n",
       "</td>\n",
       "<td>\n",
       "list\n",
       "</td>\n",
       "</tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr>\n",
       "<td>\n",
       "\"apple\"\n",
       "</td>\n",
       "<td>\n",
       "\"beetle\"\n",
       "</td>\n",
       "<td>\n",
       "\"fruits\"\n",
       "</td>\n",
       "<td>\n",
       "11\n",
       "</td>\n",
       "<td>\n",
       "4\n",
       "</td>\n",
       "<td>\n",
       "7\n",
       "</td>\n",
       "<td>\n",
       "[4, 3]\n",
       "</td>\n",
       "<td>\n",
       "[4, 3]\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td>\n",
       "\"apple\"\n",
       "</td>\n",
       "<td>\n",
       "\"beetle\"\n",
       "</td>\n",
       "<td>\n",
       "\"fruits\"\n",
       "</td>\n",
       "<td>\n",
       "11\n",
       "</td>\n",
       "<td>\n",
       "4\n",
       "</td>\n",
       "<td>\n",
       "7\n",
       "</td>\n",
       "<td>\n",
       "[4, 3]\n",
       "</td>\n",
       "<td>\n",
       "[4, 3]\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td>\n",
       "\"banana\"\n",
       "</td>\n",
       "<td>\n",
       "\"beetle\"\n",
       "</td>\n",
       "<td>\n",
       "\"fruits\"\n",
       "</td>\n",
       "<td>\n",
       "11\n",
       "</td>\n",
       "<td>\n",
       "4\n",
       "</td>\n",
       "<td>\n",
       "8\n",
       "</td>\n",
       "<td>\n",
       "[5, 2, 1]\n",
       "</td>\n",
       "<td>\n",
       "[5, 2, 1]\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td>\n",
       "\"banana\"\n",
       "</td>\n",
       "<td>\n",
       "\"audi\"\n",
       "</td>\n",
       "<td>\n",
       "\"fruits\"\n",
       "</td>\n",
       "<td>\n",
       "11\n",
       "</td>\n",
       "<td>\n",
       "2\n",
       "</td>\n",
       "<td>\n",
       "8\n",
       "</td>\n",
       "<td>\n",
       "[5, 2, 1]\n",
       "</td>\n",
       "<td>\n",
       "[5, 2, 1]\n",
       "</td>\n",
       "</tr>\n",
       "<tr>\n",
       "<td>\n",
       "\"banana\"\n",
       "</td>\n",
       "<td>\n",
       "\"beetle\"\n",
       "</td>\n",
       "<td>\n",
       "\"fruits\"\n",
       "</td>\n",
       "<td>\n",
       "11\n",
       "</td>\n",
       "<td>\n",
       "4\n",
       "</td>\n",
       "<td>\n",
       "8\n",
       "</td>\n",
       "<td>\n",
       "[5, 2, 1]\n",
       "</td>\n",
       "<td>\n",
       "[5, 2, 1]\n",
       "</td>\n",
       "</tr>\n",
       "</tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "shape: (5, 8)\n",
       "┌────────┬────────┬────────┬─────┬───┬───┬───────────┬───────────┐\n",
       "│ fruits ┆ cars   ┆ litera ┆ B   ┆ s ┆ s ┆ rev_A_by_ ┆ sort_A_by │\n",
       "│ ---    ┆ ---    ┆ l_stri ┆ --- ┆ u ┆ u ┆ fruits    ┆ _B_by_fru │\n",
       "│ str    ┆ str    ┆ ng_fru ┆ i64 ┆ m ┆ m ┆ ---       ┆ its       │\n",
       "│        ┆        ┆ its    ┆     ┆ _ ┆ _ ┆ list      ┆ ---       │\n",
       "│        ┆        ┆ ---    ┆     ┆ A ┆ A ┆ [i64]     ┆ list      │\n",
       "│        ┆        ┆ str    ┆     ┆ _ ┆ _ ┆           ┆ [i64]     │\n",
       "│        ┆        ┆        ┆     ┆ b ┆ b ┆           ┆           │\n",
       "│        ┆        ┆        ┆     ┆ y ┆ y ┆           ┆           │\n",
       "│        ┆        ┆        ┆     ┆ _ ┆ _ ┆           ┆           │\n",
       "│        ┆        ┆        ┆     ┆ c ┆ f ┆           ┆           │\n",
       "│        ┆        ┆        ┆     ┆ a ┆ r ┆           ┆           │\n",
       "│        ┆        ┆        ┆     ┆ r ┆ u ┆           ┆           │\n",
       "│        ┆        ┆        ┆     ┆ s ┆ i ┆           ┆           │\n",
       "│        ┆        ┆        ┆     ┆ - ┆ t ┆           ┆           │\n",
       "│        ┆        ┆        ┆     ┆ - ┆ s ┆           ┆           │\n",
       "│        ┆        ┆        ┆     ┆ - ┆ - ┆           ┆           │\n",
       "│        ┆        ┆        ┆     ┆ i ┆ - ┆           ┆           │\n",
       "│        ┆        ┆        ┆     ┆ 6 ┆ - ┆           ┆           │\n",
       "│        ┆        ┆        ┆     ┆ 4 ┆ i ┆           ┆           │\n",
       "│        ┆        ┆        ┆     ┆   ┆ 6 ┆           ┆           │\n",
       "│        ┆        ┆        ┆     ┆   ┆ 4 ┆           ┆           │\n",
       "╞════════╪════════╪════════╪═════╪═══╪═══╪═══════════╪═══════════╡\n",
       "│ apple  ┆ beetle ┆ fruits ┆ 11  ┆ 4 ┆ 7 ┆ [4, 3]    ┆ [4, 3]    │\n",
       "├╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌┼╌╌╌╌╌┼╌╌╌┼╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┤\n",
       "│ apple  ┆ beetle ┆ fruits ┆ 11  ┆ 4 ┆ 7 ┆ [4, 3]    ┆ [4, 3]    │\n",
       "├╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌┼╌╌╌╌╌┼╌╌╌┼╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┤\n",
       "│ banana ┆ beetle ┆ fruits ┆ 11  ┆ 4 ┆ 8 ┆ [5, 2, 1] ┆ [5, 2, 1] │\n",
       "├╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌┼╌╌╌╌╌┼╌╌╌┼╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┤\n",
       "│ banana ┆ audi   ┆ fruits ┆ 11  ┆ 2 ┆ 8 ┆ [5, 2, 1] ┆ [5, 2, 1] │\n",
       "├╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌┼╌╌╌╌╌┼╌╌╌┼╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┤\n",
       "│ banana ┆ beetle ┆ fruits ┆ 11  ┆ 4 ┆ 8 ┆ [5, 2, 1] ┆ [5, 2, 1] │\n",
       "└────────┴────────┴────────┴─────┴───┴───┴───────────┴───────────┘"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sort(\"fruits\").select(\n",
    "    [\n",
    "        \"fruits\",\n",
    "        \"cars\",\n",
    "        pl.lit(\"fruits\").alias(\"literal_string_fruits\"),\n",
    "        pl.col(\"B\").filter(pl.col(\"cars\") == \"beetle\").sum(),\n",
    "        pl.col(\"A\").filter(pl.col(\"B\") > 2).sum().over(\"cars\").alias(\"sum_A_by_cars\"),  # groups by \"cars\"\n",
    "        pl.col(\"A\").sum().over(\"fruits\").alias(\"sum_A_by_fruits\"),  # groups by \"fruits\"\n",
    "        pl.col(\"A\").reverse().over(\"fruits\").alias(\"rev_A_by_fruits\"),  # groups by \"fruits\n",
    "        pl.col(\"A\").sort_by(\"B\").over(\"fruits\").alias(\"sort_A_by_B_by_fruits\"),  # groups by \"fruits\"\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "178b2936-3345-48a8-b0f7-396380eaee63",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pandas_time:  0.2836948000000348\n",
      "Polars_time:  0.0953109999999242\n",
      "Pandas_time1:  0.343936900000017\n",
      "Polars_time1:  0.042977000000064436\n"
     ]
    }
   ],
   "source": [
    "# 读取时间对比\n",
    "start_df = timeit.default_timer()\n",
    "df = pd.read_csv(\"E:/data/a.csv\")\n",
    "df = df.sort_values(\"current\", ascending=False).head()\n",
    "stop_df = timeit.default_timer()\n",
    "print('Pandas_time: ', stop_df - start_df)\n",
    "\n",
    "start_pl = timeit.default_timer()\n",
    "data = pl.read_csv(\"E:/data/a.csv\")\n",
    "data.sort(by=\"current\", reverse=True).head()\n",
    "stop_pl = timeit.default_timer()\n",
    "print('Polars_time: ', stop_pl - start_pl)\n",
    "\n",
    "# 纵向拼接时间对比\n",
    "start_df1 = timeit.default_timer()\n",
    "df_1 = pd.read_csv('E:/data/a.csv')\n",
    "df_2 = pd.read_csv('E:/data/b.csv')\n",
    "df_1.append(df_2, ignore_index=True)\n",
    "stop_df1 = timeit.default_timer()\n",
    "print('Pandas_time1: ', stop_df1 - start_df1)\n",
    "\n",
    "start_pl1 = timeit.default_timer()\n",
    "pl_1 = pl.read_csv('E:/data/a.csv')\n",
    "pl_2 = pl.read_csv('E:/data/b.csv')\n",
    "pl_1.vstack(pl_2)\n",
    "stop_pl1 = timeit.default_timer()\n",
    "print('Polars_time1: ', stop_pl1 - start_pl1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3a84ee9-5f03-4b03-8de1-f57924b4849c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}