Merge pull request #156 from OpenEnergyPlatform/feature-155-improve-o…

…em2orm-upload-tutorial Improve oem2orm upload tutorial
OpenEnergyPlatform · Jan 5, 2023 · 29d25e6 · 29d25e6
2 parents 034116e + 5438c29
commit 29d25e6
Showing 1 changed file with 18 additions and 11 deletions.
diff --git a/upload/OEP_Upload_Process_Data_and_Metadata_oem2orm.ipynb b/upload/OEP_Upload_Process_Data_and_Metadata_oem2orm.ipynb
@@ -247,8 +247,8 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The following command will write the content of your dataframe to the table on the OEP that was created earlier.<br>\n",
-    "Have a look in the OEP after it ran succesfully!"
+    "The following command will write the content of your dataframe to the table on the OEP that was created earlier. <br> The table is uploaded in chunks to ensure a stable connection to the OEP when uploading large tables.<br>\n",
+    "Take a look at your table on the OEP after succesfull upload or for debugging in case of an exception!"
    ]
   },
   {
@@ -257,15 +257,22 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "try: \n",
-    "    example_df.to_sql(table_name, connection, schema=schema, if_exists='append', index=False)\n",
-    "    print('Inserted data to ' + schema + '.' + table_name)\n",
-    "except Exception as e:\n",
-    "    print('Writing to ' + table_name + ' failed!')\n",
-    "    print('Note that you cannot load the same data into the table twice. There will be an id conflict.')\n",
-    "    print('Delete and recreate with the commands above, if you want to test your upload again.')\n",
-    "    print('The program throws the following exception:')\n",
-    "    print(e)"
+    "# Define chunksize. The chunksize defines the number of uploaded csv-rows per request.\n",
+    "chunksize = 6000\n",
+    "\n",
+    "with pd.read_csv(filepath, encoding='utf8', sep=';', chunksize=chunksize) as reader:\n",
+    "    for chunk in reader:\n",
+    "        print(f'Uploading chunk from table: {table_name}.')\n",
+    "        try: \n",
+    "            chunk.to_sql(table_name, connection, schema=schema, if_exists='append', index=False)\n",
+    "            print(f'Appended {len(chunk)} rows to table on OEP')\n",
+    "        except Exception as e:\n",
+    "            print(f'Writing to {schema}.{table_name} failed!')\n",
+    "            print('Delete and recreate with the commands above, if you want to repeat your upload again.')\n",
+    "            print('The program throws the following exception:')\n",
+    "            print(e)\n",
+    "        finally:\n",
+    "            print(f'Take a look at your table on the OEP: https://openenergy-platform.org/dataedit/view/{schema}/{table_name}')"
    ]
   },
   {