temp table, csv copy

app/src/app/uploader/page.tsx

@@ -40,19 +40,19 @@ export default function Uploader() {
         }).then(response => {
           const {document_id} = response;
           let rowCursor = 0;
-          let assignmentTXT = 'geo_id,zone\n';
+          let uploadRows: [string, number][] = [];
 
           const partialUploadStep = () => {
             const assignments: Assignment[] = [];
             const rows = results.data as Array<Array<string>>;
             rows.slice(rowCursor, rowCursor + ROWS_PER_BATCH).forEach(row => {
               if (row.length == 2 && row[1] !== '' && !isNaN(Number(row[1]))) {
-                assignmentTXT += `${row[0]},${row[1]}\n`;
+                uploadRows.push([row[0], row[1]]);
               }
             });
-            uploadAssignments({assignmentTXT, document_id}).then(stepResult => {
+            uploadAssignments({assignments: uploadRows, document_id}).then(stepResult => {
               setProgress(rowCursor + assignments.length);
-              assignmentTXT = 'geo_id,zone\n';
+              uploadRows = [];
               rowCursor += ROWS_PER_BATCH;
               if (rowCursor > results.data.length) {
                 setMapLinks([...mapLinks, {document_id, name: file.name}]);

app/src/app/utils/api/apiHandlers.ts

@@ -595,12 +595,12 @@ export const patchUnShatterParents: (params: {
  * @returns true if upload succeeds
  */
 export const uploadAssignments: (updateData: {
-  assignmentTXT: string;
+  assignments: [string, number][];
   document_id: string;
-}) => Promise<AssignmentsCreate> = async ({assignmentTXT, document_id}) => {
+}) => Promise<AssignmentsCreate> = async ({assignments, document_id}) => {
   return await axios
     .patch(`${process.env.NEXT_PUBLIC_API_URL}/api/upload_assignments`, {
-      assignmentTXT: assignmentTXT,
+      assignments,
       document_id,
     })
     .then(res => {

backend/app/main.py

@@ -5,7 +5,6 @@
 from sqlmodel import Session, String, select, true
 from starlette.middleware.cors import CORSMiddleware
 from sqlalchemy.dialects.postgresql import insert
-import io
 import logging
 from sqlalchemy import bindparam
 from sqlmodel import ARRAY, INT
@@ -275,21 +274,23 @@ async def upload_assignments(
     data: AssignmentsBulkUpload, session: Session = Depends(get_session)
 ):
     d = data.model_dump()
-    csv_file = io.StringIO(d["assignmentTXT"])
+    csv_rows = d["assignments"]
+    document_id = d["document_id"]
 
-    session.execute(text("DROP TABLE IF EXISTS temploader"))
-    session.execute(text("""CREATE TABLE temploader (
+    session.execute(text("""CREATE TEMP TABLE temploader (
         geo_id TEXT,
         zone INT
     )"""))
-    session.connection().connection.cursor().copy_expert(
-        "COPY temploader FROM STDIN WITH CSV HEADER", csv_file
-    )
+    cursor = session.connection().connection.cursor()
+    with cursor.copy("COPY temploader (geo_id, zone) FROM STDIN") as copy:
+        for record in csv_rows:
+            copy.write_row([record[0], int(record[1])])
+
     session.execute(text("""
         INSERT INTO document.assignments (geo_id, zone, document_id)
         SELECT geo_id, zone, :document_id
         FROM temploader
-    """), {"document_id":  d["document_id"]})
+    """), {"document_id":  document_id})
     session.commit()
 
     return {"assignments_upserted": 1}

backend/app/models.py

@@ -184,7 +184,7 @@ class AssignmentsCreate(BaseModel):
 
 
 class AssignmentsBulkUpload(BaseModel):
-    assignmentTXT: str
+    assignments: list[list[str]]
     document_id: str