New API to allow uploaded files to pass through backend service

NOTICE.md

@@ -113,10 +113,6 @@ From the following locations:
 
 <https://github.com/remix-run/react-router>
 
-### @azure/storage-blob
-
-<https://github.com/Azure/azure-sdk-for-js>
-
 ### classnames
 
 <https://github.com/JedWatson/classnames>

app/backend/testsuite.py

@@ -6,6 +6,7 @@
 import os
 from fastapi.testclient import TestClient
 from dotenv import load_dotenv
+import io
 
 dir = current_working_directory = os.getcwd()
 # We're running from MAKE file, so we need to change directory to app/backend
@@ -351,7 +352,105 @@ def test_delete_item():
     assert response.status_code == 200
     assert response.json() == True
 
+def test_upload_file_one_tag():
+    with open("test_data/parts_inventory.csv", "rb") as file:
+        response = client.post(
+            "/file",
+            files={"file": ("parts_inventory.csv", file, "text/csv")},
+            data={"file_path": "parts_inventory.csv", "tags": "test"}
+        )
+        assert response.status_code == 200
+        assert response.json() == {"message": "File 'parts_inventory.csv' uploaded successfully"}
+
+
+def test_uploadfilenotagsnofolder():
+    with open("test_data/parts_inventory.csv", "rb") as file:
+        response = client.post(
+            "/file",
+            files={"file": ("parts_inventory.csv", file, "text/csv")},
+            data={"file_path": "parts_inventory.csv", "tags": ""}
+        )
+        print(response.json())
+        assert response.status_code == 200
+        assert response.json() == {"message": "File 'parts_inventory.csv' uploaded successfully"}
 
+def test_uploadfiletags():
+    with open("test_data/parts_inventory.csv", "rb") as file:
+        response = client.post(
+            "/file",
+            files={"file": ("parts_inventory.csv", file, "text/csv")},
+            data={"file_path": "parts_inventory.csv", "tags": "test,inventory"}
+        )
+        print(response.json())
+        assert response.status_code == 200
+        assert response.json() == {"message": "File 'parts_inventory.csv' uploaded successfully"}
+def test_uploadfilespecificfolder():
+    with open("test_data/parts_inventory.csv", "rb") as file:
+        response = client.post(
+            "/file",
+            files={"file": ("parts_inventory.csv", file, "text/csv")},
+            data={"file_path": "Finance/parts_inventory.csv", "tags": "test"}
+        )
+        assert response.status_code == 200
+        assert response.json() == {"message": "File 'parts_inventory.csv' uploaded successfully"}
+def test_uploadfilespecificfoldernested():
+    with open("test_data/parts_inventory.csv", "rb") as file:
+        response = client.post(
+            "/file",
+            files={"file": ("parts_inventory.csv", file, "text/csv")},
+            data={"file_path": "Finance/new/parts_inventory.csv", "tags": "test"}
+        )
+        assert response.status_code == 200
+        assert response.json() == {"message": "File 'parts_inventory.csv' uploaded successfully"}
+
+def test_upload_file_no_file():
+    response = client.post(
+        "/file",
+        data={"file_path": "parts_inventory.csv", "tags": "test"}
+    )
+    assert response.status_code == 422  # Unprocessable Entity
+
+def test_upload_file_large_file():
+    file_content = b"a" * (10 * 1024 * 1024)  # 10 MB file
+    file = io.BytesIO(file_content)
+    file.name = "large_parts_inventory.csv"
+
+    response = client.post(
+        "/file",
+        files={"file": (file.name, file, "text/csv")},
+        data={"file_path": "large_parts_inventory.csv", "tags": "test"}
+    )
+    assert response.status_code == 200
+    assert response.json() == {"message": "File 'large_parts_inventory.csv' uploaded successfully"}
+
+def test_upload_file_missing_file_path():
+    with open("test_data/parts_inventory.csv", "rb") as file:
+        response = client.post(
+            "/file",
+            files={"file": ("parts_inventory.csv", file, "text/csv")},
+            data={"tags": "test"}
+        )
+        assert response.status_code == 422  # Unprocessable Entity
+def test_upload_file_special_characters_in_file_path():
+    with open("test_data/parts_inventory.csv", "rb") as file:
+        response = client.post(
+            "/file",
+            files={"file": ("parts_inventory.csv", file, "text/csv")},
+            data={"file_path": "Finance/@new/parts_inventory.csv", "tags": "test"}
+        )
+        assert response.status_code == 200
+        assert response.json() == {"message": "File 'parts_inventory.csv' uploaded successfully"}
+
+def test_upload_file_long_tags():
+    with open("test_data/parts_inventory.csv", "rb") as file:
+        long_tags = ",".join(["tag"] * 1000)  # Very long tags string
+        response = client.post(
+            "/file",
+            files={"file": ("parts_inventory.csv", file, "text/csv")},
+            data={"file_path": "parts_inventory.csv", "tags": long_tags}
+        )
+        assert response.status_code == 200
+        assert response.json() == {"message": "File 'parts_inventory.csv' uploaded successfully"}
 # This test requires some amount of data to be present and processed in IA
 # It is commented out because processing the data takes time and the test will fail if the data is not processed
 # Change the question to a valid question that will produce citations if you want to run this test

app/frontend/package.json

@@ -9,7 +9,6 @@
     "watch": "tsc && vite build --watch"
   },
   "dependencies": {
-    "@azure/storage-blob": "^12.24.0",
     "@fluentui/react": "^8.110.7",
     "@fluentui/react-icons": "^2.0.195",
     "@react-spring/web": "^9.7.1",

app/frontend/src/api/api.ts

@@ -3,7 +3,6 @@
 
 import { ChatResponse, 
     ChatRequest, 
-    BlobClientUrlResponse, 
     AllFilesUploadStatus, 
     GetUploadStatusRequest, 
     GetInfoResponse, 
@@ -64,22 +63,6 @@ export function getCitationFilePath(citation: string): string {
     return `${encodeURIComponent(citation)}`;
 }
 
-export async function getBlobClientUrl(): Promise<string> {
-    const response = await fetch("/getblobclienturl", {
-        method: "GET",
-        headers: {
-            "Content-Type": "application/json"
-        }
-    });
-
-    const parsedResponse: BlobClientUrlResponse = await response.json();
-    if (response.status > 299 || !response.ok) {
-        throw Error(parsedResponse.error || "Unknown error");
-    }
-
-    return parsedResponse.url;
-}
-
 export async function getAllUploadStatus(options: GetUploadStatusRequest): Promise<AllFilesUploadStatus> {
     const response = await fetch("/getalluploadstatus", {
         method: "POST",

app/frontend/src/components/FolderPicker/FolderPicker.tsx

@@ -18,9 +18,8 @@ import { ITextFieldStyleProps, ITextFieldStyles, TextField } from '@fluentui/rea
 import { ILabelStyles, ILabelStyleProps } from '@fluentui/react/lib/Label';
 import { IIconProps } from '@fluentui/react';
 import { IButtonProps } from '@fluentui/react/lib/Button';
-import { ContainerClient } from "@azure/storage-blob";
 
-import { getBlobClientUrl } from "../../api";
+import { getFolders } from "../../api";
 import styles from "./FolderPicker.module.css";
 
 var allowNewFolders = false;
@@ -85,26 +84,13 @@ export const FolderPicker = ({allowFolderCreation, onSelectedKeyChange, preSelec
 
     async function fetchBlobFolderData() {
         try {
-            const blobClientUrl = await getBlobClientUrl();
-            var containerClient = new ContainerClient(blobClientUrl);
-            const delimiter = "/";
-            const prefix = "";
-            var newOptions: IComboBoxOption[] = allowNewFolders ? [] : [
+            const newOptions: IComboBoxOption[] = allowNewFolders ? [] : [
                 { key: 'selectAll', text: 'Select All', itemType: SelectableOptionMenuItemType.SelectAll },
                 { key: 'FolderHeader', text: 'Folders', itemType: SelectableOptionMenuItemType.Header }];
-            for await (const item of containerClient.listBlobsByHierarchy(delimiter, {
-                prefix,
-              })) {
-                // Check if the item is a folder
-                if (item.kind === "prefix") {
-                  // Get the folder name and add to the dropdown list
-                  var folderName = item.name.slice(0,-1);
-
-                  newOptions.push({key: folderName, text: folderName});
-                  setOptions(newOptions);
-                }
-              }
-              if (!allowNewFolders) {
+            const folders = await getFolders();
+            newOptions.push(...folders.map((folder: string) => ({ key: folder, text: folder })));
+            setOptions(newOptions);
+            if (!allowNewFolders) {
                 var filteredOptions = newOptions.filter(
                     option =>
                       (option.itemType === SelectableOptionMenuItemType.Normal || option.itemType === undefined) && !option.disabled,