Merge branch 'pre-staging' into sds3

CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to SODA will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## v15.2.2 - 2024-10-09
+
+## Bug Fixes:
+
+- Fixed an issue with the Pennsieve agent version check not being able to detect old versions of the Pennsieve agent (The agent found at https://github.com/Pennsieve/agent).
+- Fixed an issue in how SODA reads the config.ini file that made it so that if the account name 'SODA-Pennsieve' exists it is always used for authentication with Pennsieve even when a defualt_profile key exists in the config.
+- Fixed an issue in the manifest creation and banner image upload/edit features of the `Advanced Features` tab that made it so users without Pennsieve Agents were not directed to the Pennsieve Agent download page. Instead they were told 'the pennsieve agent is not running.'
+- It was possible for an upload to fail early in 'Prepare Datasets Step-by-Step' and 'Upload Dataset' such that the catch statement referenced a variable that was not yet defined. This has been fixed.
+
 ## v15.2.0 - 2024-08-21
 
 ## Feature Additions:

src/pyflask/configUtils/config.py

@@ -54,8 +54,8 @@ def lowercase_account_names(config, account_name, configpath):
 
     # add the section back with the lowercase account name
     config.add_section(formatted_account_name) 
-    config.set(formatted_account_name, "api_token", config.get(formatted_account_name, "api_token"))
-    config.set(formatted_account_name, "api_secret", config.get(formatted_account_name, "api_secret"))
+    config.set(formatted_account_name, "api_token", config.get(account_name, "api_token"))
+    config.set(formatted_account_name, "api_secret", config.get(account_name, "api_secret"))
 
     # set the global default_profile option to lowercase
     config.set("global", "default_profile", formatted_account_name)

src/pyflask/curate/curate.py

@@ -907,10 +907,7 @@ def recursive_dataset_scan(
         list_copy_files, list_move_files = recursive_dataset_scan(
             folder, folderpath, list_copy_files, list_move_files
         )
-    for file in list_copy_files:
-        namespace_logger.info(f"file to copy: {file}")
-    for file in list_move_files:
-        namespace_logger.info(f"file to move: {file}")
+
     # 3. Add high-level metadata files in the list
     if "metadata-files" in soda_json_structure.keys():
         namespace_logger.info("generate_dataset_locally (optional) step 3 handling metadata-files")
@@ -1900,6 +1897,11 @@ def build_create_folder_request(folder_name, folder_parent_id, dataset_id):
 
 bytes_file_path_dict = {}
 
+# retry variables instantiated outside function
+list_of_files_to_rename = {}
+renamed_files_counter = 0 
+
+
 def ps_upload_to_dataset(soda_json_structure, ps, ds, resume=False):
     global namespace_logger
 
@@ -1925,7 +1927,9 @@ def ps_upload_to_dataset(soda_json_structure, ps, ds, resume=False):
     global elapsed_time
     global manifest_id
     global origin_manifest_id
-    global main_curate_status 
+    global main_curate_status
+    global list_of_files_to_rename
+    global renamed_files_counter
 
 
     total_files = 0

src/pyflask/datasets/compare_local_remote_files.py

@@ -16,6 +16,7 @@
 
 def import_subfolders(subfolder, path):
     global PENNSIEVE_URL
+    global namespace_logger
     global pennsieve_dataset_paths
     folder_id = subfolder["id"]
     try:
@@ -38,7 +39,7 @@ def import_subfolders(subfolder, path):
         for folder_name, folder in subfolder["folders"].items():
             import_subfolders(folder, f"{path}/{folder_name}")
     except Exception as e:
-        print(f"Exception when calling API: {e}")
+        namespace_logger.info(f"Exception when calling API: {e}")
         raise e
 
 
@@ -72,7 +73,7 @@ def import_pennsieve_dataset(dataset_id, path):
         for folder_name, folder in pennsieve_dataset_structure["folders"].items():
             import_subfolders(folder, f"{folder_name}")
     except Exception as e:
-        print(f"Exception when calling API: {e}")
+        namespace_logger.info(f"Exception when calling API: {e}")
         raise e
 
 
@@ -100,7 +101,7 @@ def import_local_dataset(path):
         for folder_name, folder in local_dataset_structure["folders"].items():
             import_local_subfolders(folder, f"{path}/{folder_name}")
     except Exception as e:
-        print(f"Exception when calling API: {e}")
+        namespace_logger.info(f"Exception when calling API: {e}")
         raise e
 
 
@@ -125,7 +126,7 @@ def import_local_subfolders(subfolder, path):
         for folder_name, folder in subfolder["folders"].items():
             import_local_subfolders(folder, f"{path}/{folder_name}")
     except Exception as e:
-        print(f"Exception when calling API: {e}")
+        namespace_logger.info(f"Exception when calling API: {e}")
         raise e
 
 

src/pyflask/datasets/datasets.py

@@ -132,7 +132,6 @@ def reserve_dataset_doi(dataset):  # sourcery skip: extract-method
         doi_request.raise_for_status()
         return {"doi": doi_request.json()["doi"]}
     except Exception as e:
-        print(e)
         abort(e.response.status_code,  e.response.json().get('message'))
 
 

src/pyflask/manageDatasets/manage_datasets.py

@@ -265,12 +265,7 @@ def bf_get_accounts():
     sections = config.sections()
     global namespace_logger
 
-    if SODA_SPARC_API_KEY in sections:
-        lowercase_account_names(config, SODA_SPARC_API_KEY, configpath)
-        with contextlib.suppress(Exception):
-            get_access_token()
-            return SODA_SPARC_API_KEY.lower()
-    elif "global" in sections:
+    if "global" in sections:
         if "default_profile" in config["global"]:
             default_profile = config["global"]["default_profile"]
             if default_profile in sections:
@@ -282,6 +277,13 @@ def bf_get_accounts():
                 except Exception as e:
                     namespace_logger.info("Failed to authenticate the stored token")
                     abort(401, e)
+
+    elif SODA_SPARC_API_KEY in sections:
+        lowercase_account_names(config, SODA_SPARC_API_KEY, configpath)
+        with contextlib.suppress(Exception):
+            get_access_token()
+            return SODA_SPARC_API_KEY.lower()
+
     else:
         namespace_logger.info("No default account found")
         for account in sections:

src/renderer/src/assets/component-utils/addDatasetAndOrganizationCards.js

@@ -165,7 +165,7 @@ const addDatasetAndOrganizationCardComponents = () => {
     currentParentTab: "add_edit_banner_parent-tab",
     action: "delete",
     section: "individual-question manage-dataset",
-    dataNext: "div_add_edit_banner",
+    dataNext: "div_add_edit_banner_image_agent_check",
     fields: [
       FIELD_OPTIONS.ACCOUNT_EDITABLE,
 

src/renderer/src/components/backgroundServices/PennsieveAgentCheckDisplay/index.jsx

@@ -11,7 +11,7 @@ const CLOSE_SODA_BUTTON_TEXT = "Close SODA";
 const KNOWN_ERROR_MESSAGES = [
   "UNIQUE constraint failed:",
   "NotAuthorizedException: Incorrect username or password.",
-  "401 Error Creating new UserSettings",
+  "UserSettings" /* If the error message contains "UserSettings", it is likely solved by deleting the Pennsieve Agent database files */,
 ];
 
 // Utility Functions
@@ -22,13 +22,24 @@ const deletePennsieveAgentDBFilesAndRestart = async () => {
     "/.pennsieve/pennsieve_agent.db-wal",
   ];
 
-  for (const file of filesToDelete) {
-    const filePath = `${window.homeDirectory}${file}`;
-    if (window.fs.existsSync(filePath)) {
-      await window.fs.unlink(filePath);
+  try {
+    // Stop the Pennsieve agent to free up the database files
+    await window.spawn.stopPennsieveAgent();
+
+    // Delete the Pennsieve agent database files
+    for (const file of filesToDelete) {
+      const filePath = `${window.homeDirectory}${file}`;
+      if (window.fs.existsSync(filePath)) {
+        await window.fs.unlink(filePath);
+      } else {
+        console.error(`Unable to find Pennsieve agent DB file: ${filePath}`);
+      }
     }
+  } catch (error) {
+    console.error("Error deleting Pennsieve agent DB files:", error);
   }
 
+  // Restart the Pennsieve agent check
   await window.checkPennsieveAgent();
 };
 
@@ -61,8 +72,14 @@ const PennsieveAgentErrorMessageDisplay = ({ errorMessage }) => {
               <>
                 <Text>
                   This is a known issue with the Pennsieve Agent. It can typically be resolved by
-                  deleting the local Pennsieve Agent database files. Would you like SODA to do this
-                  and restart the Agent?
+                  deleting the local Pennsieve Agent database files. You can refer to the
+                  <ExternalLink
+                    href="https://docs.sodaforsparc.io/docs/common-errors/trouble-starting-the-pennsieve-agent-in-soda"
+                    buttonText="SODA documentation"
+                    buttonType="anchor"
+                  />
+                  for information on how to fix the issue manually. SODA can also attempt to fix the
+                  issue for you. Would you like to have SODA try to fix the issue?
                 </Text>
                 <Group justify="center" mt="sm">
                   <Button onClick={deletePennsieveAgentDBFilesAndRestart}>
@@ -108,6 +125,7 @@ const PennsieveAgentCheckDisplay = () => {
     postPennsieveAgentCheckAction,
   } = useGlobalStore();
 
+  // If the Pennsieve agent check is in progress, display a loading spinner
   if (pennsieveAgentCheckInProgress === true) {
     return (
       <FullWidthContainer>
@@ -121,6 +139,7 @@ const PennsieveAgentCheckDisplay = () => {
     );
   }
 
+  // If an error message title and message are present, display the error message
   if (pennsieveAgentCheckError?.title && pennsieveAgentCheckError?.message) {
     return (
       <FullWidthContainer>
@@ -142,6 +161,7 @@ const PennsieveAgentCheckDisplay = () => {
     );
   }
 
+  // If the Pennsieve agent is not installed, display a message with a download link
   if (pennsieveAgentInstalled === false) {
     return (
       <FullWidthContainer>
@@ -176,14 +196,16 @@ const PennsieveAgentCheckDisplay = () => {
     );
   }
 
-  if (pennsieveAgentOutputErrorMessage === true) {
+  // If the Pennsieve agent check returned an error message, display the error message
+  if (pennsieveAgentOutputErrorMessage != null) {
     return (
       <FullWidthContainer>
         <PennsieveAgentErrorMessageDisplay errorMessage={pennsieveAgentOutputErrorMessage} />
       </FullWidthContainer>
     );
   }
 
+  // If the Pennsieve agent is not up to date, display a message with a download link to the latest version
   if (pennsieveAgentUpToDate === false) {
     return (
       <FullWidthContainer>
@@ -223,6 +245,7 @@ const PennsieveAgentCheckDisplay = () => {
     );
   }
 
+  // If the Pennsieve agent check was successful (no flags occurred), display a success message
   return (
     <FullWidthContainer>
       <Stack mt="sm" align="center" mx="sm">

src/renderer/src/scripts/advanced-features/advanced_features.js

@@ -5,6 +5,7 @@ import { hideAllSectionsAndDeselectButtons } from "../../assets/nav";
 import { existingDataset, modifyDataset } from "../../assets/lotties/lotties";
 import lottie from "lottie-web";
 import Swal from "sweetalert2";
+import { clientError } from "../others/http-error-handler/error-handler";
 
 while (!window.baseHtmlLoaded) {
   await new Promise((resolve) => setTimeout(resolve, 100));
@@ -230,6 +231,17 @@ $("#advanced-start-over-button").on("click", async () => {
     ).children[0].children[0].style.display = "block";
     document.getElementById("input-manifest-local-folder-dataset").placeholder = "Browse here";
 
+    // hide the pennsieve agent check section
+    document
+      .getElementById("advanced-features-manifest-generation-pennsieve-agent-check")
+      .classList.add("hidden");
+    // hide the pull manifest information button's div
+    document.getElementById("div-btn-pull-ds-manifest").classList.add("hidden");
+    // set the dataset text to none
+    document.querySelector("#bf_dataset_create_manifest").textContent = "None";
+    // hide the confirm button
+    document.querySelector("#btn-confirm-dataset-manifest-page").classList.add("hidden");
+
     // Hide the all sub-questions for generating manifest
     document.getElementById("Question-prepare-manifest-2").classList.remove("show");
     document.getElementById("Question-prepare-manifest-2").classList.remove("prev");
@@ -316,3 +328,79 @@ $("#button-import-banner-image").click(async () => {
   $("#edit_banner_image_modal").modal("show");
   $("#edit_banner_image_modal").addClass("show");
 });
+
+// Pennsieve Agent check display
+document.querySelector("#btn-confirm-dataset-manifest-page").addEventListener("click", async () => {
+  // hide the confirm button
+  document.querySelector("#btn-confirm-dataset-manifest-page").classList.add("hidden");
+
+  // show the Pennsieve Agent slot
+  const pennsieveAgentCheckDivId = "advanced-features-manifest-generation-pennsieve-agent-check";
+  let pennsieveAgentCheckDiv = document.querySelector(`#${pennsieveAgentCheckDivId}`);
+
+  // start agent check
+  let agentCheckSuccessful = false;
+  try {
+    pennsieveAgentCheckDiv.classList.remove("hidden");
+    // Check to make sure the Pennsieve agent is installed
+    agentCheckSuccessful = await window.checkPennsieveAgent(pennsieveAgentCheckDivId);
+  } catch (e) {
+    console.error("Error with agent" + e);
+  }
+
+  if (!agentCheckSuccessful) {
+    return;
+  }
+
+  // scroll to the button
+  document.querySelector("#div-btn-pull-ds-manifest").scrollIntoView({ behavior: "smooth" });
+});
+
+document.querySelector("#btn-pull-ds-manifest").addEventListener("click", async function () {
+  try {
+    window.generateManifestFolderLocallyForEdit(this);
+  } catch (e) {
+    clientError(e);
+  }
+});
+
+function createAgentCHeckObserver(targetDiv, successCallback) {
+  const observer = new MutationObserver((mutations) => {
+    mutations.forEach((mutation) => {
+      const textToCheck = "The Pennsieve Agent is running and ready to upload!";
+      if (targetDiv.textContent.includes(textToCheck)) {
+        successCallback();
+      }
+    });
+  });
+
+  observer.observe(targetDiv, { childList: true, subtree: true, characterData: true });
+  return observer;
+}
+
+// create a mutation observer on this id advanced-features-manifest-generation-pennsieve-agent-check
+// to check if the agent check has been completed and we can show the div-btn-pull-ds-manifest
+const agentCheckDiv = document.querySelector(
+  "#advanced-features-manifest-generation-pennsieve-agent-check"
+);
+createAgentCHeckObserver(agentCheckDiv, () => {
+  document.querySelector("#div-btn-pull-ds-manifest").classList.remove("hidden");
+});
+
+// add the same mutatiomn observer for the banner image pennsieve agent check
+const agentCheckBannerImageDiv = document.querySelector(
+  "#advanced-features-banner-image-pennsieve-agent-check"
+);
+
+createAgentCHeckObserver(agentCheckBannerImageDiv, async () => {
+  await window.transitionFreeFormMode(
+    document.querySelector("#div_add_edit_banner_image_agent_check"),
+    "div_add_edit_banner_image_agent_check",
+    "delete",
+    "freeform"
+  );
+  await window.wait(1000);
+
+  // scroll the next section into view
+  document.querySelector("#edit_banner_image_button").scrollIntoView({ behavior: "smooth" });
+});

src/renderer/src/scripts/meta/announcements.json

@@ -1,4 +1,14 @@
 {
+  "15.2.2": {
+    "announcements": {
+      "bug-fixes": [
+        "Fixed an issue where SODA was not able to detect legacy versions of the Pennsieve Agent and suggest an update.",
+        "Fixed the Pennsieve Agent installation check that runs in the `Advanced Features` section so that users are properly notified if they do not have the Agent installed.",
+        "Fixed a Pennsieve connectivity issue legacy users were experiencing when updating to the latest version of SODA."
+      ],
+      "features": []
+    }
+  },
   "15.2.0": {
     "announcements": {
       "bug-fixes": [

src/renderer/src/scripts/metadata-files/manifest.js

@@ -119,6 +119,8 @@ $(document).ready(async function () {
   $("#bf_dataset_create_manifest").on("DOMSubtreeModified", () => {
     if ($("#bf_dataset_create_manifest").text().trim() !== "None") {
       $("#div-check-bf-create-manifest").css("display", "flex");
+      document.querySelector("#btn-confirm-dataset-manifest-page").classList.remove("hidden");
+
       $($("#div-check-bf-create-manifest").children()[0]).show();
     } else {
       $("#div-check-bf-create-manifest").css("display", "none");