Merge pull request #191 from DANS-KNAW/development

Development

apps/digitaltwins/src/config/theme.ts

@@ -8,16 +8,16 @@ import grey from "@mui/material/colors/grey";
 const customTheme = createTheme({
   palette: {
     primary: {
-      light: "#E4F9FF",
-      main: "#38a7d4",
-      dark: "#1f97c8",
+      light: "#27a56a",
+      main: "#007663",
+      dark: "#036354",
       contrastText: "#fff",
     },
     secondary: {
-      light: "#ff7961",
-      main: "#f44336",
-      dark: "#ba000d",
-      contrastText: "#000",
+      light: "#00add9",
+      main: "#004b85",
+      dark: "#00365f",
+      contrastText: "#fff",
     },
   },
   typography: {

apps/ohsmart/index.html

@@ -41,4 +41,4 @@
       To create a production bundle, use `npm run build` or `yarn build`.
     -->
   </body>
-</html>
+</html>

packages/deposit/src/deposit/depositApi.ts

@@ -7,6 +7,8 @@ export const depositApi = createApi({
   baseQuery: fetchBaseQuery({
     baseUrl: `${import.meta.env.VITE_PACKAGING_TARGET}/dataset/`,
   }),
+  // no stale forms
+  keepUnusedDataFor: 0.1,
   refetchOnMountOrArgChange: true,
   endpoints: (build) => ({
     fetchSavedMetadata: build.query({

packages/file-mapper/src/features/FileMapper.tsx

@@ -10,7 +10,7 @@ import Step from '@mui/material/Step';
 import StepLabel from '@mui/material/StepLabel';
 import { useTranslation } from "react-i18next";
 import { Step1, Step2, Step3 } from './Steps';
-import { getActiveStep, setActiveStep, getFile, getSavedMap, getMapping } from './fileMapperSlice';
+import { getActiveStep, setActiveStep, getFile, getSavedMap, getMapping, getFileError } from './fileMapperSlice';
 import { useSubmitMapMutation } from './fileMapperApi';
 import { useAppSelector, useAppDispatch } from "../redux/hooks";
 import CircularProgress from '@mui/material/CircularProgress';
@@ -32,6 +32,7 @@ const FileMapper = ({setMappedForm, page}: {
   const file = useAppSelector(getFile);
   const savedMap = useAppSelector(getSavedMap);
   const mapping = useAppSelector(getMapping);
+  const fileError = useAppSelector(getFileError);
   const [ submitMap, { isLoading, data } ] = useSubmitMapMutation();
 
   // set page title
@@ -99,15 +100,15 @@ const FileMapper = ({setMappedForm, page}: {
                 disabled={activeStep === 0}
                 onClick={handleBack}
                 sx={{ mr: 1 }}
-                variant="outlined"
+                variant="contained"
               >
                 {t("buttonBack")}
               </Button>
               <Box sx={{ flex: '1 1 auto' }} />
               <Button 
                 onClick={handleNext} 
-                disabled={!file || isLoading}
-                variant="outlined"   
+                disabled={!file || isLoading || fileError !== undefined }
+                variant="contained"   
               >
                 {
                   activeStep === steps.length - 1 || (file && savedMap)

packages/file-mapper/src/features/Steps.tsx

@@ -15,6 +15,7 @@ import TableHead from '@mui/material/TableHead';
 import TableRow from '@mui/material/TableRow';
 import TextField from '@mui/material/TextField';
 import Autocomplete from '@mui/material/Autocomplete';
+import CircularProgress from '@mui/material/CircularProgress';
 import List from '@mui/material/List';
 import ListItem from '@mui/material/ListItem';
 import ListItemButton from '@mui/material/ListItemButton';
@@ -24,14 +25,21 @@ import Checkbox from '@mui/material/Checkbox';
 import Divider from '@mui/material/Divider';
 import useMediaQuery from '@mui/material/useMediaQuery';
 import { useTheme } from '@mui/material/styles';
-import type { DarwinOptions, Saves, SerializedFile } from "../types";
+import type { DarwinOptions, Saves, SerializedFile, FileError } from "../types";
 import { 
   getFile, 
   setFile, 
   getMapping, 
   setMapping, 
   getSavedMap,
   setSavedMap,
+  resetMapping,
+  setFileCols,
+  getFileCols,
+  resetFileCols,
+  getFileError,
+  setFileError,
+  resetFileError,
 } from './fileMapperSlice';
 import { useFetchDarwinTermsQuery } from "./fileMapperApi";
 import { useAppSelector, useAppDispatch } from "../redux/hooks";
@@ -49,6 +57,9 @@ const saves = [
   },
 ];
 
+// define max number of rows a file selected for processing can contain
+const maxRows = 10;
+
 const StepWrap = ({ title, children }: { title: string; children: ReactNode }) => 
   <Box sx={{pt: 2, pb: 1}}>
     <Typography variant="h2">{title}</Typography>
@@ -62,6 +73,7 @@ export const Step1 = () => {
   const dispatch = useAppDispatch();
   const file = useAppSelector(getFile);
   const savedMap = useAppSelector(getSavedMap);
+  const fileError = useAppSelector(getFileError);
 
   const onDrop = async (files: File[]) => {
     // serialize files to store in redux
@@ -72,6 +84,11 @@ export const Step1 = () => {
     };
 
     dispatch(setFile(serializedFile));
+
+    // reset saved mapping and column values after selecting a different file
+    dispatch(resetMapping());
+    dispatch(resetFileCols());
+    dispatch(resetFileError());
   }
 
   const {  
@@ -119,6 +136,11 @@ export const Step1 = () => {
               <Alert severity="success">
                 {t("selectedFile", { name: file.name, size: (file.size / 1024).toFixed(0) })}
               </Alert>
+              {fileError && 
+                <Alert severity="error">
+                  {t(fileError, {max: maxRows})}
+                </Alert>
+              }
             </Box> 
           }
         </Box>
@@ -151,24 +173,37 @@ export const Step1 = () => {
 }
 
 export const Step2 = () => {
-  const [ fileCols, setFileCols ] = useState<string[]>([]);
   const { t } = useTranslation("steps");
+  const dispatch = useAppDispatch();
   const file = useAppSelector(getFile);
+  const fileCols = useAppSelector(getFileCols);
+  const fileError = useAppSelector(getFileError);
+  // loading indicator only if new file needs to be read
+  const [ loading, setLoading ] = useState<boolean>( !Array.isArray(fileCols) && !fileError );
 
   useEffect(() => {
     const reader = new FileReader();
 
     reader.onload = (event) => {
+      // parse the xlsx/csv file
       const workbook = XLSX.read(event.target?.result, { type: 'binary' });
       const sheetName = workbook.SheetNames[0];
       const sheet = workbook.Sheets[sheetName];
       const sheetData = XLSX.utils.sheet_to_json(sheet, { header: 1 })[0];
-
-      Array.isArray(sheetData) && setFileCols(sheetData);
+      // check to see if sheet doesn't have too many rows
+      const rowCount = XLSX.utils.sheet_to_json(sheet).length;
+      if (rowCount > maxRows) {
+        dispatch(setFileError("tooManyRows"));
+      } else {
+        // save column data to store so we only have to load and parse once
+        Array.isArray(sheetData) && dispatch(setFileCols(sheetData));
+      }
+      setLoading(false);
     };
 
-    if (file) {
-      // convert file url back to blob
+    if (file && !fileCols && !fileError) {
+      dispatch(setFileError(undefined));
+      // if no file loaded yet, convert file url back to blob
       (async () => {
         const fetchedFile = await fetch(file.url);
         const blob = await fetchedFile.blob();
@@ -177,7 +212,7 @@ export const Step2 = () => {
         }
       })();
     }
-  }, [file]);
+  }, [file, fileCols, fileError]);
 
   return (
     <StepWrap title={t("createMapping")}>
@@ -190,7 +225,20 @@ export const Step2 = () => {
             </TableRow>
           </TableHead>
           <TableBody>
-            {fileCols.length > 0 && fileCols.map((row) => (
+            { ( loading || fileError ) && 
+              <TableRow>
+                <TableCell align="center" colSpan={2}>
+                  {
+                    loading ? 
+                    <CircularProgress /> : 
+                    <Alert severity="error">
+                      {t(fileError as FileError, {max: maxRows})}
+                    </Alert>
+                  }
+                </TableCell>
+              </TableRow>
+            }
+            {fileCols && fileCols.map((row) => (
               <Row key={row} row={row} />
             ))}
           </TableBody>

packages/file-mapper/src/features/fileMapperSlice.ts

@@ -1,6 +1,6 @@
 import { createSlice, type PayloadAction } from "@reduxjs/toolkit";
 import { RootState } from "../redux/store";
-import type { SerializedFile, Mapping } from "../types";
+import type { SerializedFile, Mapping, FileError } from "../types";
 import type { FormConfig } from "@dans-framework/deposit";
 
 const initialState: {
@@ -9,12 +9,16 @@ const initialState: {
   file: SerializedFile | undefined;
   savedMap: string;
   form: FormConfig | undefined;
+  fileCols: string[] | undefined;
+  fileError: string | undefined;
 } = {
   activeStep: 0,
   mapping: {},
   file: undefined,
   savedMap: '',
   form: undefined,
+  fileCols: undefined,
+  fileError: undefined,
 };
 
 export const fileMapperSlice = createSlice({
@@ -27,6 +31,9 @@ export const fileMapperSlice = createSlice({
     setMapping: (state, action: PayloadAction<Mapping>) => {
       state.mapping = action.payload;
     },
+    resetMapping: (state) => {
+      state.mapping = initialState.mapping;
+    },
     setSavedMap: (state, action: PayloadAction<string>) => {
       state.savedMap = action.payload;
     },
@@ -36,15 +43,29 @@ export const fileMapperSlice = createSlice({
     saveData: (state, action) => {
       state.form = action.payload;
     },
+    setFileCols: (state, action: PayloadAction<string[]>) => {
+      state.fileCols = action.payload;
+    },
+    setFileError: (state, action: PayloadAction<FileError | undefined>) => {
+      state.fileError = action.payload;
+    },
+    resetFileError: (state) => {
+      state.fileError = initialState.fileError;
+    },
+    resetFileCols: (state) => {
+      state.fileCols = initialState.fileCols;
+    }
   },
 });
 
-export const { setActiveStep, setMapping, setSavedMap, setFile, saveData } = fileMapperSlice.actions;
+export const { setActiveStep, setMapping, resetMapping, setSavedMap, setFile, saveData, setFileCols, resetFileCols, setFileError, resetFileError } = fileMapperSlice.actions;
 
 export const getActiveStep = (state: RootState) => state.fileMapper.activeStep;
 export const getMapping = (state: RootState) => state.fileMapper.mapping;
 export const getSavedMap = (state: RootState) => state.fileMapper.savedMap;
 export const getFile = (state: RootState) => state.fileMapper.file;
 export const getForm = (state: RootState) => state.fileMapper.form;
+export const getFileCols = (state: RootState) => state.fileMapper.fileCols;
+export const getFileError = (state: RootState) => state.fileMapper.fileError;
 
 export default fileMapperSlice.reducer;

packages/file-mapper/src/languages/locales/en/steps.json

@@ -17,5 +17,6 @@
   "selectOption": "Value in Darwin Core",
   "saveMapping": "Save as",
   "saveMappingExtra": "You can save your mapping for future datasets for easy reference. Please enter a name below.",
-  "isLoading": "Loading..."
+  "isLoading": "Loading...",
+  "tooManyRows": "Your data file contains too many rows to process using this tool. Make sure your file has less than {{max}} rows."
 }

packages/file-mapper/src/languages/locales/nl/steps.json

@@ -17,5 +17,6 @@
   "selectOption": "Value in Darwin Core",
   "saveMapping": "Save as",
   "saveMappingExtra": "You can save your mapping for future datasets for easy reference. Please enter a name below.",
-  "isLoading": "Loading..."
+  "isLoading": "Loading...",
+  "tooManyRows": "Je data file contains too many rows to process using this tool. Make sure your file has less than {{max}} rows."
 }

packages/file-mapper/src/types/index.ts

@@ -23,4 +23,7 @@ export interface SerializedFile {
   name: string;
   size: number;
   url: string;
-}
+}
+
+export type FileError = 
+  | "tooManyRows";

packages/utils/error/index.tsx

@@ -30,8 +30,8 @@ export const errorLogger: Middleware = () => (next) => async (action) => {
     if (
       // freshdesk enabled?
       import.meta.env.VITE_FRESHDESK_API_KEY &&
-      // some conditions when not to create a ticket
-      action.meta.arg.endpointName !== "validateAllKeys"
+      // only create a ticket when something's gone wrong with the actual submission
+      action.meta.arg.endpointName === "submitData"
     ) {
       ticket = await sendTicket(action);
     }