merge updates of optinist-for-server

.gitignore

@@ -2,8 +2,8 @@
 __pycache__
 .ipynb_checkpoints
 .vscode/*
-!.vscode/settings.json
 !.vscode/extensions.json
+!.vscode/settings.example.json
 dist
 /build
 

.pre-commit-config.yaml

@@ -22,6 +22,6 @@ repos:
     hooks:
     -   id: black
 -   repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
     -   id: flake8

.vscode/settings.json → .vscode/settings.example.json

@@ -10,9 +10,8 @@
       "source.organizeImports": true
     }
   },
-  "python.linting.enabled": true,
-  "python.linting.lintOnSave": true,
-  "python.linting.pylintEnabled": false,
-  "python.linting.flake8Enabled": true,
+  "flake8.args": ["--config=.flake8"],
+  // NOTE: Uncomment following line and fix "optinist_dev" to your conda env name
+  // "flake8.path": ["conda", "run", "-n", "optinist_dev", "python", "-m", "flake8"],
   "esbonio.sphinx.confDir": ""
 }

alembic.ini

@@ -0,0 +1,103 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = studio/alembic
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to studio/alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:studio/alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

docker-compose.dev.yml

@@ -23,3 +23,5 @@ services:
     ports:
       - "3000:3000"
     command: ash -c 'yarn install && yarn start'
+    environment:
+      TZ: Asia/Tokyo

docker-compose.yml

@@ -6,9 +6,15 @@ services:
       context: .
       dockerfile: studio/config/docker/Dockerfile
     working_dir: /app
+    volumes:
+      - ../optinist-docker-volumes/.snakemake/:/app/.snakemake
+      - ../optinist-docker-volumes/logs/:/app/logs
+      - ../optinist-docker-volumes/studio_data/:/app/studio_data
     ports:
       - "8000:8000"
     command: python main.py --host 0.0.0.0 --port 8000
     environment:
       PYTHONPATH: /app/
       TZ: Asia/Tokyo
+      OPTINIST_DIR: /app/studio_data
+    restart: always

frontend/package.json

@@ -8,22 +8,25 @@
     "@mui/icons-material": "^5.4.1",
     "@mui/lab": "^5.0.0-alpha.68",
     "@mui/material": "^5.4.1",
-    "@mui/x-data-grid": "^5.5.1",
+    "@mui/x-data-grid": "^6.11.1",
     "@reduxjs/toolkit": "^1.6.1",
     "@testing-library/jest-dom": "^5.11.4",
     "@testing-library/react": "^11.1.0",
     "@testing-library/user-event": "^12.1.10",
     "@types/colormap": "^2.3.1",
     "@types/jest": "^26.0.15",
     "@types/node": "^16.0.0",
+    "@types/qs": "^6.9.7",
     "@types/react": "^17.0.19",
     "@types/react-color": "^3.0.6",
     "@types/react-dom": "^17.0.0",
     "axios": "^0.21.1",
     "colormap": "^2.3.2",
     "flexlayout-react": "^0.5.12",
+    "moment": "^2.29.4",
     "notistack": "^2.0.3",
     "plotly.js": "^2.6.0",
+    "qs": "^6.11.2",
     "react": "^17.0.2",
     "react-color": "^2.19.3",
     "react-dnd": "^16.0.1",

frontend/src/@types/index.ts

@@ -0,0 +1,10 @@
+export enum ROLE {
+  ADMIN = 1,
+  OPERATOR = 20,
+}
+
+export const enum SHARE {
+  NOSHARE = 0,
+  ORGANIZATION = 2,
+  USERS = 1,
+}

frontend/src/App.tsx

@@ -6,6 +6,7 @@ import { BrowserRouter, Navigate, Route, Routes } from 'react-router-dom'
 import Layout from 'components/Layout'
 import Dashboard from 'pages/Dashboard'
 import Account from 'pages/Account'
+import AccountManager from 'pages/AccountManager'
 import AccountDelete from 'pages/AccountDelete'
 import Login from 'pages/Login'
 import ResetPassword from 'pages/ResetPassword'
@@ -30,15 +31,18 @@ const App: React.FC = () => {
             </Routes>
           ) : (
             <Routes>
-              <Route path="/" element={<Dashboard />} />
-              <Route path="/account" element={<Account />} />
+              <Route path="/" element={<Navigate replace to="/console" />} />
               <Route path="/account-deleted" element={<AccountDelete />} />
               <Route path="/login" element={<Login />} />
               <Route path="/reset-password" element={<ResetPassword />} />
-              <Route path="/workspaces">
+              <Route path="/console" element={<Dashboard />} />
+              <Route path="/console/account" element={<Account />} />
+              <Route path="/console/account-manager" element={<AccountManager />} />
+              <Route path="/console/workspaces">
                 <Route path="" element={<Workspaces />} />
                 <Route path=":workspaceId" element={<Workspace />} />
               </Route>
+              <Route path="/console/*" element={<Navigate replace to="/console" />} />
               <Route path="*" element={<Navigate replace to="/" />} />
             </Routes>
           )}

frontend/src/api/experiments/Experiments.ts

@@ -1,8 +1,9 @@
 import axios from 'utils/axios'
 
 import { BASE_URL } from 'const/API'
-import { EdgeDict, NodeDict, OutputPathsDTO } from 'api/run/Run'
+import { OutputPathsDTO } from 'api/run/Run'
 import { EXPERIMENTS_STATUS } from 'store/slice/Experiments/ExperimentsType'
+import { WorkflowConfigDTO } from 'api/workflow/Workflow'
 
 export type ExperimentsDTO = {
   [uid: string]: ExperimentDTO
@@ -33,23 +34,23 @@ export type ExperimentDTO = {
   success?: EXPERIMENTS_STATUS
   started_at: string
   finished_at?: string
-  workspace_id: string
+  workspace_id: number
   unique_id: string
   hasNWB: boolean
-  edgeDict: EdgeDict
-  nodeDict: NodeDict
   nwb: NWBType
 }
 
+export type FetchExperimentDTO = ExperimentDTO & WorkflowConfigDTO
+
 export async function getExperimentsApi(
-  workspaceId: string,
+  workspaceId: number,
 ): Promise<ExperimentsDTO> {
   const response = await axios.get(`${BASE_URL}/experiments/${workspaceId}`)
   return response.data
 }
 
 export async function deleteExperimentByUidApi(
-  workspaceId: string,
+  workspaceId: number,
   uid: string,
 ): Promise<boolean> {
   const response = await axios.delete(
@@ -59,7 +60,7 @@ export async function deleteExperimentByUidApi(
 }
 
 export async function deleteExperimentByListApi(
-  workspaceId: string,
+  workspaceId: number,
   uidList: Array<string>,
 ): Promise<boolean> {
   const response = await axios.post(
@@ -72,7 +73,7 @@ export async function deleteExperimentByListApi(
 }
 
 export async function downloadExperimentNwbApi(
-  workspaceId: string,
+  workspaceId: number,
   uid: string,
   nodeId?: string,
 ) {
@@ -87,7 +88,7 @@ export async function downloadExperimentNwbApi(
 }
 
 export async function downloadExperimentConfigApi(
-  workspaceId: string,
+  workspaceId: number,
   uid: string,
 ) {
   const response = await axios.get(
@@ -99,8 +100,17 @@ export async function downloadExperimentConfigApi(
   return response.data
 }
 
+export async function fetchExperimentApi(
+  workspace_id: number,
+): Promise<FetchExperimentDTO> {
+  const response = await axios.get(
+    `${BASE_URL}/experiments/fetch/${workspace_id}`,
+  )
+  return response.data
+}
+
 export async function renameExperiment(
-  workspaceId: string,
+  workspaceId: number,
   uid: string,
   new_name: string,
 ) {

frontend/src/api/files/Files.ts

@@ -32,9 +32,10 @@ export interface FileNodeDTO extends NodeBaseDTO {
 }
 
 export async function getFilesTreeApi(
+  workspaceId: number,
   fileType: FILE_TREE_TYPE,
 ): Promise<TreeNodeTypeDTO[]> {
-  const response = await axios.get(`${BASE_URL}/files`, {
+  const response = await axios.get(`${BASE_URL}/files/${workspaceId}`, {
     params: {
       file_type: fileType,
     },
@@ -43,14 +44,15 @@ export async function getFilesTreeApi(
 }
 
 export async function uploadFileApi(
+  workspaceId: number,
   fileName: string,
   config: {
     onUploadProgress: (progressEvent: any) => void
   },
   formData: FormData,
 ): Promise<{ file_path: string }> {
   const response = await axios.post(
-    `${BASE_URL}/files/upload/${fileName}`,
+    `${BASE_URL}/files/${workspaceId}/upload/${fileName}`,
     formData,
     config,
   )

frontend/src/api/hdf5/HDF5.ts

@@ -19,7 +19,12 @@ export interface HDF5FileDTO {
   nbytes: string
 }
 
-export async function getHDF5TreeApi(path: string): Promise<HDF5TreeDTO[]> {
-  const response = await axios.get(`${BASE_URL}/hdf5/${path}`)
+export async function getHDF5TreeApi(
+  path: string,
+  workspaceId: number,
+): Promise<HDF5TreeDTO[]> {
+  const response = await axios.get(
+    `${BASE_URL}/hdf5/${path}?workspace_id=${workspaceId}`,
+  )
   return response.data
 }