Generic dataset module and specific s3_datasets module - part 6 (Fron…

…tend) (#1292) ### Feature or Bugfix - Feature ### Detail As explained in the design for #1123 we are trying to implement a generic `datasets_base` module that can be used by any type of datasets in a generic way. In this PR we: - Create DatasetsBase module in frontend. Depends on S3_Datasets module - Move DatasetsList view and DatasetListItem component to DatasetsBase - Add CreateDataset modal that allows multiple types of datasets creation - Fix routes and redirects to point at /datasets/ or any other /X-dataset/ - Move dataset_base services. In backend/datasets_base/api we define the following queries that are good candidates to become part of the DatasetsBase module - listDatasets - only used in DatasetsBase/DatasetList view - it should be in DatasetsBase/services - listOwnedDatasets - only used in Shares/SharesBoxList view - it should be in Shares/services - listDatasetsCreatedInEnvironment - only used in Environments/EnvDataset tab - it should be in Environment/services If we want to keep everything clean we could rename all "datasets" as "s3_datasets" or equivalent in the S3_Datasets module. Because it is a cosmetic change that would pollute the PR a lot I have decided not to include it. ⚠️ UPDATE: Next steps The Data Shared With You table in Environments>Datasets tab needs a remake. It contains references to each type of item and it is very coupled with s3-dataset-shares. For the moment I just made it work for the changes of s3-datasets, but when completing the work in #1283 we should fix this. Maybe in favor of DataGrid ### Relates - #1123 - #955 ### Security Please answer the questions below briefly where applicable, or write `N/A`. Based on [OWASP 10](https://owasp.org/Top10/en/). - Does this PR introduce or modify any input fields or queries - this includes fetching data from storage outside the application (e.g. a database, an S3 bucket)? - Is the input sanitized? - What precautions are you taking before deserializing the data you consume? - Is injection prevented by parametrizing queries? - Have you ensured no `eval` or similar functions are used? - Does this PR introduce any functionality or component that requires authorization? - How have you ensured it respects the existing AuthN/AuthZ mechanisms? - Are you logging failed auth attempts? - Are you using or adding any cryptographic features? - Do you use a standard proven implementations? - Are the used keys controlled by the customer? Where are they stored? - Are you introducing any new policies/roles/users? - Have you used the least-privilege principle? How? By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
data-dot-all · May 22, 2024 · 372ac51 · 372ac51
1 parent a61e21c
commit 372ac51
Show file tree

Hide file tree

Showing 30 changed files with 229 additions and 79 deletions.
diff --git a/frontend/src/modules/Catalog/components/GlossarySearchResultItem.js b/frontend/src/modules/Catalog/components/GlossarySearchResultItem.js
@@ -88,7 +88,7 @@ export const GlossarySearchResultItem = ({ hit }) => {
                 underline="hover"
                 color="textPrimary"
                 component={RouterLink}
-                to={`/console/datasets/${hit._id}/`} /*eslint-disable-line*/
+                to={`/console/s3-datasets/${hit._id}/`} /*eslint-disable-line*/
                 variant="h6"
               >
                 {hit.label}
@@ -99,7 +99,7 @@ export const GlossarySearchResultItem = ({ hit }) => {
                 underline="hover"
                 color="textPrimary"
                 component={RouterLink}
-                to={`/console/datasets/table/${hit._id}/`} /*eslint-disable-line*/
+                to={`/console/s3-datasets/table/${hit._id}/`} /*eslint-disable-line*/
                 variant="h6"
               >
                 {hit.label}
@@ -110,7 +110,7 @@ export const GlossarySearchResultItem = ({ hit }) => {
                 underline="hover"
                 color="textPrimary"
                 component={RouterLink}
-                to={`/console/datasets/folder/${hit._id}/`} /*eslint-disable-line*/
+                to={`/console/s3-datasets/folder/${hit._id}/`} /*eslint-disable-line*/
                 variant="h6"
               >
                 {hit.label}

diff --git a/frontend/src/modules/Catalog/index.js b/frontend/src/modules/Catalog/index.js
@@ -7,7 +7,8 @@ export const CatalogsModule = {
   resolve_dependency: () => {
     return (
       getModuleActiveStatus(ModuleNames.S3_DATASETS) ||
-      getModuleActiveStatus(ModuleNames.DASHBOARDS)
+      getModuleActiveStatus(ModuleNames.DASHBOARDS) ||
+      getModuleActiveStatus(ModuleNames.DATASETS_BASE)
     );
   }
 };
diff --git a/frontend/src/modules/Catalog/views/Catalog.js b/frontend/src/modules/Catalog/views/Catalog.js
@@ -33,6 +33,7 @@ import {
 } from 'design';
 import { GlossarySearchWrapper, GlossarySearchResultItem } from '../components';
 import config from '../../../generated/config.json';
+import { DatasetCreateWindow } from 'modules/DatasetsBase/components';
 
 const useStyles = makeStyles((theme) => ({
   mainSearch: {
@@ -174,6 +175,14 @@ const Catalog = () => {
   const [openMenu, setOpenMenu] = useState(false);
   const dataFieldList = ['label', 'name', 'description', 'region', 'tags'];
 
+  const [isCreateModalOpen, setIsCreateModalOpen] = useState(false);
+  const handleCreateModalOpen = () => {
+    setIsCreateModalOpen(true);
+  };
+  const handleCreateModalClose = () => {
+    setIsCreateModalOpen(false);
+  };
+
   if (config.modules.s3_datasets.features.topics_dropdown === true)
     dataFieldList.push('topics');
   if (config.modules.s3_datasets.features.confidentiality_dropdown === true)
@@ -302,14 +311,16 @@ const Catalog = () => {
               <Box sx={{ m: -1 }}>
                 <Button
                   color="primary"
-                  component={RouterLink}
                   startIcon={<PlusIcon fontSize="small" />}
                   sx={{ m: 1 }}
-                  to="/console/datasets/new"
+                  onClick={handleCreateModalOpen}
                   variant="contained"
                 >
                   New Dataset
                 </Button>
+                {isCreateModalOpen && (
+                  <DatasetCreateWindow open onClose={handleCreateModalClose} />
+                )}
               </Box>
             </Grid>
           </Grid>

diff --git a/frontend/src/modules/DatasetsBase/components/DatasetCreateWindow.js b/frontend/src/modules/DatasetsBase/components/DatasetCreateWindow.js
@@ -0,0 +1,107 @@
+import { CloudDownloadOutlined } from '@mui/icons-material';
+import {
+  Box,
+  Button,
+  Card,
+  CardContent,
+  CardHeader,
+  Dialog,
+  Grid,
+  Typography
+} from '@mui/material';
+import PropTypes from 'prop-types';
+import React from 'react';
+import { Link as RouterLink } from 'react-router-dom';
+import { PlusIcon } from '../../../design';
+import { isModuleEnabled, ModuleNames } from 'utils';
+
+export const DatasetCreateWindow = (props) => {
+  const { onClose, open, ...other } = props;
+  const number_grid_items = isModuleEnabled(ModuleNames.S3_DATASETS) ? 2 : 0;
+  const width_grid_item = number_grid_items > 0 ? 12 / number_grid_items : 1;
+
+  return (
+    <Dialog maxWidth="md" fullWidth onClose={onClose} open={open} {...other}>
+      <Box sx={{ m: 4 }}>
+        <Grid
+          container
+          spacing={2}
+          alignItems="flex-start"
+          justifyContent="center"
+        >
+          {isModuleEnabled(ModuleNames.S3_DATASETS) && (
+            <Grid
+              item
+              justifyContent="center"
+              md={width_grid_item}
+              lg={width_grid_item}
+              xl={width_grid_item}
+            >
+              <Card>
+                <CardHeader title="Create S3-Glue Dataset" />
+                <CardContent>
+                  <Typography
+                    color="textSecondary"
+                    gutterBottom
+                    variant="subtitle2"
+                  >
+                    Data.all will create an S3 Bucket encrypted with KMS key and
+                    a Glue database.
+                  </Typography>
+                  <Button
+                    color="primary"
+                    component={RouterLink}
+                    startIcon={<PlusIcon fontSize="small" />}
+                    sx={{ m: 1 }}
+                    to="/console/s3-datasets/new"
+                    variant="contained"
+                  >
+                    Create
+                  </Button>
+                </CardContent>
+              </Card>
+            </Grid>
+          )}
+          {isModuleEnabled(ModuleNames.S3_DATASETS) && (
+            <Grid
+              item
+              justifyContent="center"
+              md={width_grid_item}
+              lg={width_grid_item}
+              xl={width_grid_item}
+            >
+              <Card>
+                <CardHeader title="Import S3-Glue Dataset" />
+                <CardContent>
+                  <Typography
+                    color="textSecondary"
+                    gutterBottom
+                    variant="subtitle2"
+                  >
+                    Data.all will use the S3 Bucket as it is encrypted and will
+                    create a Glue database if you do not provide one.
+                  </Typography>
+                  <Button
+                    color="primary"
+                    component={RouterLink}
+                    startIcon={<CloudDownloadOutlined fontSize="small" />}
+                    sx={{ m: 1 }}
+                    to="/console/s3-datasets/import"
+                    variant="outlined"
+                  >
+                    Import
+                  </Button>
+                </CardContent>
+              </Card>
+            </Grid>
+          )}
+        </Grid>
+      </Box>
+    </Dialog>
+  );
+};
+
+DatasetCreateWindow.propTypes = {
+  onClose: PropTypes.func,
+  open: PropTypes.bool.isRequired
+};
diff --git a/...S3_Datasets/components/DatasetListItem.js → ...atasetsBase/components/DatasetListItem.js b/...S3_Datasets/components/DatasetListItem.js → ...atasetsBase/components/DatasetListItem.js
@@ -40,7 +40,11 @@ export const DatasetListItem = (props) => {
                     color="textPrimary"
                     variant="h6"
                     onClick={() => {
-                      navigate(`/console/datasets/${dataset.datasetUri}`);
+                      navigate(
+                        dataset.datasetType === 'DatasetType.S3'
+                          ? `/console/s3-datasets/${dataset.datasetUri}`
+                          : '-'
+                      );
                     }}
                     sx={{
                       width: '99%',
@@ -203,7 +207,11 @@ export const DatasetListItem = (props) => {
             <Button
               color="primary"
               component={RouterLink}
-              to={`/console/datasets/${dataset.datasetUri}`}
+              to={
+                dataset.datasetType === 'DatasetType.S3'
+                  ? `/console/s3-datasets/${dataset.datasetUri}`
+                  : '-'
+              }
             >
               Learn More
             </Button>

diff --git a/frontend/src/modules/DatasetsBase/components/index.js b/frontend/src/modules/DatasetsBase/components/index.js
@@ -0,0 +1,2 @@
+export * from './DatasetCreateWindow';
+export * from './DatasetListItem';
diff --git a/frontend/src/modules/DatasetsBase/index.js b/frontend/src/modules/DatasetsBase/index.js
@@ -0,0 +1,10 @@
+import { ModuleNames, getModuleActiveStatus } from 'utils';
+
+export const DatasetsBaseModule = {
+  moduleDefinition: true,
+  name: 'datasets_base',
+  isEnvironmentModule: false,
+  resolve_dependency: () => {
+    return getModuleActiveStatus(ModuleNames.S3_DATASETS); // Add other dataset types when needed
+  }
+};
diff --git a/frontend/src/modules/DatasetsBase/services/index.js b/frontend/src/modules/DatasetsBase/services/index.js
@@ -0,0 +1 @@
+export * from './listDatasets';
diff --git a/...ules/S3_Datasets/services/listDatasets.js → ...les/DatasetsBase/services/listDatasets.js b/...ules/S3_Datasets/services/listDatasets.js → ...les/DatasetsBase/services/listDatasets.js
diff --git a/.../modules/S3_Datasets/views/DatasetList.js → ...modules/DatasetsBase/views/DatasetList.js b/.../modules/S3_Datasets/views/DatasetList.js → ...modules/DatasetsBase/views/DatasetList.js
@@ -1,4 +1,3 @@
-import { CloudDownloadOutlined } from '@mui/icons-material';
 import {
   Box,
   Breadcrumbs,
@@ -23,9 +22,16 @@ import {
 import { SET_ERROR, useDispatch } from 'globalErrors';
 import { useClient } from 'services';
 import { listDatasets } from '../services';
-import { DatasetListItem } from '../components';
+import { DatasetCreateWindow, DatasetListItem } from '../components';
 
 function DatasetsPageHeader() {
+  const [isCreateModalOpen, setIsCreateModalOpen] = useState(false);
+  const handleCreateModalOpen = () => {
+    setIsCreateModalOpen(true);
+  };
+  const handleCreateModalClose = () => {
+    setIsCreateModalOpen(false);
+  };
   return (
     <Grid
       alignItems="center"
@@ -60,24 +66,16 @@ function DatasetsPageHeader() {
         <Box sx={{ m: -1 }}>
           <Button
             color="primary"
-            component={RouterLink}
-            startIcon={<CloudDownloadOutlined fontSize="small" />}
-            sx={{ m: 1 }}
-            to="/console/datasets/import"
-            variant="outlined"
-          >
-            Import
-          </Button>
-          <Button
-            color="primary"
-            component={RouterLink}
             startIcon={<PlusIcon fontSize="small" />}
             sx={{ m: 1 }}
-            to="/console/datasets/new"
+            onClick={handleCreateModalOpen}
             variant="contained"
           >
-            Create
+            New Dataset
           </Button>
+          {isCreateModalOpen && (
+            <DatasetCreateWindow open onClose={handleCreateModalClose} />
+          )}
         </Box>
       </Grid>
     </Grid>

diff --git a/frontend/src/modules/Environments/components/EnvironmentOwnedDatasets.js b/frontend/src/modules/Environments/components/EnvironmentOwnedDatasets.js
@@ -149,7 +149,11 @@ export const EnvironmentOwnedDatasets = ({ environment }) => {
                       <TableCell>
                         <IconButton
                           onClick={() => {
-                            navigate(`/console/datasets/${dataset.datasetUri}`);
+                            navigate(
+                              dataset.datasetType === 'DatasetType.S3'
+                                ? `/console/s3-datasets/${dataset.datasetUri}`
+                                : '-'
+                            );
                           }}
                         >
                           <ArrowRightIcon fontSize="small" />

diff --git a/frontend/src/modules/Environments/components/EnvironmentSharedDatasets.js b/frontend/src/modules/Environments/components/EnvironmentSharedDatasets.js
@@ -134,9 +134,7 @@ export const EnvironmentSharedDatasets = ({ environment }) => {
                 {items.nodes.length > 0 ? (
                   items.nodes.map((item) => (
                     <TableRow hover key={item.itemUri}>
-                      <TableCell>
-                        {item.GlueTableName ? 'Table' : 'Folder'}
-                      </TableCell>
+                      <TableCell>{item.itemType}</TableCell>
                       <TableCell>
                         {item.GlueTableName || item.S3AccessPointName}
                       </TableCell>
@@ -146,7 +144,7 @@ export const EnvironmentSharedDatasets = ({ environment }) => {
                       <TableCell>
                         <IconButton
                           onClick={() => {
-                            navigate(`/console/datasets/${item.datasetUri}`);
+                            navigate(`/console/s3-datasets/${item.datasetUri}`);
                           }}
                         >
                           <ArrowRightIcon fontSize="small" />

diff --git a/frontend/src/modules/Environments/services/listDatasetsCreatedInEnvironment.js b/frontend/src/modules/Environments/services/listDatasetsCreatedInEnvironment.js
@@ -24,6 +24,7 @@ export const listDatasetsCreatedInEnvironment = ({
         hasPrevious
         nodes {
           datasetUri
+          datasetType
           label
           AwsAccountId
           region

diff --git a/frontend/src/modules/Folders/views/FolderCreateForm.js b/frontend/src/modules/Folders/views/FolderCreateForm.js
@@ -59,7 +59,7 @@ function FolderCreateHeader(props) {
             underline="hover"
             color="textPrimary"
             component={RouterLink}
-            to={`/console/datasets/${dataset.datasetUri}`}
+            to={`/console/s3-datasets/${dataset.datasetUri}`}
             variant="subtitle2"
           >
             {dataset.name}
@@ -73,7 +73,7 @@ function FolderCreateHeader(props) {
             component={RouterLink}
             startIcon={<ArrowLeftIcon fontSize="small" />}
             sx={{ mt: 1 }}
-            to={`/console/datasets/${dataset.datasetUri}`}
+            to={`/console/s3-datasets/${dataset.datasetUri}`}
             variant="outlined"
           >
             Cancel
@@ -139,7 +139,7 @@ const FolderCreateForm = () => {
           variant: 'success'
         });
         navigate(
-          `/console/datasets/folder/${response.data.createDatasetStorageLocation.locationUri}`
+          `/console/s3-datasets/folder/${response.data.createDatasetStorageLocation.locationUri}`
         );
       } else {
         dispatch({ type: SET_ERROR, error: response.errors[0].message });

diff --git a/frontend/src/modules/Folders/views/FolderEditForm.js b/frontend/src/modules/Folders/views/FolderEditForm.js
@@ -66,7 +66,7 @@ function FolderEditHeader(props) {
             underline="hover"
             color="textPrimary"
             component={RouterLink}
-            to={`/console/datasets/${folder.dataset.datasetUri}`}
+            to={`/console/s3-datasets/${folder.dataset.datasetUri}`}
             variant="subtitle2"
           >
             {folder.dataset.label}
@@ -76,7 +76,7 @@ function FolderEditHeader(props) {
               underline="hover"
               color="textPrimary"
               component={RouterLink}
-              to={`/console/datasets/folder/${folder.locationUri}`}
+              to={`/console/s3-datasets/folder/${folder.locationUri}`}
               variant="subtitle2"
             >
               {folder.label}
@@ -94,7 +94,7 @@ function FolderEditHeader(props) {
             component={RouterLink}
             startIcon={<ArrowLeftIcon fontSize="small" />}
             sx={{ mt: 1 }}
-            to={`/console/datasets/folder/${folder.locationUri}`}
+            to={`/console/s3-datasets/folder/${folder.locationUri}`}
             variant="outlined"
           >
             Cancel
@@ -190,7 +190,7 @@ const FolderEditForm = () => {
         },
         variant: 'success'
       });
-      navigate(`/console/datasets/folder/${folder.locationUri}`);
+      navigate(`/console/s3-datasets/folder/${folder.locationUri}`);
     } catch (err) {
       setStatus({ success: false });
       setErrors({ submit: err.message });