diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java
index e4399771521..0173539e031 100644
--- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java
+++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/CatalogsPageTest.java
@@ -29,6 +29,7 @@ public class CatalogsPageTest extends AbstractWebIT {
private static final String metalakeName = "metalake_name";
String catalogName = "catalog_name";
+ String catalogType = "relational";
String modifiedCatalogName = catalogName + "_edited";
String schemaName = "default";
String tableName = "employee";
@@ -124,21 +125,22 @@ public void testEditCatalog() throws InterruptedException {
@Test
@Order(6)
public void testClickCatalogLink() {
- catalogsPage.clickCatalogLink(metalakeName, modifiedCatalogName);
+ catalogsPage.clickCatalogLink(metalakeName, modifiedCatalogName, catalogType);
Assertions.assertTrue(catalogsPage.verifyShowTableTitle("Schemas"));
}
@Test
@Order(7)
public void testClickSchemaLink() {
- catalogsPage.clickSchemaLink(metalakeName, modifiedCatalogName, schemaName);
+ catalogsPage.clickSchemaLink(metalakeName, modifiedCatalogName, catalogType, schemaName);
Assertions.assertTrue(catalogsPage.verifyShowTableTitle("Tables"));
}
@Test
@Order(8)
public void testClickTableLink() {
- catalogsPage.clickTableLink(metalakeName, modifiedCatalogName, schemaName, tableName);
+ catalogsPage.clickTableLink(
+ metalakeName, modifiedCatalogName, catalogType, schemaName, tableName);
Assertions.assertTrue(catalogsPage.verifyShowTableTitle("Columns"));
Assertions.assertTrue(catalogsPage.verifyTableColumns());
}
diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java
index 5c22a9be224..e4c68cd5fcb 100644
--- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java
+++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/web/ui/pages/CatalogsPage.java
@@ -174,9 +174,16 @@ public void clickDeleteCatalogBtn(String name) {
}
}
- public void clickCatalogLink(String metalakeName, String catalogName) {
+ public void clickCatalogLink(String metalakeName, String catalogName, String catalogType) {
try {
- String xpath = "//a[@href='?metalake=" + metalakeName + "&catalog=" + catalogName + "']";
+ String xpath =
+ "//a[@href='?metalake="
+ + metalakeName
+ + "&catalog="
+ + catalogName
+ + "&type="
+ + catalogType
+ + "']";
WebElement link = tableGrid.findElement(By.xpath(xpath));
WebDriverWait wait = new WebDriverWait(driver, MAX_TIMEOUT);
wait.until(ExpectedConditions.elementToBeClickable(By.xpath(xpath)));
@@ -186,13 +193,16 @@ public void clickCatalogLink(String metalakeName, String catalogName) {
}
}
- public void clickSchemaLink(String metalakeName, String catalogName, String schemaName) {
+ public void clickSchemaLink(
+ String metalakeName, String catalogName, String catalogType, String schemaName) {
try {
String xpath =
"//a[@href='?metalake="
+ metalakeName
+ "&catalog="
+ catalogName
+ + "&type="
+ + catalogType
+ "&schema="
+ schemaName
+ "']";
@@ -206,13 +216,19 @@ public void clickSchemaLink(String metalakeName, String catalogName, String sche
}
public void clickTableLink(
- String metalakeName, String catalogName, String schemaName, String tableName) {
+ String metalakeName,
+ String catalogName,
+ String catalogType,
+ String schemaName,
+ String tableName) {
try {
String xpath =
"//a[@href='?metalake="
+ metalakeName
+ "&catalog="
+ catalogName
+ + "&type="
+ + catalogType
+ "&schema="
+ schemaName
+ "&table="
diff --git a/web/src/app/metalakes/metalake/MetalakeTree.js b/web/src/app/metalakes/metalake/MetalakeTree.js
index f2642e9a8d1..4b807eaf15c 100644
--- a/web/src/app/metalakes/metalake/MetalakeTree.js
+++ b/web/src/app/metalakes/metalake/MetalakeTree.js
@@ -22,7 +22,8 @@ import {
removeExpandedNode,
setSelectedNodes,
setLoadedNodes,
- getTableDetails
+ getTableDetails,
+ getFilesetDetails
} from '@/lib/store/metalakes'
import { extractPlaceholder } from '@/lib/utils'
@@ -47,6 +48,12 @@ const MetalakeTree = props => {
const [metalake, catalog, schema, table] = pathArr
dispatch(getTableDetails({ init: true, metalake, catalog, schema, table }))
}
+ } else if (nodeProps.data.node === 'fileset') {
+ if (store.selectedNodes.includes(nodeProps.data.key)) {
+ const pathArr = extractPlaceholder(nodeProps.data.key)
+ const [metalake, catalog, schema, fileset] = pathArr
+ dispatch(getFilesetDetails({ init: true, metalake, catalog, schema, fileset }))
+ }
} else {
dispatch(setIntoTreeNodeWithFetch({ key: nodeProps.data.key }))
}
@@ -143,6 +150,19 @@ const MetalakeTree = props => {
)
+ case 'fileset':
+ return (
+ handleClickIcon(e, nodeProps)}
+ onMouseEnter={e => onMouseEnter(e, nodeProps)}
+ onMouseLeave={e => onMouseLeave(e, nodeProps)}
+ >
+
+
+ )
default:
return <>
diff --git a/web/src/app/metalakes/metalake/MetalakeView.js b/web/src/app/metalakes/metalake/MetalakeView.js
index 116c0428367..e700ff145fc 100644
--- a/web/src/app/metalakes/metalake/MetalakeView.js
+++ b/web/src/app/metalakes/metalake/MetalakeView.js
@@ -18,10 +18,12 @@ import {
fetchCatalogs,
fetchSchemas,
fetchTables,
+ fetchFilesets,
getMetalakeDetails,
getCatalogDetails,
getSchemaDetails,
getTableDetails,
+ getFilesetDetails,
setSelectedNodes
} from '@/lib/store/metalakes'
@@ -35,39 +37,51 @@ const MetalakeView = () => {
const routeParams = {
metalake: searchParams.get('metalake'),
catalog: searchParams.get('catalog'),
+ type: searchParams.get('type'),
schema: searchParams.get('schema'),
- table: searchParams.get('table')
+ table: searchParams.get('table'),
+ fileset: searchParams.get('fileset')
}
if ([...searchParams.keys()].length) {
- const { metalake, catalog, schema, table } = routeParams
+ const { metalake, catalog, type, schema, table, fileset } = routeParams
if (paramsSize === 1 && metalake) {
dispatch(fetchCatalogs({ init: true, page: 'metalakes', metalake }))
dispatch(getMetalakeDetails({ metalake }))
}
- if (paramsSize === 2 && catalog) {
- dispatch(fetchSchemas({ init: true, page: 'catalogs', metalake, catalog }))
- dispatch(getCatalogDetails({ metalake, catalog }))
+ if (paramsSize === 3 && catalog) {
+ dispatch(fetchSchemas({ init: true, page: 'catalogs', metalake, catalog, type }))
+ dispatch(getCatalogDetails({ metalake, catalog, type }))
}
- if (paramsSize === 3 && catalog && schema) {
- dispatch(fetchTables({ init: true, page: 'schemas', metalake, catalog, schema }))
+ if (paramsSize === 4 && catalog && type && schema) {
+ if (type === 'fileset') {
+ dispatch(fetchFilesets({ init: true, page: 'schemas', metalake, catalog, schema }))
+ } else {
+ dispatch(fetchTables({ init: true, page: 'schemas', metalake, catalog, schema }))
+ }
dispatch(getSchemaDetails({ metalake, catalog, schema }))
}
- if (paramsSize === 4 && catalog && schema && table) {
+ if (paramsSize === 5 && catalog && schema && table) {
dispatch(getTableDetails({ init: true, metalake, catalog, schema, table }))
}
+
+ if (paramsSize === 5 && catalog && schema && fileset) {
+ dispatch(getFilesetDetails({ init: true, metalake, catalog, schema, fileset }))
+ }
}
dispatch(
setSelectedNodes(
routeParams.catalog
? [
- `{{${routeParams.metalake}}}{{${routeParams.catalog}}}${
+ `{{${routeParams.metalake}}}{{${routeParams.catalog}}}{{${routeParams.type}}}${
routeParams.schema ? `{{${routeParams.schema}}}` : ''
- }${routeParams.table ? `{{${routeParams.table}}}` : ''}`
+ }${routeParams.table ? `{{${routeParams.table}}}` : ''}${
+ routeParams.fileset ? `{{${routeParams.fileset}}}` : ''
+ }`
]
: []
)
diff --git a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js
index 6580ce89aa3..63963eb1e0d 100644
--- a/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js
+++ b/web/src/app/metalakes/metalake/rightContent/CreateCatalogDialog.js
@@ -43,7 +43,7 @@ import { useSearchParams } from 'next/navigation'
const defaultValues = {
name: '',
type: 'relational',
- provider: 'hive',
+ provider: '',
comment: '',
propItems: providers[0].defaultProps
}
@@ -58,7 +58,7 @@ const schema = yup.object().shape({
nameRegex,
'This field must start with a letter or underscore, and can only contain letters, numbers, and underscores'
),
- type: yup.mixed().oneOf(['relational']).required(),
+ type: yup.mixed().oneOf(['relational', 'fileset']).required(),
provider: yup.mixed().oneOf(providerTypeValues).required(),
propItems: yup.array().of(
yup.object().shape({
@@ -87,6 +87,8 @@ const CreateCatalogDialog = props => {
const [cacheData, setCacheData] = useState()
+ const [providerTypes, setProviderTypes] = useState(providers)
+
const {
control,
reset,
@@ -103,6 +105,7 @@ const CreateCatalogDialog = props => {
})
const providerSelect = watch('provider')
+ const typeSelect = watch('type')
const handleFormChange = ({ index, event }) => {
let data = [...innerProps]
@@ -280,6 +283,18 @@ const CreateCatalogDialog = props => {
console.error('fields error', errors)
}
+ useEffect(() => {
+ if (typeSelect === 'fileset') {
+ setProviderTypes(providers.filter(p => p.value === 'hadoop'))
+ setValue('provider', 'hadoop')
+ } else {
+ setProviderTypes(providers.filter(p => p.value !== 'hadoop'))
+ setValue('provider', 'hive')
+ }
+
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [typeSelect, open])
+
useEffect(() => {
let defaultProps = []
@@ -410,6 +425,7 @@ const CreateCatalogDialog = props => {
disabled={type === 'update'}
>
+
)}
/>
@@ -436,10 +452,13 @@ const CreateCatalogDialog = props => {
labelId='select-catalog-provider'
disabled={type === 'update'}
>
-
-
-
-
+ {providerTypes.map(item => {
+ return (
+
+ )
+ })}
)}
/>
diff --git a/web/src/app/metalakes/metalake/rightContent/MetalakePath.js b/web/src/app/metalakes/metalake/rightContent/MetalakePath.js
index 58d6e3a1f17..4f7a400535a 100644
--- a/web/src/app/metalakes/metalake/rightContent/MetalakePath.js
+++ b/web/src/app/metalakes/metalake/rightContent/MetalakePath.js
@@ -24,16 +24,19 @@ const MetalakePath = props => {
const routeParams = {
metalake: searchParams.get('metalake'),
catalog: searchParams.get('catalog'),
+ type: searchParams.get('type'),
schema: searchParams.get('schema'),
- table: searchParams.get('table')
+ table: searchParams.get('table'),
+ fileset: searchParams.get('fileset')
}
- const { metalake, catalog, schema, table } = routeParams
+ const { metalake, catalog, type, schema, table, fileset } = routeParams
const metalakeUrl = `?metalake=${metalake}`
- const catalogUrl = `?metalake=${metalake}&catalog=${catalog}`
- const schemaUrl = `?metalake=${metalake}&catalog=${catalog}&schema=${schema}`
- const tableUrl = `?metalake=${metalake}&catalog=${catalog}&schema=${schema}&table=${table}`
+ const catalogUrl = `?metalake=${metalake}&catalog=${catalog}&type=${type}`
+ const schemaUrl = `?metalake=${metalake}&catalog=${catalog}&type=${type}&schema=${schema}`
+ const tableUrl = `?metalake=${metalake}&catalog=${catalog}&type=${type}&schema=${schema}&table=${table}`
+ const filesetUrl = `?metalake=${metalake}&catalog=${catalog}&type=${type}&schema=${schema}&fileset=${fileset}`
const handleClick = (event, path) => {
path === `?${searchParams.toString()}` && event.preventDefault()
@@ -91,6 +94,19 @@ const MetalakePath = props => {
)}
+ {fileset && (
+
+ handleClick(event, filesetUrl)}
+ underline='hover'
+ >
+
+ {fileset}
+
+
+ )}
)
}
diff --git a/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js b/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js
index 00ae259f367..7ae1a86543b 100644
--- a/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js
+++ b/web/src/app/metalakes/metalake/rightContent/tabsContent/TabsContent.js
@@ -5,7 +5,7 @@
'use client'
-import { useState } from 'react'
+import { useState, useEffect } from 'react'
import Box from '@mui/material/Box'
import Tab from '@mui/material/Tab'
@@ -49,44 +49,60 @@ const CustomTabPanel = props => {
}
const TabsContent = () => {
+ let tableTitle = ''
+ const searchParams = useSearchParams()
+ const paramsSize = [...searchParams.keys()].length
+ const type = searchParams.get('type')
const [tab, setTab] = useState('table')
+ const isNotNeedTableTab = type && type === 'fileset' && paramsSize === 5
const handleChangeTab = (event, newValue) => {
setTab(newValue)
}
- let tableTitle = ''
- const searchParams = useSearchParams()
- const paramsSize = [...searchParams.keys()].length
-
switch (paramsSize) {
case 1:
tableTitle = 'Catalogs'
break
- case 2:
- tableTitle = 'Schemas'
- break
case 3:
- tableTitle = 'Tables'
+ tableTitle = 'Schemas'
break
case 4:
+ tableTitle = type === 'fileset' ? 'Filesets' : 'Tables'
+ break
+ case 5:
tableTitle = 'Columns'
break
default:
break
}
+ useEffect(() => {
+ if (isNotNeedTableTab) {
+ setTab('details')
+ } else {
+ setTab('table')
+ }
+
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [searchParams])
+
return (
-
+ {!isNotNeedTableTab ? (
+
+ ) : null}
-
-
-
+ {!isNotNeedTableTab ? (
+
+
+
+ ) : null}
+
diff --git a/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js b/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js
index 6ec15defc54..4c0ee996f5b 100644
--- a/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js
+++ b/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js
@@ -170,7 +170,7 @@ const TableView = () => {
title='Delete'
size='small'
sx={{ color: theme => theme.palette.error.light }}
- onClick={() => handleDelete({ name: row.name, type: 'catalog' })}
+ onClick={() => handleDelete({ name: row.name, type: 'catalog', catalogType: row.type })}
data-refer={`delete-catalog-${row.name}`}
>
@@ -345,9 +345,9 @@ const TableView = () => {
}
}
- const handleDelete = ({ name, type }) => {
+ const handleDelete = ({ name, type, catalogType }) => {
setOpenConfirmDelete(true)
- setConfirmCacheData({ name, type })
+ setConfirmCacheData({ name, type, catalogType })
}
const handleCloseConfirm = () => {
@@ -358,7 +358,7 @@ const TableView = () => {
const handleConfirmDeleteSubmit = () => {
if (confirmCacheData) {
if (confirmCacheData.type === 'catalog') {
- dispatch(deleteCatalog({ metalake, catalog: confirmCacheData.name }))
+ dispatch(deleteCatalog({ metalake, catalog: confirmCacheData.name, type: confirmCacheData.catalogType }))
}
setOpenConfirmDelete(false)
@@ -368,7 +368,7 @@ const TableView = () => {
const checkColumns = () => {
if (paramsSize == 1 && searchParams.has('metalake')) {
return catalogsColumns
- } else if (paramsSize == 4 && searchParams.has('table')) {
+ } else if (paramsSize == 5 && searchParams.has('table')) {
return tableColumns
} else {
return columns
diff --git a/web/src/lib/api/catalogs/index.js b/web/src/lib/api/catalogs/index.js
index 45c1c6eb3fe..b8acee1fdd0 100644
--- a/web/src/lib/api/catalogs/index.js
+++ b/web/src/lib/api/catalogs/index.js
@@ -6,7 +6,7 @@
import { defHttp } from '@/lib/utils/axios'
const Apis = {
- GET: ({ metalake }) => `/api/metalakes/${metalake}/catalogs`,
+ GET: ({ metalake }) => `/api/metalakes/${metalake}/catalogs?details=true`,
GET_DETAIL: ({ metalake, catalog }) => `/api/metalakes/${metalake}/catalogs/${catalog}`,
CREATE: ({ metalake }) => `/api/metalakes/${metalake}/catalogs`,
UPDATE: ({ metalake, catalog }) => `/api/metalakes/${metalake}/catalogs/${catalog}`,
diff --git a/web/src/lib/api/filesets/index.js b/web/src/lib/api/filesets/index.js
new file mode 100644
index 00000000000..a2e03b970d8
--- /dev/null
+++ b/web/src/lib/api/filesets/index.js
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2024 Datastrato Pvt Ltd.
+ * This software is licensed under the Apache License version 2.
+ */
+
+import { defHttp } from '@/lib/utils/axios'
+
+const Apis = {
+ GET: ({ metalake, catalog, schema }) => `/api/metalakes/${metalake}/catalogs/${catalog}/schemas/${schema}/filesets`,
+ GET_DETAIL: ({ metalake, catalog, schema, fileset }) =>
+ `/api/metalakes/${metalake}/catalogs/${catalog}/schemas/${schema}/filesets/${fileset}`
+}
+
+export const getFilesetsApi = params => {
+ return defHttp.get({
+ url: `${Apis.GET(params)}`
+ })
+}
+
+export const getFilesetDetailsApi = ({ metalake, catalog, schema, fileset }) => {
+ return defHttp.get({
+ url: `${Apis.GET_DETAIL({ metalake, catalog, schema, fileset })}`
+ })
+}
diff --git a/web/src/lib/store/metalakes/index.js b/web/src/lib/store/metalakes/index.js
index 6e263996576..5c16154feb4 100644
--- a/web/src/lib/store/metalakes/index.js
+++ b/web/src/lib/store/metalakes/index.js
@@ -6,6 +6,7 @@
import { createSlice, createAsyncThunk } from '@reduxjs/toolkit'
import { to, extractPlaceholder, updateTreeData, findInTree } from '@/lib/utils'
+import toast from 'react-hot-toast'
import _ from 'lodash-es'
@@ -26,6 +27,7 @@ import {
} from '@/lib/api/catalogs'
import { getSchemasApi, getSchemaDetailsApi } from '@/lib/api/schemas'
import { getTablesApi, getTableDetailsApi } from '@/lib/api/tables'
+import { getFilesetsApi, getFilesetDetailsApi } from '@/lib/api/filesets'
export const fetchMetalakes = createAsyncThunk('appMetalakes/fetchMetalakes', async (params, { getState }) => {
const [err, res] = await to(getMetalakesApi())
@@ -85,7 +87,7 @@ export const setIntoTreeNodeWithFetch = createAsyncThunk(
}
const pathArr = extractPlaceholder(key)
- const [metalake, catalog, schema, table] = pathArr
+ const [metalake, catalog, type, schema] = pathArr
if (pathArr.length === 1) {
const [err, res] = await to(getCatalogsApi({ metalake }))
@@ -94,22 +96,23 @@ export const setIntoTreeNodeWithFetch = createAsyncThunk(
throw new Error(err)
}
- const { identifiers = [] } = res
+ const { catalogs = [] } = res
- result.data = identifiers.map(catalogItem => {
+ result.data = catalogs.map(catalogItem => {
return {
...catalogItem,
node: 'catalog',
- id: `{{${metalake}}}{{${catalogItem.name}}}`,
- key: `{{${metalake}}}{{${catalogItem.name}}}`,
- path: `?${new URLSearchParams({ metalake, catalog: catalogItem.name }).toString()}`,
+ id: `{{${metalake}}}{{${catalogItem.name}}}{{${catalogItem.type}}}`,
+ key: `{{${metalake}}}{{${catalogItem.name}}}{{${catalogItem.type}}}`,
+ path: `?${new URLSearchParams({ metalake, catalog: catalogItem.name, type: catalogItem.type }).toString()}`,
name: catalogItem.name,
title: catalogItem.name,
+ namespace: [metalake],
schemas: [],
children: []
}
})
- } else if (pathArr.length === 2) {
+ } else if (pathArr.length === 3) {
const [err, res] = await to(getSchemasApi({ metalake, catalog }))
if (err || !res) {
@@ -122,16 +125,16 @@ export const setIntoTreeNodeWithFetch = createAsyncThunk(
return {
...schemaItem,
node: 'schema',
- id: `{{${metalake}}}{{${catalog}}}{{${schemaItem.name}}}`,
- key: `{{${metalake}}}{{${catalog}}}{{${schemaItem.name}}}`,
- path: `?${new URLSearchParams({ metalake, catalog, schema: schemaItem.name }).toString()}`,
+ id: `{{${metalake}}}{{${catalog}}}{{${type}}}{{${schemaItem.name}}}`,
+ key: `{{${metalake}}}{{${catalog}}}{{${type}}}{{${schemaItem.name}}}`,
+ path: `?${new URLSearchParams({ metalake, catalog, type, schema: schemaItem.name }).toString()}`,
name: schemaItem.name,
title: schemaItem.name,
tables: [],
children: []
}
})
- } else if (pathArr.length === 3) {
+ } else if (pathArr.length === 4 && type !== 'fileset') {
const [err, res] = await to(getTablesApi({ metalake, catalog, schema }))
const { identifiers = [] } = res
@@ -144,9 +147,9 @@ export const setIntoTreeNodeWithFetch = createAsyncThunk(
return {
...tableItem,
node: 'table',
- id: `{{${metalake}}}{{${catalog}}}{{${schema}}}{{${tableItem.name}}}`,
- key: `{{${metalake}}}{{${catalog}}}{{${schema}}}{{${tableItem.name}}}`,
- path: `?${new URLSearchParams({ metalake, catalog, schema, table: tableItem.name }).toString()}`,
+ id: `{{${metalake}}}{{${catalog}}}{{${type}}}{{${schema}}}{{${tableItem.name}}}`,
+ key: `{{${metalake}}}{{${catalog}}}{{${type}}}{{${schema}}}{{${tableItem.name}}}`,
+ path: `?${new URLSearchParams({ metalake, catalog, type, schema, table: tableItem.name }).toString()}`,
name: tableItem.name,
title: tableItem.name,
isLeaf: true,
@@ -154,6 +157,27 @@ export const setIntoTreeNodeWithFetch = createAsyncThunk(
children: []
}
})
+ } else if (pathArr.length === 4 && type === 'fileset') {
+ const [err, res] = await to(getFilesetsApi({ metalake, catalog, schema }))
+
+ const { identifiers = [] } = res
+
+ if (err || !res) {
+ throw new Error(err)
+ }
+
+ result.data = identifiers.map(filesetItem => {
+ return {
+ ...filesetItem,
+ node: 'fileset',
+ id: `{{${metalake}}}{{${catalog}}}{{${type}}}{{${schema}}}{{${filesetItem.name}}}`,
+ key: `{{${metalake}}}{{${catalog}}}{{${type}}}{{${schema}}}{{${filesetItem.name}}}`,
+ path: `?${new URLSearchParams({ metalake, catalog, type, schema, fileset: filesetItem.name }).toString()}`,
+ name: filesetItem.name,
+ title: filesetItem.name,
+ isLeaf: true
+ }
+ })
}
return result
@@ -187,17 +211,18 @@ export const fetchCatalogs = createAsyncThunk(
throw new Error(err)
}
- const { identifiers = [] } = res
+ const { catalogs = [] } = res
- const catalogs = identifiers.map(catalog => {
+ const catalogsData = catalogs.map(catalog => {
return {
...catalog,
node: 'catalog',
- id: `{{${metalake}}}{{${catalog.name}}}`,
- key: `{{${metalake}}}{{${catalog.name}}}`,
- path: `?${new URLSearchParams({ metalake, catalog: catalog.name }).toString()}`,
+ id: `{{${metalake}}}{{${catalog.name}}}{{${catalog.type}}}`,
+ key: `{{${metalake}}}{{${catalog.name}}}{{${catalog.type}}}`,
+ path: `?${new URLSearchParams({ metalake, catalog: catalog.name, type: catalog.type }).toString()}`,
name: catalog.name,
title: catalog.name,
+ namespace: [metalake],
schemas: [],
children: []
}
@@ -214,11 +239,12 @@ export const fetchCatalogs = createAsyncThunk(
? schema.children.map(table => {
return {
...table,
- id: `{{${metalake}}}{{${update.newCatalog.name}}}{{${schema.name}}}{{${table.name}}}`,
- key: `{{${metalake}}}{{${update.newCatalog.name}}}{{${schema.name}}}{{${table.name}}}`,
+ id: `{{${metalake}}}{{${update.newCatalog.name}}}{{${update.newCatalog.type}}}{{${schema.name}}}{{${table.name}}}`,
+ key: `{{${metalake}}}{{${update.newCatalog.name}}}{{${update.newCatalog.type}}}{{${schema.name}}}{{${table.name}}}`,
path: `?${new URLSearchParams({
metalake,
catalog: update.newCatalog.name,
+ type: update.newCatalog.type,
schema: schema.name,
table: table.name
}).toString()}`
@@ -228,11 +254,12 @@ export const fetchCatalogs = createAsyncThunk(
return {
...schema,
- id: `{{${metalake}}}{{${update.newCatalog.name}}}{{${schema.name}}}`,
- key: `{{${metalake}}}{{${update.newCatalog.name}}}{{${schema.name}}}`,
+ id: `{{${metalake}}}{{${update.newCatalog.name}}}{{${update.newCatalog.type}}}{{${schema.name}}}`,
+ key: `{{${metalake}}}{{${update.newCatalog.name}}}{{${update.newCatalog.type}}}{{${schema.name}}}`,
path: `?${new URLSearchParams({
metalake,
catalog: update.newCatalog.name,
+ type: update.newCatalog.type,
schema: schema.name
}).toString()}`,
tables: tables,
@@ -243,9 +270,13 @@ export const fetchCatalogs = createAsyncThunk(
return {
...catalog,
- id: `{{${metalake}}}{{${update.newCatalog.name}}}`,
- key: `{{${metalake}}}{{${update.newCatalog.name}}}`,
- path: `?${new URLSearchParams({ metalake, catalog: update.newCatalog.name }).toString()}`,
+ id: `{{${metalake}}}{{${update.newCatalog.name}}}{{${update.newCatalog.type}}}`,
+ key: `{{${metalake}}}{{${update.newCatalog.name}}}{{${update.newCatalog.type}}}`,
+ path: `?${new URLSearchParams({
+ metalake,
+ catalog: update.newCatalog.name,
+ type: update.newCatalog.type
+ }).toString()}`,
name: update.newCatalog.name,
title: update.newCatalog.name,
schemas: schemas,
@@ -261,9 +292,9 @@ export const fetchCatalogs = createAsyncThunk(
const expandedNodes = getState().metalakes.expandedNodes.map(node => {
const [metalake, catalog, schema, table] = extractPlaceholder(node)
if (catalog === update.catalog) {
- const updatedNode = `{{${metalake}}}{{${update.newCatalog.name}}}${schema ? `{{${schema}}}` : ''}${
- table ? `{{${table}}}` : ''
- }`
+ const updatedNode = `{{${metalake}}}{{${update.newCatalog.name}}}{{${update.newCatalog.type}}}${
+ schema ? `{{${schema}}}` : ''
+ }${table ? `{{${table}}}` : ''}`
return updatedNode
}
@@ -277,9 +308,9 @@ export const fetchCatalogs = createAsyncThunk(
const loadedNodes = getState().metalakes.loadedNodes.map(node => {
const [metalake, catalog, schema, table] = extractPlaceholder(node)
if (catalog === update.catalog) {
- const updatedNode = `{{${metalake}}}{{${update.newCatalog.name}}}${schema ? `{{${schema}}}` : ''}${
- table ? `{{${table}}}` : ''
- }`
+ const updatedNode = `{{${metalake}}}{{${update.newCatalog.name}}}{{${update.newCatalog.type}}}${
+ schema ? `{{${schema}}}` : ''
+ }${table ? `{{${table}}}` : ''}`
return updatedNode
}
@@ -292,14 +323,14 @@ export const fetchCatalogs = createAsyncThunk(
}
} else {
const mergedTree = _.values(
- _.merge(_.keyBy(getState().metalakes.metalakeTree, 'key'), _.keyBy(catalogs, 'key'))
+ _.merge(_.keyBy(catalogsData, 'key'), _.keyBy(getState().metalakes.metalakeTree, 'key'))
)
dispatch(setMetalakeTree(mergedTree))
}
}
return {
- catalogs,
+ catalogs: catalogsData,
page,
init
}
@@ -334,16 +365,17 @@ export const createCatalog = createAsyncThunk(
const catalogData = {
...catalogItem,
node: 'catalog',
- id: `{{${metalake}}}{{${catalogItem.name}}}`,
- key: `{{${metalake}}}{{${catalogItem.name}}}`,
- path: `?${new URLSearchParams({ metalake, catalog: catalogItem.name }).toString()}`,
+ id: `{{${metalake}}}{{${catalogItem.name}}}{{${catalogItem.type}}}`,
+ key: `{{${metalake}}}{{${catalogItem.name}}}{{${catalogItem.type}}}`,
+ path: `?${new URLSearchParams({ metalake, catalog: catalogItem.name, type: catalogItem.type }).toString()}`,
name: catalogItem.name,
title: catalogItem.name,
+ namespace: [metalake],
schemas: [],
children: []
}
- dispatch(dispatch(fetchCatalogs({ metalake, init: true })))
+ dispatch(fetchCatalogs({ metalake, init: true }))
dispatch(addCatalogToTree(catalogData))
@@ -366,7 +398,7 @@ export const updateCatalog = createAsyncThunk(
export const deleteCatalog = createAsyncThunk(
'appMetalakes/deleteCatalog',
- async ({ metalake, catalog }, { dispatch }) => {
+ async ({ metalake, catalog, type }, { dispatch }) => {
dispatch(setTableLoading(true))
const [err, res] = await to(deleteCatalogApi({ metalake, catalog }))
dispatch(setTableLoading(false))
@@ -377,7 +409,7 @@ export const deleteCatalog = createAsyncThunk(
dispatch(fetchCatalogs({ metalake, catalog, page: 'metalakes', init: true }))
- dispatch(removeCatalogFromTree(`{{${metalake}}}{{${catalog}}}`))
+ dispatch(removeCatalogFromTree(`{{${metalake}}}{{${catalog}}}{{${type}}}`))
return res
}
@@ -385,7 +417,7 @@ export const deleteCatalog = createAsyncThunk(
export const fetchSchemas = createAsyncThunk(
'appMetalakes/fetchSchemas',
- async ({ init, page, metalake, catalog }, { getState, dispatch }) => {
+ async ({ init, page, metalake, catalog, type }, { getState, dispatch }) => {
if (init) {
dispatch(setTableLoading(true))
}
@@ -403,15 +435,15 @@ export const fetchSchemas = createAsyncThunk(
const schemaItem = findInTree(
getState().metalakes.metalakeTree,
'key',
- `{{${metalake}}}{{${catalog}}}{{${schema.name}}}`
+ `{{${metalake}}}{{${catalog}}}{{${type}}}{{${schema.name}}}`
)
return {
...schema,
node: 'schema',
- id: `{{${metalake}}}{{${catalog}}}{{${schema.name}}}`,
- key: `{{${metalake}}}{{${catalog}}}{{${schema.name}}}`,
- path: `?${new URLSearchParams({ metalake, catalog, schema: schema.name }).toString()}`,
+ id: `{{${metalake}}}{{${catalog}}}{{${type}}}{{${schema.name}}}`,
+ key: `{{${metalake}}}{{${catalog}}}{{${type}}}{{${schema.name}}}`,
+ path: `?${new URLSearchParams({ metalake, catalog, type, schema: schema.name }).toString()}`,
name: schema.name,
title: schema.name,
tables: schemaItem ? schemaItem.children : [],
@@ -419,10 +451,10 @@ export const fetchSchemas = createAsyncThunk(
}
})
- if (init && getState().metalakes.loadedNodes.includes(`{{${metalake}}}{{${catalog}}}`)) {
+ if (init && getState().metalakes.loadedNodes.includes(`{{${metalake}}}{{${catalog}}}{{${type}}}`)) {
dispatch(
setIntoTreeNodes({
- key: `{{${metalake}}}{{${catalog}}}`,
+ key: `{{${metalake}}}{{${catalog}}}{{${type}}}`,
data: schemas,
tree: getState().metalakes.metalakeTree
})
@@ -433,7 +465,7 @@ export const fetchSchemas = createAsyncThunk(
dispatch(fetchCatalogs({ metalake }))
}
- dispatch(setExpandedNodes([`{{${metalake}}}`, `{{${metalake}}}{{${catalog}}}`]))
+ dispatch(setExpandedNodes([`{{${metalake}}}`, `{{${metalake}}}{{${catalog}}}{{${type}}}`]))
return { schemas, page, init }
}
@@ -474,9 +506,15 @@ export const fetchTables = createAsyncThunk(
return {
...table,
node: 'table',
- id: `{{${metalake}}}{{${catalog}}}{{${schema}}}{{${table.name}}}`,
- key: `{{${metalake}}}{{${catalog}}}{{${schema}}}{{${table.name}}}`,
- path: `?${new URLSearchParams({ metalake, catalog, schema, table: table.name }).toString()}`,
+ id: `{{${metalake}}}{{${catalog}}}{{${'relational'}}}{{${schema}}}{{${table.name}}}`,
+ key: `{{${metalake}}}{{${catalog}}}{{${'relational'}}}{{${schema}}}{{${table.name}}}`,
+ path: `?${new URLSearchParams({
+ metalake,
+ catalog,
+ type: 'relational',
+ schema,
+ table: table.name
+ }).toString()}`,
name: table.name,
title: table.name,
isLeaf: true,
@@ -485,10 +523,13 @@ export const fetchTables = createAsyncThunk(
}
})
- if (init && getState().metalakes.loadedNodes.includes(`{{${metalake}}}{{${catalog}}}{{${schema}}}`)) {
+ if (
+ init &&
+ getState().metalakes.loadedNodes.includes(`{{${metalake}}}{{${catalog}}}{{${'relational'}}}{{${schema}}}`)
+ ) {
dispatch(
setIntoTreeNodes({
- key: `{{${metalake}}}{{${catalog}}}{{${schema}}}`,
+ key: `{{${metalake}}}{{${catalog}}}{{${'relational'}}}{{${schema}}}`,
data: tables,
tree: getState().metalakes.metalakeTree
})
@@ -502,8 +543,8 @@ export const fetchTables = createAsyncThunk(
dispatch(
setExpandedNodes([
`{{${metalake}}}`,
- `{{${metalake}}}{{${catalog}}}`,
- `{{${metalake}}}{{${catalog}}}{{${schema}}}`
+ `{{${metalake}}}{{${catalog}}}{{${'relational'}}}`,
+ `{{${metalake}}}{{${catalog}}}{{${'relational'}}}{{${schema}}}`
])
)
@@ -535,8 +576,8 @@ export const getTableDetails = createAsyncThunk(
dispatch(
setExpandedNodes([
`{{${metalake}}}`,
- `{{${metalake}}}{{${catalog}}}`,
- `{{${metalake}}}{{${catalog}}}{{${schema}}}`
+ `{{${metalake}}}{{${catalog}}}{{${'relational'}}}`,
+ `{{${metalake}}}{{${catalog}}}{{${'relational'}}}{{${schema}}}`
])
)
@@ -544,6 +585,104 @@ export const getTableDetails = createAsyncThunk(
}
)
+export const fetchFilesets = createAsyncThunk(
+ 'appMetalakes/fetchFilesets',
+ async ({ init, page, metalake, catalog, schema }, { getState, dispatch }) => {
+ if (init) {
+ dispatch(setTableLoading(true))
+ }
+
+ const [err, res] = await to(getFilesetsApi({ metalake, catalog, schema }))
+ dispatch(setTableLoading(false))
+
+ if (init && (err || !res)) {
+ dispatch(resetTableData())
+ throw new Error(err)
+ }
+
+ const { identifiers = [] } = res
+
+ const filesets = identifiers.map(fileset => {
+ return {
+ ...fileset,
+ node: 'fileset',
+ id: `{{${metalake}}}{{${catalog}}}{{${'fileset'}}}{{${schema}}}{{${fileset.name}}}`,
+ key: `{{${metalake}}}{{${catalog}}}{{${'fileset'}}}{{${schema}}}{{${fileset.name}}}`,
+ path: `?${new URLSearchParams({
+ metalake,
+ catalog,
+ type: 'fileset',
+ schema,
+ fileset: fileset.name
+ }).toString()}`,
+ name: fileset.name,
+ title: fileset.name,
+ isLeaf: true
+ }
+ })
+
+ if (
+ init &&
+ getState().metalakes.loadedNodes.includes(`{{${metalake}}}{{${catalog}}}{{${'fileset'}}}{{${schema}}}`)
+ ) {
+ dispatch(
+ setIntoTreeNodes({
+ key: `{{${metalake}}}{{${catalog}}}{{${'fileset'}}}{{${schema}}}`,
+ data: filesets,
+ tree: getState().metalakes.metalakeTree
+ })
+ )
+ }
+
+ if (getState().metalakes.metalakeTree.length === 0) {
+ dispatch(fetchCatalogs({ metalake }))
+ }
+
+ dispatch(
+ setExpandedNodes([
+ `{{${metalake}}}`,
+ `{{${metalake}}}{{${catalog}}}{{${'fileset'}}}`,
+ `{{${metalake}}}{{${catalog}}}{{${'fileset'}}}{{${schema}}}`
+ ])
+ )
+
+ return { filesets, page, init }
+ }
+)
+
+export const getFilesetDetails = createAsyncThunk(
+ 'appMetalakes/getFilesetDetails',
+ async ({ init, metalake, catalog, schema, fileset }, { getState, dispatch }) => {
+ dispatch(resetTableData())
+ if (init) {
+ dispatch(setTableLoading(true))
+ }
+ const [err, res] = await to(getFilesetDetailsApi({ metalake, catalog, schema, fileset }))
+ dispatch(setTableLoading(false))
+
+ if (err || !res) {
+ dispatch(resetTableData())
+ throw new Error(err)
+ }
+
+ const { fileset: resFileset } = res
+
+ if (getState().metalakes.metalakeTree.length === 0) {
+ dispatch(fetchCatalogs({ metalake }))
+ }
+
+ dispatch(
+ setExpandedNodes([
+ `{{${metalake}}}`,
+ `{{${metalake}}}{{${catalog}}}{{${'fileset'}}}`,
+ `{{${metalake}}}{{${catalog}}}{{${'fileset'}}}{{${schema}}}`
+ ])
+ )
+
+ return resFileset
+ }
+)
+
export const appMetalakesSlice = createSlice({
name: 'appMetalakes',
initialState: {
@@ -554,6 +693,7 @@ export const appMetalakesSlice = createSlice({
schemas: [],
tables: [],
columns: [],
+ filesets: [],
metalakeTree: [],
loadedNodes: [],
selectedNodes: [],
@@ -604,6 +744,7 @@ export const appMetalakesSlice = createSlice({
state.schemas = []
state.tables = []
state.columns = []
+ state.filesets = []
},
setTableLoading(state, action) {
state.tableLoading = action.payload
@@ -616,7 +757,12 @@ export const appMetalakesSlice = createSlice({
state.metalakeTree = updateTreeData(tree, key, data)
},
addCatalogToTree(state, action) {
- state.metalakeTree.push(action.payload)
+ const catalogIndex = state.metalakeTree.findIndex(c => c.key === action.payload.key)
+ if (catalogIndex === -1) {
+ state.metalakeTree.push(action.payload)
+ } else {
+ state.metalakeTree.splice(catalogIndex, 1, action.payload)
+ }
},
removeCatalogFromTree(state, action) {
state.metalakeTree = state.metalakeTree.filter(i => i.key !== action.payload)
@@ -627,13 +773,28 @@ export const appMetalakesSlice = createSlice({
builder.addCase(fetchMetalakes.fulfilled, (state, action) => {
state.metalakes = action.payload.metalakes
})
+ builder.addCase(fetchMetalakes.rejected, (state, action) => {
+ toast.error(action.error.message)
+ })
builder.addCase(setIntoTreeNodeWithFetch.fulfilled, (state, action) => {
const { key, data, tree } = action.payload
state.metalakeTree = updateTreeData(tree, key, data)
})
+ builder.addCase(setIntoTreeNodeWithFetch.rejected, (state, action) => {
+ toast.error(action.error.message)
+ })
builder.addCase(getMetalakeDetails.fulfilled, (state, action) => {
state.activatedDetails = action.payload
})
+ builder.addCase(getMetalakeDetails.rejected, (state, action) => {
+ toast.error(action.error.message)
+ })
+ builder.addCase(createCatalog.rejected, (state, action) => {
+ toast.error(action.error.message)
+ })
+ builder.addCase(updateCatalog.rejected, (state, action) => {
+ toast.error(action.error.message)
+ })
builder.addCase(fetchCatalogs.fulfilled, (state, action) => {
state.catalogs = action.payload.catalogs
@@ -645,28 +806,65 @@ export const appMetalakesSlice = createSlice({
state.metalakeTree = action.payload.catalogs
}
})
+ builder.addCase(fetchCatalogs.rejected, (state, action) => {
+ toast.error(action.error.message)
+ })
builder.addCase(getCatalogDetails.fulfilled, (state, action) => {
state.activatedDetails = action.payload
})
+ builder.addCase(getCatalogDetails.rejected, (state, action) => {
+ toast.error(action.error.message)
+ })
+ builder.addCase(deleteCatalog.rejected, (state, action) => {
+ toast.error(action.error.message)
+ })
builder.addCase(fetchSchemas.fulfilled, (state, action) => {
state.schemas = action.payload.schemas
if (action.payload.init) {
state.tableData = action.payload.schemas
}
})
+ builder.addCase(fetchSchemas.rejected, (state, action) => {
+ toast.error(action.error.message)
+ })
builder.addCase(getSchemaDetails.fulfilled, (state, action) => {
state.activatedDetails = action.payload
})
+ builder.addCase(getSchemaDetails.rejected, (state, action) => {
+ toast.error(action.error.message)
+ })
builder.addCase(fetchTables.fulfilled, (state, action) => {
state.tables = action.payload.tables
if (action.payload.init) {
state.tableData = action.payload.tables
}
})
+ builder.addCase(fetchTables.rejected, (state, action) => {
+ toast.error(action.error.message)
+ })
builder.addCase(getTableDetails.fulfilled, (state, action) => {
state.activatedDetails = action.payload
state.tableData = action.payload.columns || []
})
+ builder.addCase(getTableDetails.rejected, (state, action) => {
+ toast.error(action.error.message)
+ })
+ builder.addCase(fetchFilesets.fulfilled, (state, action) => {
+ state.filesets = action.payload.filesets
+ if (action.payload.init) {
+ state.tableData = action.payload.filesets
+ }
+ })
+ builder.addCase(fetchFilesets.rejected, (state, action) => {
+ toast.error(action.error.message)
+ })
+ builder.addCase(getFilesetDetails.fulfilled, (state, action) => {
+ state.activatedDetails = action.payload
+ state.tableData = []
+ })
+ builder.addCase(getFilesetDetails.rejected, (state, action) => {
+ toast.error(action.error.message)
+ })
}
})
diff --git a/web/src/lib/utils/initial.js b/web/src/lib/utils/initial.js
index 2eaa4d9ac76..476879b1e4b 100644
--- a/web/src/lib/utils/initial.js
+++ b/web/src/lib/utils/initial.js
@@ -4,6 +4,11 @@
*/
export const providers = [
+ {
+ label: 'hadoop',
+ value: 'hadoop',
+ defaultProps: []
+ },
{
label: 'hive',
value: 'hive',