Skip to content

Commit

Permalink
Merge branch 'main' into bz/extract-fill-id
Browse files Browse the repository at this point in the history
  • Loading branch information
@mergify
mergify[bot] authored Jan 10, 2023
2 parents e35b3bc + 5d25f9c commit e24e792
Show file tree
Hide file tree
Showing 153 changed files with 3,510 additions and 1,443 deletions.
8 changes: 4 additions & 4 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 12 additions & 0 deletions ci/scripts/docker.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
# Exits as soon as any line fails.
set -euo pipefail

# Build docker image ${BUILDKITE_COMMIT}-${arch}

date="$(date +%Y%m%d)"
ghcraddr="ghcr.io/risingwavelabs/risingwave"
dockerhubaddr="risingwavelabs/risingwave"
Expand All @@ -11,6 +13,16 @@ arch="$(uname -m)"
echo "--- docker build and tag"
docker build -f docker/Dockerfile -t "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" --target risingwave .

echo "--- check the image can start correctly"
container_id=$(docker run -d "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" risingwave)
ret_code=$(docker inspect --format='{{.State.ExitCode}}' "$container_id")
if [ "$ret_code" -ne 0 ]; then
echo "docker run failed with exit code $ret_code"
docker logs "$container_id"
exit 1
fi
docker kill "$container_id"

echo "--- docker images"
docker images

Expand Down
7 changes: 7 additions & 0 deletions ci/scripts/multi-arch-docker.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,13 @@
# Exits as soon as any line fails.
set -euo pipefail

# Create multi-arch docker images from ${BUILDKITE_COMMIT}-x86_64 and ${BUILDKITE_COMMIT}-aarch64
# They are created by ci/scripts/docker.sh
#
# Also add addtional tags to the images:
# nightly-yyyyMMdd: nightly build in main-cron
# latest: only push to ghcr. dockerhub latest is latest release

date="$(date +%Y%m%d)"
ghcraddr="ghcr.io/risingwavelabs/risingwave"
dockerhubaddr="risingwavelabs/risingwave"
Expand Down
85 changes: 47 additions & 38 deletions dashboard/components/Relations.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ export type Column<R> = {
export function Relations<R extends Relation>(
title: string,
getRelations: () => Promise<R[]>,
extraColumns: Column<R>[] = []
extraColumns: Column<R>[] = [],
isStreamingJob: boolean = true // only show metrics and graphs for streaming jobs
) {
const toast = useToast()
const [relationList, setRelationList] = useState<R[]>([])
Expand Down Expand Up @@ -83,9 +84,13 @@ export function Relations<R extends Relation>(
{c.name}
</Th>
))}
<Th width={1}>Metrics</Th>
<Th width={1}>Depends</Th>
<Th width={1}>Fragments</Th>
{isStreamingJob && (
<>
<Th width={1}>Metrics</Th>
<Th width={1}>Depends</Th>
<Th width={1}>Fragments</Th>
</>
)}
<Th>Visible Columns</Th>
</Tr>
</Thead>
Expand All @@ -98,40 +103,44 @@ export function Relations<R extends Relation>(
{extraColumns.map((c) => (
<Td key={c.name}>{c.content(r)}</Td>
))}
<Td>
<Button
size="sm"
aria-label="view metrics"
colorScheme="teal"
variant="link"
>
M
</Button>
</Td>
<Td>
<Link href={`/streaming_graph/?id=${r.id}`}>
<Button
size="sm"
aria-label="view metrics"
colorScheme="teal"
variant="link"
>
D
</Button>
</Link>
</Td>
<Td>
<Link href={`/streaming_plan/?id=${r.id}`}>
<Button
size="sm"
aria-label="view metrics"
colorScheme="teal"
variant="link"
>
F
</Button>
</Link>
</Td>
{isStreamingJob && (
<>
<Td>
<Button
size="sm"
aria-label="view metrics"
colorScheme="teal"
variant="link"
>
M
</Button>
</Td>
<Td>
<Link href={`/streaming_graph/?id=${r.id}`}>
<Button
size="sm"
aria-label="view dependents"
colorScheme="teal"
variant="link"
>
D
</Button>
</Link>
</Td>
<Td>
<Link href={`/streaming_plan/?id=${r.id}`}>
<Button
size="sm"
aria-label="view fragments"
colorScheme="teal"
variant="link"
>
F
</Button>
</Link>
</Td>
</>
)}
<Td overflowWrap="normal">
{r.columns
.filter((col) => !col.isHidden)
Expand Down
11 changes: 7 additions & 4 deletions dashboard/pages/api/streaming.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,16 @@ export interface Relation {
id: number
name: string
owner: number
dependentRelations: number[]
columns: ColumnCatalog[]
}

export async function getRelations(): Promise<Relation[]> {
const materialized_views: Relation[] = await getMaterializedViews()
const sinks: Relation[] = await getSinks()
export interface StreamingJob extends Relation {
dependentRelations: number[]
}

export async function getStreamingJobs(): Promise<StreamingJob[]> {
const materialized_views: StreamingJob[] = await getMaterializedViews()
const sinks: StreamingJob[] = await getSinks()
let relations = materialized_views.concat(sinks)
relations = sortBy(relations, (x) => x.id)
return relations
Expand Down
135 changes: 14 additions & 121 deletions dashboard/pages/data_sources.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,130 +15,23 @@
*
*/

import {
Box,
Button,
Table,
TableContainer,
Tbody,
Td,
Th,
Thead,
Tr,
useToast,
} from "@chakra-ui/react"
import Head from "next/head"

import Link from "next/link"
import { Fragment, useEffect, useState } from "react"
import Title from "../components/Title"
import extractColumnInfo from "../lib/extractInfo"
import { Column, Relations } from "../components/Relations"
import { Source } from "../proto/gen/catalog"
import { getDataSources } from "./api/streaming"

export default function DataSources() {
const toast = useToast()
const [sourceList, setSourceList] = useState<Source[]>([])

useEffect(() => {
async function doFetch() {
try {
setSourceList(await getDataSources())
} catch (e: any) {
toast({
title: "Error Occurred",
description: e.toString(),
status: "error",
duration: 5000,
isClosable: true,
})
console.error(e)
}
}
doFetch()
return () => {}
}, [toast])
const columns: Column<Source>[] = [
{
name: "Connector",
width: 3,
content: (r) => r.properties.connector ?? "unknown",
},
{
name: "Row Format",
width: 3,
content: (s) => s.info?.rowFormat ?? "unknown",
},
]

const retVal = (
<Box p={3}>
<Title>Data Sources</Title>
<TableContainer>
<Table variant="simple" size="sm" maxWidth="full">
<Thead>
<Tr>
<Th width={3}>Id</Th>
<Th width={5}>Name</Th>
<Th width={3}>Owner</Th>
<Th width={3}>Type</Th>
<Th width={1}>Metrics</Th>
<Th width={1}>Depends</Th>
<Th width={1}>Fragments</Th>
<Th>Visible Columns</Th>
</Tr>
</Thead>
<Tbody>
{sourceList
.filter((source) => !source.name.startsWith("__"))
.map((source) => (
<Tr key={source.id}>
<Td>{source.id}</Td>
<Td>{source.name}</Td>
<Td>{source.owner}</Td>
<Td>{source.info}</Td>
<Td>
<Button
size="sm"
aria-label="view metrics"
colorScheme="teal"
variant="link"
>
M
</Button>
</Td>
<Td>
<Link href={`/streaming_graph/?id=${source.id}`}>
<Button
size="sm"
aria-label="view metrics"
colorScheme="teal"
variant="link"
>
D
</Button>
</Link>
</Td>
<Td>
<Link href={`/streaming_plan/?id=${source.id}`}>
<Button
size="sm"
aria-label="view metrics"
colorScheme="teal"
variant="link"
>
F
</Button>
</Link>
</Td>
<Td overflowWrap="normal">
{(() => {
return source.columns
.map((col) => extractColumnInfo(col))
.join(", ")
})()}
</Td>
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
</Box>
)
return (
<Fragment>
<Head>
<title>Data Sources</title>
</Head>
{retVal}
</Fragment>
)
return Relations("Data Sources", getDataSources, columns, false)
}
18 changes: 9 additions & 9 deletions dashboard/pages/streaming_graph.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@ import { Fragment, useCallback, useEffect, useState } from "react"
import { StreamGraph } from "../components/StreamGraph"
import Title from "../components/Title"
import { ActorPoint } from "../lib/layout"
import { getRelations, Relation } from "./api/streaming"
import { getStreamingJobs, StreamingJob } from "./api/streaming"

const SIDEBAR_WIDTH = "200px"

function buildDependencyAsEdges(list: Relation[]): ActorPoint[] {
function buildDependencyAsEdges(list: StreamingJob[]): ActorPoint[] {
const edges = []
const relationSet = new Set(list.map((r) => r.id))
for (const r of reverse(sortBy(list, "id"))) {
Expand All @@ -48,13 +48,13 @@ function buildDependencyAsEdges(list: Relation[]): ActorPoint[] {

export default function StreamingGraph() {
const toast = useToast()
const [relationList, setRelationList] = useState<Relation[]>()
const [streamingJobList, setStreamingJobList] = useState<StreamingJob[]>()

useEffect(() => {
async function doFetch() {
try {
setRelationList(
(await getRelations()).filter((x) => !x.name.startsWith("__"))
setStreamingJobList(
(await getStreamingJobs()).filter((x) => !x.name.startsWith("__"))
)
} catch (e: any) {
toast({
Expand All @@ -72,12 +72,12 @@ export default function StreamingGraph() {
}, [toast])

const mvDependencyCallback = useCallback(() => {
if (relationList) {
return buildDependencyAsEdges(relationList)
if (streamingJobList) {
return buildDependencyAsEdges(streamingJobList)
} else {
return undefined
}
}, [relationList])
}, [streamingJobList])

const mvDependency = mvDependencyCallback()

Expand All @@ -100,7 +100,7 @@ export default function StreamingGraph() {
</Text>
<Box flex={1} overflowY="scroll">
<VStack width="full" spacing={1}>
{relationList?.map((r) => {
{streamingJobList?.map((r) => {
const match = router.query.id === r.id.toString()
return (
<Link href={`?id=${r.id}`} key={r.id}>
Expand Down
Loading

0 comments on commit e24e792

Please sign in to comment.