Web: Updates for Table and Column Lineage (#2725)

* Initial commit with new rendering engine.

* Adding prototype for column lineage view.

* Updates for zoom controls.

* Removing extra imports.

* Extra meta information.

* Adjusting application styles.

* Improving code quality, splitting components.

* Moving code around.

* Saving partial progress.

* Some progress on lineage view.

* Checkpoint for tooltips.

* Adding in side navigation.

* Fixing links.

* Fixing up lineage state.

* Fixing column lineage.

* Fixing up the back button.

* Fixing tests.

* Filtering out the non-null assertions.

* More test fixes.

* Code review comments and running formatter.

* Adding full mode to graph.

* Changing some language.

* Minor update to fix search issue.

* Removing Some text.

* Improving fonts and layout in the action bar.

* Table level node encoding.

* Column Level changes.

* Fixing reload on column lineage and adjusting colors.

---------

Co-authored-by: phix <peter.hicks@astronomer.io>

web/.gitignore

@@ -12,6 +12,7 @@
 /coverage
 /dist
 /node_modules
+/libs/graph/node_modules
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*

web/libs/graph/README.md

@@ -0,0 +1,111 @@
+# graph
+
+`graph` is a library for generating and rendering graphs. It uses elkjs to generate layouts and renders them using a custom renderer.
+
+## Installing/Using
+
+To use the library, add the following to your package.json:
+
+```json
+{
+  "dependencies": {
+    "elkjs": "^0.8.2"
+  }
+}
+```
+
+You also need to use the webpack CopyPlugin to copy the elk-worker file. Add the following to your webpack config:
+
+```js
+const CopyPlugin = require('copy-webpack-plugin');
+const path = require('path');
+
+// look for elkjs package folder
+const elkjsRoot = path.dirname(require.resolve('elkjs/package.json'));
+
+// add the CopyPlugin to the webpack config
+  plugins: [
+    ...
+    new CopyPlugin({
+      patterns: [
+        { from: path.join(elkjsRoot, 'lib/elk-worker.min.js'), to: 'elk-worker.min.js' },
+      ],
+    }),
+  ]
+```
+
+## useLayout
+
+`useLayout` provides a common interface for creating layouts with [ElkJs](https://github.com/kieler/elkjs).
+
+```ts
+import { useLayout } from 'graph'
+
+const { nodes: positionedNodes, edges: positionedEdges } = useLayout<'myKind', MyNodeDataType>({
+  id,
+  nodes,
+  edges,
+  direction: 'right',
+  webWorkerUrl,
+  getLayoutOptions
+});
+```
+
+The layout calculations are asynchronous. Once the layout is complete, the returned `nodes` will each include
+a `bottomLeftCorner: {x: number: y: number }` property, along with all the original properties.
+
+## ZoomPanSvg Component
+
+// To Add
+
+## Graph Component
+
+The Graph component is used to render a Graph. A `TaskNode` with run status is included, but custom ones are supported.
+
+```tsx
+import { Graph, Edge, Node } from 'graph';
+
+import { MyNodeComponent, MyNodeDataType, NodeRendererMap } from './';
+
+const myNodesRenderers: NodeRendererMap<'myNode', MyNodeDataType> = new Map().set(
+  'myNode',
+  MyNodeComponent,
+);
+
+// declare the nodes and edges
+const nodes: Node<'myNode', MyNodeDataType>[] = [
+  {
+    id: '1',
+    label: 'Task 1',
+    type: 'myNode',
+    data: {
+      // any additional data you want to store
+    },
+  },
+  {
+    id: '2',
+    label: 'Task 2',
+    type: 'myNode',
+    data: {
+      // any additional data you want to store
+    },
+  },
+];
+
+const edges: Edge[] = [
+  {
+    id: '1',
+    source: '1',
+    target: '2',
+    type: 'elbow',
+  },
+];
+
+// create a graph
+<Graph<'myNode', MyNodeDataType>
+  nodes={nodes}
+  edges={edges}
+  direction="right"
+  nodeRenderers={myNodesRenderers}
+/>;
+```

web/libs/graph/jest.config.js

@@ -0,0 +1,4 @@
+module.exports = {
+  ...config,
+  setupFilesAfterEnv: ['./jest.setup.js'],
+}

web/libs/graph/jest.setup.js

@@ -0,0 +1,19 @@
+/* eslint-disable no-undef */
+import '@testing-library/jest-dom'
+
+// Establish API mocking before all tests.
+beforeAll(() => {
+  Object.defineProperty(window, 'matchMedia', {
+    writable: true,
+    value: jest.fn().mockImplementation((query) => ({
+      matches: false,
+      media: query,
+      onchange: null,
+      addListener: jest.fn(), // deprecated
+      removeListener: jest.fn(), // deprecated
+      addEventListener: jest.fn(),
+      removeEventListener: jest.fn(),
+      dispatchEvent: jest.fn(),
+    })),
+  })
+})

web/libs/graph/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "graph",
+  "version": "0.0.1",
+  "private": true,
+  "main": "src/index.ts",
+  "scripts": {
+    "lint": "cd .. && yarn lint",
+    "test": "jest",
+    "test:silent": "yarn test --silent",
+    "test:ci": "yarn test:silent --coverage --runInBand",
+    "test:watch": "yarn test --watch",
+    "tsc:validate": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@react-hook/size": "^2.1.2",
+    "lodash": "^4.17.21"
+  },
+  "peerDependencies": {
+    "@chakra-ui/react": "^2",
+    "elkjs": "^0.8.2",
+    "react": "^18",
+    "react-dom": "^18",
+    "react-router-dom": "^6"
+  },
+  "devDependencies": {
+    "@testing-library/jest-dom": "^5.16.5",
+    "@testing-library/react": "^14.0.0",
+    "@testing-library/react-hooks": "^8.0.1",
+    "@testing-library/user-event": "^14.4.3",
+    "@types/jest": "^27.5.2",
+    "@types/react": "^18.0.15",
+    "@types/react-dom": "^18.0.6",
+    "d3-interpolate": "^2.0.1",
+    "d3-selection": "^2.0.0",
+    "d3-transition": "^2.0.0",
+    "d3-zoom": "^2.0.0",
+    "elkjs": "^0.8.2",
+    "jest": "^27.5.1",
+    "prettier": "^2.8.8",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-router-dom": "^6.3.0",
+    "typescript": "^5.1.3",
+    "web-worker": "^1.2.0"
+  }
+}

web/libs/graph/src/components/Edge/Edge.tsx

@@ -0,0 +1,20 @@
+import React from 'react'
+
+import { ElbowEdge } from './ElbowEdge'
+import { StraightEdge } from './StraightEdge'
+import type { PositionedEdge } from '../../types'
+
+export interface EdgeProps {
+  edge: PositionedEdge
+  isMiniMap?: boolean
+}
+
+export const Edge = (props: EdgeProps) => {
+  const { edge } = props
+  switch (edge.type) {
+    case 'straight':
+      return <StraightEdge {...props} />
+    default:
+      return <ElbowEdge {...props} />
+  }
+}

web/libs/graph/src/components/Edge/EdgeLabel.tsx

@@ -0,0 +1,26 @@
+import React from 'react'
+
+import { grey } from '@mui/material/colors'
+import type { ElkLabel } from 'elkjs'
+
+interface Props {
+  label?: ElkLabel
+  endPointY?: number
+}
+
+export const EdgeLabel = ({ label, endPointY }: Props) => {
+  const labelColor = grey['400']
+
+  if (!label || !label.y || !label.x) return null
+
+  let { y } = label
+  // The edge and label are rendering a little differently,
+  // so we need some extra magic numbers to work right
+  if (endPointY) y = label.y - 5 >= endPointY ? endPointY + 25 : endPointY - 15
+
+  return (
+    <text fill={labelColor} x={label.x} y={y}>
+      {label.text}
+    </text>
+  )
+}

web/libs/graph/src/components/Edge/ElbowEdge.tsx

@@ -0,0 +1,63 @@
+import React, { useMemo } from 'react'
+
+import { chakra, keyframes, usePrefersReducedMotion } from '@chakra-ui/react'
+
+import { EdgeLabel } from './EdgeLabel'
+import { grey } from '@mui/material/colors'
+import type { EdgeProps } from './Edge'
+
+const ChakraPolyline = chakra('polyline') // need to use animation prop
+const marchingAnts = keyframes({ from: { strokeDashoffset: 60 }, to: { strokeDashoffset: 0 } })
+
+export const ElbowEdge = ({ edge, isMiniMap }: EdgeProps) => {
+  const reducedMotion = usePrefersReducedMotion() || isMiniMap // do not animate the minimap
+
+  const points = useMemo(() => {
+    const { startPoint, bendPoints, endPoint } = edge
+
+    return [
+      // source
+      { x: startPoint.x, y: startPoint.y },
+      ...(bendPoints?.map((bendPoint) => ({ x: bendPoint.x, y: bendPoint.y })) ?? []),
+      // target
+      { x: endPoint.x, y: endPoint.y },
+    ]
+  }, [edge])
+
+  // Find the longest edge that the label would be near
+  let longestEdge: { y: number; length: number } | undefined
+  if (edge.label) {
+    points.forEach((p, i) => {
+      if (i > 0) {
+        const length = p.x - points[i - 1].x
+        if (!longestEdge || longestEdge.length < length) longestEdge = { y: p.y, length }
+      }
+    })
+  }
+  return (
+    <>
+      <polyline
+        id={`${edge.sourceNodeId}-${edge.targetNodeId}`}
+        fill='none'
+        stroke={edge.color || grey['600']}
+        strokeWidth={edge.strokeWidth || 2}
+        strokeLinejoin='round'
+        points={points.map(({ x, y }) => `${x},${y}`).join(' ')}
+      />
+      <EdgeLabel label={edge.label} endPointY={longestEdge?.y} />
+      {!reducedMotion && edge.isAnimated && (
+        <ChakraPolyline
+          id={`${edge.sourceNodeId}-${edge.targetNodeId}-animated`}
+          fill='none'
+          strokeLinecap='round'
+          stroke={edge.color || grey['600']}
+          strokeWidth={edge.strokeWidth || 5}
+          strokeLinejoin='round'
+          strokeDasharray='0px 60px'
+          animation={`${marchingAnts} infinite 2s linear`}
+          points={points.map(({ x, y }) => `${x},${y}`).join(' ')}
+        />
+      )}
+    </>
+  )
+}

web/libs/graph/src/components/Edge/StraightEdge.tsx

@@ -0,0 +1,48 @@
+import React from 'react'
+
+import { chakra, keyframes, usePrefersReducedMotion } from '@chakra-ui/react'
+
+import { EdgeLabel } from './EdgeLabel'
+import { grey } from '@mui/material/colors'
+import type { EdgeProps } from './Edge'
+
+const ChakraLine = chakra('line') // need to use animation prop
+const marchingAnts = keyframes({ from: { strokeDashoffset: 60 }, to: { strokeDashoffset: 0 } })
+
+export const StraightEdge = ({ edge, isMiniMap }: EdgeProps) => {
+  const reducedMotion = usePrefersReducedMotion() || isMiniMap // do not animate the minimap
+  const color = grey['600']
+
+  return (
+    <>
+      <line
+        id={`${edge.sourceNodeId}-${edge.targetNodeId}`}
+        fill='none'
+        stroke={edge.color || color}
+        strokeWidth={edge.strokeWidth || 2}
+        strokeLinejoin='round'
+        x1={edge.startPoint.x}
+        y1={edge.startPoint.y}
+        x2={edge.endPoint.x}
+        y2={edge.endPoint.y}
+      />
+      <EdgeLabel label={edge.label} endPointY={edge.endPoint.y} />
+      {!reducedMotion && edge.isAnimated && (
+        <ChakraLine
+          id={`${edge.sourceNodeId}-${edge.targetNodeId}-animated`}
+          fill='none'
+          strokeLinecap='round'
+          stroke={edge.color || color}
+          strokeWidth={edge.strokeWidth || 5}
+          strokeLinejoin='round'
+          strokeDasharray='0px 60px'
+          animation={`${marchingAnts} infinite 2s linear`}
+          x1={edge.startPoint.x}
+          y1={edge.startPoint.y}
+          x2={edge.endPoint.x}
+          y2={edge.endPoint.y}
+        />
+      )}
+    </>
+  )
+}

web/libs/graph/src/components/Edge/index.ts

@@ -0,0 +1 @@
+export * from './Edge'

web/libs/graph/src/components/Graph.test.tsx

@@ -0,0 +1,32 @@
+import React from 'react'
+
+import '@testing-library/jest-dom'
+
+import { PositionedNode } from '../index'
+
+/* CUSTOM NODES */
+export interface SimpleNodeData {
+  displayName?: string
+}
+
+interface SimpleNodeProps {
+  node: PositionedNode<'simple', SimpleNodeData>
+}
+
+// a task node has a name and an operator that get displayed
+const SimpleNode = ({ node }: SimpleNodeProps) => (
+  <rect
+    x={10}
+    y={20}
+    width={node.width}
+    height={node.height}
+    data-testid={node.id}
+    data-custom='true'
+  />
+)
+
+SimpleNode.getLayoutOptions = (node: SimpleNodeProps['node']) => ({
+  ...node,
+  width: node.width ?? 100,
+  height: node.height ?? 100,
+})