Zed Table (#2626)

package.json

@@ -121,6 +121,7 @@
     "@electron/remote": "^2.0.1",
     "@js-joda/core": "^3.2.0",
     "@reduxjs/toolkit": "^1.5.0",
+    "@tanstack/react-table": "^8.7.0",
     "acorn": "^7.4.1",
     "ajv": "^6.9.1",
     "animejs": "^3.2.0",
@@ -145,6 +146,7 @@
     "keytar": "^7.7.0",
     "lodash": "^4.17.21",
     "md5": "^2.2.1",
+    "memoize-one": "^6.0.0",
     "moment": "^2.27.0",
     "moment-timezone": "^0.5.31",
     "mousetrap": "^1.6.5",
@@ -155,7 +157,7 @@
     "polished": "^3.6.5",
     "prismjs": "^1.25.0",
     "react": "^18.0.0",
-    "react-arborist": "2.1.0",
+    "react-arborist": "2.2.0",
     "react-dnd": "^14.0.5",
     "react-dnd-html5-backend": "^14.0.2",
     "react-dom": "^18.0.0",

packages/e2e-tests/helpers/test-app.ts

@@ -93,12 +93,12 @@ export default class TestApp {
 
   // TODO: this method is a wip, it still needs to wait for cells to populate first
   async getViewerResults(includeHeaders = true): Promise<string[]> {
-    const fields = await this.mainWin.locator(".viewer .field-cell")
+    const fields = await this.mainWin.locator(".zed-table__cell")
     let results = await fields.evaluateAll<string[], HTMLElement>((nodes) =>
       nodes.map((n) => n.innerText.trim())
     )
     if (includeHeaders) {
-      const headers = await this.mainWin.locator(".viewer .header-cell")
+      const headers = await this.mainWin.locator(".zed-table__header-cell")
       const headerResults = await headers.evaluateAll<string[], HTMLElement>(
         (headerCells) => headerCells.map((hc) => hc.innerText.trim())
       )

packages/e2e-tests/tests/export.spec.ts

@@ -25,10 +25,7 @@ test.describe("Export tests", () => {
     await app.createPool([
       path.normalize(path.join(testDataDir(), "sample.tsv")),
     ])
-    await app.mainWin
-      .locator('#app-root button:above(:text("Query Pool"))')
-      .first()
-      .click()
+    await app.mainWin.getByRole("button", {name: "Query Pool"}).click()
     await app.query("sort ts")
   })
 
@@ -45,12 +42,11 @@ test.describe("Export tests", () => {
           Promise.resolve({canceled: false, filePath})
       }, file)
 
-      await app.mainWin
-        .locator('#app-root button:above(:text("Export"))')
-        .first()
-        .click()
-      await app.mainWin.getByRole("radio", {name: `${label}`}).click()
-      await app.mainWin.getByRole("button").filter({hasText: "Export"}).click()
+      const menu = app.mainWin.getByRole("list", {name: "resultsToolbarMenu"})
+      await menu.getByRole("button", {name: "Export"}).click()
+      const dialog = app.mainWin.getByRole("dialog")
+      await dialog.getByRole("radio", {name: `${label}`}).click()
+      await dialog.getByRole("button").filter({hasText: "Export"}).click()
 
       await expect(
         await app.mainWin.locator("text=Export Complete").first()

packages/e2e-tests/tests/ingest.spec.ts

@@ -35,10 +35,7 @@ test.describe("Ingest tests", () => {
     await app.createPool([
       path.normalize(path.join(testDataDir(), "sample.tsv")),
     ])
-    await app.mainWin
-      .locator('#app-root button:above(:text("Query Pool"))')
-      .first()
-      .click()
+    await app.mainWin.getByRole("button", {name: "Query Pool"}).click()
   })
 
   test.afterAll(async () => {

packages/e2e-tests/tests/queries.spec.ts

@@ -11,10 +11,7 @@ test.describe("Query tests", () => {
     await app.createPool([
       path.normalize(path.join(testDataDir(), "sample.tsv")),
     ])
-    await app.mainWin
-      .locator('#app-root button:above(:text("Query Pool"))')
-      .first()
-      .click()
+    await app.mainWin.getByRole("button", {name: "Query Pool"}).click()
   })
 
   test.afterAll(async () => {
@@ -38,27 +35,28 @@ test.describe("Query tests", () => {
   })
 
   test("named queries' creation, modification, update/save, proper outdated status display", async () => {
+    const titleBar = app.mainWin.getByTestId("title-bar")
     // creation
-    await app.mainWin.locator('button :text-is("Save")').click()
-    await app.mainWin
+    await titleBar.getByRole("button", {name: "Save"}).click()
+    await titleBar
       .locator('[placeholder="Query name\\.\\.\\."]')
       .fill("Test Query Name")
-    await app.mainWin.locator('button :text-is("Create")').click()
+    await titleBar.getByRole("button", {name: "Create"}).click()
     await expect(await app.mainWin.locator("_react=HeadingSaved")).toBeVisible()
     await expect(
-      await app.mainWin.locator('button :text-is("Test Query Name")')
+      await titleBar.getByRole("button", {name: "Test Query Name"})
     ).toBeVisible()
 
     // modification
     await app.query("4")
     await expect(
-      await app.mainWin.locator('button :text-is("Test Query Name *")')
+      await titleBar.getByRole("button", {name: "Test Query Name *"})
     ).toBeVisible()
 
     // update
-    await app.mainWin.locator('button :text-is("Save")').click()
+    await titleBar.getByRole("button", {name: "Save"}).click()
     await expect(
-      await app.mainWin.locator('button :text-is("Test Query Name")')
+      await titleBar.getByRole("button", {name: "Test Query Name"})
     ).toBeVisible()
 
     // outdated display
@@ -71,14 +69,15 @@ test.describe("Query tests", () => {
   })
 
   test("named query, save as => new named query", async () => {
-    await app.mainWin.locator('button :text-is("Save As")').click()
-    await app.mainWin
+    const titleBar = app.mainWin.getByTestId("title-bar")
+    await titleBar.getByRole("button", {name: "Save As"}).click()
+    await titleBar
       .locator('[placeholder="Query name\\.\\.\\."]')
       .fill("Another Test Query")
     await app.mainWin
       .locator('div[role="textbox"]')
       .fill("another test query zed value")
-    await app.mainWin.locator('button :text-is("Create")').click()
+    await titleBar.getByRole("button", {name: "Create"}).click()
     await expect(
       await app.mainWin.locator('button :text-is("Another Test Query")')
     ).toBeVisible()

packages/zealot/src/index.ts

@@ -7,7 +7,7 @@ import {ResultStream} from "./query/result-stream"
 import {DefaultContext, TypeDefs} from "./zed/context"
 import * as zed from "./zed/index"
 import * as zjson from "./zjson"
-
+export * from "./test/factory"
 export * from "./client/types"
 export * from "./types"
 export {

packages/zealot/src/query/channel.ts

@@ -17,7 +17,7 @@ export class Channel extends EventEmitter {
     this.emit("row", row)
   }
 
-  addShape(id: number, type: zed.Type) {
+  addShape(id: number | string, type: zed.Type) {
     this.shapesMap[id] = type
     this.emit("shape", type)
   }
@@ -34,6 +34,8 @@ export class Channel extends EventEmitter {
     const value = this.stream.decode(json)
     if ("id" in json.type && !this.hasShape(json.type.id)) {
       this.addShape(json.type.id, value.type)
+    } else if (json.type.kind === "primitive") {
+      this.addShape(json.type.name, value.type)
     }
     this.addRow(value)
   }

packages/zealot/src/query/result-stream.ts

@@ -94,7 +94,6 @@ export class ResultStream {
         }
       }
     })
-    return this._promise
   }
 
   private consumeLine(json: zjson.QueryObject) {

packages/zealot/src/test/factory.ts

@@ -10,10 +10,7 @@ export function createRecord(object: {[key: string]: unknown}): zed.Record {
   for (let name in object) {
     fields.push(createField(name, object[name]))
   }
-  const typeFields: zed.TypeField[] = fields.map((f) => ({
-    name: f.name,
-    type: f.value.type,
-  }))
+  const typeFields = fields.map((f) => new zed.TypeField(f.name, f.value.type))
 
   // This could be more efficient
   const type: TypeRecord = DefaultContext.lookupTypeRecord(typeFields)
@@ -28,7 +25,7 @@ export function createField(name: string, value: unknown): zed.Field {
 }
 
 export function createData(value: unknown): zed.Value {
-  if (value instanceof zed.Primitive) {
+  if (zed.isValue(value)) {
     return value as zed.Value
   }
 
@@ -56,7 +53,7 @@ export function createData(value: unknown): zed.Value {
     return new zed.String(value)
   }
 
-  if (isObject(value)) {
+  if (isObject(value) && value?.constructor === Object) {
     return createRecord(value as {[k: string]: unknown})
   }
 

packages/zealot/src/zed/context.ts

@@ -6,12 +6,13 @@ import {TypeAlias} from "./types/type-alias"
 import {TypeArray} from "./types/type-array"
 import {TypeError} from "./types/type-error"
 import {TypeMap} from "./types/type-map"
-import {TypeField, TypeRecord} from "./types/type-record"
+import {TypeRecord} from "./types/type-record"
 import {TypeSet} from "./types/type-set"
 import {TypeUnion} from "./types/type-union"
 import {Type} from "./types/types"
 import {Field} from "./values/field"
 import {Value} from "./values/types"
+import {TypeField} from "./types/type-field"
 
 export type TypeDefs = {[key: string]: Type}
 
@@ -69,7 +70,9 @@ export class ZedContext {
   lookupTypeRecord(fields: TypeField[] | null): TypeRecord {
     const key = TypeRecord.stringify(fields)
     if (key in this.typeByShape) {
-      return this.typeByShape[key] as TypeRecord
+      const record = this.typeByShape[key] as TypeRecord
+      record.fields = fields
+      return record
     } else {
       return this.alloc(key, new TypeRecord(fields))
     }

packages/zealot/src/zed/decode-stream.ts

@@ -1,6 +1,7 @@
 import {isNull} from "lodash"
 import * as zjson from "../zjson"
 import {TypeDefs, ZedContext} from "./context"
+import {TypeField} from "./types/type-field"
 import {PrimitiveName} from "./types/type-primitives"
 import {Type} from "./types/types"
 import {getPrimitiveType} from "./utils/get-primitive-type"
@@ -50,10 +51,9 @@ export class DecodeStream {
         return this.context.lookupTypeRecord(
           isNull(obj.fields)
             ? null
-            : obj.fields.map(({name, type}) => ({
-                name,
-                type: this.decodeType(type),
-              }))
+            : obj.fields.map(({name, type}) => {
+                return new TypeField(name, this.decodeType(type))
+              })
         )
       default:
         throw `Implement decoding: ${obj.kind}`

packages/zealot/src/zed/index.ts

@@ -1,6 +1,6 @@
 import {Type} from "./types/types"
 import {Value} from "./values/types"
-export type Any = Type | Value
+
 export {ZedContext as Context} from "./context"
 export {TypeAlias} from "./types/type-alias"
 export {TypeArray} from "./types/type-array"
@@ -18,7 +18,8 @@ export {TypeIp} from "./types/type-ip"
 export {TypeMap} from "./types/type-map"
 export {TypeNet} from "./types/type-net"
 export {TypeNull} from "./types/type-null"
-export {TypeRecord, TypeField} from "./types/type-record"
+export {TypeRecord, FieldData} from "./types/type-record"
+export {TypeField} from "./types/type-field"
 export {TypeSet} from "./types/type-set"
 export {TypeString} from "./types/type-string"
 export {TypeTime} from "./types/type-time"
@@ -76,3 +77,7 @@ export * from "./utils/is-string"
 export * from "./utils/is-time"
 export * from "./utils/is-type-alias"
 export * from "./utils/true-type"
+export * from "./utils/base-value"
+export * from "./utils/is-container"
+export * from "./utils/is-value"
+export type Any = Type | Value

packages/zealot/src/zed/types/type-field.ts

@@ -0,0 +1,5 @@
+import {Type} from "./types"
+
+export class TypeField {
+  constructor(public name: string, public type: Type) {}
+}

packages/zealot/src/zed/types/type-record.ts

@@ -6,13 +6,10 @@ import {trueType} from "../utils/true-type"
 import {Field} from "../values/field"
 import {Null} from "../values/null"
 import {Record} from "../values/record"
+import {TypeField} from "./type-field"
 import {Type} from "./types"
 
-export type TypeField = {
-  name: string
-  type: Type
-}
-
+export type FieldData = {name: string; type: Type}
 export class TypeRecord implements Type {
   kind = "record"
 
@@ -22,7 +19,7 @@ export class TypeRecord implements Type {
     return !!this.fields?.find((f) => f.name === name)
   }
 
-  static stringify(fields: TypeField[] | null) {
+  static stringify(fields: FieldData[] | null) {
     if (isNull(fields)) return "null"
     let s = "{"
     let sep = ""

packages/zealot/src/zed/types/types.ts

@@ -28,4 +28,5 @@ export interface Type {
   toString(): string
   serialize(stream: EncodeStream): zjson.NoId<zjson.Type> | zjson.PrimitiveType
   create(value: zjson.Value, stream: DecodeStream, parent?: Field): Value
+  kind: string
 }

packages/zealot/src/zed/utils/base-value.ts

@@ -0,0 +1,9 @@
+import {Any} from ".."
+import {Union} from "../values/union"
+
+export function baseValue(value: Any | null): Any | null {
+  if (value instanceof Union) {
+    return baseValue(value.value)
+  }
+  return value
+}

packages/zealot/src/zed/utils/is-container.ts

@@ -0,0 +1,15 @@
+import {Record} from "../values/record"
+import {Array} from "../values/array"
+import {ZedMap} from "../values/map"
+import {Set} from "../values/set"
+import {Union} from "../values/union"
+import {Error} from "../values/error"
+
+const containers = [Record, Array, Set, Union, ZedMap, Error]
+
+export function isContainer(value: unknown) {
+  for (let name of containers) {
+    if (value instanceof name) return true
+  }
+  return false
+}

packages/zealot/src/zed/utils/is-type-alias.ts

@@ -1,6 +1,5 @@
 import {TypeAlias} from "../types/type-alias"
-import {Type} from "../types/types"
 
-export function isTypeAlias(type: Type): type is TypeAlias {
+export function isTypeAlias(type: unknown): type is TypeAlias {
   return type instanceof TypeAlias
 }

packages/zealot/src/zed/utils/is-value.ts

@@ -0,0 +1,6 @@
+import {isContainer} from "./is-container"
+import {isPrimitive} from "./is-primitive"
+
+export function isValue(value: unknown) {
+  return isPrimitive(value) || isContainer(value)
+}

packages/zealot/src/zed/utils/true-type.ts

@@ -1,10 +1,9 @@
-import {Type} from "../types/types"
 import {isTypeAlias} from "./is-type-alias"
 
-export function trueType<T extends Type>(start: Type): T {
+export function trueType(start: unknown) {
   let t = start
   while (isTypeAlias(t)) {
     t = t.type
   }
-  return t as T
+  return t
 }

packages/zealot/src/zed/values/field.ts

@@ -1,3 +1,4 @@
+import {zed} from "../.."
 import {Record} from "./record"
 import {Value} from "./types"
 
@@ -33,4 +34,8 @@ export class Field {
     if (parent instanceof Record) return parent
     else return null
   }
+
+  get baseValue() {
+    return zed.baseValue(this.value)
+  }
 }