Histogram is Back in Zui (#2472)

The Histogram is Back in Zui

In an effort to ship sooner, I've dropped some of my original requirements. The histogram can be shown and hidden manually for each tab using "App Menu" => "View" => "Show Histogram". The histogram will also be hidden if we can't find a "time" range for the pool. Other than that, we don't try to determine if a pool is a zeek pool or not. If a user want's non-zeek data and are annoyed with the histogram, they can hide it.

Brushing the histogram will never update the Zed text directly, it will only update the time range pin. Updating the Zed is prone to errors as we do not have a robust way to do it. This is a restraint that simplifies the app by making it predictable.

This histogram is just like the old one. It counts by the "ts" time field, and the string "_path" field.

package.json

@@ -100,7 +100,8 @@
     "typescript": "^4.6.2",
     "web-streams-polyfill": "^3.2.0",
     "whatwg-fetch": "^3.2.0",
-    "win-7zip": "^0.1.0"
+    "win-7zip": "^0.1.0",
+    "zui-test-data": "workspace:*"
   },
   "dependencies": {
     "@babel/core": "^7.17.9",

packages/e2e-tests/helpers/test-app.ts

@@ -22,6 +22,10 @@ export default class TestApp {
     this.zealot = new Client("http://localhost:9867")
   }
 
+  find(text: string) {
+    return this.mainWin.locator(text)
+  }
+
   async init() {
     const userDataDir = path.resolve(
       path.join(itestDir(), this.name, (this.testNdx++).toString())

packages/e2e-tests/tests/histogram.spec.ts

@@ -0,0 +1,37 @@
+import {test, expect} from "@playwright/test"
+import TestApp from "../helpers/test-app"
+import {getPath} from "zui-test-data"
+
+test.describe("Histogram Spec", () => {
+  const app = new TestApp("Histogram Spec")
+
+  test.beforeAll(async () => {
+    await app.init()
+  })
+
+  test.afterAll(async () => {
+    await app.shutdown()
+  })
+
+  test("Histogram appears for zeek data", async () => {
+    await app.createPool([getPath("small-zeek.zng")])
+    await app.find(`role=button[name="Query Pool"]`).click()
+
+    const results = app.find(`role=status[name="results"]`)
+    await expect(results).toHaveText(/Results:/)
+
+    const chart = app.find(`[aria-label="histogram"]`)
+    await expect(chart).toBeVisible()
+  })
+
+  test("Histogram does not appears for non-zeek data", async () => {
+    await app.createPool([getPath("prs.json")])
+    await app.find(`role=button[name="Query Pool"]`).click()
+
+    const results = app.find(`role=status[name="results"]`)
+    await expect(results).toHaveText(/Results:/)
+
+    const chart = app.find(`[aria-label="histogram"]`)
+    await expect(chart).toBeHidden()
+  })
+})

packages/zealot/src/client/client.test.ts

@@ -59,9 +59,7 @@ test("#query collector", async () => {
 
   const fn = jest.fn()
   await resp.collect(fn)
-  // It calls when the first 30 are returned, then when they
-  // are all returned
-  expect(fn).toHaveBeenCalledTimes(2)
+  expect(fn).toHaveBeenCalledTimes(1)
 })
 
 test("curl", () => {

packages/zealot/src/query/channel.ts

@@ -47,8 +47,8 @@ export class Channel extends EventEmitter {
      */
     let first = true
     let count = 0
-    let countThresh = 30
-    let timeThresh = 500
+    let countThresh = 2000
+    let timeThresh = 2000
     let timeId = 0
 
     const flush = () => {

packages/zui-test-data/data/brimcap-queries.json

@@ -0,0 +1 @@
+{ "name": "Brimcap", "items": [ { "name": "Activity Overview", "value": "count() by _path | sort -r", "description": "Shows a list of all Zeek streams in the data set, with a count of associated records" }, { "name": "Unique DNS Queries", "value": "_path==\"dns\" | count() by query | sort -r", "description": "Shows all unique DNS queries in the data set with count" }, { "name": "Windows Networking Activity", "value": "grep(smb*,_path) OR _path==\"dce_rpc\"", "description": "Filters and displays smb_files, smb_mapping and DCE_RPC activity" }, { "name": "HTTP Requests", "value": "_path==\"http\" | cut id.orig_h, id.resp_h, id.resp_p, method, host, uri | uniq -c", "description": "Displays a list of the count of unique HTTP requests including source and destination" }, { "name": "Unique Network Connections", "value": "_path==\"conn\" | cut id.orig_h, id.resp_p, id.resp_h | sort | uniq", "description": "Displays a table showing all unique source:port:destination connections pairings" }, { "name": "Connection Received Data", "value": "_path==\"conn\" | put total_bytes := orig_bytes + resp_bytes | sort -r total_bytes | cut uid, id, orig_bytes, resp_bytes, total_bytes", "description": "Shows the connections between hosts, sorted by data received" }, { "name": "File Activity", "value": "filename!=null | cut _path, tx_hosts, rx_hosts, conn_uids, mime_type, filename, md5, sha1", "description": "Displays a curated view of file data including md5 and sha1 for complete file transfers" }, { "name": "HTTP Post Requests", "value": "method==\"POST\" | cut ts, uid, id, method, uri, status_code", "description": "Displays all HTTP Post requests including the URI and HTTP status code" }, { "name": "Show IP Subnets", "value": "_path==\"conn\" | put classnet := network_of(id.resp_h) | cut classnet | count() by classnet | sort -r", "description": "Enumerates the classful networks for all destination IP addresses including count of connections" }, { "name": "Suricata Alerts by Category", "value": "event_type==\"alert\" | count() by alert.severity,alert.category | sort count", "description": "Shows all Suricata alert counts by category and severity" }, { "name": "Suricata Alerts by Source and Destination", "value": "event_type==\"alert\" | alerts := union(alert.category) by src_ip, dest_ip", "description": "Shows all Suricata alerts in a list by unique source and destination IP addresses" }, { "name": "Suricata Alerts by Subnet", "value": "event_type==\"alert\" | alerts := union(alert.category) by network_of(dest_ip)", "description": "Displays a list of Suricata alerts by CIDR network" } ] }

packages/zui-test-data/index.js

@@ -0,0 +1,11 @@
+const path = require("path")
+
+module.exports = {
+  /**
+   * @param {string} name
+   * @returns string
+   */
+  getPath(name) {
+    return path.join(__dirname, "data", name)
+  },
+}

packages/zui-test-data/package.json

@@ -0,0 +1,5 @@
+{
+  "version": "1.0.0",
+  "name": "zui-test-data",
+  "main": "index.js"
+}

src/app/commands/command.ts

@@ -0,0 +1,58 @@
+import BrimApi from "src/js/api"
+import {Dispatch, GetState, Store} from "src/js/state/types"
+
+type CommandMeta = {
+  id: string
+}
+
+type CommandContext = {
+  dispatch: Dispatch
+  getState: GetState
+  api: BrimApi
+}
+
+type CommandExecutor = (context: CommandContext) => void
+
+export class Commands {
+  private map = new Map<string, Command>()
+  private store: Store | null
+  private api: BrimApi | null
+
+  add(command: Command) {
+    this.map.set(command.id, command)
+    return command
+  }
+
+  get context() {
+    if (!this.store || !this.api)
+      throw new Error("Must set command context before accessing")
+    return {
+      dispatch: this.store.dispatch,
+      getState: this.store.getState,
+      api: this.api,
+    }
+  }
+
+  setContext(store: Store, api: BrimApi) {
+    this.store = store
+    this.api = api
+  }
+}
+
+export const commands = new Commands()
+
+export class Command {
+  constructor(private meta: CommandMeta, private exec: CommandExecutor) {}
+
+  get id() {
+    return this.meta.id
+  }
+
+  run() {
+    return this.exec(commands.context)
+  }
+}
+
+export const createCommand = (meta: CommandMeta, exec: CommandExecutor) => {
+  return commands.add(new Command(meta, exec))
+}

src/app/commands/pins.ts

@@ -0,0 +1,29 @@
+import Editor from "src/js/state/Editor"
+import Pools from "src/js/state/Pools"
+import {createCommand} from "./command"
+
+export const createTimeRange = createCommand(
+  {id: "pins.createTimeRange"},
+  async ({dispatch, api, getState}) => {
+    const pins = Editor.getPins(getState())
+    const range = await dispatch(Pools.getTimeRange(api.current.poolName))
+    const now = new Date()
+    const defaultFrom = new Date(
+      Date.UTC(now.getFullYear(), now.getMonth(), now.getDate(), 0, 0, 0, 0)
+    )
+    const defaultTo = new Date(
+      Date.UTC(now.getFullYear(), now.getMonth(), now.getDate() + 1, 0, 0, 0, 0)
+    )
+    const from = (range && range[0]) || defaultFrom
+    const to = (range && range[1]) || defaultTo
+    dispatch(
+      Editor.addPin({
+        type: "time-range",
+        field: "ts",
+        from: from.toISOString(),
+        to: to.toISOString(),
+      })
+    )
+    dispatch(Editor.editPin(pins.length))
+  }
+)

src/app/query-home/title-bar/active-query.ts → src/app/core/models/active-query.ts

@@ -1,19 +1,20 @@
+import {ZedAst} from "src/app/core/models/zed-ast"
+import {BrimQuery} from "src/app/query-home/utils/brim-query"
 import {QueryVersion} from "src/js/state/QueryVersions/types"
-import {BrimQuery} from "../utils/brim-query"
 
 export class ActiveQuery {
   constructor(
     public session: BrimQuery, // the singleton for the tab
     public query: BrimQuery | null, // the query from the url param
-    public version: QueryVersion | null // the version from the url param
+    public version: QueryVersion // the version from the url param
   ) {}
 
   id() {
     return this.query?.id || this.session.id
   }
 
   versionId() {
-    return this.version?.version || null
+    return this.version.version || null
   }
 
   isDeleted() {
@@ -43,6 +44,10 @@ export class ActiveQuery {
     return !this.isAnonymous() && !this.isModified() && !this.isLatest()
   }
 
+  isReadOnly() {
+    return this.isSaved() && !!this.query.isReadOnly
+  }
+
   name() {
     if (this.isAnonymous()) return ""
     return this.query.name
@@ -55,4 +60,12 @@ export class ActiveQuery {
   ts() {
     return this.version.ts
   }
+
+  toZed() {
+    return BrimQuery.versionToZed(this.version)
+  }
+
+  toAst() {
+    return new ZedAst(BrimQuery.versionToZed(this.version))
+  }
 }

src/app/core/models/zed-ast.ts

@@ -0,0 +1,52 @@
+import * as zealot from "@brimdata/zealot"
+
+export class ZedAst {
+  public tree: any
+
+  constructor(public script: string) {
+    this.tree = zealot.parseAst(script)
+  }
+
+  get poolName() {
+    const from = this.from
+    if (!from) return null
+    const trunk = from.trunks.find((t) => t.source.kind === "Pool")
+    if (!trunk) return null
+    const name = trunk.source.spec.pool
+    if (!name) return null
+    return name
+  }
+
+  get from() {
+    return this.ops.find((o) => o.kind === "From")
+  }
+
+  get pools() {
+    const trunks = this.from?.trunks || []
+    return trunks.filter((t) => t.source.kind === "Pool").map((t) => t.source)
+  }
+
+  private _ops: any[]
+  get ops() {
+    if (this._ops) return this._ops
+    if (!this.tree || this.tree.error) return []
+    const list = []
+
+    function collectOps(op, list) {
+      list.push(op)
+      if (COMPOUND_PROCS.includes(op.kind)) {
+        for (const p of op.ops) collectOps(p, list)
+      } else if (op.kind === OP_EXPR_PROC) {
+        collectOps(op.expr, list)
+      }
+    }
+
+    collectOps(this.tree, list)
+    return (this._ops = list)
+  }
+}
+
+export const OP_EXPR_PROC = "OpExpr"
+export const PARALLEL_PROC = "Parallel"
+export const SEQUENTIAL_PROC = "Sequential"
+export const COMPOUND_PROCS = [PARALLEL_PROC, SEQUENTIAL_PROC]

src/app/core/models/zed-script.test.ts

@@ -0,0 +1,30 @@
+import {ZedScript} from "./zed-script"
+
+test("time range", () => {
+  const script = new ZedScript(
+    "from sample.pcap range 2022-01-01T00:00:00Z to 2022-02-01T00:00:00Z"
+  )
+
+  expect(script.range).toEqual([
+    new Date(Date.UTC(2022, 0, 1, 0, 0, 0)),
+    new Date(Date.UTC(2022, 1, 1, 0, 0, 0)),
+  ])
+})
+
+test("number range range", () => {
+  const script = new ZedScript("from sample.pcap range 0 to 100")
+
+  expect(script.range).toEqual([0, 100])
+})
+
+test("no range", () => {
+  const script = new ZedScript("from sample.pcap")
+
+  expect(script.range).toEqual(null)
+})
+
+test("no pool", () => {
+  const script = new ZedScript("hello world")
+
+  expect(script.range).toEqual(null)
+})

src/app/core/models/zed-script.ts

@@ -0,0 +1,28 @@
+import {ZedAst} from "./zed-ast"
+
+export class ZedScript {
+  constructor(public script: string) {}
+
+  private _ast: ZedAst
+  get ast() {
+    return this._ast || (this._ast = new ZedAst(this.script))
+  }
+
+  get range() {
+    const pool = this.ast.pools[0]
+    if (!pool) return null
+    const range = pool.range
+    if (!range) return null
+
+    return [parseRangeItem(range.lower), parseRangeItem(range.upper)]
+  }
+}
+
+function parseRangeItem({type, text}) {
+  switch (type) {
+    case "int64":
+      return parseInt(text)
+    case "time":
+      return new Date(text)
+  }
+}

src/app/core/pools/pool.ts

@@ -59,7 +59,7 @@ export class Pool {
     if (!this.stats) throw new Error("Pool has no stats")
     if (!this.stats.span) throw new Error("Pool has no span")
     const date = new Date(
-      this.minTime().getTime() + Math.ceil(this.stats.span.dur / 1e6)
+      this.minTime().getTime() + Math.ceil(this.stats.span.dur / 1e6) + 1
     )
     if (isNaN(date.getTime())) throw new Error("Invalid Date")
     return date

src/app/features/right-pane/history/history-item.tsx

@@ -1,13 +1,15 @@
 import {formatDistanceToNowStrict} from "date-fns"
-import React from "react"
+import React, {useMemo} from "react"
 import {useSelector} from "react-redux"
 import {useBrimApi} from "src/app/core/context"
-import {ActiveQuery} from "src/app/query-home/title-bar/active-query"
 import Current from "src/js/state/Current"
+import Queries from "src/js/state/Queries"
 import QueryVersions from "src/js/state/QueryVersions"
 import styled from "styled-components"
 import {Timeline} from "./timeline"
 import {useEntryMenu} from "./use-entry-menu"
+import {State} from "src/js/state/types"
+import {ActiveQuery} from "src/app/core/models/active-query"
 
 const Wrap = styled.div`
   height: 28px;
@@ -112,10 +114,15 @@ export function HistoryItem({version, queryId, index}: Props) {
   const api = useBrimApi()
   const onContextMenu = useEntryMenu(index)
   const sessionId = useSelector(Current.getSessionId)
-  const session = useSelector(Current.getQueryById(sessionId))
-  const query = useSelector(Current.getQueryById(queryId))
-  const sVersion = useSelector(QueryVersions.getByVersion(sessionId, version))
-  const qVersion = useSelector(QueryVersions.getByVersion(queryId, version))
+  const session = useSelector(Current.getSession)
+  const build = useMemo(Queries.makeBuildSelector, [])
+  const query = useSelector((state: State) => build(state, queryId))
+  const sVersion = useSelector((state) =>
+    QueryVersions.at(sessionId).find(state, version)
+  )
+  const qVersion = useSelector((state) =>
+    QueryVersions.at(queryId).find(state, version)
+  )
   const versionObj = sVersion || qVersion
   const active = new ActiveQuery(session, query, versionObj)
   const onClick = () => {