From dc95f8912570d371ca7bc0e05d9e4714172e59e5 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Wed, 3 Aug 2022 16:47:14 -0700 Subject: [PATCH] Run the correct histogram query --- src/app/core/models/zed-ast.ts | 41 ++++++++++++ src/app/core/models/zed-script.test.ts | 30 +++++++++ src/app/core/models/zed-script.ts | 28 ++++++++ .../histogram/MainHistogram/Chart.tsx | 36 +++++++---- .../MainHistogram/useMainHistogram.tsx | 57 +++++++++++++++-- .../histogram/build-histogram-query.ts | 64 +++++++++++++++++++ .../histogram/run-histogram-query.ts | 39 +++++++++++ src/app/query-home/loader.ts | 12 ++-- src/app/query-home/search-area/Input.tsx | 2 +- .../query-home/toolbar/results-toolbar.tsx | 2 +- src/js/lib/histogramInterval.ts | 10 +++ 11 files changed, 294 insertions(+), 27 deletions(-) create mode 100644 src/app/core/models/zed-ast.ts create mode 100644 src/app/core/models/zed-script.test.ts create mode 100644 src/app/core/models/zed-script.ts create mode 100644 src/app/query-home/histogram/build-histogram-query.ts create mode 100644 src/app/query-home/histogram/run-histogram-query.ts diff --git a/src/app/core/models/zed-ast.ts b/src/app/core/models/zed-ast.ts new file mode 100644 index 0000000000..203b911087 --- /dev/null +++ b/src/app/core/models/zed-ast.ts @@ -0,0 +1,41 @@ +import * as zealot from "@brimdata/zealot" + +export class ZedAst { + public tree: any + constructor(public script: string) { + this.tree = zealot.parseAst(script) + } + + get from() { + return this.ops.find((o) => o.kind === "From") + } + + get pools() { + const trunks = this.from?.trunks || [] + return trunks.filter((t) => t.source.kind === "Pool").map((t) => t.source) + } + + private _ops: any[] + get ops() { + if (this._ops) return this._ops + if (!this.tree || this.tree.error) return [] + const list = [] + + function collectOps(op, list) { + list.push(op) + if (COMPOUND_PROCS.includes(op.kind)) { + for (const p of op.ops) collectOps(p, list) + } else if (op.kind === OP_EXPR_PROC) { + collectOps(op.expr, list) + } + } + + collectOps(this.tree, list) + return (this._ops = list) + } +} + +export const OP_EXPR_PROC = "OpExpr" +export const PARALLEL_PROC = "Parallel" +export const SEQUENTIAL_PROC = "Sequential" +export const COMPOUND_PROCS = [PARALLEL_PROC, SEQUENTIAL_PROC] diff --git a/src/app/core/models/zed-script.test.ts b/src/app/core/models/zed-script.test.ts new file mode 100644 index 0000000000..386ac68324 --- /dev/null +++ b/src/app/core/models/zed-script.test.ts @@ -0,0 +1,30 @@ +import {ZedScript} from "./zed-script" + +test("time range", () => { + const script = new ZedScript( + "from sample.pcap range 2022-01-01T00:00:00Z to 2022-02-01T00:00:00Z" + ) + + expect(script.range).toEqual([ + new Date(Date.UTC(2022, 0, 1, 0, 0, 0)), + new Date(Date.UTC(2022, 1, 1, 0, 0, 0)), + ]) +}) + +test("number range range", () => { + const script = new ZedScript("from sample.pcap range 0 to 100") + + expect(script.range).toEqual([0, 100]) +}) + +test("no range", () => { + const script = new ZedScript("from sample.pcap") + + expect(script.range).toEqual(null) +}) + +test("no pool", () => { + const script = new ZedScript("hello world") + + expect(script.range).toEqual(null) +}) diff --git a/src/app/core/models/zed-script.ts b/src/app/core/models/zed-script.ts new file mode 100644 index 0000000000..93fae3dac9 --- /dev/null +++ b/src/app/core/models/zed-script.ts @@ -0,0 +1,28 @@ +import {ZedAst} from "./zed-ast" + +export class ZedScript { + constructor(public script: string) {} + + private _ast: ZedAst + get ast() { + return this._ast || (this._ast = new ZedAst(this.script)) + } + + get range() { + const pool = this.ast.pools[0] + if (!pool) return null + const range = pool.range + if (!range) return null + + return [parseRangeItem(range.lower), parseRangeItem(range.upper)] + } +} + +function parseRangeItem({type, text}) { + switch (type) { + case "int64": + return parseInt(text) + case "time": + return new Date(text) + } +} diff --git a/src/app/query-home/histogram/MainHistogram/Chart.tsx b/src/app/query-home/histogram/MainHistogram/Chart.tsx index 7c8c6520ad..79d035a287 100644 --- a/src/app/query-home/histogram/MainHistogram/Chart.tsx +++ b/src/app/query-home/histogram/MainHistogram/Chart.tsx @@ -1,27 +1,39 @@ import React from "react" +import {useSelector} from "react-redux" +import {ZedScript} from "src/app/core/models/zed-script" import Dimens from "src/js/components/Dimens" +import Results from "src/js/state/Results" +import styled from "styled-components" import ChartSVG from "../ChartSVG" +import {HISTOGRAM_RESULTS} from "../run-histogram-query" import useMainHistogram from "./useMainHistogram" +const BG = styled.div` + height: 80px; + margin: 24px 16px 16px 16px; +` + export default function MainHistogramChart() { + const query = useSelector(Results.getQuery(HISTOGRAM_RESULTS)) + const range = new ZedScript(query).range + if (!range) return null return ( - ( - - )} - /> + + ( + + )} + /> + ) } type Props = {height: number; width: number} -const MainHistogramSvg = React.memo(function MainHistogramSvg({ - width, - height, -}: Props) { +function MainHistogramSvg({width, height}: Props) { const chart = useMainHistogram(width, height) return -}) +} diff --git a/src/app/query-home/histogram/MainHistogram/useMainHistogram.tsx b/src/app/query-home/histogram/MainHistogram/useMainHistogram.tsx index 4d8019c98c..16616dbb9b 100644 --- a/src/app/query-home/histogram/MainHistogram/useMainHistogram.tsx +++ b/src/app/query-home/histogram/MainHistogram/useMainHistogram.tsx @@ -6,7 +6,6 @@ import {DateTuple} from "src/js/lib/TimeWindow" import {Pen, HistogramChart} from "../types" import {innerHeight, innerWidth} from "../dimens" -import Chart from "src/js/state/Chart" import EmptyMessage from "src/js/components/EmptyMessage" import HistogramTooltip from "src/js/components/HistogramTooltip" import LoadingMessage from "src/js/components/LoadingMessage" @@ -22,15 +21,30 @@ import xAxisTime from "../pens/xAxisTime" import xPositionTooltip from "../pens/xPositionTooltip" import yAxisSingleTick from "../pens/yAxisSingleTick" import submitSearch from "../../flows/submit-search" +import Results from "src/js/state/Results" +import {HISTOGRAM_RESULTS} from "../run-histogram-query" +import {ChartData} from "src/js/state/Chart/types" +import {zed} from "packages/zealot/src" +import UniqArray from "src/js/models/UniqArray" +import MergeHash from "src/js/models/MergeHash" +import {ZedScript} from "src/app/core/models/zed-script" + +const id = HISTOGRAM_RESULTS + +// get pool +// make a new brim query with the values, +// get the pool name +// get the pool +// get the full pool range export default function useMainHistogram( width: number, height: number ): HistogramChart { - const chartData = useSelector(Chart.getData) - const status = useSelector(Chart.getStatus) - const span = [new Date(0), new Date()] //useSelector(tab.getSpanAsDates) - + const chartData = useSelector(Results.getValues(id)) as zed.Record[] + const status = useSelector(Results.getStatus(id)) + const query = useSelector(Results.getQuery(id)) + const range = new ZedScript(query).range const dispatch = useDispatch() const pens = useConst([], () => { function onDragEnd(span: DateTuple) { @@ -63,7 +77,7 @@ export default function useMainHistogram( }) return useMemo(() => { - const data = format(chartData, span) + const data = format(histogramFormat(chartData), range) const maxY = d3.max(data.points, (d: {count: number}) => d.count) || 0 const oneCharWidth = 5.5366666667 const chars = d3.format(",")(maxY).length @@ -96,5 +110,34 @@ export default function useMainHistogram( .domain(data.span), pens, } - }, [chartData, status, span, width, height]) + }, [chartData, status, range, width, height]) +} + +function histogramFormat(records: zed.Record[]): ChartData { + const paths = new UniqArray() + const table = new MergeHash() + + records.forEach((r) => { + const [ts, path, count] = r.fields.map((f) => f.data) as [ + zed.Time, + zed.String, + zed.Uint64 + ] + + try { + const pathName = path.toString() + const key = ts.toDate().getTime() + const val = {[path.toString()]: count.toInt()} + + table.merge(key, val) + paths.push(pathName) + } catch (e) { + console.log("Error rendering histogram: " + e.toString()) + } + }) + + return { + table: table.toJSON(), + keys: paths.toArray(), + } } diff --git a/src/app/query-home/histogram/build-histogram-query.ts b/src/app/query-home/histogram/build-histogram-query.ts new file mode 100644 index 0000000000..4be9af94b6 --- /dev/null +++ b/src/app/query-home/histogram/build-histogram-query.ts @@ -0,0 +1,64 @@ +import {ZedScript} from "src/app/core/models/zed-script" +import {Pool} from "src/app/core/pools/pool" +import {syncPool} from "src/app/core/pools/sync-pool" +import span from "src/js/brim/span" +import histogramInterval, {timeUnits} from "src/js/lib/histogramInterval" +import {DateTuple} from "src/js/lib/TimeWindow" +import Current from "src/js/state/Current" +import Editor from "src/js/state/Editor" +import Pools from "src/js/state/Pools" +import {Thunk} from "src/js/state/types" +import zql from "src/js/zql" +import {BrimQuery} from "../utils/brim-query" + +export const buildHistogramQuery = + (): Thunk> => + async (dispatch, getState, {api}) => { + const poolName = api.current.poolName + const range = await dispatch(getRange(poolName)) + console.log(range) + return histogramZed(poolName, range) + } + +export const getRange = + (name: string): Thunk | DateTuple> => + (dispatch) => { + const queryRange = dispatch(getRangeFromQuery()) + if (queryRange) return queryRange + else return dispatch(getRangeFromPool(name)) + } + +function histogramZed(pool: string, range: DateTuple | null) { + if (!range) return null + const {number, unit} = histogramInterval(range) + const interval = `${number}${timeUnits[unit]}` + return `from "${pool}" range ${zql`${range[0]}`} to ${zql`${range[1]}`} | count() by every(${interval}), _path` +} + +const getRangeFromPool = + (poolName: string): Thunk> => + async (dispatch) => { + if (!poolName) return null + const pool = await dispatch(ensurePoolLoaded(poolName)) + if (!pool) return + return span(pool.everythingSpan()).toDateTuple() + } + +const getRangeFromQuery = (): Thunk => (_, getState) => { + const snapshot = Editor.getSnapshot(getState()) + const inputs = new ZedScript(BrimQuery.versionToZed(snapshot)) + return inputs.range as DateTuple +} + +const ensurePoolLoaded = + (name: string): Thunk> => + (dispatch, getState) => { + const lakeId = Current.getLakeId(getState()) + const pool = Pools.getByName(lakeId, name)(getState()) + if (!pool) return Promise.resolve(null) + if (pool.hasStats()) { + return Promise.resolve(pool) + } else { + return dispatch(syncPool(pool.id, lakeId)) + } + } diff --git a/src/app/query-home/histogram/run-histogram-query.ts b/src/app/query-home/histogram/run-histogram-query.ts new file mode 100644 index 0000000000..77bc49a361 --- /dev/null +++ b/src/app/query-home/histogram/run-histogram-query.ts @@ -0,0 +1,39 @@ +import {Collector} from "@brimdata/zealot" +import Results from "src/js/state/Results" +import {Thunk} from "src/js/state/types" +import {buildHistogramQuery} from "./build-histogram-query" + +export const HISTOGRAM_RESULTS = "zui/histogram" +const id = HISTOGRAM_RESULTS + +// This is looking very similar to Results/flows.ts fetchResults() +// Maybe this can be part of the api. It automatically saves it to +// the results reducer, and it paginates the query for you? +export const runHistogramQuery = + (): Thunk => + async (dispatch, getState, {api}) => { + const tabId = api.current.tabId + const key = api.current.location.key + const query = await dispatch(buildHistogramQuery()) + if (!query) return + console.log(query) + dispatch(Results.init({id, tabId, query, key})) + const collect: Collector = ({rows, shapesMap}) => { + dispatch(Results.setValues({id, tabId, values: rows})) + dispatch(Results.setShapes({id, tabId, shapes: shapesMap})) + } + + try { + const res = await api.query(query, {tabId, id, collect}) + await res.promise + dispatch(Results.success({id, tabId, count: res.rows.length})) + } catch (error) { + if ( + error instanceof DOMException && + error.message.match(/user aborted/) + ) { + return + } + dispatch(Results.error({id, error, tabId})) + } + } diff --git a/src/app/query-home/loader.ts b/src/app/query-home/loader.ts index b287e6a9e5..fdcb348762 100644 --- a/src/app/query-home/loader.ts +++ b/src/app/query-home/loader.ts @@ -9,6 +9,7 @@ import Notice from "src/js/state/Notice" import Tabs from "src/js/state/Tabs" import {Thunk} from "src/js/state/types" import {Location} from "history" +import {runHistogramQuery} from "./histogram/run-histogram-query" export function loadRoute(location: Location): Thunk { return (dispatch) => { @@ -16,7 +17,7 @@ export function loadRoute(location: Location): Thunk { dispatch(Notice.dismiss()) dispatch(Results.error({id: MAIN_RESULTS, error: null, tabId: ""})) dispatch(syncEditor) - dispatch(fetchData(location)) + dispatch(fetchData()) } } @@ -34,16 +35,15 @@ function syncEditor(dispatch, getState) { }) } -function fetchData(location) { +function fetchData() { return (dispatch, getState) => { - const key = Results.getKey(MAIN_RESULTS)(getState()) const version = Current.getVersion(getState()) - if (key === location.key) return - startTransition(() => { - version && + if (version) { dispatch(Results.fetchFirstPage(BrimQuery.versionToZed(version))) + dispatch(runHistogramQuery()) + } }) } } diff --git a/src/app/query-home/search-area/Input.tsx b/src/app/query-home/search-area/Input.tsx index bb711deb90..280fb51fc3 100644 --- a/src/app/query-home/search-area/Input.tsx +++ b/src/app/query-home/search-area/Input.tsx @@ -28,7 +28,7 @@ const InputBackdrop = styled.div<{height: number}>` const Submit = styled(SubmitButton)` position: absolute; - right: 20px; + right: 16px; bottom: 10px; ` diff --git a/src/app/query-home/toolbar/results-toolbar.tsx b/src/app/query-home/toolbar/results-toolbar.tsx index a159802b09..0089074ed0 100644 --- a/src/app/query-home/toolbar/results-toolbar.tsx +++ b/src/app/query-home/toolbar/results-toolbar.tsx @@ -7,7 +7,7 @@ const BG = styled.section` display: flex; justify-content: space-between; align-items: flex-start; - padding: 6px 20px; + padding: 6px 16px; ` export function ResultsToolbar() { diff --git a/src/js/lib/histogramInterval.ts b/src/js/lib/histogramInterval.ts index 169fd0311c..b47b29ef98 100644 --- a/src/js/lib/histogramInterval.ts +++ b/src/js/lib/histogramInterval.ts @@ -3,6 +3,16 @@ import moment from "moment" import {DateTuple} from "../lib/TimeWindow" import {Interval} from "../types" +export const timeUnits = { + millisecond: "ms", + second: "s", + minute: "m", + hour: "h", + day: "d", + week: "w", + year: "y", +} + export default function histogramInterval([start, end]: DateTuple): Interval { const duration = moment.duration(moment(end).diff(moment(start)))