query annotation and test fixes (#1836)

* check for from before annotating, fix test date

Signed-off-by: Mason Fish <mason@brimsecurity.com>

* update test snapshots

Signed-off-by: Mason Fish <mason@brimsecurity.com>

* Update src/js/flows/search/mod.ts

Co-authored-by: Noah Treuhaft <noah.treuhaft@gmail.com>

Co-authored-by: Mason Fish <mason@brimsecurity.com>
Co-authored-by: Noah Treuhaft <noah.treuhaft@gmail.com>

app/search/flows/histogram-search.test.ts

@@ -40,7 +40,7 @@ test("zealot gets the request", async () => {
   const calls = zealot.calls("query")
   expect(calls.length).toBe(1)
   expect(calls[0].args).toEqual(
-    "from '1' | ts >= 2015-03-05T14:15:00Z | ts <= 2015-04-13T09:36:33.751Z | * | every 12h count() by _path"
+    "from '1' | ts >= 2015-03-05T14:15:00.000Z | ts <= 2015-04-13T09:36:33.751Z | * | every 12h count() by _path"
   )
 })
 

app/search/flows/viewer-search.test.ts

@@ -38,6 +38,27 @@ beforeEach(() => {
   store.dispatch(tabHistory.push(lakePath(pool.id, "1")))
 })
 
+const getQueryCallChecker = () => {
+  let callCount = 0
+  return async (query: string, expectAnnotation = false) => {
+    zealot.stubStream("query", dnsResp)
+    await dispatch(
+      viewerSearch({
+        query,
+        from,
+        to
+      })
+    )
+    callCount++
+    const calls = zealot.calls("query")
+    expect(calls.length).toBe(callCount)
+    const expected = expectAnnotation
+      ? `from '1' | ts >= ${from.toISOString()} | ts <= 1970-01-01T00:00:00.001Z | ${query}`
+      : query
+    expect(calls[callCount - 1].args).toEqual(expected)
+  }
+}
+
 const from = new Date()
 const to = new Date(1)
 const submit = () =>
@@ -55,13 +76,18 @@ describe("a normal response", () => {
   })
 
   test("zealot gets the request", async () => {
-    await submit()
-    const calls = zealot.calls("query")
-    expect(calls.length).toBe(1)
-    expect(calls[0].args).toEqual(
-      `from '1' | ts >= ${from.toISOString()} | ts <= 1970-01-01T00:00:00.001Z | dns query | head 500`
-    )
+    const checkQueryCall = getQueryCallChecker()
+    await checkQueryCall("dns query | head 500", true)
   })
+
+  test("zealot does not annotate requests beginning with variations of 'from'", async () => {
+    const checkQueryCall = getQueryCallChecker()
+    await checkQueryCall("from 'test' | test")
+    await checkQueryCall("from('test') | test")
+    await checkQueryCall("from ('test') | test")
+    await checkQueryCall("from    ('test') | test")
+  })
+
   test("the table gets populated", async () => {
     await submit()
     expect(select(Viewer.getViewerRecords).length).toBe(2)

ppl/detail/flows/fetch.test.ts

@@ -44,10 +44,10 @@ describe("zeek log when community_id is found", () => {
     const searches = zealot.calls("query")
     const len = searches.length
     expect(searches[len - 2].args).toMatchInlineSnapshot(
-      `"from '1' | ts >= 2015-03-05T14:15:00Z | ts <= 2015-04-13T09:36:33.751Z | uid==\\"CbOjYpkXn9LfqV51c\\" or \\"CbOjYpkXn9LfqV51c\\" in conn_uids or \\"CbOjYpkXn9LfqV51c\\" in uids or referenced_file.uid==\\"CbOjYpkXn9LfqV51c\\" | head 100"`
+      `"from '1' | ts >= 2015-03-05T14:15:00.000Z | ts <= 2015-04-13T09:36:33.751Z | uid==\\"CbOjYpkXn9LfqV51c\\" or \\"CbOjYpkXn9LfqV51c\\" in conn_uids or \\"CbOjYpkXn9LfqV51c\\" in uids or referenced_file.uid==\\"CbOjYpkXn9LfqV51c\\" | head 100"`
     )
     expect(searches[len - 1].args).toMatchInlineSnapshot(
-      `"from '1' | ts >= 2015-03-05T14:15:00Z | ts <= 2015-04-13T09:36:33.751Z | uid==\\"CbOjYpkXn9LfqV51c\\" or \\"CbOjYpkXn9LfqV51c\\" in conn_uids or \\"CbOjYpkXn9LfqV51c\\" in uids or referenced_file.uid==\\"CbOjYpkXn9LfqV51c\\" or (community_id == \\"1:N7YGmWjwTmMKNhsZHBR618n3ReA=\\" and ts >= 1582646593.978 and ts < 1582646683.994) | head 100"`
+      `"from '1' | ts >= 2015-03-05T14:15:00.000Z | ts <= 2015-04-13T09:36:33.751Z | uid==\\"CbOjYpkXn9LfqV51c\\" or \\"CbOjYpkXn9LfqV51c\\" in conn_uids or \\"CbOjYpkXn9LfqV51c\\" in uids or referenced_file.uid==\\"CbOjYpkXn9LfqV51c\\" or (community_id == \\"1:N7YGmWjwTmMKNhsZHBR618n3ReA=\\" and ts >= 1582646593.978 and ts < 1582646683.994) | head 100"`
     )
   })
 
@@ -76,7 +76,7 @@ describe("zeek log when community_id is not found", () => {
     const {zealot, store} = setup
     await store.dispatch(fetchCorrelation(zeek))
     expect(last<any>(zealot.calls("query")).args).toMatchInlineSnapshot(
-      `"from '1' | ts >= 2015-03-05T14:15:00Z | ts <= 2015-04-13T09:36:33.751Z | uid==\\"CbOjYpkXn9LfqV51c\\" or \\"CbOjYpkXn9LfqV51c\\" in conn_uids or \\"CbOjYpkXn9LfqV51c\\" in uids or referenced_file.uid==\\"CbOjYpkXn9LfqV51c\\" | head 100"`
+      `"from '1' | ts >= 2015-03-05T14:15:00.000Z | ts <= 2015-04-13T09:36:33.751Z | uid==\\"CbOjYpkXn9LfqV51c\\" or \\"CbOjYpkXn9LfqV51c\\" in conn_uids or \\"CbOjYpkXn9LfqV51c\\" in uids or referenced_file.uid==\\"CbOjYpkXn9LfqV51c\\" | head 100"`
     )
   })
 
@@ -106,7 +106,7 @@ describe("suricata alert when community_id found", () => {
     const {zealot, store} = setup
     await store.dispatch(fetchCorrelation(suricata))
     expect(last<any>(zealot.calls("query")).args).toMatchInlineSnapshot(
-      `"from '1' | ts >= 2015-03-05T14:15:00Z | ts <= 2015-04-13T09:36:33.751Z | community_id==\\"1:N7YGmWjwTmMKNhsZHBR618n3ReA=\\" | head 100"`
+      `"from '1' | ts >= 2015-03-05T14:15:00.000Z | ts <= 2015-04-13T09:36:33.751Z | community_id==\\"1:N7YGmWjwTmMKNhsZHBR618n3ReA=\\" | head 100"`
     )
   })
 

src/js/flows/search/mod.ts

@@ -27,6 +27,8 @@ type annotateArgs = {
 }
 
 export const annotateQuery = (query: string, args: annotateArgs) => {
+  // if query already starts with 'from', we do not annotate it further
+  if (/^from[\s(]/i.test(query)) return query
   const {
     poolId,
     from = new Date(new Date().getTime() - 30 * 24 * 60 * 60 * 1000), // 30 days
@@ -52,10 +54,10 @@ const isZeroDefaultSpan = (
   )
 }
 
-const dateToNanoTs = (date: Date | Ts | bigint): string => {
+export const dateToNanoTs = (date: Date | Ts | bigint): string => {
   const NanoFormat = new DateTimeFormatterBuilder()
     .appendPattern("yyyy-MM-dd'T'HH:mm:ss")
-    .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
+    .appendFraction(ChronoField.NANO_OF_SECOND, 3, 9, true)
     .appendLiteral("Z")
     .toFormatter()