If its not a pcap, its a log

src/js/brim/ingest/detectFileType.js

@@ -1,40 +1,21 @@
 /* @flow */
 import fs from "fs"
-import readline from "readline"
 
 const PCAP_1_HEX = "d4c3b2a1"
 const PCAP_2_HEX = "a1b2c3d4"
 const PCAPNG_HEX = "0a0d0d0a"
 const PCAP_HEXES = [PCAP_1_HEX, PCAP_2_HEX, PCAPNG_HEX]
 
-export type IngestFileType = "pcap" | "json" | "zeek" | "unknown"
+export type IngestFileType = "pcap" | "log"
 
 export default async function(path: string): Promise<IngestFileType> {
   if (await isPcap(path)) {
     return "pcap"
-  } else if (await isZeekAscii(path)) {
-    return "zeek"
-  } else if (await isZeekJson(path)) {
-    return "json"
   } else {
-    return "unknown"
+    return "log"
   }
 }
 
-async function isZeekJson(file) {
-  for await (let line of firstLines(file, 1)) {
-    if (!isJson(line)) return false
-  }
-  return true
-}
-
-async function isZeekAscii(file) {
-  for await (let line of firstLines(file, 1)) {
-    if (!isZeekHeader(line)) return false
-  }
-  return true
-}
-
 async function isPcap(file) {
   let bytes = await firstBytes(file, 4)
   for (let hex of PCAP_HEXES) {
@@ -44,19 +25,6 @@ async function isPcap(file) {
   return false
 }
 
-function isZeekHeader(line) {
-  return /^#separator \S+/.test(line)
-}
-
-function isJson(line) {
-  try {
-    JSON.parse(line)
-    return true
-  } catch {
-    return false
-  }
-}
-
 function firstBytes(file, n) {
   return new Promise((res, rej) => {
     let stream = fs.createReadStream(file, {start: 0, end: n - 1})
@@ -69,16 +37,3 @@ function firstBytes(file, n) {
       .on("error", rej)
   })
 }
-
-async function* firstLines(file, n) {
-  let i = 0
-  let input = fs.createReadStream(file, "utf-8")
-  let rl = readline.createInterface({input})
-
-  // $FlowFixMe I think we need to upgrade flow to use the asyncIterator
-  for await (let line of rl) {
-    yield line
-    i++
-    if (i === n) return
-  }
-}

src/js/brim/ingest/detectFileTypes.test.js

@@ -16,8 +16,8 @@ test("add file types", async () => {
   expect(types).toEqual([
     {type: "pcap", path: pcap},
     {type: "pcap", path: pcapng},
-    {type: "zeek", path: zeek},
-    {type: "json", path: json},
-    {type: "unknown", path: unknown}
+    {type: "log", path: zeek},
+    {type: "log", path: json},
+    {type: "log", path: unknown}
   ])
 })

src/js/brim/ingest/fileList.js

@@ -23,6 +23,10 @@ export default function fileList(files: FileListData) {
       return files.map((f) => f.path)
     },
 
+    any(type: string) {
+      return !!files.find((f) => f.type === type)
+    },
+
     allPcap() {
       return files.every((f) => f.type === "pcap")
     },

src/js/brim/ingest/getParams.js

@@ -23,18 +23,12 @@ export default function getParams(
 ): IngestParams | IngestParamsError {
   let files = fileList(data)
 
-  if (files.multiple() && files.allPcap()) {
+  if (files.multiple() && files.any("pcap")) {
     return {
       error: "Only one pcap can be opened at a time."
     }
   }
 
-  if (files.multiple() && files.mixed()) {
-    return {
-      error: "Only files of a single type (zeek or pcap) can be opened."
-    }
-  }
-
   function getDataDir() {
     if (files.oneFile()) return path.normalize(files.first().path)
 

src/js/brim/ingest/test.js

@@ -14,44 +14,44 @@ test("one pcap", () => {
 })
 
 test("one zeek log", () => {
-  let data = ingest.getParams([{type: "zeek", path: "/work/zeek.log"}])
+  let data = ingest.getParams([{type: "log", path: "/work/zeek.log"}])
 
   expect(data).toEqual({
     dataDir: path.join("/work", "zeek.log.brim"),
-    endpoint: "zeek",
+    endpoint: "log",
     paths: ["/work/zeek.log"]
   })
 })
 
 test("two zeek logs in same dir", () => {
   let data = ingest.getParams(
     [
-      {type: "zeek", path: "/work/zeek-1.log"},
-      {type: "zeek", path: "/work/zeek-2.log"}
+      {type: "log", path: "/work/zeek-1.log"},
+      {type: "log", path: "/work/zeek-2.log"}
     ],
     "/home"
   )
 
   expect(data).toEqual({
     dataDir: path.join("/home", ".brim", "work.brim"),
-    endpoint: "zeek",
+    endpoint: "log",
     paths: ["/work/zeek-1.log", "/work/zeek-2.log"]
   })
 })
 
 test("two zeek logs in different dir", () => {
   let data = ingest.getParams(
     [
-      {type: "zeek", path: "/work/day-1/zeek.log"},
-      {type: "zeek", path: "/work/day-2/zeek.log"}
+      {type: "log", path: "/work/day-1/zeek.log"},
+      {type: "log", path: "/work/day-2/zeek.log"}
     ],
     "/home",
     new Date(0)
   )
 
   expect(data).toEqual({
     dataDir: path.join("/home", ".brim", "zeek_1969-12-31_16:00:00.brim"),
-    endpoint: "zeek",
+    endpoint: "log",
     paths: ["/work/day-1/zeek.log", "/work/day-2/zeek.log"]
   })
 })
@@ -70,10 +70,10 @@ test("two pcaps", () => {
 test("1 pcap and 1 zeek", () => {
   let data = ingest.getParams([
     {type: "pcap", path: "/pcap-1"},
-    {type: "zeek", path: "/zeek-1"}
+    {type: "log", path: "/zeek-1"}
   ])
 
   expect(data).toEqual({
-    error: "Only files of a single type (zeek or pcap) can be opened."
+    error: "Only one pcap can be opened at a time."
   })
 })

src/js/components/StatusBar.js

@@ -47,7 +47,9 @@ export default function StatusBar() {
             <label>Ingest failed with warnings.</label>
           )}
           <div
-            className={classNames("warnings", {disabled: isEmpty(warnings)})}
+            className={classNames("warnings", {
+              disabled: isEmpty(warnings) || !s.queryable()
+            })}
             onClick={onWarningsClick}
           >
             <Warning />

src/js/flows/ingestFiles.js

@@ -90,12 +90,8 @@ const postFiles = (client) => ({
     let stream
     if (endpoint === "pcap") {
       stream = client.pcaps.post({space: name, path: paths[0]})
-    } else if (endpoint === "zeek") {
-      stream = client.logs.post({space: name, paths})
-    } else if (endpoint === "json") {
-      stream = client.logs.post({space: name, paths, types: "default"})
     } else {
-      throw new Error("Unknown file types")
+      stream = client.logs.post({space: name, paths, types: "default"})
     }
     return {...params, stream}
   }

src/js/state/Spaces/reducer.js

@@ -1,8 +1,10 @@
 /* @flow */
 
-import type {SpacesAction, SpacesState} from "./types"
 import produce from "immer"
 
+import type {SpacesAction, SpacesState} from "./types"
+import {isNumber} from "../../lib/is"
+
 const init: SpacesState = {}
 
 const spacesReducer = produce((draft, action: SpacesAction) => {
@@ -49,7 +51,7 @@ export default function reducer(
 function getSpace(state, name) {
   let space = state[name] ? state[name] : {}
   if (!space.ingest_warnings) space.ingest_warnings = []
-  if (!space.ingest_progress) space.ingest_progress = null
+  if (!isNumber(space.ingest_progress)) space.ingest_progress = null
   state[name] = space
   return state[name]
 }