From 04822715b32663a904d801ac600a08a65bdaa3d7 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Wed, 29 Apr 2020 14:43:35 -0700 Subject: [PATCH] fix: If its not a pcap, its a log --- src/js/brim/ingest/detectFileType.js | 49 +--------------------- src/js/brim/ingest/detectFileTypes.test.js | 6 +-- src/js/brim/ingest/fileList.js | 4 ++ src/js/brim/ingest/getParams.js | 8 +--- src/js/brim/ingest/test.js | 20 ++++----- src/js/components/StatusBar.js | 4 +- src/js/flows/ingestFiles.js | 6 +-- src/js/state/Spaces/reducer.js | 6 ++- 8 files changed, 28 insertions(+), 75 deletions(-) diff --git a/src/js/brim/ingest/detectFileType.js b/src/js/brim/ingest/detectFileType.js index fded2b97f0..8488bfa586 100644 --- a/src/js/brim/ingest/detectFileType.js +++ b/src/js/brim/ingest/detectFileType.js @@ -1,40 +1,21 @@ /* @flow */ import fs from "fs" -import readline from "readline" const PCAP_1_HEX = "d4c3b2a1" const PCAP_2_HEX = "a1b2c3d4" const PCAPNG_HEX = "0a0d0d0a" const PCAP_HEXES = [PCAP_1_HEX, PCAP_2_HEX, PCAPNG_HEX] -export type IngestFileType = "pcap" | "json" | "zeek" | "unknown" +export type IngestFileType = "pcap" | "log" export default async function(path: string): Promise { if (await isPcap(path)) { return "pcap" - } else if (await isZeekAscii(path)) { - return "zeek" - } else if (await isZeekJson(path)) { - return "json" } else { - return "unknown" + return "log" } } -async function isZeekJson(file) { - for await (let line of firstLines(file, 1)) { - if (!isJson(line)) return false - } - return true -} - -async function isZeekAscii(file) { - for await (let line of firstLines(file, 1)) { - if (!isZeekHeader(line)) return false - } - return true -} - async function isPcap(file) { let bytes = await firstBytes(file, 4) for (let hex of PCAP_HEXES) { @@ -44,19 +25,6 @@ async function isPcap(file) { return false } -function isZeekHeader(line) { - return /^#separator \S+/.test(line) -} - -function isJson(line) { - try { - JSON.parse(line) - return true - } catch { - return false - } -} - function firstBytes(file, n) { return new Promise((res, rej) => { let stream = fs.createReadStream(file, {start: 0, end: n - 1}) @@ -69,16 +37,3 @@ function firstBytes(file, n) { .on("error", rej) }) } - -async function* firstLines(file, n) { - let i = 0 - let input = fs.createReadStream(file, "utf-8") - let rl = readline.createInterface({input}) - - // $FlowFixMe I think we need to upgrade flow to use the asyncIterator - for await (let line of rl) { - yield line - i++ - if (i === n) return - } -} diff --git a/src/js/brim/ingest/detectFileTypes.test.js b/src/js/brim/ingest/detectFileTypes.test.js index beb025b994..ed18d47e0f 100644 --- a/src/js/brim/ingest/detectFileTypes.test.js +++ b/src/js/brim/ingest/detectFileTypes.test.js @@ -16,8 +16,8 @@ test("add file types", async () => { expect(types).toEqual([ {type: "pcap", path: pcap}, {type: "pcap", path: pcapng}, - {type: "zeek", path: zeek}, - {type: "json", path: json}, - {type: "unknown", path: unknown} + {type: "log", path: zeek}, + {type: "log", path: json}, + {type: "log", path: unknown} ]) }) diff --git a/src/js/brim/ingest/fileList.js b/src/js/brim/ingest/fileList.js index 048747b033..d8d5644632 100644 --- a/src/js/brim/ingest/fileList.js +++ b/src/js/brim/ingest/fileList.js @@ -23,6 +23,10 @@ export default function fileList(files: FileListData) { return files.map((f) => f.path) }, + any(type: string) { + return !!files.find((f) => f.type === type) + }, + allPcap() { return files.every((f) => f.type === "pcap") }, diff --git a/src/js/brim/ingest/getParams.js b/src/js/brim/ingest/getParams.js index df5d918dec..b660050880 100644 --- a/src/js/brim/ingest/getParams.js +++ b/src/js/brim/ingest/getParams.js @@ -23,18 +23,12 @@ export default function getParams( ): IngestParams | IngestParamsError { let files = fileList(data) - if (files.multiple() && files.allPcap()) { + if (files.multiple() && files.any("pcap")) { return { error: "Only one pcap can be opened at a time." } } - if (files.multiple() && files.mixed()) { - return { - error: "Only files of a single type (zeek or pcap) can be opened." - } - } - function getDataDir() { if (files.oneFile()) return path.normalize(files.first().path) diff --git a/src/js/brim/ingest/test.js b/src/js/brim/ingest/test.js index 1d07f8708f..69ff8d70ab 100644 --- a/src/js/brim/ingest/test.js +++ b/src/js/brim/ingest/test.js @@ -14,11 +14,11 @@ test("one pcap", () => { }) test("one zeek log", () => { - let data = ingest.getParams([{type: "zeek", path: "/work/zeek.log"}]) + let data = ingest.getParams([{type: "log", path: "/work/zeek.log"}]) expect(data).toEqual({ dataDir: path.join("/work", "zeek.log.brim"), - endpoint: "zeek", + endpoint: "log", paths: ["/work/zeek.log"] }) }) @@ -26,15 +26,15 @@ test("one zeek log", () => { test("two zeek logs in same dir", () => { let data = ingest.getParams( [ - {type: "zeek", path: "/work/zeek-1.log"}, - {type: "zeek", path: "/work/zeek-2.log"} + {type: "log", path: "/work/zeek-1.log"}, + {type: "log", path: "/work/zeek-2.log"} ], "/home" ) expect(data).toEqual({ dataDir: path.join("/home", ".brim", "work.brim"), - endpoint: "zeek", + endpoint: "log", paths: ["/work/zeek-1.log", "/work/zeek-2.log"] }) }) @@ -42,8 +42,8 @@ test("two zeek logs in same dir", () => { test("two zeek logs in different dir", () => { let data = ingest.getParams( [ - {type: "zeek", path: "/work/day-1/zeek.log"}, - {type: "zeek", path: "/work/day-2/zeek.log"} + {type: "log", path: "/work/day-1/zeek.log"}, + {type: "log", path: "/work/day-2/zeek.log"} ], "/home", new Date(0) @@ -51,7 +51,7 @@ test("two zeek logs in different dir", () => { expect(data).toEqual({ dataDir: path.join("/home", ".brim", "zeek_1969-12-31_16:00:00.brim"), - endpoint: "zeek", + endpoint: "log", paths: ["/work/day-1/zeek.log", "/work/day-2/zeek.log"] }) }) @@ -70,10 +70,10 @@ test("two pcaps", () => { test("1 pcap and 1 zeek", () => { let data = ingest.getParams([ {type: "pcap", path: "/pcap-1"}, - {type: "zeek", path: "/zeek-1"} + {type: "log", path: "/zeek-1"} ]) expect(data).toEqual({ - error: "Only files of a single type (zeek or pcap) can be opened." + error: "Only one pcap can be opened at a time." }) }) diff --git a/src/js/components/StatusBar.js b/src/js/components/StatusBar.js index fe53ea589d..6ab9369ed4 100644 --- a/src/js/components/StatusBar.js +++ b/src/js/components/StatusBar.js @@ -47,7 +47,9 @@ export default function StatusBar() { )}
diff --git a/src/js/flows/ingestFiles.js b/src/js/flows/ingestFiles.js index f7867b8620..63fd6ac276 100644 --- a/src/js/flows/ingestFiles.js +++ b/src/js/flows/ingestFiles.js @@ -90,12 +90,8 @@ const postFiles = (client) => ({ let stream if (endpoint === "pcap") { stream = client.pcaps.post({space: name, path: paths[0]}) - } else if (endpoint === "zeek") { - stream = client.logs.post({space: name, paths}) - } else if (endpoint === "json") { - stream = client.logs.post({space: name, paths, types: "default"}) } else { - throw new Error("Unknown file types") + stream = client.logs.post({space: name, paths, types: "default"}) } return {...params, stream} } diff --git a/src/js/state/Spaces/reducer.js b/src/js/state/Spaces/reducer.js index f948757d64..84a19a910f 100644 --- a/src/js/state/Spaces/reducer.js +++ b/src/js/state/Spaces/reducer.js @@ -1,8 +1,10 @@ /* @flow */ -import type {SpacesAction, SpacesState} from "./types" import produce from "immer" +import type {SpacesAction, SpacesState} from "./types" +import {isNumber} from "../../lib/is" + const init: SpacesState = {} const spacesReducer = produce((draft, action: SpacesAction) => { @@ -49,7 +51,7 @@ export default function reducer( function getSpace(state, name) { let space = state[name] ? state[name] : {} if (!space.ingest_warnings) space.ingest_warnings = [] - if (!space.ingest_progress) space.ingest_progress = null + if (!isNumber(space.ingest_progress)) space.ingest_progress = null state[name] = space return state[name] }