enhancement: created new lookup library (#7087)

* Created new lookup library Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Rudimentary form of lookup working with Vrl. Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Lookup can parse paths itself. Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Restored VRL coalesce Signed-off-by: Stephen Wakely <stephen.wakely@datadoghq.com> * Updated property tests. Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Fixed issue with inserting a path that changes the type. Plus a number of other tidy ups. Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Clippy and formatting. Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Removed unneeded file Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Removed commented out Path. Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Formatting Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Vrl needs to include the . when displaying paths. Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Feedback from blt. Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Fixes following feedback from Jean. Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Fixes following feedback from Bruce. Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Clippy Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * More clippy fixes Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Renamed clone_lookup to be to_lookup Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Enable negative indexing Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Added quickcheck tests for lookups Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Fixed setting zero index Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Add limit to alternative component path Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Removed irrelevant tests Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Clippy Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Move rerun-if-changed instruction Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Tidied some comments. Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Don't serialize metadata Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Update types when array index is changed. Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Removed insert_null. Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Clippy Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Minor comment fixes Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Remove serialize test as it is incorrect Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Feedback from Bruce. Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com> * Don't wipe out metadata on vrl remove Signed-off-by: Stephen Wakely <fungus.humungus@gmail.com>
vectordotdev · Apr 29, 2021 · 0240ae6 · 0240ae6
1 parent ffaa9e4
commit 0240ae6
Show file tree

Hide file tree

Showing 59 changed files with 3,698 additions and 1,200 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -72,6 +72,7 @@ members = [
   "lib/tracing-limit",
   "lib/vector-api-client",
   "lib/vector-wasm",
+  "lib/lookup",
   "lib/vrl/cli",
   "lib/vrl/compiler",
   "lib/vrl/core",
@@ -164,6 +165,9 @@ tui = { version = "0.14.0", optional = true, default-features = false, features
 vrl = { path = "lib/vrl/core" }
 vrl-stdlib = { path = "lib/vrl/stdlib" }
 
+# Lookup
+lookup = { path = "lib/lookup" }
+
 # External libs
 anyhow = { version = "1.0.40", default-features = false }
 async-compression = { version = "0.3.7", default-features = false, features = ["tokio", "gzip", "zstd"] }
@@ -284,6 +288,7 @@ tokio01-test = "0.1.1"
 tower-test = "0.4.0"
 walkdir = "2.3.2"
 quickcheck = "1.0.3"
+lookup = { path = "lib/lookup", features = ["arbitrary"] }
 
 [features]
 # Default features for *-unknown-linux-gnu and *-apple-darwin

diff --git a/lib/lookup/Cargo.toml b/lib/lookup/Cargo.toml
@@ -0,0 +1,27 @@
+[package]
+name = "lookup"
+version = "0.1.0"
+authors = ["Vector Contributors <vector@timber.io>"]
+edition = "2018"
+publish = false
+
+[dependencies]
+async-graphql = { version = "=2.6.4", default-features = false, features = ["chrono"] }
+serde = { version = "1.0.117", default-features = false, features = ["derive"] }
+tracing = { version = "0.1", default-features = false, features = ["attributes"] }
+snafu = { version = "0.6", default-features = false }
+regex = { version = "1.4.5", default-features = false, features = ["std", "perf"] }
+inherent = "0.1"
+lalrpop-util = "0.19.5"
+lazy_static = "1.3.0"
+quickcheck = { version = "1.0.3", optional = true }
+
+[dev-dependencies]
+serde_json = { version = "1.0.64", features = ["raw_value"] }
+quickcheck = { version = "1.0.3" }
+
+[build-dependencies]
+lalrpop = "0.19"
+
+[features]
+arbitrary = ["quickcheck"]
diff --git a/lib/lookup/build.rs b/lib/lookup/build.rs
@@ -0,0 +1,10 @@
+extern crate lalrpop;
+
+fn main() {
+    println!("cargo:rerun-if-changed=src/path.lalrpop");
+
+    lalrpop::Configuration::new()
+        .always_use_colors()
+        .process_current_dir()
+        .unwrap();
+}
diff --git a/lib/lookup/src/error.rs b/lib/lookup/src/error.rs
@@ -0,0 +1,7 @@
+use snafu::Snafu;
+
+#[derive(Debug, Snafu)]
+pub enum LookupError {
+    #[snafu(display("Invalid path: {}.", message))]
+    Invalid { message: String },
+}
diff --git a/lib/lookup/src/field.rs b/lib/lookup/src/field.rs
@@ -0,0 +1,13 @@
+use lazy_static::lazy_static;
+use regex::Regex;
+
+lazy_static! {
+    static ref VALID_FIELD: Regex = Regex::new("^[0-9]*[a-zA-Z_][0-9a-zA-Z_]*$").unwrap();
+}
+
+/// A valid fieldname can contain alphanumeric characters and an underscore.
+/// It may start with a number, but has to consist of more than just a number.
+/// Fields that have other characters can be used, but need to be quoted.
+pub(crate) fn is_valid_fieldname(name: &str) -> bool {
+    VALID_FIELD.is_match(&name)
+}
diff --git a/lib/lookup/src/lib.rs b/lib/lookup/src/lib.rs
@@ -0,0 +1,69 @@
+use std::fmt::{Debug, Display};
+use std::hash::Hash;
+
+pub use error::LookupError;
+pub use lookup_buf::{FieldBuf, LookupBuf, SegmentBuf};
+pub use lookup_view::{Field, Lookup, Segment};
+
+mod error;
+mod field;
+mod lookup_buf;
+mod lookup_view;
+mod parser;
+
+/// This trait, while it is not necessarily imported and used, exists
+/// to enforce parity among view/buf types.
+pub trait Look<'a>:
+    Debug + Display + PartialEq + Eq + PartialOrd + Ord + Clone + Hash + Sized + ToString
+{
+    type Segment: LookSegment<'a>;
+
+    fn get(&mut self, index: usize) -> Option<&Self::Segment>;
+
+    fn push_back(&mut self, segment: impl Into<Self::Segment>);
+
+    fn pop_back(&mut self) -> Option<Self::Segment>;
+
+    fn push_front(&mut self, segment: impl Into<Self::Segment>);
+
+    fn pop_front(&mut self) -> Option<Self::Segment>;
+
+    fn len(&self) -> usize;
+
+    fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    fn from_str(input: &'a str) -> Result<Self, LookupError>;
+
+    fn extend(&mut self, other: Self);
+
+    fn starts_with(&self, needle: &Self) -> bool;
+
+    fn is_root(&self) -> bool;
+}
+
+// This trait, while it is not necessarily imported and used, exists
+// to enforce parity among view/buf types.
+//
+// It is convention to implement these functions on the bare type itself,
+// then have the implementation proxy to this **without modification**.
+//
+// This is so the functions are always available to users, without needing an import.
+pub trait LookSegment<'a>:
+    Debug + PartialEq + Eq + PartialOrd + Ord + Clone + Hash + Sized
+{
+    type Field: Debug + PartialEq + Eq + PartialOrd + Ord + Clone + Hash + Sized;
+
+    fn field(field: Self::Field) -> Self;
+
+    fn is_field(&self) -> bool;
+
+    fn index(v: isize) -> Self;
+
+    fn is_index(&self) -> bool;
+
+    fn coalesce(v: Vec<Self::Field>) -> Self;
+
+    fn is_coalesce(&self) -> bool;
+}