Add local dictionary support

cli/Cargo.toml

@@ -21,6 +21,7 @@ tracing-futures = "0.2.5"
 hyper = {version = "0.14.0", features = ["full"]}
 hyper-tls = "0.5.0"
 wat = "1.0.38"
+serde_json = "1.0.66"
 
 [dev-dependencies]
 anyhow = "1.0.31"

cli/src/main.rs

@@ -43,10 +43,12 @@ pub async fn serve(opts: Opts) -> Result<(), Error> {
     if let Some(config_path) = opts.config_path() {
         let config = FastlyConfig::from_file(config_path)?;
         let backends = config.backends();
+        let dictionaries = config.dictionaries();
         let backend_names = itertools::join(backends.keys(), ", ");
 
         ctx = ctx
             .with_backends(backends.clone())
+            .with_dictionaries(dictionaries.clone())
             .with_config_path(config_path.into());
 
         if backend_names.is_empty() {

lib/Cargo.toml

@@ -30,6 +30,7 @@ regex = "1.3.9"
 semver = "0.10.0"
 serde = "1.0.114"
 serde_derive = "1.0.114"
+serde_json = "1.0.59"
 thiserror = "1.0.20"
 tokio = {version = "1.2", features = ["full"]}
 toml = "0.5.6"
@@ -41,6 +42,9 @@ wasmtime = "0.29.0"
 wasmtime-wasi = {version = "0.29.0", features = ["tokio"]}
 wiggle = {version = "0.29.0", features = ["wasmtime_async"]}
 
+[dev-dependencies]
+tempfile = "3.2.0"
+
 [features]
 default = []
 test-fatalerror-config = []

lib/src/config.rs

@@ -1,7 +1,10 @@
 //! Fastly-specific configuration utilities.
 
 use {
-    self::backends::BackendsConfig,
+    self::{
+        backends::BackendsConfig,
+        dictionaries::DictionariesConfig
+    },
     crate::error::FastlyConfigError,
     serde_derive::Deserialize,
     std::{collections::HashMap, convert::TryInto, fs, path::Path, str::FromStr, sync::Arc},
@@ -12,6 +15,15 @@ use {
 #[cfg(test)]
 mod unit_tests;
 
+/// Fastly limits
+mod limits;
+
+/// Types and deserializers for dictionaries configuration settings.
+mod dictionaries;
+pub use self::dictionaries::Dictionary;
+pub use self::dictionaries::DictionaryName;
+pub type Dictionaries = HashMap<DictionaryName, Dictionary>;
+
 /// Types and deserializers for backend configuration settings.
 mod backends;
 pub use self::backends::Backend;
@@ -55,6 +67,11 @@ impl FastlyConfig {
         &self.local_server.backends.0
     }
 
+    /// Get the dictionaries configuration.
+    pub fn dictionaries(&self) -> &Dictionaries {
+        &self.local_server.dictionaries.0
+    }
+
     /// Parse a `fastly.toml` file into a `FastlyConfig`.
     pub fn from_file(path: impl AsRef<Path>) -> Result<Self, FastlyConfigError> {
         fs::read_to_string(path.as_ref())
@@ -130,6 +147,7 @@ impl TryInto<FastlyConfig> for TomlFastlyConfig {
 #[derive(Clone, Debug, Default)]
 pub struct LocalServerConfig {
     backends: BackendsConfig,
+    dictionaries: DictionariesConfig,
 }
 
 /// Internal deserializer used to read the `[testing]` section of a `fastly.toml` file.
@@ -138,15 +156,31 @@ pub struct LocalServerConfig {
 /// a [`LocalServerConfig`] with [`TryInto::try_into`].
 #[derive(Deserialize)]
 struct RawLocalServerConfig {
-    backends: Table,
+    backends: Option<Table>,
+    dictionaries: Option<Table>,
 }
 
 impl TryInto<LocalServerConfig> for RawLocalServerConfig {
     type Error = FastlyConfigError;
     fn try_into(self) -> Result<LocalServerConfig, Self::Error> {
-        let Self { backends } = self;
-        backends
-            .try_into()
-            .map(|backends| LocalServerConfig { backends })
+        let Self {
+            backends,
+            dictionaries,
+        } = self;
+        let backends = if let Some(backends) = backends {
+            backends.try_into()?
+        } else {
+            BackendsConfig::default()
+        };
+        let dictionaries = if let Some(dictionaries) = dictionaries {
+            dictionaries.try_into()?
+        } else {
+            DictionariesConfig::default()
+        };
+
+        Ok(LocalServerConfig {
+            backends,
+            dictionaries,
+        })
     }
 }

lib/src/config/dictionaries.rs

@@ -0,0 +1,222 @@
+use std::{collections::HashMap, fmt, path::PathBuf};
+
+#[derive(Clone, Debug, Eq, PartialEq, Hash)]
+pub struct DictionaryName(String);
+
+impl fmt::Display for DictionaryName {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+impl DictionaryName {
+    pub fn new(name: String) -> Self {
+        Self(name)
+    }
+}
+
+/// A single Dictionary definition.
+///
+/// A Dictionary consists of a file and format, but more fields may be added in the future.
+#[derive(Clone, Debug)]
+pub struct Dictionary {
+    pub file: PathBuf,
+    pub format: DictionaryFormat,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum DictionaryFormat {
+    JSON,
+}
+
+/// A map of [`Dictionary`] definitions, keyed by their name.
+#[derive(Clone, Debug, Default)]
+pub struct DictionariesConfig(pub HashMap<DictionaryName, Dictionary>);
+
+/// This module contains [`TryFrom`] implementations used when deserializing a `fastly.toml`.
+///
+/// These implementations are called indirectly by [`FastlyConfig::from_file`][super::FastlyConfig],
+/// and help validate that we have been given an appropriate TOML schema. If the configuration is
+/// not valid, a [`FastlyConfigError`] will be returned.
+mod deserialization {
+
+    use {
+        super::{DictionariesConfig, Dictionary, DictionaryFormat, DictionaryName},
+        crate::{
+            config::limits::{
+                DICTIONARY_ITEM_KEY_MAX_LEN, DICTIONARY_ITEM_VALUE_MAX_LEN, DICTIONARY_MAX_LEN,
+            },
+            error::{DictionaryConfigError, FastlyConfigError},
+        },
+        std::{convert::TryFrom, convert::TryInto, fs, str::FromStr},
+        toml::value::{Table, Value},
+        tracing::{event, Level},
+    };
+
+    /// Helper function for converting a TOML [`Value`] into a [`Table`].
+    ///
+    /// This function checks that a value is a [`Value::Table`] variant and returns the underlying
+    /// [`Table`], or returns an error if the given value was not of the right type — e.g., a
+    /// [`Boolean`][Value::Boolean] or a [`String`][Value::String]).
+    fn into_table(value: Value) -> Result<Table, DictionaryConfigError> {
+        match value {
+            Value::Table(table) => Ok(table),
+            _ => Err(DictionaryConfigError::InvalidEntryType),
+        }
+    }
+
+    /// Return an [`DictionaryConfigError::UnrecognizedKey`] error if any unrecognized keys are found.
+    ///
+    /// This should be called after we have removed and validated the keys we expect in a [`Table`].
+    fn check_for_unrecognized_keys(table: &Table) -> Result<(), DictionaryConfigError> {
+        if let Some(key) = table.keys().next() {
+            // While other keys might still exist, we can at least return a helpful error including
+            // the name of *one* unrecognized keys we found.
+            Err(DictionaryConfigError::UnrecognizedKey(key.to_owned()))
+        } else {
+            Ok(())
+        }
+    }
+
+    impl TryFrom<Table> for DictionariesConfig {
+        type Error = FastlyConfigError;
+        fn try_from(toml: Table) -> Result<Self, Self::Error> {
+            /// Process a dictionary's definitions, or return a [`FastlyConfigError`].
+            fn process_entry(
+                (name, defs): (String, Value),
+            ) -> Result<(DictionaryName, Dictionary), FastlyConfigError> {
+                into_table(defs)
+                    .and_then(|mut toml| {
+                        let format = toml
+                            .remove("format")
+                            .ok_or(DictionaryConfigError::MissingFormat)
+                            .and_then(|format| match format {
+                                Value::String(format) => Ok(format.parse()?),
+                                _ => Err(DictionaryConfigError::InvalidFormatEntry),
+                            })?;
+
+                        let file = toml
+                            .remove("file")
+                            .ok_or(DictionaryConfigError::MissingFile)
+                            .and_then(|file| match file {
+                                Value::String(file) => {
+                                    if file.is_empty() {
+                                        Err(DictionaryConfigError::EmptyFileEntry)
+                                    } else {
+                                        Ok(file.into())
+                                    }
+                                }
+                                _ => Err(DictionaryConfigError::InvalidFileEntry),
+                            })?;
+                        check_for_unrecognized_keys(&toml)?;
+                        event!(
+                            Level::INFO,
+                            "checking if the dictionary '{}' adheres to Fastly's API",
+                            name
+                        );
+                        let data = fs::read_to_string(&file).map_err(|err| {
+                            DictionaryConfigError::IoError {
+                                name: name.to_string(),
+                                error: err.to_string(),
+                            }
+                        })?;
+
+                        match format {
+                            DictionaryFormat::JSON => parse_dict_as_json(&name, &data)?,
+                        }
+
+                        let name = name.parse()?;
+                        Ok((name, Dictionary { file, format }))
+                    })
+                    .map_err(|err| FastlyConfigError::InvalidDictionaryDefinition {
+                        name: name.clone(),
+                        err,
+                    })
+            }
+
+            toml.into_iter()
+                .map(process_entry)
+                .collect::<Result<_, _>>()
+                .map(Self)
+        }
+    }
+
+    fn parse_dict_as_json(name: &str, data: &str) -> Result<(), DictionaryConfigError> {
+        let json: serde_json::Value = serde_json::from_str(data).map_err(|_| {
+            DictionaryConfigError::DictionaryFileWrongFormat {
+                name: name.to_string(),
+            }
+        })?;
+        let dict =
+            json.as_object()
+                .ok_or_else(|| DictionaryConfigError::DictionaryFileWrongFormat {
+                    name: name.to_string(),
+                })?;
+        if dict.len() > DICTIONARY_MAX_LEN {
+            return Err(DictionaryConfigError::DictionaryCountTooLong {
+                name: name.to_string(),
+                size: DICTIONARY_MAX_LEN.try_into().unwrap(),
+            });
+        }
+
+        event!(
+            Level::INFO,
+            "checking if the items in dictionary '{}' adhere to Fastly's API",
+            name
+        );
+        for (key, value) in dict.iter() {
+            if key.chars().count() > DICTIONARY_ITEM_KEY_MAX_LEN {
+                return Err(DictionaryConfigError::DictionaryItemKeyTooLong {
+                    name: name.to_string(),
+                    key: key.clone(),
+                    size: DICTIONARY_ITEM_KEY_MAX_LEN.try_into().unwrap(),
+                });
+            }
+            let value = value.as_str().ok_or_else(|| {
+                DictionaryConfigError::DictionaryItemValueWrongFormat {
+                    name: name.to_string(),
+                    key: key.clone(),
+                }
+            })?;
+            if value.chars().count() > DICTIONARY_ITEM_VALUE_MAX_LEN {
+                return Err(DictionaryConfigError::DictionaryItemValueTooLong {
+                    name: name.to_string(),
+                    key: key.clone(),
+                    size: DICTIONARY_ITEM_VALUE_MAX_LEN.try_into().unwrap(),
+                });
+            }
+        }
+        Ok(())
+    }
+
+    impl FromStr for DictionaryName {
+        type Err = DictionaryConfigError;
+        fn from_str(name: &str) -> Result<Self, Self::Err> {
+            // Name must start with alphabetical and contain only alphanumeric, underscore, and whitespace
+            if name.starts_with(char::is_alphabetic)
+                && name
+                    .chars()
+                    .all(|c| char::is_alphanumeric(c) || c == '_' || char::is_whitespace(c))
+            {
+                Ok(Self(name.to_owned()))
+            } else {
+                Err(DictionaryConfigError::InvalidName(name.to_owned()))
+            }
+        }
+    }
+
+    impl FromStr for DictionaryFormat {
+        type Err = DictionaryConfigError;
+        fn from_str(name: &str) -> Result<Self, Self::Err> {
+            if name.is_empty() {
+                return Err(DictionaryConfigError::EmptyFormatEntry);
+            }
+            match name {
+                "json" => Ok(DictionaryFormat::JSON),
+                _ => Err(DictionaryConfigError::InvalidDictionaryFileFormat(
+                    name.to_owned(),
+                )),
+            }
+        }
+    }
+}

lib/src/config/limits.rs

@@ -0,0 +1,4 @@
+// From https://docs.fastly.com/en/guides/resource-limits#vcl-and-configuration-limits
+pub const DICTIONARY_MAX_LEN: usize = 1000;
+pub const DICTIONARY_ITEM_KEY_MAX_LEN: usize = 256;
+pub const DICTIONARY_ITEM_VALUE_MAX_LEN: usize = 8000;