datastore: changes to match changes in Core kit datastore

Updated the `commit_transaction` function to enable committing metadata
from pending transactions. In commit transaction we will first commit
metadata and then pending keys to correctly perform the check to
identify
if key exists or not.

The strength handling among pending and committed transaction is as:
If pending metadata is strong and committed metadata is weak, commit the
pending setting.

If pending metadata is weak, committed metadata is strong and
the setting is already available, do not downgrade from strong to weak.
Otherwise add the strength file with value as weak.

If pending and committed metadata are the same, no action is performed.

Additionally, made minor changes to metadata functions for improved
access and flexibility:
Introduced a `committed` field to dynamically access metadata in pending
transactions.
Replaced the hardcoded use of live committed metadata with this
committed variable ans pass Committed::Live from previous usages.

Refer commit:
bottlerocket-os/bottlerocket-core-kit@20a435e
Refer PR:
bottlerocket-os/bottlerocket-core-kit#294

sources/api/datastore/Cargo.toml

@@ -16,6 +16,7 @@ serde = { workspace = true, features = ["derive"] }
 serde_json.workspace = true
 snafu.workspace = true
 walkdir.workspace = true
+serde_plain.workspace = true
 
 [build-dependencies]
 generate-readme.workspace = true

sources/api/datastore/src/constraints_check.rs

@@ -0,0 +1,52 @@
+//! The outcome of the constraint check determines whether the transaction can proceed to commit.
+//! A ‘rejected’ result means that one or more constraints have not been satisfied,
+//! preventing the transaction from being committed. On the other hand, an ‘approved’
+//! result confirms that all constraints are satisfied and provides the required
+//! settings and metadata for the commit.
+//! Constraint checks can alter the write.
+
+use std::collections::HashMap;
+
+use crate::{error, Key};
+
+type RejectReason = String;
+
+/// Represents a successful write operation after constraints have been approved.
+/// Contains the following fields:
+/// - `settings`: A collection of key-value pairs representing the settings to be committed.
+/// - `metadata`: A collection of metadata entries.
+#[derive(PartialEq)]
+pub struct ApprovedWrite {
+    pub settings: HashMap<Key, String>,
+    pub metadata: Vec<(Key, Key, String)>,
+}
+
+/// Represents the result of a constraint check.
+/// The result can either reject the operation or approve it with the required data.
+#[derive(PartialEq)]
+pub enum ConstraintCheckResult {
+    Reject(RejectReason),
+    Approve(ApprovedWrite),
+}
+
+impl TryFrom<ConstraintCheckResult> for ApprovedWrite {
+    type Error = error::Error;
+
+    fn try_from(constraint_check_result: ConstraintCheckResult) -> Result<Self, Self::Error> {
+        match constraint_check_result {
+            ConstraintCheckResult::Reject(err) => error::ConstraintCheckRejectSnafu { err }.fail(),
+            ConstraintCheckResult::Approve(approved_write) => Ok(approved_write),
+        }
+    }
+}
+
+impl From<Option<ApprovedWrite>> for ConstraintCheckResult {
+    fn from(approved_write: Option<ApprovedWrite>) -> Self {
+        match approved_write {
+            None => ConstraintCheckResult::Reject(
+                "The write for the given transaction is rejected".to_string(),
+            ),
+            Some(approved_write) => ConstraintCheckResult::Approve(approved_write),
+        }
+    }
+}

sources/api/datastore/src/error.rs

@@ -59,6 +59,20 @@ pub enum Error {
 
     #[snafu(display("Key name beyond maximum length {}: {}", name, max))]
     KeyTooLong { name: String, max: usize },
+
+    #[snafu(display("Unable to serialize data: {}", source))]
+    Serialize { source: serde_json::Error },
+
+    #[snafu(display("Unable to run the check constraint function: {}", source))]
+    CheckConstraintExecution {
+        source: Box<dyn std::error::Error + Send + Sync>,
+    },
+
+    #[snafu(display(
+        "Check constraint function rejected the transaction. Aborting commit : {}",
+        err
+    ))]
+    ConstraintCheckReject { err: String },
 }
 
-pub type Result<T> = std::result::Result<T, Error>;
+pub type Result<T, E = Error> = std::result::Result<T, E>;

sources/api/datastore/src/filesystem.rs

@@ -13,6 +13,8 @@ use std::io;
 use std::path::{self, Path, PathBuf};
 use walkdir::{DirEntry, WalkDir};
 
+use crate::constraints_check::{ApprovedWrite, ConstraintCheckResult};
+
 use super::key::{Key, KeyType};
 use super::{error, Committed, DataStore, Result};
 
@@ -413,14 +415,15 @@ impl DataStore for FilesystemDataStore {
     fn list_populated_metadata<S1, S2>(
         &self,
         prefix: S1,
+        committed: &Committed,
         metadata_key_name: &Option<S2>,
     ) -> Result<HashMap<Key, HashSet<Key>>>
     where
         S1: AsRef<str>,
         S2: AsRef<str>,
     {
         // Find metadata key paths on disk
-        let key_paths = find_populated_key_paths(self, KeyType::Meta, prefix, &Committed::Live)?;
+        let key_paths = find_populated_key_paths(self, KeyType::Meta, prefix, committed)?;
 
         // For each file on disk, check the user's conditions, and add it to our output
         let mut result = HashMap::new();
@@ -460,8 +463,13 @@ impl DataStore for FilesystemDataStore {
         self.delete_key_path(path, committed)
     }
 
-    fn get_metadata_raw(&self, metadata_key: &Key, data_key: &Key) -> Result<Option<String>> {
-        let path = self.metadata_path(metadata_key, data_key, &Committed::Live)?;
+    fn get_metadata_raw(
+        &self,
+        metadata_key: &Key,
+        data_key: &Key,
+        committed: &Committed,
+    ) -> Result<Option<String>> {
+        let path = self.metadata_path(metadata_key, data_key, committed)?;
         read_file_for_key(metadata_key, &path)
     }
 
@@ -470,8 +478,9 @@ impl DataStore for FilesystemDataStore {
         metadata_key: &Key,
         data_key: &Key,
         value: S,
+        committed: &Committed,
     ) -> Result<()> {
-        let path = self.metadata_path(metadata_key, data_key, &Committed::Live)?;
+        let path = self.metadata_path(metadata_key, data_key, committed)?;
         write_file_mkdir(path, value)
     }
 
@@ -482,32 +491,57 @@ impl DataStore for FilesystemDataStore {
 
     /// We commit by copying pending keys to live, then removing pending.  Something smarter (lock,
     /// atomic flip, etc.) will be required to make the server concurrent.
-    fn commit_transaction<S>(&mut self, transaction: S) -> Result<HashSet<Key>>
+    fn commit_transaction<S, C>(
+        &mut self,
+        transaction: S,
+        constraint_check: &C,
+    ) -> Result<HashSet<Key>>
     where
         S: Into<String> + AsRef<str>,
+        C: Fn(
+            &mut Self,
+            &Committed,
+        ) -> std::result::Result<
+            ConstraintCheckResult,
+            Box<dyn std::error::Error + Send + Sync + 'static>,
+        >,
     {
         let pending = Committed::Pending {
             tx: transaction.into(),
         };
-        // Get data for changed keys
-        let pending_data = self.get_prefix("settings.", &pending)?;
 
-        // Nothing to do if no keys are present in pending
-        if pending_data.is_empty() {
-            return Ok(Default::default());
+        let constraints_check_result =
+            constraint_check(self, &pending).context(error::CheckConstraintExecutionSnafu)?;
+
+        let approved_write = ApprovedWrite::try_from(constraints_check_result)?;
+
+        let mut pending_keys: HashSet<Key> = Default::default();
+
+        trace!(
+            "commit_transaction: transaction_metadata: {:?}",
+            approved_write.metadata
+        );
+
+        // write the metadata.
+        for (metadata_key, data_key, value) in approved_write.metadata {
+            self.set_metadata(&metadata_key, &data_key, value, &Committed::Live)?;
         }
 
-        // Save Keys for return value
-        let pending_keys: HashSet<Key> = pending_data.keys().cloned().collect();
+        let pending_data = approved_write.settings;
 
-        // Apply changes to live
-        debug!("Writing pending keys to live");
-        self.set_keys(&pending_data, &Committed::Live)?;
+        if !pending_data.is_empty() {
+            // Save Keys for return value
+            pending_keys = pending_data.keys().cloned().collect();
 
-        // Remove pending
-        debug!("Removing old pending keys");
-        let path = self.base_path(&pending);
-        fs::remove_dir_all(&path).context(error::IoSnafu { path })?;
+            // Apply changes to live
+            debug!("Writing pending keys to live");
+            self.set_keys(&pending_data, &Committed::Live)?;
+
+            // Remove pending
+            debug!("Removing old pending keys");
+            let path = self.base_path(&pending);
+            fs::remove_dir_all(&path).context(error::IoSnafu { path })?;
+        }
 
         Ok(pending_keys)
     }