Fetch YANG modules dynamically via GetSchema RPC

The CLI previously depended on the holo-yang crate to load all YANG
modules used by holod. This dependency caused issues with keeping
both projects in sync, requiring a new holo-yang release to crates.io
whenever YANG modules were added or modified.

This commit removes the holo-yang crate dependency. Instead, the CLI
now uses the recently introduced `GetSchema` gRPC RPC to fetch missing
YANG modules dynamically.  These modules are cached on the filesystem
and only fetched when needed.

Signed-off-by: Renato Westphal <renato@opensourcerouting.org>

Cargo.toml

@@ -21,8 +21,6 @@ tokio = { version = "1.0", features = ["full"] }
 tonic = { version = "0.11", features = ["tls"] }
 yang3 = { version = "0.9", features = ["bundled"] }
 
-holo-yang = "0.5.0"
-
 [build-dependencies]
 tonic-build = "0.11"
 

proto/holo.proto

@@ -29,6 +29,9 @@ service Northbound {
   // Retrieve the capabilities supported by the target.
   rpc Capabilities(CapabilitiesRequest) returns (CapabilitiesResponse) {}
 
+  // Retrieve the specified schema data from the target.
+  rpc GetSchema(GetSchemaRequest) returns (GetSchemaResponse) {}
+
   // Retrieve configuration data, state data or both from the target.
   rpc Get(GetRequest) returns (GetResponse) {}
 
@@ -71,6 +74,34 @@ message CapabilitiesResponse {
   repeated Encoding supported_encodings = 3;
 }
 
+//
+// RPC: GetSchema()
+//
+message GetSchemaRequest {
+  // The name of the module to retrieve.
+  string module_name = 1;
+
+  // The specific revision of the module (optional).
+  string module_revision = 2;
+
+  // The name of the submodule to retrieve (optional).
+  string submodule_name = 3;
+
+  // The specific revision of the submodule (optional).
+  string submodule_revision = 4;
+
+  // The desired format of the schema output (optional).
+  SchemaFormat format = 5;
+}
+
+message GetSchemaResponse {
+  // - grpc::StatusCode::OK: Success.
+  // - grpc::StatusCode::NOT_FOUND: Schema module not found.
+
+  // The requested schema data.
+  string data = 1;
+}
+
 //
 // RPC: Get()
 //
@@ -231,6 +262,9 @@ message ModuleData {
 
   // Latest revision of the module;
   string revision = 3;
+
+  // Supported features.
+  repeated string supported_features = 4;
 }
 
 // Supported encodings for YANG instance data.
@@ -240,6 +274,12 @@ enum Encoding {
   LYB = 2;
 }
 
+// Supported schema formats.
+enum SchemaFormat {
+  YANG = 0;
+  YIN = 1;
+}
+
 // YANG instance data.
 message DataTree {
   // The encoding format of the data.

src/client/grpc.rs

@@ -4,14 +4,18 @@
 // SPDX-License-Identifier: MIT
 //
 
-use holo_yang as yang;
+use std::ffi::{CStr, CString};
+use std::os::raw::{c_char, c_void};
+
 use proto::northbound_client::NorthboundClient;
 use yang3::data::{
     Data, DataDiffFlags, DataFormat, DataPrinterFlags, DataTree,
 };
+use yang3::ffi;
 
 use crate::client::{Client, DataType, DataValue};
 use crate::error::Error;
+use crate::YANG_MODULES_DIR;
 
 pub mod proto {
     tonic::include_proto!("holo");
@@ -40,6 +44,14 @@ impl GrpcClient {
         self.runtime.block_on(self.client.capabilities(request))
     }
 
+    fn rpc_sync_get_schema(
+        &mut self,
+        request: proto::GetSchemaRequest,
+    ) -> Result<tonic::Response<proto::GetSchemaResponse>, tonic::Status> {
+        let request = tonic::Request::new(request);
+        self.runtime.block_on(self.client.get_schema(request))
+    }
+
     fn rpc_sync_get(
         &mut self,
         request: proto::GetRequest,
@@ -82,15 +94,46 @@ impl Client for GrpcClient {
         Ok(GrpcClient { client, runtime })
     }
 
-    fn load_modules(&mut self, yang_ctx: &mut yang3::context::Context) {
+    fn load_modules(
+        &mut self,
+        dest: &'static str,
+        yang_ctx: &mut yang3::context::Context,
+    ) {
         // Retrieve the set of capabilities supported by the daemon.
         let capabilities = self
             .rpc_sync_capabilities()
             .expect("Failed to parse gRPC Capabilities() response");
 
+        // Establish a separate connection to holod for libyang to fetch any
+        // missing YANG modules or submodules using the `GetSchema` RPC.
+        let client = Self::connect(dest).expect("Connection to holod failed");
+        unsafe {
+            yang_ctx.set_module_import_callback(
+                ly_module_import_cb,
+                Box::into_raw(Box::new(client)) as _,
+            )
+        };
+
         // Load YANG modules dynamically.
         for module in capabilities.into_inner().supported_modules {
-            yang::load_module(yang_ctx, &module.name);
+            let revision = if module.revision.is_empty() {
+                None
+            } else {
+                Some(module.revision.as_ref())
+            };
+            let features = &module
+                .supported_features
+                .iter()
+                .map(String::as_str)
+                .collect::<Vec<_>>();
+            if let Err(error) =
+                yang_ctx.load_module(&module.name, revision, features)
+            {
+                panic!(
+                    "failed to load YANG module ({}): {}",
+                    module.name, error
+                );
+            }
         }
     }
 
@@ -200,3 +243,85 @@ impl From<DataFormat> for proto::Encoding {
         }
     }
 }
+
+// ===== helper functions =====
+
+unsafe extern "C" fn ly_module_import_cb(
+    module_name: *const c_char,
+    module_revision: *const c_char,
+    submodule_name: *const c_char,
+    submodule_revision: *const c_char,
+    user_data: *mut c_void,
+    format: *mut ffi::LYS_INFORMAT::Type,
+    module_data: *mut *const c_char,
+    _free_module_data: *mut ffi::ly_module_imp_data_free_clb,
+) -> ffi::LY_ERR::Type {
+    let module_name = char_ptr_to_string(module_name);
+    let module_revision = char_ptr_to_opt_string(module_revision);
+    let submodule_name = char_ptr_to_opt_string(submodule_name);
+    let submodule_revision = char_ptr_to_opt_string(submodule_revision);
+
+    // Retrive module or submodule via gRPC.
+    let client = unsafe { &mut *(user_data as *mut GrpcClient) };
+    if let Ok(response) = client.rpc_sync_get_schema(proto::GetSchemaRequest {
+        module_name: module_name.clone(),
+        module_revision: module_revision.clone().unwrap_or_default(),
+        submodule_name: submodule_name.clone().unwrap_or_default(),
+        submodule_revision: submodule_revision.clone().unwrap_or_default(),
+        format: proto::SchemaFormat::Yang.into(),
+    }) {
+        // Cache the module in the filesystem.
+        let data = response.into_inner().data;
+        let path = match (module_revision, submodule_name, submodule_revision) {
+            (None, None, _) => build_cache_path(&module_name, None),
+            (Some(module_revision), None, _) => {
+                build_cache_path(&module_name, Some(&module_revision))
+            }
+            (_, Some(submodule_name), None) => {
+                build_cache_path(&submodule_name, None)
+            }
+            (_, Some(submodule_name), Some(submodule_revision)) => {
+                build_cache_path(&submodule_name, Some(&submodule_revision))
+            }
+        };
+        if let Err(error) = std::fs::write(&path, &data) {
+            eprintln!(
+                "Failed to save YANG module in the cache ({}): {}",
+                module_name, error
+            );
+        }
+
+        // Return the retrieved module or submodule.
+        let data = CString::new(data).unwrap();
+        *format = ffi::LYS_INFORMAT::LYS_IN_YANG;
+        *module_data = data.as_ptr();
+        std::mem::forget(data);
+        return ffi::LY_ERR::LY_SUCCESS;
+    }
+
+    ffi::LY_ERR::LY_ENOTFOUND
+}
+
+// Builds the file path for caching a YANG module or submodule.
+fn build_cache_path(name: &str, revision: Option<&str>) -> String {
+    match revision {
+        Some(revision) => {
+            format!("{}/{}@{}.yang", YANG_MODULES_DIR, name, revision)
+        }
+        None => format!("{}/{}.yang", YANG_MODULES_DIR, name),
+    }
+}
+
+// Converts C String to owned string.
+fn char_ptr_to_string(c_str: *const c_char) -> String {
+    unsafe { CStr::from_ptr(c_str).to_string_lossy().into_owned() }
+}
+
+// Converts C String to optional owned string.
+fn char_ptr_to_opt_string(c_str: *const c_char) -> Option<String> {
+    if c_str.is_null() {
+        None
+    } else {
+        Some(char_ptr_to_string(c_str))
+    }
+}

src/client/mod.rs

@@ -30,7 +30,11 @@ pub trait Client: Send + std::fmt::Debug {
         Self: Sized;
 
     // Retrieve and load all supported YANG modules.
-    fn load_modules(&mut self, yang_ctx: &mut yang3::context::Context);
+    fn load_modules(
+        &mut self,
+        dest: &'static str,
+        yang_ctx: &mut yang3::context::Context,
+    );
 
     // Retrieve configuration data, state data or both.
     fn get(

src/internal_commands.rs

@@ -7,7 +7,6 @@
 use std::fmt::Write;
 use std::process::{Child, Command, Stdio};
 
-use holo_yang::YANG_CTX;
 use indextree::NodeId;
 use prettytable::{format, row, Table};
 use similar::TextDiff;
@@ -21,6 +20,7 @@ use crate::client::{DataType, DataValue};
 use crate::parser::ParsedArgs;
 use crate::session::{CommandMode, ConfigurationType, Session};
 use crate::token::{Commands, TokenKind};
+use crate::YANG_CTX;
 
 const XPATH_PROTOCOL: &str =
     "/ietf-routing:routing/control-plane-protocols/control-plane-protocol";

src/main.rs

@@ -14,12 +14,11 @@ mod token;
 mod token_xml;
 mod token_yang;
 
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc, Mutex, OnceLock};
 
 use clap::{App, Arg};
-use holo_yang as yang;
-use holo_yang::YANG_CTX;
 use reedline::Signal;
+use yang3::context::{Context, ContextFlags};
 
 use crate::client::grpc::GrpcClient;
 use crate::client::Client;
@@ -28,6 +27,12 @@ use crate::session::{CommandMode, Session};
 use crate::terminal::CliPrompt;
 use crate::token::{Action, Commands};
 
+// Global YANG context.
+pub static YANG_CTX: OnceLock<Arc<Context>> = OnceLock::new();
+
+// Default YANG modules cache directory.
+pub const YANG_MODULES_DIR: &str = "/usr/local/share/holo-cli/modules";
+
 pub struct Cli {
     commands: Commands,
     session: Session,
@@ -152,14 +157,12 @@ fn main() {
         )
         .get_matches();
 
+    // Connect to the daemon.
     let addr = matches
         .value_of("address")
         .unwrap_or("http://[::1]:50051")
         .to_string();
     let grpc_addr: &'static str = Box::leak(addr.into_boxed_str());
-
-    // Initialize YANG context and gRPC client.
-    let mut yang_ctx = yang::new_context();
     let mut client = match GrpcClient::connect(grpc_addr) {
         Ok(client) => client,
         Err(error) => {
@@ -168,7 +171,25 @@ fn main() {
             std::process::exit(1);
         }
     };
-    client.load_modules(&mut yang_ctx);
+
+    // Initialize YANG context.
+    let mut yang_ctx = Context::new(
+        ContextFlags::NO_YANGLIBRARY | ContextFlags::PREFER_SEARCHDIRS,
+    )
+    .unwrap();
+    yang_ctx.set_searchdir(YANG_MODULES_DIR).unwrap();
+
+    // Ensure the YANG modules cache directory exists, creating it if necessary.
+    if let Err(error) = std::fs::create_dir_all(YANG_MODULES_DIR) {
+        eprintln!(
+            "Failed to create YANG modules directory ({}): {}",
+            YANG_MODULES_DIR, error
+        );
+        std::process::exit(1);
+    }
+
+    // Load YANG modules.
+    client.load_modules(grpc_addr, &mut yang_ctx);
     YANG_CTX.set(Arc::new(yang_ctx)).unwrap();
 
     // Initialize CLI master structure.

src/session.rs

@@ -6,7 +6,6 @@
 
 use derive_new::new;
 use enum_as_inner::EnumAsInner;
-use holo_yang::YANG_CTX;
 use indextree::NodeId;
 use yang3::data::{
     Data, DataFormat, DataParserFlags, DataTree, DataValidationFlags,
@@ -17,7 +16,7 @@ use crate::client::{Client, DataType, DataValue};
 use crate::error::Error;
 use crate::parser::ParsedArgs;
 use crate::token::Commands;
-use crate::token_yang;
+use crate::{token_yang, YANG_CTX};
 
 static DEFAULT_HOSTNAME: &str = "holo";
 

src/token_yang.rs

@@ -7,13 +7,13 @@
 use std::fmt::Write;
 use std::os::raw::c_void;
 
-use holo_yang::YANG_CTX;
 use indextree::NodeId;
 use itertools::Itertools;
 use yang3::schema::{DataValueType, SchemaNode, SchemaNodeKind};
 
 use crate::parser::ParsedArgs;
 use crate::token::{Action, Commands, Token, TokenKind};
+use crate::YANG_CTX;
 
 pub(crate) fn gen_cmds(commands: &mut Commands) {
     // Iterate over top-level YANG nodes.