[WIP] graph-builder: add dynamic plugins configuration

cincinnati/Cargo.toml

@@ -15,6 +15,7 @@ url = "^1.7.2"
 log = "^0.4.3"
 futures = "0.1"
 tokio = "0.1"
+toml = "^0.4.10"
 try_from = "0.3.2"
 
 [dev-dependencies]

cincinnati/src/lib.rs

@@ -22,6 +22,7 @@ extern crate commons;
 #[macro_use]
 extern crate log;
 extern crate protobuf;
+extern crate toml;
 extern crate try_from;
 extern crate url;
 

cincinnati/src/plugins/internal/edge_add_remove.rs

@@ -2,9 +2,12 @@
 
 use crate as cincinnati;
 use failure::Fallible;
-use plugins::{InternalIO, InternalPlugin};
+use plugins::{InternalIO, InternalPlugin, InternalPluginWrapper, Plugin, PluginIO};
+use std::collections::HashMap;
 use ReleaseId;
 
+static DEFAULT_KEY_FILTER: &str = "com.openshift.upgrades.graph";
+
 pub struct EdgeAddRemovePlugin {
     pub key_prefix: String,
 }
@@ -25,6 +28,32 @@ impl InternalPlugin for EdgeAddRemovePlugin {
 ///
 /// The labels are assumed to have the syntax `<prefix>.(previous|next).(remove|add)=(<Version>,)*<Version>`
 impl EdgeAddRemovePlugin {
+    /// Plugin name, for configuration.
+    pub(crate) const PLUGIN_NAME: &'static str = "edge-add-remove";
+
+    /// Validate plugin configuration and fill in defaults.
+    pub fn sanitize_config(cfg: &mut HashMap<String, String>) -> Fallible<()> {
+        let name = cfg.get("name").cloned().unwrap_or_default();
+        ensure!(name == Self::PLUGIN_NAME, "unexpected plugin name");
+
+        cfg.entry("key_prefix".to_string())
+            .or_insert_with(|| DEFAULT_KEY_FILTER.to_string());
+        // TODO(lucab): perform semantic validation.
+
+        Ok(())
+    }
+
+    /// Try to build a plugin from configuration.
+    pub fn from_settings(cfg: &HashMap<String, String>) -> Fallible<Box<Plugin<PluginIO>>> {
+        let key_prefix = cfg
+            .get("key_prefix")
+            .ok_or_else(|| format_err!("empty key_prefix"))?
+            .to_string();
+        let plugin = Self { key_prefix };
+
+        Ok(Box::new(InternalPluginWrapper(plugin)))
+    }
+
     /// Remove next and previous releases from quay labels
     ///
     /// The labels are assumed to have the syntax `<prefix>.(previous|next).remove=(<Version>,)*<Version>`

cincinnati/src/plugins/internal/metadata_fetch_quay.rs

@@ -10,12 +10,14 @@ extern crate tokio;
 
 use self::futures::future::Future;
 use failure::{Fallible, ResultExt};
-use plugins::{InternalIO, InternalPlugin};
-use std::path::PathBuf;
+use plugins::{InternalIO, InternalPlugin, InternalPluginWrapper, Plugin, PluginIO};
+use std::collections::HashMap;
+use toml::value::Value;
 use ReleaseId;
 
 pub static DEFAULT_QUAY_LABEL_FILTER: &str = "com.openshift.upgrades.graph";
 pub static DEFAULT_QUAY_MANIFESTREF_KEY: &str = "com.openshift.upgrades.graph.release.manifestref";
+pub static DEFAULT_QUAY_REPOSITORY: &str = "openshift";
 
 pub struct QuayMetadataFetchPlugin {
     client: quay::v1::Client,
@@ -24,6 +26,82 @@ pub struct QuayMetadataFetchPlugin {
     manifestref_key: String,
 }
 
+/// Plugin settings.
+#[derive(Deserialize)]
+struct PluginSettings {
+    api_base: String,
+    api_credentials_path: Option<String>,
+    repository: String,
+    label_filter: String,
+    manifestref_key: String,
+}
+
+impl QuayMetadataFetchPlugin {
+    /// Plugin name, for configuration.
+    pub(crate) const PLUGIN_NAME: &'static str = "quay-metadata";
+
+    /// Validate plugin configuration and fill in defaults.
+    pub fn sanitize_config(cfg: &mut HashMap<String, String>) -> Fallible<()> {
+        let name = cfg.get("name").cloned().unwrap_or_default();
+        ensure!(name == Self::PLUGIN_NAME, "unexpected plugin name");
+
+        cfg.entry("api_base".to_string())
+            .or_insert_with(|| quay::v1::DEFAULT_API_BASE.to_string());
+        cfg.entry("repository".to_string())
+            .or_insert_with(|| DEFAULT_QUAY_REPOSITORY.to_string());
+        cfg.entry("label_filter".to_string())
+            .or_insert_with(|| DEFAULT_QUAY_LABEL_FILTER.to_string());
+        cfg.entry("manifestref_key".to_string())
+            .or_insert_with(|| DEFAULT_QUAY_MANIFESTREF_KEY.to_string());
+        // TODO(lucab): perform validation.
+
+        Ok(())
+    }
+
+    /// Try to build a plugin from settings.
+    pub fn from_settings(cfg: &HashMap<String, String>) -> Fallible<Box<Plugin<PluginIO>>> {
+        let cfg: PluginSettings = Value::try_from(cfg)?.try_into()?;
+
+        let plugin = Self::try_new(
+            cfg.repository,
+            cfg.label_filter,
+            cfg.manifestref_key,
+            cfg.api_credentials_path,
+            cfg.api_base,
+        )?;
+        Ok(Box::new(InternalPluginWrapper(plugin)))
+    }
+
+    fn try_new(
+        repo: String,
+        label_filter: String,
+        manifestref_key: String,
+        api_token_path: Option<String>,
+        api_base: String,
+    ) -> Fallible<Self> {
+        let api_token = match api_token_path {
+            Some(p) => {
+                let token =
+                    quay::read_credentials(p).context("could not read quay API credentials")?;
+                Some(token)
+            }
+            None => None,
+        };
+
+        let client: quay::v1::Client = quay::v1::Client::builder()
+            .access_token(api_token)
+            .api_base(Some(api_base))
+            .build()?;
+
+        Ok(Self {
+            client,
+            repo,
+            label_filter,
+            manifestref_key,
+        })
+    }
+}
+
 impl InternalPlugin for QuayMetadataFetchPlugin {
     fn run_internal(&self, io: InternalIO) -> Fallible<InternalIO> {
         let (mut graph, parameters) = (io.graph, io.parameters);
@@ -100,28 +178,3 @@ impl InternalPlugin for QuayMetadataFetchPlugin {
         Ok(InternalIO { graph, parameters })
     }
 }
-
-impl QuayMetadataFetchPlugin {
-    pub fn try_new(
-        repo: String,
-        label_filter: String,
-        manifestref_key: String,
-        api_token_path: Option<&PathBuf>,
-        api_base: String,
-    ) -> Fallible<Self> {
-        let api_token =
-            quay::read_credentials(api_token_path).expect("could not read quay API credentials");
-
-        let client: quay::v1::Client = quay::v1::Client::builder()
-            .access_token(api_token.map(|s| s.to_string()))
-            .api_base(Some(api_base.to_string()))
-            .build()?;
-
-        Ok(Self {
-            client,
-            repo,
-            label_filter,
-            manifestref_key,
-        })
-    }
-}

cincinnati/src/plugins/internal/node_remove.rs

@@ -1,12 +1,43 @@
 //! This plugin removes releases according to its metadata
 
 use failure::Fallible;
-use plugins::{InternalIO, InternalPlugin};
+use plugins::{InternalIO, InternalPlugin, InternalPluginWrapper, Plugin, PluginIO};
+use std::collections::HashMap;
+
+static DEFAULT_KEY_FILTER: &str = "com.openshift.upgrades.graph";
 
 pub struct NodeRemovePlugin {
     pub key_prefix: String,
 }
 
+impl NodeRemovePlugin {
+    /// Plugin name, for configuration.
+    pub(crate) const PLUGIN_NAME: &'static str = "node-remove";
+
+    /// Validate plugin configuration and fill in defaults.
+    pub fn sanitize_config(cfg: &mut HashMap<String, String>) -> Fallible<()> {
+        let name = cfg.get("name").cloned().unwrap_or_default();
+        ensure!(name == Self::PLUGIN_NAME, "unexpected plugin name");
+
+        cfg.entry("key_prefix".to_string())
+            .or_insert_with(|| DEFAULT_KEY_FILTER.to_string());
+        // TODO(lucab): perform semantic validation.
+
+        Ok(())
+    }
+
+    /// Try to build a plugin from settings.
+    pub fn from_settings(cfg: &HashMap<String, String>) -> Fallible<Box<Plugin<PluginIO>>> {
+        let key_prefix = cfg
+            .get("key_prefix")
+            .ok_or_else(|| format_err!("empty key_prefix"))?
+            .to_string();
+        let plugin = Self { key_prefix };
+
+        Ok(Box::new(InternalPluginWrapper(plugin)))
+    }
+}
+
 impl InternalPlugin for NodeRemovePlugin {
     fn run_internal(&self, io: InternalIO) -> Fallible<InternalIO> {
         let mut graph = io.graph;

cincinnati/src/plugins/mod.rs

@@ -4,7 +4,9 @@
 pub mod external;
 pub mod interface;
 pub mod internal;
+mod registry;
 
+pub use self::registry::{sanitize_config, try_from_settings};
 use crate as cincinnati;
 use failure::{Error, Fallible, ResultExt};
 use plugins::interface::{PluginError, PluginExchange};

cincinnati/src/plugins/registry.rs

@@ -0,0 +1,35 @@
+//! Plugin registry.
+//!
+//! This registry relies on a static list of all available plugin,
+//! referenced by name. It is used for configuration purposes.
+
+#![allow(clippy::implicit_hasher)]
+
+use super::internal::edge_add_remove::EdgeAddRemovePlugin;
+use super::internal::metadata_fetch_quay::QuayMetadataFetchPlugin;
+use super::internal::node_remove::NodeRemovePlugin;
+use super::{Plugin, PluginIO};
+use failure::Fallible;
+use std::collections::HashMap;
+
+/// Validate configuration for a plugin and fill in defaults.
+pub fn sanitize_config(cfg: &mut HashMap<String, String>) -> Fallible<()> {
+    let kind = cfg.get("name").cloned().unwrap_or_default();
+    match kind.as_str() {
+        EdgeAddRemovePlugin::PLUGIN_NAME => EdgeAddRemovePlugin::sanitize_config(cfg),
+        NodeRemovePlugin::PLUGIN_NAME => NodeRemovePlugin::sanitize_config(cfg),
+        QuayMetadataFetchPlugin::PLUGIN_NAME => QuayMetadataFetchPlugin::sanitize_config(cfg),
+        x => bail!("unknown plugin '{}'", x),
+    }
+}
+
+/// Try to build a plugin from runtime settings.
+pub fn try_from_settings(settings: &HashMap<String, String>) -> Fallible<Box<Plugin<PluginIO>>> {
+    let kind = settings.get("name").cloned().unwrap_or_default();
+    match kind.as_str() {
+        EdgeAddRemovePlugin::PLUGIN_NAME => EdgeAddRemovePlugin::from_settings(settings),
+        NodeRemovePlugin::PLUGIN_NAME => NodeRemovePlugin::from_settings(settings),
+        QuayMetadataFetchPlugin::PLUGIN_NAME => QuayMetadataFetchPlugin::from_settings(settings),
+        x => bail!("unknown plugin '{}'", x),
+    }
+}

dist/openshift/cincinnati.configmap.yaml

@@ -5,6 +5,7 @@ metadata:
 type: Opaque
 data:
   gb.address: "0.0.0.0"
+  gb.status.address: "0.0.0.0"
   gb.registry: "quay.io"
   gb.repository: "steveej/cincinnati-test"
   gb.log.verbosity: "vvv"

dist/openshift/cincinnati.yaml

@@ -39,6 +39,11 @@ objects:
                 configMapKeyRef:
                     key: gb.address
                     name: cincinnati
+            - name: STATUS_ADDRESS
+              valueFrom:
+                configMapKeyRef:
+                    key: gb.status.address
+                    name: cincinnati
             - name: REGISTRY
               valueFrom:
                 configMapKeyRef:
@@ -54,7 +59,14 @@ objects:
                 configMapKeyRef:
                   key: gb.log.verbosity
                   name: cincinnati
-          args: ["-$(GB_LOG_VERBOSITY)", "--address", "$(ADDRESS)", "--port", "${GB_PORT}", "--registry", "$(REGISTRY)", "--repository", "$(REPOSITORY)", "--credentials-file=/etc/secrets/registry-credentials"]
+          args: ["-$(GB_LOG_VERBOSITY)",
+                 "--service.address", "$(ADDRESS)",
+                 "--service.port", "${GB_PORT}",
+                 "--status.address", "$(STATUS_ADDRESS)",
+                 "--status.port", "${GB_STATUS_PORT}",
+                 "--upstream.registry.url", "$(REGISTRY)",
+                 "--upstream.registry.repository", "$(REPOSITORY)",
+                 "--upstream.registry.credentials_path", "/etc/secrets/registry-credentials"]
           ports:
           - name: graph-builder
             containerPort: ${{GB_PORT}}

graph-builder/Cargo.toml

@@ -27,6 +27,7 @@ serde_json = "^1.0.22"
 structopt = "^0.2.10"
 tar = "^0.4.16"
 tokio = "0.1"
+toml = "^0.4.10"
 url = "^1.7.2"
 
 [features]