Support custom syntax highlighting themes (#1499)

Related to #419

Gruvbox tmTheme added to test_site, it is taken from
https://github.com/Colorsublime/Colorsublime-Themes (MIT licensed)

components/config/src/config/markup.rs

@@ -1,9 +1,15 @@
-use std::path::Path;
+use std::{path::Path, sync::Arc};
 
 use serde_derive::{Deserialize, Serialize};
-use syntect::parsing::{SyntaxSet, SyntaxSetBuilder};
+use syntect::{
+    highlighting::{Theme, ThemeSet},
+    html::css_for_theme_with_class_style,
+    parsing::{SyntaxSet, SyntaxSetBuilder},
+};
 
-use errors::Result;
+use errors::{bail, Result};
+
+use crate::highlighting::{CLASS_STYLE, THEME_SET};
 
 pub const DEFAULT_HIGHLIGHT_THEME: &str = "base16-ocean-dark";
 
@@ -43,26 +49,92 @@ pub struct Markdown {
     pub external_links_no_referrer: bool,
     /// Whether smart punctuation is enabled (changing quotes, dashes, dots etc in their typographic form)
     pub smart_punctuation: bool,
-
-    /// A list of directories to search for additional `.sublime-syntax` files in.
-    pub extra_syntaxes: Vec<String>,
+    /// A list of directories to search for additional `.sublime-syntax` and `.tmTheme` files in.
+    pub extra_syntaxes_and_themes: Vec<String>,
     /// The compiled extra syntaxes into a syntax set
     #[serde(skip_serializing, skip_deserializing)] // not a typo, 2 are need
     pub extra_syntax_set: Option<SyntaxSet>,
+    /// The compiled extra themes into a theme set
+    #[serde(skip_serializing, skip_deserializing)] // not a typo, 2 are need
+    pub extra_theme_set: Arc<Option<ThemeSet>>,
 }
 
 impl Markdown {
-    /// Attempt to load any extra syntax found in the extra syntaxes of the config
-    pub fn load_extra_syntaxes(&mut self, base_path: &Path) -> Result<()> {
-        if self.extra_syntaxes.is_empty() {
-            return Ok(());
+    /// Gets the configured highlight theme from the THEME_SET or the config's extra_theme_set
+    /// Returns None if the configured highlighting theme is set to use css
+    pub fn get_highlight_theme(&self) -> Option<&Theme> {
+        if self.highlight_theme == "css" {
+            None
+        } else {
+            Some(self.get_highlight_theme_by_name(&self.highlight_theme))
+        }
+    }
+
+    /// Gets an arbitrary theme from the THEME_SET or the extra_theme_set
+    pub fn get_highlight_theme_by_name<'config>(&'config self, theme_name: &str) -> &'config Theme {
+        (*self.extra_theme_set)
+            .as_ref()
+            .and_then(|ts| ts.themes.get(theme_name))
+            .unwrap_or_else(|| &THEME_SET.themes[theme_name])
+    }
+
+    /// Attempt to load any extra syntaxes and themes found in the extra_syntaxes_and_themes folders
+    pub fn load_extra_syntaxes_and_highlight_themes(
+        &self,
+        base_path: &Path,
+    ) -> Result<(Option<SyntaxSet>, Option<ThemeSet>)> {
+        if self.extra_syntaxes_and_themes.is_empty() {
+            return Ok((None, None));
         }
 
         let mut ss = SyntaxSetBuilder::new();
-        for dir in &self.extra_syntaxes {
+        let mut ts = ThemeSet::new();
+        for dir in &self.extra_syntaxes_and_themes {
             ss.add_from_folder(base_path.join(dir), true)?;
+            ts.add_from_folder(base_path.join(dir))?;
+        }
+        let ss = ss.build();
+
+        Ok((
+            if ss.syntaxes().is_empty() { None } else { Some(ss) },
+            if ts.themes.is_empty() { None } else { Some(ts) },
+        ))
+    }
+
+    pub fn export_theme_css(&self, theme_name: &str) -> String {
+        let theme = self.get_highlight_theme_by_name(theme_name);
+        css_for_theme_with_class_style(theme, CLASS_STYLE)
+    }
+
+    pub fn init_extra_syntaxes_and_highlight_themes(&mut self, path: &Path) -> Result<()> {
+        if self.highlight_theme == "css" {
+            return Ok(());
+        }
+
+        let (loaded_extra_syntaxes, loaded_extra_highlight_themes) =
+            self.load_extra_syntaxes_and_highlight_themes(path)?;
+
+        if let Some(extra_syntax_set) = loaded_extra_syntaxes {
+            self.extra_syntax_set = Some(extra_syntax_set);
+        }
+        if let Some(extra_theme_set) = loaded_extra_highlight_themes {
+            self.extra_theme_set = Arc::new(Some(extra_theme_set));
+        }
+
+        // validate that the chosen highlight_theme exists in the loaded highlight theme sets
+        if !THEME_SET.themes.contains_key(&self.highlight_theme) {
+            if let Some(extra) = &*self.extra_theme_set {
+                if !extra.themes.contains_key(&self.highlight_theme) {
+                    bail!(
+                        "Highlight theme {} not found in the extra theme set",
+                        self.highlight_theme
+                    )
+                }
+            } else {
+                bail!("Highlight theme {} not available.\n\
+                You can load custom themes by configuring `extra_syntaxes_and_themes` to include a list of folders containing '.tmTheme' files", self.highlight_theme)
+            }
         }
-        self.extra_syntax_set = Some(ss.build());
 
         Ok(())
     }
@@ -110,8 +182,9 @@ impl Default for Markdown {
             external_links_no_follow: false,
             external_links_no_referrer: false,
             smart_punctuation: false,
-            extra_syntaxes: Vec::new(),
+            extra_syntaxes_and_themes: vec![],
             extra_syntax_set: None,
+            extra_theme_set: Arc::new(None),
         }
     }
 }

components/config/src/config/mod.rs

@@ -12,7 +12,6 @@ use globset::{Glob, GlobSet, GlobSetBuilder};
 use serde_derive::{Deserialize, Serialize};
 use toml::Value as Toml;
 
-use crate::highlighting::THEME_SET;
 use crate::theme::Theme;
 use errors::{bail, Error, Result};
 use utils::fs::read_file;
@@ -106,6 +105,7 @@ pub struct SerializedConfig<'a> {
 }
 
 impl Config {
+    // any extra syntax and highlight themes have been loaded and validated already by the from_file method before parsing the config
     /// Parses a string containing TOML to our Config struct
     /// Any extra parameter will end up in the extra field
     pub fn parse(content: &str) -> Result<Config> {
@@ -118,15 +118,6 @@ impl Config {
             bail!("A base URL is required in config.toml with key `base_url`");
         }
 
-        if config.markdown.highlight_theme != "css"
-            && !THEME_SET.themes.contains_key(&config.markdown.highlight_theme)
-        {
-            bail!(
-                "Highlight theme {} defined in config does not exist.",
-                config.markdown.highlight_theme
-            );
-        }
-
         languages::validate_code(&config.default_language)?;
         for code in config.languages.keys() {
             languages::validate_code(code)?;
@@ -166,7 +157,16 @@ impl Config {
         let path = path.as_ref();
         let content =
             read_file(path).map_err(|e| errors::Error::chain("Failed to load config", e))?;
-        Config::parse(&content)
+
+        let mut config = Config::parse(&content)?;
+        let config_dir = path
+            .parent()
+            .ok_or(Error::msg("Failed to find directory containing the config file."))?;
+
+        // this is the step at which missing extra syntax and highlighting themes are raised as errors
+        config.markdown.init_extra_syntaxes_and_highlight_themes(config_dir)?;
+
+        Ok(config)
     }
 
     /// Makes a url, taking into account that the base url might have a trailing slash

components/config/src/highlighting.rs

@@ -1,10 +1,12 @@
 use lazy_static::lazy_static;
 use syntect::dumps::from_binary;
 use syntect::highlighting::{Theme, ThemeSet};
+use syntect::html::ClassStyle;
 use syntect::parsing::{SyntaxReference, SyntaxSet};
 
 use crate::config::Config;
-use syntect::html::{css_for_theme_with_class_style, ClassStyle};
+
+pub const CLASS_STYLE: ClassStyle = ClassStyle::SpacedPrefixed { prefix: "z-" };
 
 lazy_static! {
     pub static ref SYNTAX_SET: SyntaxSet = {
@@ -16,8 +18,6 @@ lazy_static! {
         from_binary(include_bytes!("../../../sublime/themes/all.themedump"));
 }
 
-pub const CLASS_STYLE: ClassStyle = ClassStyle::SpacedPrefixed { prefix: "z-" };
-
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum HighlightSource {
     /// One of the built-in Zola syntaxes
@@ -42,11 +42,7 @@ pub fn resolve_syntax_and_theme<'config>(
     language: Option<&'_ str>,
     config: &'config Config,
 ) -> SyntaxAndTheme<'config> {
-    let theme = if config.markdown.highlight_theme != "css" {
-        Some(&THEME_SET.themes[&config.markdown.highlight_theme])
-    } else {
-        None
-    };
+    let theme = config.markdown.get_highlight_theme();
 
     if let Some(ref lang) = language {
         if let Some(ref extra_syntaxes) = config.markdown.extra_syntax_set {
@@ -88,8 +84,3 @@ pub fn resolve_syntax_and_theme<'config>(
         }
     }
 }
-
-pub fn export_theme_css(theme_name: &str) -> String {
-    let theme = &THEME_SET.themes[theme_name];
-    css_for_theme_with_class_style(theme, CLASS_STYLE)
-}

components/rendering/benches/all.rs

@@ -106,29 +106,30 @@ fn bench_render_content_without_highlighting(b: &mut test::Bencher) {
     let mut config = Config::default();
     config.markdown.highlight_code = false;
     let current_page_permalink = "";
+    let lang = "";
     let context = RenderContext::new(
         &tera,
         &config,
-        "",
+        lang,
         current_page_permalink,
         &permalinks_ctx,
         InsertAnchor::None,
     );
     b.iter(|| render_content(CONTENT, &context).unwrap());
 }
 
-#[bench]
 fn bench_render_content_no_shortcode(b: &mut test::Bencher) {
     let tera = Tera::default();
     let content2 = CONTENT.replace(r#"{{ youtube(id="my_youtube_id") }}"#, "");
     let mut config = Config::default();
     config.markdown.highlight_code = false;
     let permalinks_ctx = HashMap::new();
     let current_page_permalink = "";
+    let lang = "";
     let context = RenderContext::new(
         &tera,
         &config,
-        "",
+        lang,
         current_page_permalink,
         &permalinks_ctx,
         InsertAnchor::None,
@@ -144,16 +145,15 @@ fn bench_render_shortcodes_one_present(b: &mut test::Bencher) {
     let config = Config::default();
     let permalinks_ctx = HashMap::new();
     let current_page_permalink = "";
+    let lang = "";
     let context = RenderContext::new(
         &tera,
         &config,
-        "",
+        lang,
         current_page_permalink,
         &permalinks_ctx,
         InsertAnchor::None,
     );
-
-    b.iter(|| render_shortcodes(CONTENT, &context));
 }
 
 #[bench]
@@ -165,10 +165,11 @@ fn bench_render_content_no_shortcode_with_emoji(b: &mut test::Bencher) {
     config.markdown.render_emoji = true;
     let permalinks_ctx = HashMap::new();
     let current_page_permalink = "";
+    let lang = "";
     let context = RenderContext::new(
         &tera,
         &config,
-        "",
+        lang,
         current_page_permalink,
         &permalinks_ctx,
         InsertAnchor::None,

components/rendering/src/codeblock/highlight.rs

@@ -26,7 +26,7 @@ pub(crate) struct ClassHighlighter<'config> {
 }
 
 impl<'config> ClassHighlighter<'config> {
-    pub fn new(syntax: &'config SyntaxReference, syntax_set: &'config SyntaxSet) -> Self {
+    pub fn new(syntax: &SyntaxReference, syntax_set: &'config SyntaxSet) -> Self {
         let parse_state = ParseState::new(syntax);
         Self { syntax_set, open_spans: 0, parse_state, scope_stack: ScopeStack::new() }
     }

components/site/src/lib.rs

@@ -14,7 +14,6 @@ use rayon::prelude::*;
 use tera::{Context, Tera};
 use walkdir::{DirEntry, WalkDir};
 
-use config::highlighting::export_theme_css;
 use config::{get_config, Config};
 use errors::{bail, Error, Result};
 use front_matter::InsertAnchor;
@@ -74,7 +73,7 @@ impl Site {
         let path = path.as_ref();
         let config_file = config_file.as_ref();
         let mut config = get_config(config_file)?;
-        config.markdown.load_extra_syntaxes(path)?;
+        config.markdown.load_extra_syntaxes_and_highlight_themes(path)?;
 
         if let Some(theme) = config.theme.clone() {
             // Grab data from the extra section of the theme
@@ -691,7 +690,7 @@ impl Site {
         for t in &self.config.markdown.highlight_themes_css {
             let p = self.static_path.join(&t.filename);
             if !p.exists() {
-                let content = export_theme_css(&t.theme);
+                let content = &self.config.markdown.export_theme_css(&t.theme);
                 create_file(&p, &content)?;
             }
         }

docs/content/documentation/content/syntax-highlighting.md

@@ -150,7 +150,7 @@ Here is a full list of supported languages and their short names:
 Note: due to some issues with the JavaScript syntax, the TypeScript syntax will be used instead.
 
 If you want to highlight a language not on this list, please open an issue or a pull request on the [Zola repo](https://github.com/getzola/zola).
-Alternatively, the `extra_syntaxes` configuration option can be used to add additional syntax files.
+Alternatively, the `extra_syntaxes_and_themes` configuration option can be used to add additional syntax (and theme) files.
 
 If your site source is laid out as follows:
 
@@ -169,7 +169,7 @@ If your site source is laid out as follows:
     └── ...
 ```
 
-you would set your `extra_syntaxes` to `["syntaxes", "syntaxes/Sublime-Language1"]` to load `lang1.sublime-syntax` and `lang2.sublime-syntax`.
+you would set your `extra_syntaxes_and_themes` to `["syntaxes", "syntaxes/Sublime-Language1"]` to load `lang1.sublime-syntax` and `lang2.sublime-syntax`.
 
 ## Inline VS classed highlighting
 
@@ -347,3 +347,40 @@ Line 2 and 7 are comments that are not shown in the final output.
 
 When line numbers are active, the code block is turned into a table with one row and two cells. The first cell contains the line number and the second cell contains the code.
 Highlights are done via the `<mark>` HTML tag. When a line with line number is highlighted two `<mark>` tags are created: one around the line number(s) and one around the code.
+
+## Custom Highlighting Themes
+
+The default *theme* for syntax highlighting is called `base16-ocean-dark`, you can choose another theme from the built in set of highlight themes using the `highlight_theme` configuration option.
+For example, this documentation site currently uses the `kronuz` theme, which is built in.
+
+```
+[markdown]
+highlight_code = true
+highlight_theme = "kronuz"
+```
+
+Alternatively, the `extra_syntaxes_and_themes` configuration option can be used to add additional theme files.
+You can load your own highlight theme from a TextMate `.tmTheme` file.
+
+It works the same way as adding extra syntaxes. It should contain a list of paths to folders containing the .tmTheme files you want to include.
+You would then set `highlight_theme` to the name of one of these files, without the `.tmTheme` extension.
+
+If your site source is laid out as follows:
+
+```
+.
+├── config.toml
+├── content/
+│   └── ...
+├── static/
+│   └── ...
+├── highlight_themes/
+│   ├── MyGroovyTheme/
+│   │   └── theme1.tmTheme
+│   ├── theme2.tmTheme
+└── templates/
+    └── ...
+```
+
+you would set your `extra_highlight_themes` to `["highlight_themes", "highlight_themes/MyGroovyTheme"]` to load `theme1.tmTheme` and `theme2.tmTheme`.
+Then choose one of them to use, say theme1, by setting `highlight_theme = theme1`.

docs/content/documentation/getting-started/configuration.md

@@ -236,6 +236,9 @@ Zola currently has the following highlight themes available:
 Zola uses the Sublime Text themes, making it very easy to add more.
 If you want a theme not listed above, please open an issue or a pull request on the [Zola repo](https://github.com/getzola/zola).
 
+Alternatively you can use the `extra_syntaxes_and_themes` configuration option to load your own custom themes from a .tmTheme file.
+See [Syntax Highlighting](@/syntax-highlighting.md) for more details.
+
 ## Slugification strategies
 
 By default, Zola will turn every path, taxonomies and anchors to a slug, an ASCII representation with no special characters.