resolves asciidoctor#247 add support for the Rouge source highlighter

- add integration with Rouge for highlighting source listings
- organize the code to setup source highlighting
- enable line number support when highlighting with Rouge
- add pastie theme for Rouge
- patch Rouge style lookup (see rouge-ruby/rouge#280)

lib/asciidoctor-pdf/converter.rb

@@ -32,7 +32,8 @@ def self.unicode_char number
   end
 
   # NOTE require_library doesn't support require_relative and we don't modify the load path for this gem
-  CodeRayRequirePath = ::File.join((::File.dirname __FILE__), 'prawn_ext/coderay_encoder')
+  CodeRayRequirePath = ::File.join (::File.dirname __FILE__), 'prawn_ext/coderay_encoder'
+  RougeRequirePath = ::File.join (::File.dirname __FILE__), 'rouge_ext'
 
   AdmonitionIcons = {
     caution:   { key: 'fa-fire', color: 'BF3400' },
@@ -70,6 +71,7 @@ def self.unicode_char number
   CalloutExtractRx = /(?:(?:\/\/|#|--|;;) ?)?(\\)?<!?(--|)(\d+)\2>(?=(?: ?\\?<!?\2\d+\2>)*$)/
   ImageAttributeValueRx = /^image:{1,2}(.*?)\[(.*?)\]$/
   LineScanRx = /\n|.+/
+  SourceHighlighters = ['coderay', 'pygments', 'rouge']
 
   def initialize backend, opts
     super
@@ -870,18 +872,37 @@ def convert_image node
   # QUESTION can we avoid arranging fragments multiple times (conums & autofit) by eagerly preparing arranger?
   def convert_listing_or_literal node
     add_dest_for_block node if node.id
+
     # HACK disable built-in syntax highlighter; must be done before calling node.content!
-    # NOTE the highlight sub is only set for coderay and pygments
-    if node.style == 'source' && !scratch? && ((subs = node.subs).include? :highlight)
-      highlighter = node.document.attr 'source-highlighter'
-      # NOTE the source highlighter logic below handles the callouts and highlight subs
-      prev_subs = subs.dup
-      subs.delete_all :highlight, :callouts
+    if node.style == 'source' && node.attributes['language'] &&
+        (highlighter = node.document.attributes['source-highlighter']) &&
+        (SourceHighlighters.include? highlighter)
+      prev_subs = (subs = node.subs).dup
+      # NOTE the highlight sub is only set for coderay and pygments atm
+      highlight_idx = subs.index :highlight
+      # NOTE scratch? here only applies if listing block is nested inside another block
+      if scratch?
+        highlighter = nil
+        if highlight_idx
+          subs[highlight_idx] = :specialcharacters
+        else
+          prev_subs = nil
+        end
+      else
+        # NOTE the source highlighter logic below handles the callouts and highlight subs
+        if highlight_idx
+          subs.delete_all :highlight, :callouts
+        else
+          subs.delete_all :specialcharacters, :callouts
+        end
+      end
     else
       highlighter = nil
       prev_subs = nil
     end
+
     source_string = preserve_indentation node.content
+
     source_chunks = case highlighter
     when 'coderay'
       Helpers.require_library CodeRayRequirePath, 'coderay' unless defined? ::Asciidoctor::Prawn::CodeRayEncoder
@@ -890,14 +911,22 @@ def convert_listing_or_literal node
       conum_mapping ? (restore_conums fragments, conum_mapping) : fragments
     when 'pygments'
       Helpers.require_library 'pygments', 'pygments.rb' unless defined? ::Pygments
-      source_string, conum_mapping = extract_conums source_string
       lexer = ::Pygments::Lexer[node.attr 'language', 'text', false] || ::Pygments::Lexer['text']
       pygments_config = { nowrap: true, noclasses: true, style: (node.document.attr 'pygments-style') || 'pastie' }
+      source_string, conum_mapping = extract_conums source_string
       result = lexer.highlight source_string, options: pygments_config
       fragments = text_formatter.format result
       conum_mapping ? (restore_conums fragments, conum_mapping) : fragments
+    when 'rouge'
+      Helpers.require_library RougeRequirePath, 'rouge' unless defined? ::Rouge::Formatters::Prawn
+      lexer = ::Rouge::Lexer.find(node.attr 'language', 'text', false) || ::Rouge::Lexers::PlainText
+      formatter = (@rouge_formatter ||= ::Rouge::Formatters::Prawn.new theme: (node.document.attr 'rouge-style'))
+      source_string, conum_mapping = extract_conums source_string
+      # NOTE trailing endline is added to address https://github.com/jneen/rouge/issues/279
+      fragments = formatter.format (lexer.lex %(#{source_string}#{EOL})), line_numbers: (node.attr? 'linenums')
+      conum_mapping ? (restore_conums fragments, conum_mapping) : fragments
     else
-      # NOTE only format if we detect a need
+      # NOTE only format if we detect a need (callouts or inline formatting)
       if source_string =~ BuiltInEntityCharOrTagRx
         text_formatter.format source_string
       else

lib/asciidoctor-pdf/rouge_ext.rb

@@ -0,0 +1,4 @@
+require 'rouge'
+require_relative 'rouge_ext/formatters/prawn'
+require_relative 'rouge_ext/css_theme'
+require_relative 'rouge_ext/themes/pastie'

lib/asciidoctor-pdf/rouge_ext/css_theme.rb

@@ -0,0 +1,14 @@
+module Rouge
+class CSSTheme
+  # Patch style_for to return most specific style first
+  # See https://github.com/jneen/rouge/issues/280 (fix pending)
+  def style_for token
+    token.token_chain.reverse_each do |t|
+      if (s = styles[t])
+        return s
+      end
+    end
+    nil
+  end
+end
+end

lib/asciidoctor-pdf/rouge_ext/formatters/prawn.rb

@@ -0,0 +1,111 @@
+module Rouge
+module Formatters
+# Transforms a token stream into an array of 
+# formatted text fragments for use with Prawn.
+class Prawn < Formatter
+  tag 'prawn'
+
+  EOL = "\n"
+  NoBreakSpace = [0x00a0].pack 'U*'
+  BoldStyle = [:bold].to_set
+  ItalicStyle = [:italic].to_set
+  BoldItalicStyle = [:bold, :italic].to_set
+
+  def initialize opts = {}
+    unless ::Rouge::Theme === (theme = opts[:theme])
+      unless theme && (theme = ::Rouge::Theme.find theme)
+        theme = ::Rouge::Themes::Pastie
+      end
+      theme = theme.new
+    end
+    @theme = theme
+    @normalized_colors = {}
+    @linenum_fragment_base = (create_fragment Token['Generic.Lineno']).merge linenum: true
+  end
+
+  # Override format method so fragments don't get flatted to a string
+  # and to add an options Hash.
+  def format tokens, opts = {}
+    stream tokens, opts
+
+    # ...we could strip trailing endline added to address https://github.com/jneen/rouge/issues/279
+    #fragments = stream tokens, opts
+    #if (last_fragment = fragments[-1])
+    #  last_fragment[:text] = last_fragment[:text].chomp
+    #end
+    #fragments
+  end
+
+  def stream tokens, opts = {}
+    if opts[:line_numbers]
+      # TODO implement line number start (offset)
+      linenum = 0
+      fragments = []
+      fragments << (create_linenum_fragment linenum += 1)
+      tokens.each do |tok, val|
+        fragment = create_fragment tok, val
+        if val == EOL
+          fragments << fragment
+          fragments << (create_linenum_fragment linenum += 1)
+        elsif val.include? EOL
+          val.each_line do |line|
+            fragments << (fragment.merge text: line)
+            # NOTE append linenum fragment if there's a next line; only works if source doesn't have trailing endline
+            if line.end_with? EOL
+              fragments << (create_linenum_fragment linenum += 1)
+            end
+          end
+        else
+          fragments << fragment
+        end
+      end
+      # NOTE drop orphaned linenum fragment (due to trailing endline in source)
+      fragments.pop if (last_fragment = fragments[-1]) && last_fragment[:linenum]
+      # NOTE pad numbers with less digits than the highest line number
+      if (linenum_w = (linenum / 10) + 1) > 1
+        # NOTE extra column is the trailing space after the line number
+        linenum_w += 1
+        fragments.each do |fragment|
+          fragment[:text] = %(#{fragment[:text].rjust linenum_w, NoBreakSpace}) if fragment[:linenum]
+        end
+      end
+      fragments
+    else
+      tokens.map {|tok, val| create_fragment tok, val }
+    end
+  end
+
+  def create_fragment tok, val = nil
+    fragment = val ? { text: val } : {}
+    if (style_rules = @theme.style_for tok)
+      # TODO support background color
+      if (fg = normalize_color style_rules.fg)
+        fragment[:color] = fg
+      end
+      if style_rules[:bold]
+        fragment[:styles] = style_rules[:italic] ? BoldItalicStyle : BoldStyle
+      elsif style_rules[:italic]
+        fragment[:styles] = ItalicStyle
+      end
+    end
+    fragment
+  end
+
+  def create_linenum_fragment linenum
+    @linenum_fragment_base.merge text: %(#{linenum} )
+  end
+
+  def normalize_color raw
+    return unless raw
+    if (normalized = @normalized_colors[raw])
+      normalized
+    else
+      normalized = raw
+      normalized = normalized[1..-1] if normalized.start_with? '#'
+      normalized = normalized.each_char.map {|c| c * 2 }.join if normalized.size == 3
+      @normalized_colors[raw] = normalized
+    end
+  end
+end
+end
+end

lib/asciidoctor-pdf/rouge_ext/themes/pastie.rb

@@ -0,0 +1,60 @@
+module Rouge
+module Themes
+# A Rouge theme that matches the pastie style from Pygments.
+# See https://bitbucket.org/birkenfeld/pygments-main/src/default/pygments/styles/pastie.py
+class Pastie < CSSTheme
+  name 'pastie' 
+
+  style Text::Whitespace,          fg: '#bbbbbb'
+
+  style Comment,                   fg: '#888888'
+  style Comment::Preproc,          fg: '#cc0000', bold: true
+  style Comment::Special,          fg: '#cc0000', bg: '#fff0f0', bold: true
+
+  style Error,                     fg: '#a61717', bg: '#e3d2d2'
+  style Generic::Error,            fg: '#aa0000'
+  style Generic::Traceback,        fg: '#aa0000'
+
+  style Generic::Deleted,          fg: '#000000', bg: '#ffdddd'
+  style Generic::Emph,             italic: true
+  style Generic::Inserted,         fg: '#000000', bg: '#ddffdd'
+  style Generic::Heading,          fg: '#333333'
+  #style Generic::Lineno,           fg: '#555555'
+  style Generic::Lineno,           fg: '#888888'
+  style Generic::Output,           fg: '#888888'
+  style Generic::Prompt,           fg: '#555555'
+  style Generic::Strong,           bold: true
+  style Generic::Subheading,       fg: '#666666'
+
+  style Keyword,                   fg: '#008800', bold: true
+  style Keyword::Pseudo,           fg: '#008800'
+  style Keyword::Type,             fg: '#888888', bold: true
+
+  style Literal::Number,           fg: '#0000dd', bold: true
+
+  style Literal::String,           fg: '#dd2200', bg: '#fff0f0'
+  style Literal::String::Escape,   fg: '#0044dd'
+  style Literal::String::Interpol, fg: '#3333bb'
+  style Literal::String::Other,    fg: '#22bb22', bg: '#f0fff0'
+  style Literal::String::Regex,    fg: '#008800', bg: '#fff0ff'
+  style Literal::String::Symbol,   fg: '#aa6600'
+
+  style Name::Attribute,           fg: '#336699'
+  style Name::Builtin,             fg: '#003388'
+  style Name::Class,               fg: '#bb0066', bold: true
+  style Name::Constant,            fg: '#003366', bold: true
+  style Name::Decorator,           fg: '#555555'
+  style Name::Exception,           fg: '#bb0066', bold: true
+  style Name::Function,            fg: '#0066bb', bold: true
+  style Name::Label,               fg: '#336699', italic: true
+  style Name::Namespace,           fg: '#bb0066', bold: true
+  style Name::Property,            fg: '#336699', bold: true
+  style Name::Tag,                 fg: '#bb0066', bold: true
+  style Name::Variable::Global,    fg: '#dd7700'
+  style Name::Variable::Instance,  fg: '#3333bb'
+  style Name::Variable,            fg: '#336699'
+
+  style Operator::Word,            fg: '#008800'
+end
+end
+end