Implement the tex formatter and TexThemeRenderer

lib/rouge.rb

@@ -71,9 +71,11 @@ def lexer_dir(path = '')
 load_relative 'rouge/formatters/html_linewise'
 load_relative 'rouge/formatters/html_inline'
 load_relative 'rouge/formatters/terminal256'
+load_relative 'rouge/formatters/tex'
 load_relative 'rouge/formatters/null'
 
 load_relative 'rouge/theme'
+load_relative 'rouge/tex_theme_renderer'
 load_relative 'rouge/themes/thankful_eyes'
 load_relative 'rouge/themes/colorful'
 load_relative 'rouge/themes/base16'

lib/rouge/cli.rb

@@ -299,6 +299,7 @@ def initialize(opts={})
         when 'html-inline' then Formatters::HTMLInline.new(theme)
         when 'html-table' then Formatters::HTMLTable.new(Formatters::HTML.new)
         when 'null', 'raw', 'tokens' then Formatters::Null.new
+        when 'tex' then Formatters::Tex.new
         else
           error! "unknown formatter preset #{opts[:formatter]}"
         end
@@ -334,18 +335,30 @@ def self.doc
         yield %|respectively. Theme defaults to thankful_eyes.|
         yield %||
         yield %|options:|
-        yield %|  --scope	(default: .highlight) a css selector to scope by|
+        yield %|  --scope     	(default: .highlight) a css selector to scope by|
+        yield %|  --tex       	(default: false) render as TeX|
+        yield %|  --tex-prefix	(default: RG) a command prefix for TeX|
+        yield %|              	implies --tex if specified|
         yield %||
         yield %|available themes:|
         yield %|  #{Theme.registry.keys.sort.join(', ')}|
       end
 
       def self.parse(argv)
-        opts = { :theme_name => 'thankful_eyes' }
+        opts = {
+          :theme_name => 'thankful_eyes',
+          :tex => false,
+          :tex_prefix => 'RG'
+        }
 
         until argv.empty?
           arg = argv.shift
           case arg
+          when '--tex'
+            opts[:tex] = true
+          when '--tex-prefix'
+            opts[:tex] = true
+            opts[:tex_prefix] = argv.shift
           when /--(\w+)/
             opts[$1.tr('-', '_').to_sym] = argv.shift
           else
@@ -362,6 +375,10 @@ def initialize(opts)
           or error! "unknown theme: #{theme_name}"
 
         @theme = theme_class.new(opts)
+        if opts[:tex]
+          tex_prefix = opts[:tex_prefix]
+          @theme = TexThemeRenderer.new(@theme, prefix: tex_prefix)
+        end
       end
 
       def run

lib/rouge/formatters/tex.rb

@@ -0,0 +1,90 @@
+module Rouge
+  module Formatters
+    class Tex < Formatter
+      tag 'tex'
+
+      # A map of TeX escape characters.
+      # Newlines are handled specially by using #token_lines
+      # spaces are preserved as long as they aren't at the beginning
+      # of a line. see #tag_first for our initial-space strategy
+      ESCAPE = {
+        '&' => '\&',
+        '%' => '\%',
+        '$' => '\$',
+        '#' => '\#',
+        '_' => '\_',
+        '{' => '\{',
+        '}' => '\}',
+        '~' => '{\textasciitilde}',
+        '^' => '{\textasciicircum}',
+        '|' => '{\textbar}',
+        '\\' => '{\textbackslash}',
+        "\t" => '{\tab}',
+      }
+
+      ESCAPE_REGEX = /[#{ESCAPE.keys.map(&Regexp.method(:escape)).join}]/om
+
+      def initialize(opts={})
+        @prefix = opts.fetch(:prefix) { 'RG' }
+      end
+
+      def escape_tex(str)
+        str.gsub(ESCAPE_REGEX, ESCAPE)
+      end
+
+      def stream(tokens, &b)
+        # surround the output with \begin{RG*}...\end{RG*}
+        yield "\\begin{#{@prefix}*}%\n"
+
+        # we strip the newline off the last line to avoid
+        # an extra line being rendered. we do this by yielding
+        # the \newline tag *before* every line group except
+        # the first.
+        first = true
+
+        token_lines tokens do |line|
+          if first
+            first = false
+          else
+            yield "\\newline%\n"
+          end
+
+          render_line(line, &b)
+        end
+
+        yield "%\n\\end{#{@prefix}*}%\n"
+      end
+
+      def render_line(line, &b)
+        head, *rest = line
+        return unless head
+
+        tag_first(*head, &b)
+        rest.each do |(tok, val)|
+          yield tag(tok, val)
+        end
+      end
+
+      # special handling for the first token
+      # of a line. we replace all initial spaces
+      # with \hphantom{xxxx}, which renders an
+      # empty space equal to the size of the x's.
+      def tag_first(tok, val)
+        leading = nil
+        val.sub!(/^[ ]+/) { leading = $&.size; '' }
+        yield "\\hphantom{#{'x' * leading}}" if leading
+        yield tag(tok, val)
+      end
+
+      def tag(tok, val)
+        if escape?(tok)
+          val
+        elsif tok == Token::Tokens::Text
+          escape_tex(val)
+        else
+          "\\#@prefix{#{tok.shortname}}{#{escape_tex(val)}}"
+        end
+      end
+    end
+  end
+end

lib/rouge/tex_theme_renderer.rb

@@ -0,0 +1,128 @@
+module Rouge
+  class TexThemeRenderer
+    def initialize(theme, opts={})
+      @theme = theme
+      @prefix = opts.fetch(:prefix) { 'RG' }
+    end
+
+    # Our general strategy is this:
+    #
+    # * First, define the \RG{tokname}{content} command, which will
+    #   expand into \RG@tok@tokname{content}. We use \csname...\endcsname
+    #   to interpolate into a command.
+    #
+    # * Define the default RG* environment, which will enclose the whole
+    #   thing. By default this will simply set \ttfamily (select monospace font)
+    #   but it can be overridden with \renewcommand by the user to be
+    #   any other formatting.
+    #
+    # * Define all the colors using xcolors \definecolor command. First we define
+    #   every palette color with a name such as RG@palette@themneame@colorname.
+    #   Then we find all foreground and background colors that have literal html
+    #   colors embedded in them and define them with names such as
+    #   RG@palette@themename@000000. While html allows three-letter colors such
+    #   as #FFF, xcolor requires all six characters to be present, so we make sure
+    #   to normalize that as well as the case convention in #inline_name.
+    #
+    # * Define the token commands RG@tok@xx. These will take the content as the
+    #   argument and format it according to the theme, referring to the color
+    #   in the palette.
+    def render(&b)
+      yield <<'END'.gsub('RG', @prefix)
+\makeatletter
+\def\RG#1#2{\csname RG@tok@#1\endcsname{#2}}%
+\newenvironment{RG*}{\ttfamily}{\relax}%
+END
+
+      base = @theme.class.base_style
+      yield "\\definecolor{#{@prefix}@fgcolor}{HTML}{#{inline_name(base.fg)}}"
+      yield "\\definecolor{#{@prefix}@bgcolor}{HTML}{#{inline_name(base.bg)}}"
+
+      render_palette(@theme.palette, &b)
+
+      @theme.styles.each do |tok, style|
+        render_inline_pallete(style, &b)
+      end
+
+      Token.each_token do |tok|
+        style = @theme.class.get_own_style(tok)
+        style ? render_style(tok, style, &b) : render_blank(tok, &b)
+      end
+      yield '\makeatother'
+    end
+
+    def render_palette(palette, &b)
+      palette.each do |name, color|
+        hex = inline_name(color)
+
+        yield "\\definecolor{#{palette_name(name)}}{HTML}{#{hex}}%"
+      end
+    end
+
+    def render_inline_pallete(style, &b)
+      gen_inline(style[:fg], &b)
+      gen_inline(style[:bg], &b)
+    end
+
+    def inline_name(color)
+      color =~ /^#(\h+)/ or return nil
+
+      # xcolor does not support 3-character HTML colors,
+      # so we convert them here
+      case $1.size
+      when 6
+        $1
+      when 3
+        # duplicate every character: abc -> aabbcc
+        $1.gsub(/\h/, '\0\0')
+      else
+        raise "invalid HTML color: #{$1}"
+      end.upcase
+    end
+
+    def gen_inline(name, &b)
+      # detect inline colors
+      hex = inline_name(name)
+      return unless hex
+
+      @gen_inline ||= {}
+      @gen_inline[hex] ||= begin
+        yield "\\definecolor{#{palette_name(hex)}}{HTML}{#{hex}}%"
+      end
+    end
+
+    def camelize(name)
+      name.gsub(/_(.)/) { $1.upcase }
+    end
+
+    def palette_name(name)
+      name = inline_name(name) || name.to_s
+
+      "#{@prefix}@palette@#{camelize(@theme.name)}@#{camelize(name.to_s)}"
+    end
+
+    def token_name(tok)
+      "\\csname #@prefix@tok@#{tok.shortname}\\endcsname"
+    end
+
+    def render_blank(tok, &b)
+      out = "\\expandafter\\def#{token_name(tok)}#1{#1}"
+    end
+
+    def render_style(tok, style, &b)
+      out = "\\expandafter\\def#{token_name(tok)}#1{"
+      out << "\\fboxsep=0pt\\colorbox{#{palette_name(style[:bg])}}{" if style[:bg]
+      out << '\\textbf{' if style[:bold]
+      out << '\\textit{' if style[:italic]
+      out << "\\textcolor{#{palette_name(style[:fg])}}{" if style[:fg]
+      out << "#1"
+      # close the right number of curlies
+      out << "}" if style[:bold]
+      out << "}" if style[:italic]
+      out << "}" if style[:fg]
+      out << "}" if style[:bg]
+      out << "}%"
+      yield out
+    end
+  end
+end

lib/rouge/theme.rb

@@ -83,6 +83,10 @@ def get_style(token)
       self.class.get_style(token)
     end
 
+    def name
+      self.class.name
+    end
+
     class << self
       def style(*tokens)
         style = tokens.last.is_a?(Hash) ? tokens.pop : {}

spec/formatters/tex_spec.rb

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+describe Rouge::Formatters::Tex do
+  let(:subject) { Rouge::Formatters::Tex.new }
+  let(:options) { {} }
+  let(:tokens) { [] }
+  let(:output) { subject.format(tokens) }
+  let(:expected) { '' }
+
+  describe 'a basic example' do
+    let(:tokens) { [[Token['Name'], 'foo']] }
+    let(:options) { { :wrap => false } }
+
+    let(:expected) do
+<<-'OUT'
+\begin{RG*}%
+\RG{n}{foo}%
+\end{RG*}%
+    OUT
+    end
+
+    it 'renders' do
+      assert { output == expected }
+    end
+  end
+
+  describe "escaping strategies" do
+    # some fake code that might look like:
+    #
+    # foo {
+    #   ~100%
+    # }
+    #
+    # we must escape the braces, the percent sign, the tilde, 
+    # and the initial space on the second line.
+    let(:tokens) do
+      [[Token['Keyword'], 'foo'],
+       [Token['Text'], ' '],
+       [Token['Punctuation'], '{'],
+       [Token['Text'], "\n  "],
+       [Token['Name.Constant'], '~100%'],
+       [Token['Text'], "\n"],
+       [Token['Punctuation'], '}'],
+       [Token['Text'], "\n"]]
+    end
+
+    let(:expected) do
+      <<-'OUT'
+\begin{RG*}%
+\RG{k}{foo} \RG{p}{\{}\newline%
+\hphantom{xx}\RG{no}{{\textasciitilde}100\%}\newline%
+\RG{p}{\}}%
+\end{RG*}%
+      OUT
+    end
+
+    it 'renders' do
+      assert { output == expected }
+    end
+  end
+end