Add Language Detection endpoint

Close #247

Test

README.md

@@ -343,6 +343,30 @@ Response:
 
   ... PDF document body ...
 
+### Detect language
+
+This is a foreground document language detection request. The detected language
+will be returned as the response body.
+
+ POST /convert
+
+Params *(suggest using `multipart/form-data`)*:
+
+* `file` - the file to detect
+* `action` - `detect_language`
+
+#### Example:
+
+ POST /convert
+  file=... foo.docx ...
+  action=detect_language
+
+Response:
+
+  Content-Type: text/plain
+
+  en
+
 ## Callbacks
 
 When a document conversion is completed, an attempt will be made to POST a

config/app.yml

@@ -33,3 +33,4 @@ wkhtmltopdf_path: <%= ENV['WKHTMLTOPDF_PATH'] %>
 # Other settings
 tika_config_directory: <%= ENV['TIKA_CONFIG_DIRECTORY'] %>
 wkhtmltopdf_params: '-d 100 --encoding UTF-8'
+tesseract_available_languages: <%= ENV['TESSERACT_AVAILABLE_LANGUAGES'] %>

lib/app.rb

@@ -192,6 +192,28 @@ class App < Sinatra::Base
  respond_with_error e
  end
 
+ #
+ # Detect document language
+ #
+ # POST params:
+ # file - the file to detect language
+ post '/detect-language' do
+ unless params[:file]
+ return respond 400, "missing file parameter"
+ end
+
+ unless params[:file].respond_to?(:fetch) and params[:file].fetch(:tempfile, nil).respond_to?(:read)
+ return respond 400, "invalid file parameter"
+ end
+
+ body = params[:file][:tempfile].read
+ content = Converter.new(logger: @logger).convert_file('detect-language', body)
+ content_type content.mime_type
+ content
+ rescue StandardError => e
+ respond_with_error e
+ end
+
  # Legacy method to convert files
  # Brought over from Heathen
  #

lib/config.rb

@@ -45,6 +45,8 @@ class C_
  attr_accessor :tika_config_directory
  # @return [String] Params for wkhtmltopdf
  attr_accessor :wkhtmltopdf_params
+ # @return [Array<String>] Languages available to Tesseract for OCR. Defaults to `["eng"]`
+ attr_accessor :tesseract_available_languages
 
  def self.config_file_path
  Pathname.new File.expand_path('../config/app.yml', __dir__)
@@ -70,6 +72,7 @@ def self.config
 
  c.tika_config_directory = yaml['tika_config_directory'] || '../tmp/tika'
  c.wkhtmltopdf_params = yaml['wkhtmltopdf_params'] || ''
+ c.tesseract_available_languages = (yaml['tesseract_available_languages'] || 'eng').split(',')
 
  c
  end

lib/heathen.rb

@@ -12,6 +12,7 @@
 require_relative 'heathen/processor'
 
 require_relative 'heathen/processor_methods/convert_image'
+require_relative 'heathen/processor_methods/detect_language'
 require_relative 'heathen/processor_methods/htmltotext'
 require_relative 'heathen/processor_methods/libreoffice'
 require_relative 'heathen/processor_methods/pdftotext'

lib/heathen/processor_methods/detect_language.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Heathen
+ class Processor
+ def detect_language
+ executioner.execute(
+ Colore::C_.tika_path,
+ "--config=#{Colore::TikaConfig.path_for_language_detection}",
+ '--language',
+ job.content_file,
+ binary: true
+ )
+ raise ConversionFailed.new if executioner.last_exit_status != 0
+
+ job.content = executioner.stdout
+ end
+ end
+end

lib/heathen/task.rb

@@ -103,3 +103,7 @@ def task_key(action, mime_type)
 Heathen::Task.register 'doc', '.*' do
  perform_task 'msoffice'
 end
+
+Heathen::Task.register 'detect_language', '.*' do
+ detect_language
+end

lib/tika_config.rb

@@ -20,7 +20,7 @@ module TikaConfig
  <parser class="org.apache.tika.parser.DefaultParser"></parser>
  <parser class="org.apache.tika.parser.ocr.TesseractOCRParser">
  <params>
- <param name="language" type="string">%<language_alpha3>s</param>
+ <param name="language" type="string">%<alpha3_languages>s</param>
  </params>
  </parser>
  </parsers>
@@ -34,12 +34,12 @@ def tika_config_path
  Pathname.new File.expand_path(Colore::C_.tika_config_directory, __dir__)
  end
 
- def path_for!(language_alpha3)
- file = tika_config_path.join('ocr', VERSION, "tika.#{language_alpha3}.xml")
+ def path_for!(alpha3_languages)
+ file = tika_config_path.join('ocr', VERSION, "tika.#{alpha3_languages.sort.join('-')}.xml")
  return file if file.file?
 
  FileUtils.mkdir_p(tika_config_path.join('ocr', VERSION))
- file.write format(TEMPLATE, language_alpha3: language_alpha3)
+ file.write format(TEMPLATE, alpha3_languages: alpha3_languages.join('+'))
  file
  end
  end
@@ -55,7 +55,15 @@ def path_for!(language_alpha3)
  def self.path_for(language)
  language_alpha3 = Colore::Utils.language_alpha3(language) || DEFAULT_LANGUAGE
 
- path_for!(language_alpha3)
+ path_for!([language_alpha3])
+ end
+
+ # Returns the file path of the Tika configuration for performing language
+ # detection.
+ #
+ # @return [Pathname] The path to the Tika configuration file for language detection
+ def self.path_for_language_detection
+ path_for!(Colore::C_.tesseract_available_languages)
  end
  end
 end

spec/heathen/processor_methods/detect_language_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Heathen::Processor do
+ let(:content) { fixture('heathen/quickfox.ar.jpg').read }
+ let(:job) { Heathen::Job.new 'foo', content }
+ let(:processor) { described_class.new job: job, logger: spec_logger }
+
+ before do
+ setup_tika_config
+ end
+
+ after do
+ processor.clean_up
+ delete_tika_config
+ end
+
+ describe '#detect_language' do
+ before do
+ processor.detect_language
+ end
+
+ context 'with English documents' do
+ let(:content) { fixture('heathen/quickfox.jpg').read }
+
+ it 'detects English input file language' do
+ expect(job.content).to eq 'en'
+ expect(job.content.mime_type).to eq 'text/plain; charset=us-ascii'
+ end
+ end
+
+ context 'with Arabic documents' do
+ let(:content) { fixture('heathen/quickfox.ar.jpg').read }
+
+ it 'detects Arabic input file language' do
+ expect(job.content).to eq 'ar'
+ expect(job.content.mime_type).to eq 'text/plain; charset=us-ascii'
+ end
+ end
+ end
+end

spec/integration/standard_tasks_spec.rb

@@ -21,6 +21,14 @@
  end
  end
 
+ describe 'detect_language' do
+ it 'runs' do
+ content = fixture('heathen/quickfox.jpg').read
+ new_content = converter.convert 'detect_language', content
+ expect(new_content.mime_type).to eq 'text/plain; charset=us-ascii'
+ end
+ end
+
  describe 'ocr_text' do
  it 'converts jpeg' do
  content = fixture('heathen/quickfox.jpg').read