Merge pull request #241 from sul-dlss/suggester

adds in autocomplete and spellcheck to application and Solr config

Gemfile

@@ -71,4 +71,4 @@ gem 'squash_ruby', require: 'squash/ruby'
 gem 'squash_rails', require: 'squash/rails'
 gem 'sitemap_generator', '~> 5.0.5'
 gem 'newrelic_rpm'
-
+gem 'twitter-typeahead-rails'

Gemfile.lock

@@ -294,6 +294,10 @@ GEM
     thread_safe (0.3.5)
     tilt (1.4.1)
     tins (1.3.4)
+    twitter-typeahead-rails (0.10.5)
+      actionpack (>= 3.1)
+      jquery-rails
+      railties (>= 3.1)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
     uglifier (2.7.0)
@@ -343,4 +347,5 @@ DEPENDENCIES
   squash_rails
   squash_ruby
   therubyracer
+  twitter-typeahead-rails
   uglifier (>= 1.3.0)

Rakefile

@@ -5,7 +5,7 @@ require File.expand_path('../config/application', __FILE__)
 
 Rails.application.load_tasks
 
-BLACKLIGHT_JETTY_VERSION = '4.10.2'
+BLACKLIGHT_JETTY_VERSION = '4.10.4'
 ZIP_URL = "https://github.com/projectblacklight/blacklight-jetty/archive/v#{BLACKLIGHT_JETTY_VERSION}.zip"
 require 'jettywrapper'
 

app/assets/javascripts/application.js

@@ -17,3 +17,5 @@
 // Required by Blacklight
 //= require blacklight/blacklight
 //= require_tree .
+
+//= require twitter/typeahead.min

app/assets/javascripts/modules/autocomplete.js

@@ -0,0 +1,22 @@
+$(document).on('ready page:load', function() {
+  var terms = new Bloodhound({
+    datumTokenizer: Bloodhound.tokenizers.obj.whitespace('value'),
+    queryTokenizer: Bloodhound.tokenizers.whitespace,
+    remote: {
+      url: '/suggest?q=%QUERY'
+    }
+  });
+
+  terms.initialize();
+
+  $('input.search_q').typeahead({
+    hint: true,
+    highlight: true,
+    minLength: 2
+  },
+  {
+    name: 'terms',
+    displayKey: 'term',
+    source: terms.ttAdapter()
+  });
+});

app/assets/stylesheets/earthworks.css.scss

@@ -10,6 +10,7 @@
 @import 'modules/show';
 @import 'modules/sul_footer';
 @import 'modules/top_navbar';
+@import 'modules/typeahead';
 @import 'modules/zero_results';
 
 

app/assets/stylesheets/modules/typeahead.scss

@@ -0,0 +1,31 @@
+.twitter-typeahead {
+  float: left;
+  width: 100%;
+  z-index: 10000;
+
+  .tt-input.form-control {
+    width: 100%;
+  }
+
+  .tt-hint.form-control {
+    width: 100%;
+  }
+
+  .tt-dropdown-menu {
+    @extend .dropdown-menu;
+    font-family: Arial, 'Helvetica Neue', Helvetica, sans-serif; 
+
+    width: 100%;
+
+    .tt-suggestion p{
+      font-size: 14px;
+      padding-left: 10px;
+    }
+
+    .tt-cursor {
+      background-color: $dropdown-link-hover-bg;
+      color: $dropdown-link-hover-color;
+      text-decoration: none;
+    }
+  }
+}

app/controllers/suggest_controller.rb

@@ -0,0 +1,3 @@
+class SuggestController < ApplicationController
+  include Earthworks::Suggest
+end

config/application.rb

@@ -11,6 +11,9 @@ class Application < Rails::Application
     config.application_name = 'EarthWorks'
 
     require 'rights_metadata'
+    require 'suggest/response'
+    require 'suggest/search_helper'
+    require 'suggest'
     # Settings in config/environments/* take precedence over those specified here.
     # Application configuration should go into files in config/initializers
     # -- all .rb files in that directory are automatically loaded.

config/blacklight.yml

@@ -11,8 +11,8 @@
 # how to start up solr, generally for automated testing. 
 
 development:
-  url: <%= ENV['SOLR_URL'] || "http://127.0.0.1:8983/solr" %>
+  url: <%= ENV['SOLR_URL'] || "http://127.0.0.1:8983/solr/blacklight-core" %>
 test: &test
-  url: <%= ENV['TEST_SOLR_URL'] || "http://127.0.0.1:8888/solr" %>
+  url: <%= ENV['TEST_SOLR_URL'] || "http://127.0.0.1:8888/solr/blacklight-core" %>
 production:
-  url: <%= ENV['SOLR_URL'] || "http://127.0.0.1:8983/solr" %>
+  url: <%= ENV['SOLR_URL'] || "http://127.0.0.1:8983/solr/blacklight-core" %>

config/routes.rb

@@ -7,6 +7,8 @@
     match 'users/auth/webauth/logout' => 'devise/sessions#destroy', :as => :destroy_user_session, :via => Devise.mappings[:user].sign_out_via
   end
 
+  resources :suggest, only: :index, defaults: { format: 'json' }
+
   resource :feedback_form, path: 'feedback', only: [:new, :create]
   get 'feedback' => 'feedback_forms#new'
   # The priority is based upon order of creation: first created -> highest priority.

config/solr_configs/schema.xml

@@ -35,6 +35,10 @@
     <dynamicField name="*_tmi"  type="text_en" stored="false" indexed="true" multiValued="true"
                                 termVectors="true" termPositions="true" termOffsets="true"/>
     <dynamicField name="*_sort" type="text_sort" stored="false" indexed="true" multiValued="false"/>
+
+    <dynamicField name="*spell" type="textSpell" indexed="true" stored="false" multiValued="true" />
+
+    <dynamicField name="*suggest" type="textSuggest" indexed="true" stored="false" multiValued="true" />
 
     <!-- Spatial field types:
 
@@ -112,6 +116,25 @@
       </analyzer>
     </fieldType>
 
+    <fieldType class="solr.TextField" name="textSuggest" positionIncrementGap="100">
+       <analyzer>
+          <tokenizer class="solr.KeywordTokenizerFactory"/>
+          <filter class="solr.StandardFilterFactory"/>
+          <filter class="solr.LowerCaseFilterFactory"/>
+          <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+       </analyzer>
+    </fieldType>
+
+    <fieldType name="textSpell" class="solr.TextField" positionIncrementGap="100" >
+      <analyzer>
+        <tokenizer class="solr.StandardTokenizerFactory"/>
+        <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
+        <filter class="solr.StandardFilterFactory"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+        <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+      </analyzer>
+    </fieldType>
+
     <!-- Spatial field types -->
     <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_d"/>
 
@@ -154,5 +177,20 @@
   <copyField source="dct_provenance_s"   dest="dct_provenance_sort"/>
   <copyField source="dc_publisher_s"     dest="dc_publisher_sort"/>
   <copyField source="dc_title_s"         dest="dc_title_sort"/>
-
+
+  <!-- for spell checking -->
+  <copyField source="dc_title_s" dest="spell"/>
+  <copyField source="dc_creator_sm" dest="spell"/>
+  <copyField source="dc_publisher_s" dest="spell"/>
+  <copyField source="dct_provenance_s" dest="spell"/>
+  <copyField source="dc_subject_sm" dest="spell"/>
+  <copyField source="dct_spatial_sm" dest="spell"/>
+
+  <!-- for suggestions -->
+  <copyField source="dc_title_s" dest="suggest"/>
+  <copyField source="dc_creator_sm" dest="suggest"/>
+  <copyField source="dc_publisher_s" dest="suggest"/>
+  <copyField source="dct_provenance_s" dest="suggest"/>
+  <copyField source="dc_subject_sm" dest="suggest"/>
+  <copyField source="dct_spatial_sm" dest="suggest"/>
 </schema>

config/solr_configs/solrconfig.xml

@@ -147,7 +147,13 @@
       <str name="facet.field">dc_subject_sm</str>
       <str name="facet.field">layer_geom_type_s</str>
       <str name="facet.field">solr_year_i</str>
+
+      <str name="spellcheck">true</str>
     </lst>
+
+    <arr name="last-components">
+      <str>spellcheck</str>
+    </arr>
   </requestHandler>
 
   <requestHandler name="/update" class="solr.UpdateRequestHandler"/>
@@ -177,4 +183,51 @@
   <admin>
     <defaultQuery>*:*</defaultQuery>
   </admin>
-</config>
+
+  <searchComponent name="spellcheck" class="solr.SpellCheckComponent">
+    <!-- a spellchecker built from a field of the main index -->
+    <lst name="spellchecker">
+      <str name="name">default</str>
+      <str name="field">spell</str>
+      <str name="classname">solr.DirectSolrSpellChecker</str>
+      <!-- the spellcheck distance measure used, the default is the internal levenshtein -->
+      <str name="distanceMeasure">internal</str>
+      <!-- minimum accuracy needed to be considered a valid spellcheck suggestion -->
+      <float name="accuracy">0.5</float>
+      <!-- the maximum #edits we consider when enumerating terms: can be 1 or 2 -->
+      <int name="maxEdits">2</int>
+      <!-- the minimum shared prefix when enumerating terms -->
+      <int name="minPrefix">1</int>
+      <!-- maximum number of inspections per result. -->
+      <int name="maxInspections">5</int>
+      <!-- minimum length of a query term to be considered for correction -->
+      <int name="minQueryLength">4</int>
+      <!-- maximum threshold of documents a query term can appear to be considered for correction -->
+      <float name="maxQueryFrequency">0.01</float>
+      <!-- uncomment this to require suggestions to occur in 1% of the documents
+      	<float name="thresholdTokenFrequency">.01</float>
+      -->
+    </lst>
+  </searchComponent>
+
+  <searchComponent name="suggest" class="solr.SuggestComponent">
+    <lst name="suggester">
+      <str name="name">mySuggester</str>
+      <str name="lookupImpl">FuzzyLookupFactory</str>
+      <str name="suggestAnalyzerFieldType">textSuggest</str>
+      <str name="buildOnCommit">true</str>
+      <str name="field">suggest</str>
+    </lst>
+  </searchComponent>
+
+  <requestHandler name="/suggest" class="solr.SearchHandler" startup="lazy">
+    <lst name="defaults">
+      <str name="suggest">true</str>
+      <str name="suggest.count">5</str>
+      <str name="suggest.dictionary">mySuggester</str>
+    </lst>
+    <arr name="components">
+      <str>suggest</str>
+    </arr>
+  </requestHandler>
+</config>

lib/suggest.rb

@@ -0,0 +1,17 @@
+module Earthworks
+  module Suggest
+    extend ActiveSupport::Concern
+    include Suggest::SearchHelper
+
+    ##
+    # Get suggestion results from the Solr index
+    def index
+      @response = get_suggestions params
+      respond_to do |format|
+        format.json do
+          render json: @response.suggestions
+        end
+      end
+    end
+  end
+end

lib/suggest/response.rb

@@ -0,0 +1,24 @@
+module Earthworks
+  module Suggest
+    class Response
+      attr_reader :response, :request_params
+
+      ##
+      # Creates a suggest response
+      # @param [RSolr::HashWithResponse] response
+      # @param [Hash] request_params
+      def initialize(response, request_params)
+        @response = response
+        @request_params = request_params
+      end
+
+      ##
+      # Trys the suggestor response to return suggestions if they are
+      # present
+      # @return [Array]
+      def suggestions
+        response.try(:[], 'suggest').try(:[], 'mySuggester').try(:[], request_params[:q]).try(:[], 'suggestions') || []
+      end
+    end
+  end
+end

lib/suggest/search_helper.rb

@@ -0,0 +1,26 @@
+module Earthworks
+  module Suggest
+    module SearchHelper
+      extend ActiveSupport::Concern
+      include Blacklight::SearchHelper
+
+      ##
+      # For now, only use the q parameter to create a
+      # Earthworks::Suggest::Response
+      # @param [Hash] params
+      # @return [Earthworks::Suggest::Response]
+      def get_suggestions(params)
+        request_params = { q: params[:q] }
+        Earthworks::Suggest::Response.new suggest_results(request_params), request_params
+      end
+
+      ##
+      # Query the suggest handler using RSolr::Client::send_and_receive
+      # @param [Hash] request_params
+      # @return [RSolr::HashWithResponse]
+      def suggest_results(request_params)
+        repository.connection.send_and_receive('suggest', params: request_params)
+      end
+    end
+  end
+end

spec/controllers/suggest_controller_spec.rb

@@ -0,0 +1,20 @@
+require 'rails_helper'
+
+describe SuggestController do
+  describe 'GET index' do
+    it 'assigns @response' do
+      get :index, format: 'json'
+      expect(assigns(:response)).to be_an Earthworks::Suggest::Response
+    end
+    it 'renders json' do
+      get :index, format: 'json'
+      expect(response.body).to eq [].to_json
+    end
+    it 'returns suggestions' do
+      get :index, format: 'json', q: 'st'
+      json = JSON.parse(response.body)
+      expect(json.count).to eq 2
+      expect(json.first['term']).to eq 'stanford'
+    end
+  end
+end

spec/features/search_spec.rb

@@ -0,0 +1,12 @@
+require 'rails_helper'
+
+feature 'Search' do
+  feature 'spelling suggestions' do
+    scenario 'are turned on' do
+      visit root_path
+      fill_in 'q', with: 'standford'
+      click_button 'search'
+      expect(page).to have_content 'Did you mean to type:'
+    end
+  end
+end