104 fixes (#527)

* added stemmed synonym searching to synonym field in ms Signed-off-by: Kial Jinnah <kialj876@gmail.com> * pytest added Signed-off-by: Kial Jinnah <kialj876@gmail.com> * pytest update Signed-off-by: Kial Jinnah <kialj876@gmail.com>
bcgov · Feb 20, 2019 · 59bba39 · 59bba39
1 parent a110c7d
commit 59bba39
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 7 deletions.
diff --git a/api/tests/python/end_points/test_synonym_match.py b/api/tests/python/end_points/test_synonym_match.py
@@ -550,3 +550,20 @@ def test_number_synonyms(client, jwt, app, query, expected_list):
  seed_database_with(client, jwt, 'FIFTH ZZZZZ', id='1', source='2')
  verify_synonym_match(client, jwt, query=query, expected_list=expected_list)
 
+@integration_postgres_solr
+@integration_synonym_api
+@integration_solr
+@pytest.mark.parametrize("query, ordered_list", [
+ ('PACIFIC WEST CONSTRUCTION', ['----PACIFIC WEST CONSTRUCTION - PROXIMITY SEARCH',
+ '----PACIFIC WEST CONSTRUCTION* - EXACT WORD ORDER',
+ '----PACIFIC WEST synonyms:(CONSTRUCT) - PROXIMITY SEARCH',
+ '----PACIFIC WEST* synonyms:(CONSTRUCT) - EXACT WORD ORDER',
+ '----PACIFIC synonyms:(CONSTRUCT) - PROXIMITY SEARCH',
+ 'PACIFIC DEVELOPMENT',
+ ]),
+])
+def test_synonym_clause_stemmed(client, jwt, app, query, ordered_list):
+ # for loop didn't work for seeding so manual
+ seed_database_with(client, jwt, 'PACIFIC DEVELOPMENT', id='1', source='2')
+ verify_order(client, jwt, query=query, expected_order=ordered_list)
+
diff --git a/solr/cores/possible.conflicts/conf/managed-schema b/solr/cores/possible.conflicts/conf/managed-schema
@@ -687,20 +687,18 @@
  <tokenizer class="solr.WhitespaceTokenizerFactory"/>
  <filter class="solr.StopFilterFactory" words="lang/stopwords_en.txt" ignoreCase="true"/>
  <filter class="solr.LowerCaseFilterFactory"/>
- <filter class="solr.PorterStemFilterFactory"/>
- <filter class="solr.WordDelimiterGraphFilterFactory" catenateNumbers="1" generateNumberParts="0"
- splitOnCaseChange="0" generateWordParts="0" catenateAll="1" catenateWords="1"/>
+ <filter class="solr.SnowballPorterFilterFactory" protected="protected_syns.txt" language="English"/>
  <filter class="solr.FlattenGraphFilterFactory"/>
  <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
  </analyzer>
  <analyzer type="query">
  <tokenizer class="solr.WhitespaceTokenizerFactory"/>
  <filter class="solr.StopFilterFactory" words="lang/stopwords_en.txt" ignoreCase="true"/>
  <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.SnowballPorterFilterFactory" protected="protected_syns.txt" language="English"/>
  <filter class="com.s24.search.solr.analysis.jdbc.JdbcSynonymFilterFactory" dataSource="jdbc/synonyms"
- sql="select synonyms_text from synonym where enabled = true" ignoreMissingDatabase="true"
+ sql="select stems_text from synonym where enabled = true" ignoreMissingDatabase="true"
  ignoreCase="true" expand="true"/>
- <filter class="solr.PorterStemFilterFactory"/>
  <filter class="solr.WordDelimiterGraphFilterFactory" catenateNumbers="1" generateNumberParts="0" splitOnCaseChange="0" generateWordParts="0" catenateAll="1" catenateWords="1"/>
  <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
  <filter class="solr.FlattenGraphFilterFactory"/>
@@ -851,7 +849,6 @@
  ignoreCase="true"
  words="lang/stopwords_en.txt"
  />
- <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
  <filter class="solr.LowerCaseFilterFactory"/>
  <filter class="solr.FlattenGraphFilterFactory"/>
  </analyzer>
@@ -880,7 +877,6 @@
  ignoreCase="true"
  words="lang/stopwords_en.txt"
  />
- <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
  <filter class="solr.LowerCaseFilterFactory"/>
  <filter class="solr.FlattenGraphFilterFactory"/>
  </analyzer>