Feature: Add Virtuoso, Allegrograph and Graphdb integration to ontolo…

…gies_api (#54) * set up multiple triple store test environment * optimize api tests
ontoportal-lirmm · Jan 30, 2024 · 40b784f · 40b784f
1 parent 2cabca4
commit 40b784f
Show file tree

Hide file tree

Showing 26 changed files with 517 additions and 209 deletions.
diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml
@@ -7,25 +7,37 @@ on:
 jobs:
   test:
     strategy:
+      fail-fast: false
       matrix:
-        backend: ['api'] # api runs tests with 4store backend and api-agraph runs with AllegroGraph backend
+        goo-slice: [ '20', '100', '500' ]
+        ruby-version: [ '2.7' ]
+        triplestore: [ 'fs', 'ag', 'vo', 'gb' ]
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - name: copy-env-config
-      run: cp .env.sample .env
-    - name: Build docker-compose
-      run: docker-compose --profile 4store build #profile flag is set in order to build all containers in this step
-    - name: Run unit tests
-      # unit tests are run inside a container
-      # http://docs.codecov.io/docs/testing-with-docker
-      run: |
-        ci_env=`bash <(curl -s https://codecov.io/env)`
-        docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} wait-for-it solr-ut:8983 -- bundle install 
-        docker-compose run $ci_env -e CI --rm ${{ matrix.backend }} wait-for-it solr-ut:8983 -- bundle exec rake test TESTOPTS='-v'
-    - name: Upload coverage reports to Codecov
-      uses: codecov/codecov-action@v3
-      with:
-        flags: unittests
-        verbose: true
-        fail_ci_if_error: false # optional (default = false)
+      - uses: actions/checkout@v3
+      - name: Set up solr configsets
+        run: ./test/solr/generate_ncbo_configsets.sh
+      - name: Install Dependencies
+        run: sudo apt-get update && sudo apt-get -y install raptor2-utils
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Run unit tests
+        # unit tests are run inside a container
+        # http://docs.codecov.io/docs/testing-with-docker
+        run: |
+          ci_env=`bash <(curl -s https://codecov.io/env)`
+          GOO_SLICES=${{ matrix.goo-slice }} bundle exec rake test:docker:${{ matrix.triplestore }} TESTOPTS="-v"
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: unittests
+          verbose: true
+          fail_ci_if_error: false # optional (default = false)
diff --git a/Gemfile b/Gemfile
@@ -44,7 +44,7 @@ gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107
 gem 'redcarpet'
 
 # NCBO gems (can be from a local dev path or from rubygems/git)
-gem 'goo', git: 'https://github.com/ontoportal-lirmm/goo.git', branch: 'development'
+gem 'goo', github: 'ontoportal-lirmm/goo', branch: 'development'
 gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'master'
 gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master'
 gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master'

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -11,8 +11,8 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/goo.git
-  revision: 03da25b671d2ffa515b5dce51c6bd35980ae60c7
-  branch: master
+  revision: f1a452ef7728f36719d3f5ddd869e67e2a768ff9
+  branch: development
   specs:
     goo (0.0.2)
       addressable (~> 2.8)
@@ -37,12 +37,13 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ncbo_cron.git
-  revision: 155db7a33794f03858893d2367cb119f27726a31
+  revision: f33a917aae2000270d5b560cadf7044bff822779
   branch: master
   specs:
     ncbo_cron (0.0.1)
       dante
       goo
+      google-analytics-data
       google-apis-analytics_v3
       mlanett-redis-lock
       multi_json
@@ -53,8 +54,8 @@ GIT
 
 GIT
   remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git
-  revision: e98b884999e5ce917a8be5fdc37f7b4797a1559e
-  branch: master
+  revision: 0de5f3b2db02a5d2c008ace73a34d586c24e67a7
+  branch: development
   specs:
     ontologies_linked_data (0.0.1)
       activesupport
@@ -105,10 +106,9 @@ GEM
       multi_json (~> 1.0)
     addressable (2.8.6)
       public_suffix (>= 2.0.2, < 6.0)
-    airbrussh (1.5.0)
+    airbrussh (1.5.1)
       sshkit (>= 1.6.1, != 1.7.0)
     backports (3.24.1)
-    base64 (0.2.0)
     bcrypt (3.1.20)
     bcrypt_pbkdf (1.1.0)
     bigdecimal (1.4.2)
@@ -126,15 +126,16 @@ GEM
       capistrano (~> 3.1)
       sshkit (~> 1.3)
     coderay (1.1.3)
-    concurrent-ruby (1.2.2)
-    crack (0.4.5)
+    concurrent-ruby (1.2.3)
+    crack (0.4.6)
+      bigdecimal
       rexml
     cube-ruby (0.0.3)
     dante (0.2.0)
     date (3.3.4)
     declarative (0.0.20)
     docile (1.4.0)
-    domain_name (0.6.20231109)
+    domain_name (0.6.20240107)
     ed25519 (1.3.0)
     faraday (1.10.3)
       faraday-em_http (~> 1.0)
@@ -160,28 +161,59 @@ GEM
     faraday-rack (1.0.0)
     faraday-retry (1.0.3)
     ffi (1.16.3)
+    gapic-common (0.21.1)
+      faraday (>= 1.9, < 3.a)
+      faraday-retry (>= 1.0, < 3.a)
+      google-protobuf (~> 3.18)
+      googleapis-common-protos (>= 1.4.0, < 2.a)
+      googleapis-common-protos-types (>= 1.11.0, < 2.a)
+      googleauth (~> 1.9)
+      grpc (~> 1.59)
     get_process_mem (0.2.7)
       ffi (~> 1.0)
-    google-apis-analytics_v3 (0.13.0)
-      google-apis-core (>= 0.11.0, < 2.a)
-    google-apis-core (0.11.2)
+    google-analytics-data (0.5.0)
+      google-analytics-data-v1beta (>= 0.11, < 2.a)
+      google-cloud-core (~> 1.6)
+    google-analytics-data-v1beta (0.11.1)
+      gapic-common (>= 0.21.1, < 2.a)
+      google-cloud-errors (~> 1.0)
+    google-apis-analytics_v3 (0.14.0)
+      google-apis-core (>= 0.12.0, < 2.a)
+    google-apis-core (0.13.0)
       addressable (~> 2.5, >= 2.5.1)
-      googleauth (>= 0.16.2, < 2.a)
+      googleauth (~> 1.9)
       httpclient (>= 2.8.1, < 3.a)
       mini_mime (~> 1.0)
       representable (~> 3.0)
       retriable (>= 2.0, < 4.a)
       rexml
-      webrick
+    google-cloud-core (1.6.1)
+      google-cloud-env (>= 1.0, < 3.a)
+      google-cloud-errors (~> 1.0)
     google-cloud-env (2.1.0)
       faraday (>= 1.0, < 3.a)
-    googleauth (1.9.1)
+    google-cloud-errors (1.3.1)
+    google-protobuf (3.25.2-x86_64-darwin)
+    google-protobuf (3.25.2-x86_64-linux)
+    googleapis-common-protos (1.4.0)
+      google-protobuf (~> 3.14)
+      googleapis-common-protos-types (~> 1.2)
+      grpc (~> 1.27)
+    googleapis-common-protos-types (1.11.0)
+      google-protobuf (~> 3.18)
+    googleauth (1.9.2)
       faraday (>= 1.0, < 3.a)
       google-cloud-env (~> 2.1)
       jwt (>= 1.4, < 3.0)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
+    grpc (1.60.0-x86_64-darwin)
+      google-protobuf (~> 3.25)
+      googleapis-common-protos-types (~> 1.0)
+    grpc (1.60.0-x86_64-linux)
+      google-protobuf (~> 3.25)
+      googleapis-common-protos-types (~> 1.0)
     haml (5.2.2)
       temple (>= 0.8.0)
       tilt
@@ -198,7 +230,7 @@ GEM
     json_pure (2.7.1)
     jwt (2.7.1)
     kgio (2.11.4)
-    libxml-ruby (4.1.2)
+    libxml-ruby (5.0.2)
     logger (1.6.0)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
@@ -208,7 +240,7 @@ GEM
       net-pop
       net-smtp
     method_source (1.0.0)
-    mime-types (3.5.1)
+    mime-types (3.5.2)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2023.1205)
     mini_mime (1.1.5)
@@ -218,8 +250,9 @@ GEM
       redis
     multi_json (1.15.0)
     multipart-post (2.3.0)
+    mutex_m (0.2.0)
     net-http-persistent (2.9.4)
-    net-imap (0.4.8)
+    net-imap (0.4.9.1)
       date
       net-protocol
     net-pop (0.1.2)
@@ -228,12 +261,13 @@ GEM
       timeout
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
-    net-smtp (0.4.0)
+    net-sftp (4.0.0)
+      net-ssh (>= 5.0.0, < 8.0.0)
+    net-smtp (0.4.0.1)
       net-protocol
     net-ssh (7.2.1)
     netrc (0.11.0)
-    newrelic_rpm (9.6.0)
-      base64
+    newrelic_rpm (9.7.1)
     oj (2.18.5)
     omni_logger (0.1.4)
       logger
@@ -323,8 +357,10 @@ GEM
       rack-test
       sinatra (~> 1.4.0)
       tilt (>= 1.3, < 3)
-    sshkit (1.21.6)
+    sshkit (1.22.0)
+      mutex_m
       net-scp (>= 1.1.2)
+      net-sftp (>= 2.1.2)
       net-ssh (>= 2.8.0)
     systemu (2.6.5)
     temple (0.10.3)
@@ -346,7 +382,6 @@ GEM
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    webrick (1.8.1)
 
 PLATFORMS
   x86_64-darwin-23
@@ -406,4 +441,4 @@ DEPENDENCIES
   webmock
 
 BUNDLED WITH
-   2.3.23
+   2.4.21
diff --git a/config/environments/config.rb.sample b/config/environments/config.rb.sample
@@ -48,6 +48,7 @@ LinkedData.config do |config|
   config.property_search_server_url    = SOLR_PROP_SEARCH_URL.to_s
   config.replace_url_prefix            = true
   config.rest_url_prefix               = REST_URL_PREFIX.to_s
+  config.sparql_endpoint_url = "http://sparql.bioontology.org"
 #  config.enable_notifications          = false
 
     config.interportal_hash = {

diff --git a/config/environments/test.rb b/config/environments/test.rb
@@ -9,15 +9,17 @@
 REDIS_HOST = ENV.include?("REDIS_HOST") ? ENV["REDIS_HOST"] : "localhost"
 REDIS_PORT = ENV.include?("REDIS_PORT") ? ENV["REDIS_PORT"] : 6379
 SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr/term_search_core1"
-SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr/prop_search_core1"
+SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8984/solr/prop_search_core1"
 MGREP_HOST = ENV.include?("MGREP_HOST") ? ENV["MGREP_HOST"] : "localhost"
-MGREP_PORT = ENV.include?("MGREP_PORT") ? ENV["MGREP_PORT"] : 55555
+MGREP_PORT = ENV.include?("MGREP_PORT") ? ENV["MGREP_PORT"] : 55556
+GOO_SLICES = ENV["GOO_SLICES"] || 500
 
 begin
   # For prefLabel extract main_lang first, or anything if no main found.
   # For other properties only properties with a lang that is included in main_lang are used
   Goo.main_languages = ['en']
   Goo.use_cache = false
+  Goo.slice_loading_size = GOO_SLICES.to_i
 rescue NoMethodError
   puts "(CNFG) >> Goo.main_lang not available"
 end
@@ -37,6 +39,7 @@
   config.ontology_analytics_redis_port = REDIS_PORT.to_i
   config.search_server_url = SOLR_TERM_SEARCH_URL.to_s
   config.property_search_server_url = SOLR_PROP_SEARCH_URL.to_s
+  config.sparql_endpoint_url = "http://sparql.bioontology.org"
   #  config.enable_notifications          = false
   config.interportal_hash = {
     "agroportal" => {