From 2414aba9f6515acbad3f015a55d2300b6ba119a9 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Fri, 29 Jul 2022 13:54:57 -0700 Subject: [PATCH 1/6] Add caching for analytics for 24 hours. --- controllers/ontology_analytics_controller.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/controllers/ontology_analytics_controller.rb b/controllers/ontology_analytics_controller.rb index 5113c926..8ecd77d5 100644 --- a/controllers/ontology_analytics_controller.rb +++ b/controllers/ontology_analytics_controller.rb @@ -7,6 +7,7 @@ class OntologyAnalyticsController < ApplicationController namespace "/analytics" do get do + expires 86400, :public year = year_param(params) error 400, "The year you supplied is invalid. Valid years start with 2 and contain 4 digits." if params["year"] && !year month = month_param(params) @@ -24,6 +25,7 @@ class OntologyAnalyticsController < ApplicationController namespace "/ontologies/:acronym/analytics" do get do + expires 86400, :public ont = Ontology.find(params["acronym"]).first error 404, "No ontology exists with the acronym: #{params["acronym"]}" if ont.nil? analytics = ont.analytics From 6094bcd244d92f3da2beec1d9cd5a22d32d120d9 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Fri, 26 Aug 2022 11:00:53 -0700 Subject: [PATCH 2/6] Fix for #97. Check for ontology existence before brining attributes --- controllers/ontology_submissions_controller.rb | 2 +- test/controllers/test_ontology_submissions_controller.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index daa8bb66..0cf1d3e1 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -93,8 +93,8 @@ class OntologySubmissionsController < ApplicationController submission_attributes = [:submissionId, :submissionStatus, :uploadFilePath, :pullLocation] included = Ontology.goo_attrs_to_load.concat([submissions: submission_attributes]) ont = Ontology.find(acronym).include(included).first - ont.bring(:viewingRestriction) if ont.bring?(:viewingRestriction) error 422, "You must provide an existing `acronym` to download" if ont.nil? + ont.bring(:viewingRestriction) if ont.bring?(:viewingRestriction) check_access(ont) ont_restrict_downloads = LinkedData::OntologiesAPI.settings.restrict_download error 403, "License restrictions on download for #{acronym}" if ont_restrict_downloads.include? acronym diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 0098969e..7500dce4 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -125,6 +125,10 @@ def test_download_submission # Clear restrictions on downloads LinkedData::OntologiesAPI.settings.restrict_download = [] # see also test_ontologies_controller::test_download_ontology + + # Test downloads of nonexistent ontology + get "/ontologies/BOGUS66/submissions/55/download" + assert_equal(422, last_response.status, "failed to handle downloads of nonexistent ontology" + get_errors(last_response)) end def test_download_ontology_submission_rdf From 80e562fec991e6b030eb14e4cb639897412d1954 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Fri, 26 Aug 2022 16:24:23 -0700 Subject: [PATCH 3/6] Handle edge case for submission downloads which do not have UploadFilePath set Fixes #98 --- controllers/ontology_submissions_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index 0cf1d3e1..6c1e0add 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -101,7 +101,8 @@ class OntologySubmissionsController < ApplicationController submission = ont.submission(params['ontology_submission_id'].to_i) error 404, "There is no such submission for download" if submission.nil? file_path = submission.uploadFilePath - + # handle edge case where uploadFilePath is not set + error 422, "Upload File Path is not set for this submission" if file_path.to_s.empty? download_format = params["download_format"].to_s.downcase allowed_formats = ["csv", "rdf"] if download_format.empty? From 940cc3f809c4a05994b9bbc145e94d94fd688dbe Mon Sep 17 00:00:00 2001 From: mdorf Date: Thu, 3 Nov 2022 13:49:20 -0700 Subject: [PATCH 4/6] Gemfile.lock update --- Gemfile.lock | 49 ++++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 52a51568..27bf1567 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 7fe22f03b56a944822c986269076a2e2aa64d6a2 + revision: 091e0ca001244a7dbaed0644b0b1087a3a24d16a branch: develop specs: goo (0.0.2) @@ -26,7 +26,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: 7954f243ff2b76d9393e44e435f30529e6e697db + revision: 4d409a998760b7d9bbb42b575813f8ffdd5c0169 branch: develop specs: ncbo_cron (0.0.1) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 53cd329b4a4a5fa4c4294cf4fe2dd43b0d748948 + revision: a00dac9a0825697cb7f6ac5e0f4db6ab8f4bae79 branch: develop specs: ontologies_linked_data (0.0.1) @@ -103,15 +103,15 @@ GEM activesupport (3.2.22.5) i18n (~> 0.6, >= 0.6.4) multi_json (~> 1.0) - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) + addressable (2.8.1) + public_suffix (>= 2.0.2, < 6.0) airbrussh (1.4.1) sshkit (>= 1.6.1, != 1.7.0) backports (3.23.0) bcrypt (3.1.18) bigdecimal (1.4.2) builder (3.2.4) - capistrano (3.17.0) + capistrano (3.17.1) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) @@ -131,7 +131,7 @@ GEM docile (1.4.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) - faraday (1.10.0) + faraday (1.10.2) faraday-em_http (~> 1.0) faraday-em_synchrony (~> 1.0) faraday-excon (~> 1.1) @@ -157,9 +157,9 @@ GEM ffi (1.15.5) get_process_mem (0.2.7) ffi (~> 1.0) - google-apis-analytics_v3 (0.10.0) - google-apis-core (>= 0.7, < 2.a) - google-apis-core (0.7.0) + google-apis-analytics_v3 (0.12.0) + google-apis-core (>= 0.9.1, < 2.a) + google-apis-core (0.9.1) addressable (~> 2.5, >= 2.5.1) googleauth (>= 0.16.2, < 2.a) httpclient (>= 2.8.1, < 3.a) @@ -168,15 +168,16 @@ GEM retriable (>= 2.0, < 4.a) rexml webrick - googleauth (1.2.0) + googleauth (1.3.0) faraday (>= 0.17.3, < 3.a) jwt (>= 1.4, < 3.0) memoist (~> 0.16) multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - haml (5.2.2) - temple (>= 0.8.0) + haml (6.0.8) + temple (>= 0.8.2) + thor tilt http-accept (1.7.0) http-cookie (1.0.5) @@ -188,9 +189,9 @@ GEM json-schema (2.8.1) addressable (>= 2.4) json_pure (2.6.2) - jwt (2.4.1) + jwt (2.5.0) kgio (2.11.4) - libxml-ruby (3.2.3) + libxml-ruby (3.2.4) logger (1.5.1) macaddr (1.7.2) systemu (~> 2.6.5) @@ -209,11 +210,11 @@ GEM multi_json (1.15.0) multipart-post (2.2.3) net-http-persistent (2.9.4) - net-scp (1.2.1) - net-ssh (>= 2.6.5) + net-scp (4.0.0) + net-ssh (>= 2.6.5, < 8.0.0) net-ssh (7.0.1) netrc (0.11.0) - newrelic_rpm (8.9.0) + newrelic_rpm (8.12.0) oj (2.18.5) omni_logger (0.1.4) logger @@ -224,7 +225,7 @@ GEM pry (0.14.1) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (4.0.7) + public_suffix (5.0.0) rack (1.6.13) rack-accept (0.4.5) rack (>= 0.4) @@ -246,7 +247,7 @@ GEM rdf (1.0.8) addressable (>= 2.2) redcarpet (3.5.1) - redis (4.7.1) + redis (4.8.0) redis-activesupport (5.3.0) activesupport (>= 3, < 8) redis-store (>= 1.3, < 2) @@ -301,11 +302,12 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.21.2) + sshkit (1.21.3) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) systemu (2.6.5) - temple (0.8.2) + temple (0.9.1) + thor (1.2.1) tilt (2.0.11) trailblazer-option (0.1.2) tzinfo (2.0.5) @@ -325,6 +327,7 @@ GEM webrick (1.7.0) PLATFORMS + x86_64-darwin-18 x86_64-darwin-21 x86_64-linux @@ -377,4 +380,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.11 + 2.3.15 From 8875ec5ebd59b1c4a4d30baf771f333fbc4510b7 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Tue, 15 Nov 2022 15:06:30 -0800 Subject: [PATCH 5/6] Add GH workflow for capistrano deployments --- .github/workflows/deploy.yml | 85 ++++++++++++++++++++++++++++++++++++ Gemfile | 3 ++ Gemfile.lock | 18 +++++--- config/deploy.rb | 4 +- 4 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..a2404a4e --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,85 @@ +# Workflow for deploying ontologies_api to stage/prod systems via capistrano. +# This workflow runs after a successeful execution of the unit test workflow and it +# can also be triggered manually. +# +# Required github secrets: +# +# CONFIG_REPO - github repo containing config and customizations for the API. Format 'author/private_config_repo' +# it is used for getting capistrano deployment configuration for stages on the github actions runner and +# PRIVATE_CONFIG_REPO env var is constructed from it which is used by capistrano on the remote servers for pulling configs. +# +# GH_PAT - github Personal Access Token for accessing PRIVATE_CONFIG_REPO +# +# SSH_JUMPHOST - ssh jump/proxy host though which deployments have to though if app servers are hosted on private network. +# +# DEPLOY_ENC_KEY - key for decrypting deploymnet ssh key residing in config/deploy_id_rsa_enc (see miloserdow/capistrano-deploy) +# this SSH key is used for accessing jump host, UI nodes, and private github repo. + +name: Capistrano Deployment +# Controls when the action will run. +on: + # Trigger deployment to staging after unit test action completes + workflow_run: + workflows: ["Ruby Unit Tests"] + types: + - completed + branches: [master, develop] + # Allows running this workflow manually from the Actions tab + workflow_dispatch: + branches: [master, develop] + inputs: + BRANCH: + description: 'Branch/tag to deploy' + default: develop + required: true + environment: + description: 'target environment to deploy to' + type: choice + options: + - staging + - production + default: staging + +jobs: + deploy: + runs-on: ubuntu-latest + # run deployment only if "Ruby Unit Tests" workflow completes sucessefully + if: ${{ github.event.workflow_run.conclusion == 'success' }} + env: + BUNDLE_WITHOUT: default #install gems required primarily for the deployment in order to speed this workflow + PRIVATE_CONFIG_REPO: ${{ format('git@github.com:{0}.git', secrets.CONFIG_REPO) }} + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: set branch/tag and environment to deploy from inputs + run: | + # workflow_dispatch default input doesn't get set on push so we need to set defaults + # via shell parameter expansion + # https://dev.to/mrmike/github-action-handling-input-default-value-5f2g + USER_INPUT_BRANCH=${{ inputs.branch }} + echo "BRANCH=${USER_INPUT_BRANCH:-develop}" >> $GITHUB_ENV + USER_INPUT_ENVIRONMENT=${{ inputs.environment }} + echo "TARGET=${USER_INPUT_ENVIRONMENT:-staging}" >> $GITHUB_ENV + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: 2.7.6 # Not needed with a .ruby-version file + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + - name: get-deployment-config + uses: actions/checkout@v3 + with: + repository: ${{ secrets.CONFIG_REPO }} # repository containing deployment settings + token: ${{ secrets.GH_PAT }} # `GH_PAT` is a secret that contains your PAT + path: deploy_config + - name: copy-deployment-config + run: cp -r deploy_config/ontologies_api/* . + # add ssh hostkey so that capistrano doesn't complain + - name: Add jumphost's hostkey to Known Hosts + run: | + mkdir -p ~/.ssh + ssh-keyscan -H ${{ secrets.SSH_JUMPHOST }} > ~/.ssh/known_hosts + shell: bash + - uses: miloserdow/capistrano-deploy@master + with: + target: ${{ env.TARGET }} # which environment to deploy + deploy_key: ${{ secrets.DEPLOY_ENC_KEY }} # Name of the variable configured in Settings/Secrets of your github project diff --git a/Gemfile b/Gemfile index 003c81af..6c12eb18 100644 --- a/Gemfile +++ b/Gemfile @@ -50,10 +50,13 @@ gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'de gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop' group :development do + # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 + gem 'bcrypt_pbkdf', '>= 1.0', '< 2.0', require: false gem 'capistrano', '~> 3', require: false gem 'capistrano-bundler', require: false gem 'capistrano-locally', require: false gem 'capistrano-rbenv', require: false + gem 'ed25519', '>= 1.2', '< 2.0', require: false gem 'pry' gem 'shotgun', github: 'palexander/shotgun', branch: 'ncbo' end diff --git a/Gemfile.lock b/Gemfile.lock index 27bf1567..c94c9751 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 091e0ca001244a7dbaed0644b0b1087a3a24d16a + revision: 2b4a123f2535d66cf161a67b043ec13c3588b7f3 branch: develop specs: goo (0.0.2) @@ -15,7 +15,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: bad8c646205014b1fc82793fb9c07d7075b4c068 + revision: 75d1a8f302c8af4060bf62fc619f230084da521e branch: develop specs: ncbo_annotator (0.0.1) @@ -26,7 +26,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: 4d409a998760b7d9bbb42b575813f8ffdd5c0169 + revision: 8ad6ad0eeca284adfadaa01bded1d039318d6dca branch: develop specs: ncbo_cron (0.0.1) @@ -42,7 +42,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: 27d4b63963343349fccd78b78e1957509ac685c5 + revision: 4ce8fe8a8ea330b5cdd246287016f71032a4bf7a branch: develop specs: ncbo_ontology_recommender (0.0.1) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: a00dac9a0825697cb7f6ac5e0f4db6ab8f4bae79 + revision: 60807531cb3d068e879a5a43c85d837bf200c17a branch: develop specs: ontologies_linked_data (0.0.1) @@ -109,6 +109,7 @@ GEM sshkit (>= 1.6.1, != 1.7.0) backports (3.23.0) bcrypt (3.1.18) + bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) capistrano (3.17.1) @@ -131,6 +132,7 @@ GEM docile (1.4.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) + ed25519 (1.3.0) faraday (1.10.2) faraday-em_http (~> 1.0) faraday-em_synchrony (~> 1.0) @@ -175,7 +177,7 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - haml (6.0.8) + haml (6.0.10) temple (>= 0.8.2) thor tilt @@ -214,7 +216,7 @@ GEM net-ssh (>= 2.6.5, < 8.0.0) net-ssh (7.0.1) netrc (0.11.0) - newrelic_rpm (8.12.0) + newrelic_rpm (8.13.0) oj (2.18.5) omni_logger (0.1.4) logger @@ -333,12 +335,14 @@ PLATFORMS DEPENDENCIES activesupport (~> 3.0) + bcrypt_pbkdf (>= 1.0, < 2.0) bigdecimal (= 1.4.2) capistrano (~> 3) capistrano-bundler capistrano-locally capistrano-rbenv cube-ruby + ed25519 (>= 1.2, < 2.0) faraday (~> 1.9) ffi goo! diff --git a/config/deploy.rb b/config/deploy.rb index 441be857..23a982cd 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -1,6 +1,6 @@ # config valid only for Capistrano 3 -APP_PATH = '/srv/ncbo' +APP_PATH = '/srv/ontoportal' set :application, 'ontologies_api' set :repo_url, "https://github.com/ncbo/#{fetch(:application)}.git" @@ -77,7 +77,7 @@ namespace :deploy do - desc 'Incorporate the bioportal_conf private repository content' + desc 'Incorporate the private repository content' # Get cofiguration from repo if PRIVATE_CONFIG_REPO env var is set # or get config from local directory if LOCAL_CONFIG_PATH env var is set task :get_config do From 14929357c271e6f025cd83fbffa2f1f5f9c5d48b Mon Sep 17 00:00:00 2001 From: Jennifer Vendetti Date: Thu, 17 Nov 2022 12:49:39 -0800 Subject: [PATCH 6/6] Update Gemfile.lock --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index c94c9751..4ad78ef3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 2b4a123f2535d66cf161a67b043ec13c3588b7f3 + revision: 204cecf98bedb30f8b4b6e6afb1984f7659175d8 branch: develop specs: goo (0.0.2) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 60807531cb3d068e879a5a43c85d837bf200c17a + revision: 8628abe3c349052fd389751db203b27e6e541087 branch: develop specs: ontologies_linked_data (0.0.1) @@ -384,4 +384,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.15 + 2.3.14