From c921084caed1c9f0142a9271159d791109a682dc Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 3 Apr 2023 13:01:04 -0500 Subject: [PATCH 01/11] intial commit --- .secrets.baseline | 13 ++----------- indexd/blueprint.py | 33 ++++++++++++++++++++++++++++++++- indexd/default_settings.py | 8 ++++++++ indexd/utils.py | 19 ++++++++++++++++++- tests/default_test_settings.py | 15 ++++++++++++++- tests/test_drs.py | 8 ++++++++ 6 files changed, 82 insertions(+), 14 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index bf3ef796..f3928946 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -128,15 +128,6 @@ "line_number": 104 } ], - "indexd/utils.py": [ - { - "type": "Basic Auth Credentials", - "filename": "indexd/utils.py", - "hashed_secret": "5d0fa74acf95d1d6bebd0d37f76a94e77d604fd9", - "is_verified": false, - "line_number": 87 - } - ], "tests/test_aliases_endpoints.py": [ { "type": "Hex High Entropy String", @@ -256,7 +247,7 @@ "filename": "tests/test_drs.py", "hashed_secret": "5666c088b494f26cd8f63ace013992f5fc391ce0", "is_verified": false, - "line_number": 31 + "line_number": 33 } ], "tests/test_schema_migration.py": [ @@ -269,5 +260,5 @@ } ] }, - "generated_at": "2023-03-09T21:25:06Z" + "generated_at": "2023-04-03T18:00:49Z" } diff --git a/indexd/blueprint.py b/indexd/blueprint.py index f31e3a3a..85a549c2 100644 --- a/indexd/blueprint.py +++ b/indexd/blueprint.py @@ -1,3 +1,6 @@ +import os +from crypt import methods +import re import flask from indexclient.client import IndexClient @@ -65,7 +68,6 @@ def get_record(record): def dist_get_record(record): - # Sort the list of distributed ID services # Ones with which the request matches a hint will be first # Followed by those that don't match the hint @@ -102,6 +104,35 @@ def dist_get_record(record): raise IndexNoRecordFound("no record found") +@blueprint.route("/service-info", methods=["GET"]) +def get_drs_service_info(): + drs_dist = {} + + for dist in blueprint.dist: + if dist.get("type").lower() == "drs": + drs_dist = dist + if drs_dist == {}: + drs_dist = dist[0] + + ret = { + "id": drs_dist.get("id"), + "name": drs_dist.get("name"), + "type": { + "group": "org.ga4gh", + "artifact": "drs", + "version": drs_dist.get("version", "1.0.0"), + }, + "organization": { + "name": "Gen3", + "url": drs_dist.get( + "organization_url", "https://" + os.environ["HOSTNAME"] + ), + }, + } + + return flask.jsonify(ret), 200 + + @blueprint.errorhandler(UserError) def handle_user_error(err): return flask.jsonify(error=str(err)), 400 diff --git a/indexd/default_settings.py b/indexd/default_settings.py index 3eed3fdb..6a5b3d70 100644 --- a/indexd/default_settings.py +++ b/indexd/default_settings.py @@ -1,6 +1,8 @@ +from os import environ from .index.drivers.alchemy import SQLAlchemyIndexDriver from .alias.drivers.alchemy import SQLAlchemyAliasDriver from .auth.drivers.alchemy import SQLAlchemyAuthDriver +from .utils import drs_service_info_id_url_reversal CONFIG = {} @@ -52,6 +54,12 @@ "host": "https://example.com/api/ga4gh/drs/v1/", "hints": [], "type": "drs", + "drs_version": "1.0.0", + "id": drs_service_info_id_url_reversal(url="example.com"), + "organization": { + "name": "Gen3", + "url": "http://example.com/", + }, }, ] diff --git a/indexd/utils.py b/indexd/utils.py index 171bf52b..033c4408 100644 --- a/indexd/utils.py +++ b/indexd/utils.py @@ -16,7 +16,6 @@ def hint_match(record, hints): def try_drop_test_data( user, database, root_user="postgres", host="" ): # pragma: no cover - engine = create_engine( "postgres://{user}@{host}/postgres".format(user=root_user, host=host) ) @@ -198,3 +197,21 @@ def is_empty_database(driver): table_list = Inspector.from_engine(driver.engine).get_table_names() return len(table_list) == 0 + + +def drs_service_info_id_url_reversal(url): + """ + Reverse the domain name for drs service-info IDs + Args: + url (str): url of the domain + example: drs.example.org + + returns: + id (str): DRS service-info ID + example: org.example.drs + """ + + segments = url.split(".") + reversed_segments = reversed(segments) + res = ".".join(reversed_segments) + return res diff --git a/tests/default_test_settings.py b/tests/default_test_settings.py index 54494eaa..641c147c 100644 --- a/tests/default_test_settings.py +++ b/tests/default_test_settings.py @@ -1,5 +1,6 @@ import os +from indexd.utils import drs_service_info_id_url_reversal from indexd.default_settings import * from indexd.index.drivers.alchemy import SQLAlchemyIndexDriver @@ -9,7 +10,19 @@ "host": "https://fictitious-commons.io/index/", "hints": [".*dg\\.4503.*"], "type": "indexd", - } + }, + { + "name": "DRS System", + "type": "drs", + "host": "https://fictitious-commons.io/", + "artifact": "drs", + "id": drs_service_info_id_url_reversal("fictitious-commons.io"), + "version": "1.3.0", + "organization": { + "name": "Gen3", + "url": "https://fictitious-commons.io/", + }, + }, ] os.environ["PRESIGNED_FENCE_URL"] = "https://fictitious-commons.io/" diff --git a/tests/test_drs.py b/tests/test_drs.py index 20613467..b925b160 100644 --- a/tests/test_drs.py +++ b/tests/test_drs.py @@ -199,3 +199,11 @@ def test_get_drs_with_encoded_slash(client, user): assert rec_2["checksums"][0]["type"] == k assert rec_2["version"] assert rec_2["self_uri"] == "drs://testprefix:" + rec_1["did"].split(":")[1] + + +def test_drs_service_info_endpoint(client, user): + res = client.get("/service-info") + + print("--------------------------------") + print(res.status_code) + print(res.json) From ed06a8c4c5148b485c4239106a5a63c940106d29 Mon Sep 17 00:00:00 2001 From: BinamB Date: Thu, 6 Apr 2023 10:18:03 -0500 Subject: [PATCH 02/11] Add unit tests --- indexd/blueprint.py | 36 +++++++++++++------- indexd/default_settings.py | 10 ++++-- tests/default_test_settings.py | 12 ------- tests/test_drs.py | 60 ++++++++++++++++++++++++++++++++-- 4 files changed, 89 insertions(+), 29 deletions(-) diff --git a/indexd/blueprint.py b/indexd/blueprint.py index 85a549c2..c0539ba3 100644 --- a/indexd/blueprint.py +++ b/indexd/blueprint.py @@ -8,7 +8,7 @@ from dosclient.client import DOSClient from hsclient.client import HSClient -from indexd.utils import hint_match +from indexd.utils import hint_match, drs_service_info_id_url_reversal from indexd.errors import AuthError from indexd.errors import UserError @@ -109,27 +109,41 @@ def get_drs_service_info(): drs_dist = {} for dist in blueprint.dist: - if dist.get("type").lower() == "drs": + if ( + "type" in dist + and isinstance(dist["type"], dict) + and "artifact" in dist["type"] + and dist["type"]["artifact"] == "drs" + ): drs_dist = dist if drs_dist == {}: - drs_dist = dist[0] + drs_dist = blueprint.dist[0] + + reverse_domain_name = drs_service_info_id_url_reversal(url=os.environ["HOSTNAME"]) ret = { - "id": drs_dist.get("id"), - "name": drs_dist.get("name"), + "id": drs_dist.get("id", reverse_domain_name), + "name": drs_dist.get("name", "DRS System"), + "version": drs_dist.get("version", "1.0.0"), "type": { - "group": "org.ga4gh", - "artifact": "drs", - "version": drs_dist.get("version", "1.0.0"), + "group": drs_dist.get("group", "org.ga4gh"), + "artifact": drs_dist.get("artifact", "drs"), }, "organization": { "name": "Gen3", - "url": drs_dist.get( - "organization_url", "https://" + os.environ["HOSTNAME"] - ), }, } + if "type" in drs_dist and isinstance(drs_dist["type"], dict): + ret["type"]["version"] = drs_dist.get("type").get("version", "1.0.0") + else: + ret["type"]["version"] = "1.0.0" + + if "organization" in drs_dist and "url" in drs_dist["organization"]: + ret["organization"]["url"] = drs_dist["organization"]["url"] + else: + ret["organization"]["url"] = "https://" + os.environ["HOSTNAME"] + return flask.jsonify(ret), 200 diff --git a/indexd/default_settings.py b/indexd/default_settings.py index 6a5b3d70..537a1f1c 100644 --- a/indexd/default_settings.py +++ b/indexd/default_settings.py @@ -53,9 +53,13 @@ "name": "DRS System", "host": "https://example.com/api/ga4gh/drs/v1/", "hints": [], - "type": "drs", - "drs_version": "1.0.0", - "id": drs_service_info_id_url_reversal(url="example.com"), + "type": { + "group": "org.ga4gh", + "artifact": "drs", + "version": "1.0.0", + }, + "version": "1.0.0", + "id": "com.example", "organization": { "name": "Gen3", "url": "http://example.com/", diff --git a/tests/default_test_settings.py b/tests/default_test_settings.py index 641c147c..552b0b8b 100644 --- a/tests/default_test_settings.py +++ b/tests/default_test_settings.py @@ -11,18 +11,6 @@ "hints": [".*dg\\.4503.*"], "type": "indexd", }, - { - "name": "DRS System", - "type": "drs", - "host": "https://fictitious-commons.io/", - "artifact": "drs", - "id": drs_service_info_id_url_reversal("fictitious-commons.io"), - "version": "1.3.0", - "organization": { - "name": "Gen3", - "url": "https://fictitious-commons.io/", - }, - }, ] os.environ["PRESIGNED_FENCE_URL"] = "https://fictitious-commons.io/" diff --git a/tests/test_drs.py b/tests/test_drs.py index b925b160..d9fb029c 100644 --- a/tests/test_drs.py +++ b/tests/test_drs.py @@ -201,9 +201,63 @@ def test_get_drs_with_encoded_slash(client, user): assert rec_2["self_uri"] == "drs://testprefix:" + rec_1["did"].split(":")[1] -def test_drs_service_info_endpoint(client, user): +def test_drs_service_info_without_drs_block_in_config(client): + expected_info = { + "id": "io.fictitious-commons", + "name": "testStage", + "type": { + "group": "org.ga4gh", + "artifact": "drs", + "version": "1.0.0", + }, + "organization": { + "name": "Gen3", + "url": "https://fictitious-commons.io", + }, + "version": "1.0.0", + } + res = client.get("/service-info") - print("--------------------------------") - print(res.status_code) + assert res.status_code == 200 + assert res.json == expected_info print(res.json) + + +def test_drs_service_info_endpoint(client): + expected_info = { + "id": "io.fictitious-commons", + "name": "DRS System", + "type": { + "group": "org.ga4gh", + "artifact": "drs", + "version": "1.0.0", + }, + "version": "1.3.0", + "organization": { + "name": "Gen3", + "url": "https://fictitious-commons.io", + }, + } + + settings["config"]["DIST"].append( + { + "name": "DRS System", + "type": { + "group": "org.ga4gh", + "artifact": "drs", + "version": "1.0.0", + }, + "host": "https://fictitious-commons.io/", + "version": "1.3.0", + "organization": { + "name": "Gen3", + "url": "https://fictitious-commons.io", + }, + } + ) + + res = client.get("/service-info") + + assert res.status_code == 200 + assert res.json == expected_info From c30447d68dc658496a8b50c8f8ee9534b10b25e0 Mon Sep 17 00:00:00 2001 From: BinamB Date: Fri, 7 Apr 2023 09:48:46 -0500 Subject: [PATCH 03/11] Add docstring --- indexd/blueprint.py | 4 ++++ tests/test_drs.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/indexd/blueprint.py b/indexd/blueprint.py index c0539ba3..6616f2bd 100644 --- a/indexd/blueprint.py +++ b/indexd/blueprint.py @@ -106,8 +106,12 @@ def dist_get_record(record): @blueprint.route("/service-info", methods=["GET"]) def get_drs_service_info(): + """ + Returns DRS compliant service information + """ drs_dist = {} + # Check to see if the information is of type drs. If not, use the available information to return DRS compliant service information for dist in blueprint.dist: if ( "type" in dist diff --git a/tests/test_drs.py b/tests/test_drs.py index d9fb029c..119b743f 100644 --- a/tests/test_drs.py +++ b/tests/test_drs.py @@ -202,6 +202,9 @@ def test_get_drs_with_encoded_slash(client, user): def test_drs_service_info_without_drs_block_in_config(client): + """ + Test that the non-drs related, defualt distributions convert to drs service info friendly format + """ expected_info = { "id": "io.fictitious-commons", "name": "testStage", @@ -225,6 +228,9 @@ def test_drs_service_info_without_drs_block_in_config(client): def test_drs_service_info_endpoint(client): + """ + Test drs service endpoint with drs service info friendly distribution information + """ expected_info = { "id": "io.fictitious-commons", "name": "DRS System", From ed1eecb9bf93fd1a978516ff647e3d9580200d5f Mon Sep 17 00:00:00 2001 From: BinamB Date: Fri, 7 Apr 2023 09:50:18 -0500 Subject: [PATCH 04/11] Remove unused import --- indexd/default_settings.py | 1 - tests/default_test_settings.py | 1 - 2 files changed, 2 deletions(-) diff --git a/indexd/default_settings.py b/indexd/default_settings.py index 537a1f1c..82304769 100644 --- a/indexd/default_settings.py +++ b/indexd/default_settings.py @@ -2,7 +2,6 @@ from .index.drivers.alchemy import SQLAlchemyIndexDriver from .alias.drivers.alchemy import SQLAlchemyAliasDriver from .auth.drivers.alchemy import SQLAlchemyAuthDriver -from .utils import drs_service_info_id_url_reversal CONFIG = {} diff --git a/tests/default_test_settings.py b/tests/default_test_settings.py index 552b0b8b..410532f3 100644 --- a/tests/default_test_settings.py +++ b/tests/default_test_settings.py @@ -1,6 +1,5 @@ import os -from indexd.utils import drs_service_info_id_url_reversal from indexd.default_settings import * from indexd.index.drivers.alchemy import SQLAlchemyIndexDriver From 3ff73f92a9d292cb1229a2e7501fbd56c89ce8df Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 10 Apr 2023 11:06:09 -0500 Subject: [PATCH 05/11] move to drs --- indexd/blueprint.py | 49 +----------------------------------- indexd/drs/blueprint.py | 56 +++++++++++++++++++++++++++++++++++++++++ tests/test_drs.py | 4 +-- 3 files changed, 59 insertions(+), 50 deletions(-) diff --git a/indexd/blueprint.py b/indexd/blueprint.py index 6616f2bd..96708d77 100644 --- a/indexd/blueprint.py +++ b/indexd/blueprint.py @@ -8,7 +8,7 @@ from dosclient.client import DOSClient from hsclient.client import HSClient -from indexd.utils import hint_match, drs_service_info_id_url_reversal +from indexd.utils import hint_match from indexd.errors import AuthError from indexd.errors import UserError @@ -104,53 +104,6 @@ def dist_get_record(record): raise IndexNoRecordFound("no record found") -@blueprint.route("/service-info", methods=["GET"]) -def get_drs_service_info(): - """ - Returns DRS compliant service information - """ - drs_dist = {} - - # Check to see if the information is of type drs. If not, use the available information to return DRS compliant service information - for dist in blueprint.dist: - if ( - "type" in dist - and isinstance(dist["type"], dict) - and "artifact" in dist["type"] - and dist["type"]["artifact"] == "drs" - ): - drs_dist = dist - if drs_dist == {}: - drs_dist = blueprint.dist[0] - - reverse_domain_name = drs_service_info_id_url_reversal(url=os.environ["HOSTNAME"]) - - ret = { - "id": drs_dist.get("id", reverse_domain_name), - "name": drs_dist.get("name", "DRS System"), - "version": drs_dist.get("version", "1.0.0"), - "type": { - "group": drs_dist.get("group", "org.ga4gh"), - "artifact": drs_dist.get("artifact", "drs"), - }, - "organization": { - "name": "Gen3", - }, - } - - if "type" in drs_dist and isinstance(drs_dist["type"], dict): - ret["type"]["version"] = drs_dist.get("type").get("version", "1.0.0") - else: - ret["type"]["version"] = "1.0.0" - - if "organization" in drs_dist and "url" in drs_dist["organization"]: - ret["organization"]["url"] = drs_dist["organization"]["url"] - else: - ret["organization"]["url"] = "https://" + os.environ["HOSTNAME"] - - return flask.jsonify(ret), 200 - - @blueprint.errorhandler(UserError) def handle_user_error(err): return flask.jsonify(error=str(err)), 400 diff --git a/indexd/drs/blueprint.py b/indexd/drs/blueprint.py index 024fb85d..e9135244 100644 --- a/indexd/drs/blueprint.py +++ b/indexd/drs/blueprint.py @@ -1,14 +1,17 @@ +import os import flask import json from indexd.errors import AuthError, AuthzError from indexd.errors import UserError from indexd.index.errors import NoRecordFound as IndexNoRecordFound from indexd.errors import IndexdUnexpectedError +from indexd.utils import drs_service_info_id_url_reversal blueprint = flask.Blueprint("drs", __name__) blueprint.config = dict() blueprint.index_driver = None +blueprint.dist = [] @blueprint.route("/ga4gh/drs/v1/objects/", methods=["GET"]) @@ -291,6 +294,53 @@ def parse_checksums(record, drs_object): return ret_checksum +@blueprint.route("/ga4gh/drs/v1/service-info", methods=["GET"]) +def get_drs_service_info(): + """ + Returns DRS compliant service information + """ + drs_dist = {} + + # Check to see if the information is of type drs. If not, use the available information to return DRS compliant service information + for dist in blueprint.dist: + if ( + "type" in dist + and isinstance(dist["type"], dict) + and "artifact" in dist["type"] + and dist["type"]["artifact"] == "drs" + ): + drs_dist = dist + if drs_dist == {}: + drs_dist = blueprint.dist[0] + + reverse_domain_name = drs_service_info_id_url_reversal(url=os.environ["HOSTNAME"]) + + ret = { + "id": drs_dist.get("id", reverse_domain_name), + "name": drs_dist.get("name", "DRS System"), + "version": drs_dist.get("version", "1.0.0"), + "type": { + "group": drs_dist.get("group", "org.ga4gh"), + "artifact": drs_dist.get("artifact", "drs"), + }, + "organization": { + "name": "Gen3", + }, + } + + if "type" in drs_dist and isinstance(drs_dist["type"], dict): + ret["type"]["version"] = drs_dist.get("type").get("version", "1.0.0") + else: + ret["type"]["version"] = "1.0.0" + + if "organization" in drs_dist and "url" in drs_dist["organization"]: + ret["organization"]["url"] = drs_dist["organization"]["url"] + else: + ret["organization"]["url"] = "https://" + os.environ["HOSTNAME"] + + return flask.jsonify(ret), 200 + + @blueprint.errorhandler(UserError) def handle_user_error(err): ret = {"msg": str(err), "status_code": 400} @@ -325,3 +375,9 @@ def handle_unexpected_error(err): def get_config(setup_state): index_config = setup_state.app.config["INDEX"] blueprint.index_driver = index_config["driver"] + + +@blueprint.record +def get_config(setup_state): + if "DIST" in setup_state.app.config: + blueprint.dist = setup_state.app.config["DIST"] diff --git a/tests/test_drs.py b/tests/test_drs.py index 119b743f..4e7979a1 100644 --- a/tests/test_drs.py +++ b/tests/test_drs.py @@ -220,7 +220,7 @@ def test_drs_service_info_without_drs_block_in_config(client): "version": "1.0.0", } - res = client.get("/service-info") + res = client.get("/ga4gh/drs/v1/service-info") assert res.status_code == 200 assert res.json == expected_info @@ -263,7 +263,7 @@ def test_drs_service_info_endpoint(client): } ) - res = client.get("/service-info") + res = client.get("/ga4gh/drs/v1/service-info") assert res.status_code == 200 assert res.json == expected_info From 6446dbcb6378b64693f9a7edecb501087e0e23b0 Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 10 Apr 2023 11:23:12 -0500 Subject: [PATCH 06/11] move service-info --- indexd/drs/blueprint.py | 94 ++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/indexd/drs/blueprint.py b/indexd/drs/blueprint.py index e9135244..9cc1f16b 100644 --- a/indexd/drs/blueprint.py +++ b/indexd/drs/blueprint.py @@ -14,6 +14,53 @@ blueprint.dist = [] +@blueprint.route("/ga4gh/drs/v1/service-info", methods=["GET"]) +def get_drs_service_info(): + """ + Returns DRS compliant service information + """ + drs_dist = {} + + # Check to see if the information is of type drs. If not, use the available information to return DRS compliant service information + for dist in blueprint.dist: + if ( + "type" in dist + and isinstance(dist["type"], dict) + and "artifact" in dist["type"] + and dist["type"]["artifact"] == "drs" + ): + drs_dist = dist + if drs_dist == {}: + drs_dist = blueprint.dist[0] + + reverse_domain_name = drs_service_info_id_url_reversal(url=os.environ["HOSTNAME"]) + + ret = { + "id": drs_dist.get("id", reverse_domain_name), + "name": drs_dist.get("name", "DRS System"), + "version": drs_dist.get("version", "1.0.0"), + "type": { + "group": drs_dist.get("group", "org.ga4gh"), + "artifact": drs_dist.get("artifact", "drs"), + }, + "organization": { + "name": "Gen3", + }, + } + + if "type" in drs_dist and isinstance(drs_dist["type"], dict): + ret["type"]["version"] = drs_dist.get("type").get("version", "1.0.0") + else: + ret["type"]["version"] = "1.0.0" + + if "organization" in drs_dist and "url" in drs_dist["organization"]: + ret["organization"]["url"] = drs_dist["organization"]["url"] + else: + ret["organization"]["url"] = "https://" + os.environ["HOSTNAME"] + + return flask.jsonify(ret), 200 + + @blueprint.route("/ga4gh/drs/v1/objects/", methods=["GET"]) def get_drs_object(object_id): """ @@ -294,53 +341,6 @@ def parse_checksums(record, drs_object): return ret_checksum -@blueprint.route("/ga4gh/drs/v1/service-info", methods=["GET"]) -def get_drs_service_info(): - """ - Returns DRS compliant service information - """ - drs_dist = {} - - # Check to see if the information is of type drs. If not, use the available information to return DRS compliant service information - for dist in blueprint.dist: - if ( - "type" in dist - and isinstance(dist["type"], dict) - and "artifact" in dist["type"] - and dist["type"]["artifact"] == "drs" - ): - drs_dist = dist - if drs_dist == {}: - drs_dist = blueprint.dist[0] - - reverse_domain_name = drs_service_info_id_url_reversal(url=os.environ["HOSTNAME"]) - - ret = { - "id": drs_dist.get("id", reverse_domain_name), - "name": drs_dist.get("name", "DRS System"), - "version": drs_dist.get("version", "1.0.0"), - "type": { - "group": drs_dist.get("group", "org.ga4gh"), - "artifact": drs_dist.get("artifact", "drs"), - }, - "organization": { - "name": "Gen3", - }, - } - - if "type" in drs_dist and isinstance(drs_dist["type"], dict): - ret["type"]["version"] = drs_dist.get("type").get("version", "1.0.0") - else: - ret["type"]["version"] = "1.0.0" - - if "organization" in drs_dist and "url" in drs_dist["organization"]: - ret["organization"]["url"] = drs_dist["organization"]["url"] - else: - ret["organization"]["url"] = "https://" + os.environ["HOSTNAME"] - - return flask.jsonify(ret), 200 - - @blueprint.errorhandler(UserError) def handle_user_error(err): ret = {"msg": str(err), "status_code": 400} From c8d1e256b5b792f1a0d1bd8f33d4948407be073e Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 10 Apr 2023 12:05:45 -0500 Subject: [PATCH 07/11] remove unused imports --- indexd/blueprint.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/indexd/blueprint.py b/indexd/blueprint.py index 96708d77..608062a7 100644 --- a/indexd/blueprint.py +++ b/indexd/blueprint.py @@ -1,6 +1,3 @@ -import os -from crypt import methods -import re import flask from indexclient.client import IndexClient From 6fc37b9c6a73886ba281ecc21211672647ffd000 Mon Sep 17 00:00:00 2001 From: BinamB Date: Mon, 10 Apr 2023 14:35:38 -0500 Subject: [PATCH 08/11] Add more test and handle edge case --- indexd/drs/blueprint.py | 25 +++++++++++++------------ tests/test_drs.py | 28 +++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/indexd/drs/blueprint.py b/indexd/drs/blueprint.py index 9cc1f16b..6dc8e010 100644 --- a/indexd/drs/blueprint.py +++ b/indexd/drs/blueprint.py @@ -21,20 +21,21 @@ def get_drs_service_info(): """ drs_dist = {} - # Check to see if the information is of type drs. If not, use the available information to return DRS compliant service information - for dist in blueprint.dist: - if ( - "type" in dist - and isinstance(dist["type"], dict) - and "artifact" in dist["type"] - and dist["type"]["artifact"] == "drs" - ): - drs_dist = dist - if drs_dist == {}: - drs_dist = blueprint.dist[0] - reverse_domain_name = drs_service_info_id_url_reversal(url=os.environ["HOSTNAME"]) + if len(blueprint.dist) > 0: + # Check to see if the information is of type drs. If not, use the available information to return DRS compliant service information + for dist in blueprint.dist: + if ( + "type" in dist + and isinstance(dist["type"], dict) + and "artifact" in dist["type"] + and dist["type"]["artifact"] == "drs" + ): + drs_dist = dist + if drs_dist == {}: + drs_dist = blueprint.dist[0] + ret = { "id": drs_dist.get("id", reverse_domain_name), "name": drs_dist.get("name", "DRS System"), diff --git a/tests/test_drs.py b/tests/test_drs.py index 4e7979a1..aebf03f2 100644 --- a/tests/test_drs.py +++ b/tests/test_drs.py @@ -224,7 +224,6 @@ def test_drs_service_info_without_drs_block_in_config(client): assert res.status_code == 200 assert res.json == expected_info - print(res.json) def test_drs_service_info_endpoint(client): @@ -267,3 +266,30 @@ def test_drs_service_info_endpoint(client): assert res.status_code == 200 assert res.json == expected_info + + +def test_drs_service_info_no_dist_in_config(client): + """ + Test drs service info endpoint when dist is not configured in the indexd config file + """ + expected_info = { + "id": "io.fictitious-commons", + "name": "DRS System", + "type": { + "group": "org.ga4gh", + "artifact": "drs", + "version": "1.0.0", + }, + "version": "1.0.0", + "organization": { + "name": "Gen3", + "url": "https://fictitious-commons.io", + }, + } + + settings["config"]["DIST"].clear() + + res = client.get("/ga4gh/drs/v1/service-info") + + assert res.status_code == 200 + assert res.json == expected_info From e2f6f9618e2844a07f490791d2534cdb52134548 Mon Sep 17 00:00:00 2001 From: BinamB Date: Tue, 11 Apr 2023 12:50:22 -0500 Subject: [PATCH 09/11] remove utils stuff --- indexd/utils.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/indexd/utils.py b/indexd/utils.py index 1368abe5..0dae276b 100644 --- a/indexd/utils.py +++ b/indexd/utils.py @@ -191,20 +191,6 @@ def migrate_database(driver, migrate_functions, current_schema_version, model): s.add(schema_version) -def is_empty_database(driver): - """ - check if the database is empty or not - Args: - driver (object): an alias or index driver instance - - Returns: - Boolean - """ - table_list = Inspector.from_engine(driver.engine).get_table_names() - - return len(table_list) == 0 - - def drs_service_info_id_url_reversal(url): """ Reverse the domain name for drs service-info IDs From 0836cfad50fd187e561ddfa5303aa687cdf85e39 Mon Sep 17 00:00:00 2001 From: BinamB Date: Wed, 12 Apr 2023 15:10:47 -0500 Subject: [PATCH 10/11] Create separate config --- .secrets.baseline | 6 +-- bin/indexd_settings.py | 4 ++ deployment/Secrets/indexd_settings.py | 4 ++ indexd/default_settings.py | 33 +++++++------- indexd/drs/blueprint.py | 66 +++++++++++++-------------- indexd/utils.py | 2 +- tests/default_test_settings.py | 15 ++++++ tests/test_drs.py | 61 +++++-------------------- 8 files changed, 88 insertions(+), 103 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 68004bce..d1f8f1fa 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -230,7 +230,7 @@ "filename": "tests/default_test_settings.py", "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", "is_verified": false, - "line_number": 26 + "line_number": 41 } ], "tests/postgres/migrations/test_15f2e9345ade_create_tables.py": [ @@ -391,9 +391,9 @@ "filename": "tests/test_drs.py", "hashed_secret": "5666c088b494f26cd8f63ace013992f5fc391ce0", "is_verified": false, - "line_number": 31 + "line_number": 32 } ] }, - "generated_at": "2023-04-10T19:43:22Z" + "generated_at": "2023-04-12T20:10:41Z" } diff --git a/bin/indexd_settings.py b/bin/indexd_settings.py index 364aac2b..bcc67985 100644 --- a/bin/indexd_settings.py +++ b/bin/indexd_settings.py @@ -28,6 +28,10 @@ def load_json(file_name): if dist: CONFIG["DIST"] = json.loads(dist) +drs_service_info = environ.get("DRS_SERVICE_INFO", None) +if drs_service_info: + CONFIG["DRS_SERVICE_INFO"] = json.loads(drs_service_info) + CONFIG["INDEX"] = { "driver": SQLAlchemyIndexDriver( "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format( diff --git a/deployment/Secrets/indexd_settings.py b/deployment/Secrets/indexd_settings.py index 4b35c567..e7572d4f 100644 --- a/deployment/Secrets/indexd_settings.py +++ b/deployment/Secrets/indexd_settings.py @@ -29,6 +29,10 @@ def load_json(file_name): if dist: CONFIG["DIST"] = json.loads(dist) +drs_service_info = environ.get("DRS_SERVICE_INFO", None) +if drs_service_info: + CONFIG["DRS_SERVICE_INFO"] = json.loads(drs_service_info) + CONFIG["INDEX"] = { "driver": SQLAlchemyIndexDriver( "postgresql+psycopg2://{usr}:{psw}@{pghost}:{pgport}/{db}".format( diff --git a/indexd/default_settings.py b/indexd/default_settings.py index 18520fae..7b999bb7 100644 --- a/indexd/default_settings.py +++ b/indexd/default_settings.py @@ -45,24 +45,25 @@ "hints": [], "type": "dos", }, - { - "name": "DRS System", - "host": "https://example.com/api/ga4gh/drs/v1/", - "hints": [], - "type": { - "group": "org.ga4gh", - "artifact": "drs", - "version": "1.0.0", - }, - "version": "1.0.0", - "id": "com.example", - "organization": { - "name": "Gen3", - "url": "http://example.com/", - }, - }, ] +CONFIG["DRS_SERVICE_INFO"] = { + "name": "DRS System", + "host": "https://example.com/api/ga4gh/drs/v1/", + "hints": [], + "type": { + "group": "org.ga4gh", + "artifact": "drs", + "version": "1.0.3", + }, + "version": "1.0.3", + "id": "com.example", + "organization": { + "name": "CTDS", + "url": "http://example.com/", + }, +} + AUTH = SQLAlchemyAuthDriver("sqlite:///auth.sq3") settings = {"config": CONFIG, "auth": AUTH} diff --git a/indexd/drs/blueprint.py b/indexd/drs/blueprint.py index 6f42a2a5..2c9a8a46 100644 --- a/indexd/drs/blueprint.py +++ b/indexd/drs/blueprint.py @@ -5,13 +5,13 @@ from indexd.errors import UserError from indexd.index.errors import NoRecordFound as IndexNoRecordFound from indexd.errors import IndexdUnexpectedError -from indexd.utils import drs_service_info_id_url_reversal +from indexd.utils import reverse_url blueprint = flask.Blueprint("drs", __name__) blueprint.config = dict() blueprint.index_driver = None -blueprint.dist = [] +blueprint.service_info = {} @blueprint.route("/ga4gh/drs/v1/service-info", methods=["GET"]) @@ -19,46 +19,46 @@ def get_drs_service_info(): """ Returns DRS compliant service information """ - drs_dist = {} - - reverse_domain_name = drs_service_info_id_url_reversal(url=os.environ["HOSTNAME"]) - - if len(blueprint.dist) > 0: - # Check to see if the information is of type drs. If not, use the available information to return DRS compliant service information - for dist in blueprint.dist: - if ( - "type" in dist - and isinstance(dist["type"], dict) - and "artifact" in dist["type"] - and dist["type"]["artifact"] == "drs" - ): - drs_dist = dist - if drs_dist == {}: - drs_dist = blueprint.dist[0] + + reverse_domain_name = reverse_url(url=os.environ["HOSTNAME"]) ret = { - "id": drs_dist.get("id", reverse_domain_name), - "name": drs_dist.get("name", "DRS System"), - "version": drs_dist.get("version", "1.0.0"), + "id": blueprint.service_info.get("id", reverse_domain_name), + "name": blueprint.service_info.get("name", "DRS System"), + "version": blueprint.service_info.get("version", "1.0.3"), "type": { - "group": drs_dist.get("group", "org.ga4gh"), - "artifact": drs_dist.get("artifact", "drs"), - }, - "organization": { - "name": "Gen3", + "group": blueprint.service_info.get("group", "org.ga4gh"), + "artifact": blueprint.service_info.get("artifact", "drs"), }, } - if "type" in drs_dist and isinstance(drs_dist["type"], dict): - ret["type"]["version"] = drs_dist.get("type").get("version", "1.0.0") + ret["organization"] = {} + + if "type" in blueprint.service_info and isinstance( + blueprint.service_info["type"], dict + ): + ret["type"]["version"] = blueprint.service_info.get("type").get( + "version", "1.0.3" + ) else: - ret["type"]["version"] = "1.0.0" + ret["type"]["version"] = "1.0.3" - if "organization" in drs_dist and "url" in drs_dist["organization"]: - ret["organization"]["url"] = drs_dist["organization"]["url"] + if ( + "organization" in blueprint.service_info + and "url" in blueprint.service_info["organization"] + ): + ret["organization"]["url"] = blueprint.service_info["organization"]["url"] else: ret["organization"]["url"] = "https://" + os.environ["HOSTNAME"] + if ( + "organization" in blueprint.service_info + and "name" in blueprint.service_info["organization"] + ): + ret["organization"]["name"] = blueprint.service_info["organization"]["name"] + else: + ret["organization"]["name"] = "CTDS" + return flask.jsonify(ret), 200 @@ -384,5 +384,5 @@ def get_config(setup_state): @blueprint.record def get_config(setup_state): - if "DIST" in setup_state.app.config: - blueprint.dist = setup_state.app.config["DIST"] + if "DRS_SERVICE_INFO" in setup_state.app.config: + blueprint.service_info = setup_state.app.config["DRS_SERVICE_INFO"] diff --git a/indexd/utils.py b/indexd/utils.py index 0dae276b..d17bfc0d 100644 --- a/indexd/utils.py +++ b/indexd/utils.py @@ -191,7 +191,7 @@ def migrate_database(driver, migrate_functions, current_schema_version, model): s.add(schema_version) -def drs_service_info_id_url_reversal(url): +def reverse_url(url): """ Reverse the domain name for drs service-info IDs Args: diff --git a/tests/default_test_settings.py b/tests/default_test_settings.py index ddc60962..b3ec4496 100644 --- a/tests/default_test_settings.py +++ b/tests/default_test_settings.py @@ -12,6 +12,21 @@ }, ] +CONFIG["DRS_SERVICE_INFO"] = { + "name": "DRS System", + "type": { + "group": "org.ga4gh", + "artifact": "drs", + "version": "1.0.3", + }, + "host": "https://fictitious-commons.io/", + "version": "1.0.3", + "organization": { + "name": "CTDS", + "url": "https://fictitious-commons.io", + }, +} + os.environ["PRESIGNED_FENCE_URL"] = "https://fictitious-commons.io/" os.environ["HOSTNAME"] = "fictitious-commons.io" settings = {"config": CONFIG, "auth": AUTH} diff --git a/tests/test_drs.py b/tests/test_drs.py index 3cd5b7fc..4dc6d47f 100644 --- a/tests/test_drs.py +++ b/tests/test_drs.py @@ -1,3 +1,4 @@ +import flask import json import tests.conftest import requests @@ -157,74 +158,34 @@ def test_get_drs_with_encoded_slash(client, user): assert rec_2["self_uri"] == "drs://testprefix:" + rec_1["did"].split(":")[1] -def test_drs_service_info_without_drs_block_in_config(client): - """ - Test that the non-drs related, defualt distributions convert to drs service info friendly format - """ - expected_info = { - "id": "io.fictitious-commons", - "name": "testStage", - "type": { - "group": "org.ga4gh", - "artifact": "drs", - "version": "1.0.0", - }, - "organization": { - "name": "Gen3", - "url": "https://fictitious-commons.io", - }, - "version": "1.0.0", - } - - res = client.get("/ga4gh/drs/v1/service-info") - - assert res.status_code == 200 - assert res.json == expected_info - - def test_drs_service_info_endpoint(client): """ Test drs service endpoint with drs service info friendly distribution information """ + app = flask.Flask(__name__) + expected_info = { "id": "io.fictitious-commons", "name": "DRS System", "type": { "group": "org.ga4gh", "artifact": "drs", - "version": "1.0.0", + "version": "1.0.3", }, - "version": "1.3.0", + "version": "1.0.3", "organization": { - "name": "Gen3", + "name": "CTDS", "url": "https://fictitious-commons.io", }, } - settings["config"]["DIST"].append( - { - "name": "DRS System", - "type": { - "group": "org.ga4gh", - "artifact": "drs", - "version": "1.0.0", - }, - "host": "https://fictitious-commons.io/", - "version": "1.3.0", - "organization": { - "name": "Gen3", - "url": "https://fictitious-commons.io", - }, - } - ) - res = client.get("/ga4gh/drs/v1/service-info") assert res.status_code == 200 assert res.json == expected_info -def test_drs_service_info_no_dist_in_config(client): +def test_drs_service_info_no_information_configured(client): """ Test drs service info endpoint when dist is not configured in the indexd config file """ @@ -234,16 +195,16 @@ def test_drs_service_info_no_dist_in_config(client): "type": { "group": "org.ga4gh", "artifact": "drs", - "version": "1.0.0", + "version": "1.0.3", }, - "version": "1.0.0", + "version": "1.0.3", "organization": { - "name": "Gen3", + "name": "CTDS", "url": "https://fictitious-commons.io", }, } - settings["config"]["DIST"].clear() + settings["config"]["DRS_SERVICE_INFO"].clear() res = client.get("/ga4gh/drs/v1/service-info") From e17a9fc2921c9f10f1a04602548e04911136c52c Mon Sep 17 00:00:00 2001 From: BinamB Date: Thu, 13 Apr 2023 12:00:20 -0500 Subject: [PATCH 11/11] fix things --- .secrets.baseline | 4 +-- indexd/default_settings.py | 3 -- indexd/drs/blueprint.py | 50 +++++++++++++--------------------- indexd/utils.py | 5 +++- tests/default_test_settings.py | 1 - tests/test_drs.py | 12 +++++--- 6 files changed, 33 insertions(+), 42 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index d1f8f1fa..be14f3d8 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -230,7 +230,7 @@ "filename": "tests/default_test_settings.py", "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3", "is_verified": false, - "line_number": 41 + "line_number": 40 } ], "tests/postgres/migrations/test_15f2e9345ade_create_tables.py": [ @@ -395,5 +395,5 @@ } ] }, - "generated_at": "2023-04-12T20:10:41Z" + "generated_at": "2023-04-13T17:00:13Z" } diff --git a/indexd/default_settings.py b/indexd/default_settings.py index 7b999bb7..f57f1054 100644 --- a/indexd/default_settings.py +++ b/indexd/default_settings.py @@ -1,4 +1,3 @@ -from os import environ from .index.drivers.alchemy import SQLAlchemyIndexDriver from .alias.drivers.alchemy import SQLAlchemyAliasDriver from .auth.drivers.alchemy import SQLAlchemyAuthDriver @@ -49,8 +48,6 @@ CONFIG["DRS_SERVICE_INFO"] = { "name": "DRS System", - "host": "https://example.com/api/ga4gh/drs/v1/", - "hints": [], "type": { "group": "org.ga4gh", "artifact": "drs", diff --git a/indexd/drs/blueprint.py b/indexd/drs/blueprint.py index 2c9a8a46..053a5045 100644 --- a/indexd/drs/blueprint.py +++ b/indexd/drs/blueprint.py @@ -1,4 +1,5 @@ import os +import re import flask import json from indexd.errors import AuthError, AuthzError @@ -23,41 +24,28 @@ def get_drs_service_info(): reverse_domain_name = reverse_url(url=os.environ["HOSTNAME"]) ret = { - "id": blueprint.service_info.get("id", reverse_domain_name), - "name": blueprint.service_info.get("name", "DRS System"), - "version": blueprint.service_info.get("version", "1.0.3"), + "id": reverse_domain_name, + "name": "DRS System", + "version": "1.0.3", "type": { - "group": blueprint.service_info.get("group", "org.ga4gh"), - "artifact": blueprint.service_info.get("artifact", "drs"), + "group": "org.ga4gh", + "artifact": "drs", + "version": "1.0.3", + }, + "organization": { + "name": "CTDS", + "url": "https://" + os.environ["HOSTNAME"], }, } - ret["organization"] = {} - - if "type" in blueprint.service_info and isinstance( - blueprint.service_info["type"], dict - ): - ret["type"]["version"] = blueprint.service_info.get("type").get( - "version", "1.0.3" - ) - else: - ret["type"]["version"] = "1.0.3" - - if ( - "organization" in blueprint.service_info - and "url" in blueprint.service_info["organization"] - ): - ret["organization"]["url"] = blueprint.service_info["organization"]["url"] - else: - ret["organization"]["url"] = "https://" + os.environ["HOSTNAME"] - - if ( - "organization" in blueprint.service_info - and "name" in blueprint.service_info["organization"] - ): - ret["organization"]["name"] = blueprint.service_info["organization"]["name"] - else: - ret["organization"]["name"] = "CTDS" + if blueprint.service_info: + for key, value in blueprint.service_info.items(): + if key in ret: + if isinstance(value, dict): + for inner_key, inner_value in value.items(): + ret[key][inner_key] = inner_value + else: + ret[key] = value return flask.jsonify(ret), 200 diff --git a/indexd/utils.py b/indexd/utils.py index d17bfc0d..29a467ab 100644 --- a/indexd/utils.py +++ b/indexd/utils.py @@ -1,5 +1,6 @@ import logging import re +from urllib.parse import urlparse def hint_match(record, hints): @@ -202,7 +203,9 @@ def reverse_url(url): id (str): DRS service-info ID example: org.example.drs """ - + parsed_url = urlparse(url) + if parsed_url.scheme in ["http", "https"]: + url = parsed_url.hostname segments = url.split(".") reversed_segments = reversed(segments) res = ".".join(reversed_segments) diff --git a/tests/default_test_settings.py b/tests/default_test_settings.py index b3ec4496..8dee48ea 100644 --- a/tests/default_test_settings.py +++ b/tests/default_test_settings.py @@ -19,7 +19,6 @@ "artifact": "drs", "version": "1.0.3", }, - "host": "https://fictitious-commons.io/", "version": "1.0.3", "organization": { "name": "CTDS", diff --git a/tests/test_drs.py b/tests/test_drs.py index 4dc6d47f..45454688 100644 --- a/tests/test_drs.py +++ b/tests/test_drs.py @@ -203,10 +203,14 @@ def test_drs_service_info_no_information_configured(client): "url": "https://fictitious-commons.io", }, } + backup = settings["config"]["DRS_SERVICE_INFO"].copy() - settings["config"]["DRS_SERVICE_INFO"].clear() + try: + settings["config"]["DRS_SERVICE_INFO"].clear() - res = client.get("/ga4gh/drs/v1/service-info") + res = client.get("/ga4gh/drs/v1/service-info") - assert res.status_code == 200 - assert res.json == expected_info + assert res.status_code == 200 + assert res.json == expected_info + finally: + settings["config"]["DRS_SERVICE_INFO"] = backup