diff --git a/docs/_build/doctrees/auth.doctree b/docs/_build/doctrees/auth.doctree index 2457bd297..eed30afbd 100644 Binary files a/docs/_build/doctrees/auth.doctree and b/docs/_build/doctrees/auth.doctree differ diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index 1b3685cad..969ab158f 100644 Binary files a/docs/_build/doctrees/environment.pickle and b/docs/_build/doctrees/environment.pickle differ diff --git a/docs/_build/doctrees/file.doctree b/docs/_build/doctrees/file.doctree index 9c191309d..140d5ec1c 100644 Binary files a/docs/_build/doctrees/file.doctree and b/docs/_build/doctrees/file.doctree differ diff --git a/docs/_build/doctrees/indexing.doctree b/docs/_build/doctrees/indexing.doctree index 7533c883f..0bea79157 100644 Binary files a/docs/_build/doctrees/indexing.doctree and b/docs/_build/doctrees/indexing.doctree differ diff --git a/docs/_build/doctrees/jobs.doctree b/docs/_build/doctrees/jobs.doctree index 5d407094f..5bb03b228 100644 Binary files a/docs/_build/doctrees/jobs.doctree and b/docs/_build/doctrees/jobs.doctree differ diff --git a/docs/_build/doctrees/metadata.doctree b/docs/_build/doctrees/metadata.doctree index 5649d18cc..ee4713b5d 100644 Binary files a/docs/_build/doctrees/metadata.doctree and b/docs/_build/doctrees/metadata.doctree differ diff --git a/docs/_build/doctrees/object.doctree b/docs/_build/doctrees/object.doctree index 19948c058..3e16cd546 100644 Binary files a/docs/_build/doctrees/object.doctree and b/docs/_build/doctrees/object.doctree differ diff --git a/docs/_build/doctrees/query.doctree b/docs/_build/doctrees/query.doctree index 9ffcc41da..854706e1a 100644 Binary files a/docs/_build/doctrees/query.doctree and b/docs/_build/doctrees/query.doctree differ diff --git a/docs/_build/doctrees/submission.doctree b/docs/_build/doctrees/submission.doctree index 37bf26090..5178acd18 100644 Binary files a/docs/_build/doctrees/submission.doctree and b/docs/_build/doctrees/submission.doctree differ diff --git a/docs/_build/doctrees/tools/indexing.doctree b/docs/_build/doctrees/tools/indexing.doctree index 9945e6869..a5707c665 100644 Binary files a/docs/_build/doctrees/tools/indexing.doctree and b/docs/_build/doctrees/tools/indexing.doctree differ diff --git a/docs/_build/doctrees/tools/metadata.doctree b/docs/_build/doctrees/tools/metadata.doctree index 9be9131a5..26ebd997b 100644 Binary files a/docs/_build/doctrees/tools/metadata.doctree and b/docs/_build/doctrees/tools/metadata.doctree differ diff --git a/docs/_build/doctrees/wss.doctree b/docs/_build/doctrees/wss.doctree index 9517a7338..8d68519f3 100644 Binary files a/docs/_build/doctrees/wss.doctree and b/docs/_build/doctrees/wss.doctree differ diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index d6b971a97..58e81928f 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "terms": {"class": [0, 2, 10, 11, 13], "gen3auth": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "endpoint": [0, 1, 2, 3, 4, 5, 7, 8, 13], "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "idp": 0, "client_credenti": 0, "client_scop": 0, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "authbas": 0, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "request": [0, 1, 3, 5, 8, 11, 12], "implement": 0, "order": [0, 8], "support": [0, 1, 5, 8, 11], "jwt": 0, "authent": 0, "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "access": [0, 1, 3, 7, 10], "token": [0, 10], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "provid": [0, 1, 3, 5, 7, 8, 12], "refresh": [0, 10], "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "string": [0, 3, 5, 11, 13], "automat": 0, "when": [0, 3, 7, 10, 12], "thei": [0, 10], "expir": [0, 1], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "opt": 0, "The": [0, 1, 2, 3, 5, 8, 10, 11], "contain": [0, 2, 5, 8, 9, 10, 11, 12], "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "web": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "work": [0, 10], "workspac": [0, 2], "default": [0, 1, 3, 7, 8, 11, 12], "env": 0, "gen3_api_kei": 0, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "set": [0, 1, 5, 10], "includ": [0, 3], "search": [0, 2, 3], "path": [0, 1, 5, 10, 11, 13], "valu": [0, 1, 3, 5, 7, 10, 11], "doe": [0, 12], "interpret": 0, "wt": [0, 10], "an": [0, 3, 6, 8, 10, 11], "accesstoken": 0, "If": [0, 1, 7, 11, 12], "can": [0, 3, 4, 8, 11, 12], "specifi": [0, 1, 3, 11, 13], "local": [0, 13], "indic": [0, 11], "environ": 0, "fenc": [0, 1], "tupl": [0, 3, 11, 12], "client_id": 0, "client_secret": 0, "oidc": 0, "client": [0, 3], "ha": [0, 11], "grant": 0, "allow": [0, 6, 8, 10, 11, 12], "obtain": [0, 10], "space": [0, 11], "separ": [0, 11], "list": [0, 1, 3, 4, 5, 7, 8, 10, 11, 13], "scope": [0, 1], "user": [0, 10, 12], "data": [0, 1, 3, 5, 7, 8, 10, 11], "openid": 0, "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "instal": [0, 2, 11], "crdc": 0, "some": [0, 2], "arbitrari": 0, "kei": [0, 3, 5, 13], "variabl": [0, 7, 8], "rather": 0, "than": [0, 5], "pass": [0, 7, 8, 10], "argument": [0, 13], "constructor": 0, "id": [0, 1, 3, 5, 10, 11], "secret": 0, "note": [0, 3, 11, 12], "should": [0, 8, 11], "never": 0, "hardcod": 0, "http": [0, 7, 11, 12], "datacommon": 0, "o": 0, "gen3_oidc_client_creds_secret": 0, "initi": [0, 10], "follow": [0, 11], "curl": [0, 2], "given": [0, 3, 4, 5, 8, 10, 12, 13], "ex": [0, 11, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "respons": [0, 1, 3, 4, 5], "under": [0, 8, 13], "index": [0, 2, 5, 9], "authz": [0, 1, 2, 3, 9, 10, 11], "map": [0, 11], "get": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12, 13], "post": [0, 11], "put": 0, "delet": [0, 1, 2, 3, 5, 6, 8, 10, 11], "i": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12], "els": [0, 12], "filenam": [0, 8, 10, 11, 12], "get_access_token": [0, 2], "auto": [0, 2], "within": [0, 2, 9], "5": [0, 5], "minut": 0, "get_access_token_from_wt": [0, 2], "try": 0, "fetch": 0, "namespac": [0, 12], "refresh_access_token": [0, 2], "new": [0, 3], "gen3fil": [1, 2], "auth_provid": [1, 2, 3, 4, 5, 6, 7, 8, 13], "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "For": [1, 5, 6, 7, 8, 9, 11], "interact": [1, 3, 4, 5, 6, 8, 13], "manag": [1, 5, 10], "featur": [1, 6], "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "presign": 1, "url": [1, 2, 3, 9, 10, 11, 12, 13], "right": 1, "now": [1, 8], "instanc": [1, 3, 6, 7, 8, 9, 10], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "delete_fil": [1, 2], "guid": [1, 2, 3, 5, 6, 9, 11, 12], "method": [1, 7, 10], "deprec": 1, "delete_file_loc": [1, 2, 6], "instead": [1, 7, 11], "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "locat": [1, 6], "store": [1, 3, 10], "remov": [1, 6, 11, 13], "its": [1, 3], "record": [1, 3, 5, 7, 8, 11, 12], "indexd": [1, 3, 6, 10, 11, 12], "uuid": [1, 3, 8], "text": [1, 7, 8], "result": [1, 8, 10, 11], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "download_singl": [1, 2], "object_id": [1, 2, 9, 10], "singl": [1, 5, 8, 10], "": [1, 4, 8, 10, 11], "uniqu": [1, 5], "get_presigned_url": [1, 2], "protocol": 1, "retriev": [1, 8, 10, 12], "give": 1, "limit": [1, 3, 5, 12], "time": [1, 8, 10, 11], "pick": 1, "avail": [1, 2, 10, 11], "queri": [1, 2, 3, 5, 8, 11, 12], "upload_fil": [1, 2], "file_nam": [1, 2, 3, 9, 10, 11], "expires_in": 1, "bucket": [1, 6], "upload": [1, 2, 3, 8, 13], "author": 1, "storag": [1, 2, 6], "s3": [1, 10, 11], "az": 1, "isn": 1, "t": [1, 5, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "amount": [1, 9], "second": [1, 4], "sign": 1, "datetim": [1, 10], "utcnow": 1, "Be": 1, "sure": 1, "posit": [1, 7], "integ": [1, 3, 8], "also": 1, "treat": [1, 5], "max_presigned_url_ttl": 1, "configur": 1, "must": [1, 5], "allowed_data_upload_bucket": 1, "data_upload_bucket": 1, "represent": [1, 3], "document": [1, 3], "upload_file_to_guid": [1, 2], "exist": [1, 3, 5, 6, 9, 12], "readm": 2, "python": [2, 9, 11], "github": [2, 7], "repo": 2, "setup": 2, "inform": [2, 3, 10], "along": 2, "quickstart": 2, "script": 2, "more": [2, 5, 7, 9, 10], "detail": [2, 7, 10], "about": [2, 3], "variou": 2, "function": [2, 3, 4, 5, 9, 10, 11, 12], "mostli": 2, "docstr": 2, "code": [2, 8], "helper": 2, "gen3index": [2, 3], "async_create_record": [2, 3], "async_get_record": [2, 3], "async_get_records_from_checksum": [2, 3], "async_get_records_on_pag": [2, 3], "async_get_with_param": [2, 3], "async_query_url": [2, 3], "async_update_record": [2, 3], "create_blank": [2, 3], "create_new_vers": [2, 3], "create_record": [2, 3], "delete_record": [2, 3, 8], "get_all_record": [2, 3], "get_guids_prefix": [2, 3], "get_latest_vers": [2, 3], "get_record": [2, 3], "get_record_doc": [2, 3], "get_records_on_pag": [2, 3], "get_stat": [2, 3], "get_url": [2, 3], "get_valid_guid": [2, 3], "get_vers": [2, 3, 4, 5], "get_with_param": [2, 3], "is_healthi": [2, 3, 4, 5], "query_url": [2, 3], "update_blank": [2, 3], "update_record": [2, 3], "job": 2, "gen3job": [2, 4, 10], "async_run_job_and_wait": [2, 4], "create_job": [2, 4], "get_output": [2, 4], "get_statu": [2, 4], "list_job": [2, 4], "metadata": [2, 3, 6, 9, 11], "gen3metadata": [2, 5], "async_cr": [2, 5], "async_create_alias": [2, 5], "async_delete_alia": [2, 5], "async_delete_alias": [2, 5], "async_get": [2, 5], "async_get_alias": [2, 5], "async_upd": [2, 5], "async_update_alias": [2, 5], "batch_creat": [2, 5], "creat": [2, 3, 4, 5, 6, 8, 10, 11], "create_alias": [2, 5], "create_index_key_path": [2, 5], "delete_alia": [2, 5], "delete_alias": [2, 5], "delete_index_key_path": [2, 5], "get_alias": [2, 5], "get_index_key_path": [2, 5], "updat": [2, 3, 5, 10, 11], "update_alias": [2, 5], "gen3object": [2, 6], "delete_object": [2, 6], "gen3queri": [2, 7], "graphql_queri": [2, 7], "raw_data_download": [2, 7], "submiss": 2, "gen3submiss": [2, 8], "create_program": [2, 8], "create_project": [2, 8], "delete_nod": [2, 8], "delete_program": [2, 8], "delete_project": [2, 8], "export_nod": [2, 8], "export_record": [2, 8], "get_dictionary_al": [2, 8], "get_dictionary_nod": [2, 8], "get_graphql_schema": [2, 8], "get_program": [2, 8], "get_project_dictionari": [2, 8], "get_project_manifest": [2, 8], "get_project": [2, 8], "open_project": [2, 8], "submit_fil": [2, 8], "submit_record": [2, 8], "tool": 2, "dr": [2, 9], "downloadmanag": [2, 9, 10], "cache_hosts_wts_token": [2, 9, 10], "get_fresh_token": [2, 9, 10], "resolve_object": [2, 9, 10], "user_access": [2, 9, 10], "downloadstatu": [2, 9, 10], "statu": [2, 4, 9, 10], "start_tim": [2, 9, 10], "end_tim": [2, 9, 10], "object_typ": [2, 9, 10], "hostnam": [2, 9, 10], "file_s": [2, 9, 10, 11], "updated_tim": [2, 9, 10], "created_tim": [2, 9, 10], "access_method": [2, 9, 10], "children": [2, 9, 10], "_manag": [2, 9, 10], "pprint": [2, 9, 10], "manifest": [2, 8, 9, 10, 11, 12], "md5sum": [2, 9, 10], "commons_url": [2, 9, 10, 11, 12], "create_object_list": [2, 9, 10], "load": [2, 9, 10], "load_manifest": [2, 9, 10], "download_drs_object": [2, 9, 10], "download_files_in_drs_manifest": [2, 9, 10], "list_access_in_drs_manifest": [2, 9, 10], "list_drs_object": [2, 9, 10], "list_files_in_drs_manifest": [2, 9, 10], "current_dir": [2, 9, 11], "indexd_record_page_s": [2, 9, 11], "max_concurrent_request": [2, 9, 11, 12], "tmp_folder": [2, 9, 11], "async_download_object_manifest": [2, 9, 11], "size": [2, 3, 9, 10, 11], "md5": [2, 3, 9, 11], "acl": [2, 3, 9, 11], "prev_guid": [2, 9, 11], "threadcontrol": [2, 9, 11], "delete_all_guid": [2, 9, 11], "index_object_manifest": [2, 9, 11], "verifi": [2, 9], "async_verify_object_manifest": [2, 9, 11], "ingest": [2, 9], "column_to_use_as_guid": [2, 9, 12], "guid_type_for_indexed_file_object": [2, 9, 12], "guid_type_for_non_indexed_file_object": [2, 9, 12], "async_ingest_metadata_manifest": [2, 9, 12], "async_query_urls_from_indexd": [2, 9, 12], "gen3wsstorag": [2, 13], "copi": [2, 13], "download_url": [2, 13], "l": [2, 13], "ls_path": [2, 13], "rm": [2, 13], "rm_path": [2, 13], "upload_url": [2, 13], "modul": [2, 10, 11], "service_loc": [3, 4, 5], "public": [3, 5], "read": [3, 5, 11], "onli": [3, 5, 7, 8, 10, 11], "necessari": [3, 5], "basic": [3, 11, 12], "cred": 3, "async": [3, 4, 5, 9, 11, 12], "hash": [3, 11], "did": 3, "baseid": 3, "urls_metadata": 3, "version": [3, 4, 5], "_ssl": [3, 4, 5], "asynchron": [3, 4, 5], "dict": [3, 4, 5, 10, 11, 12], "eg": 3, "ab167e49d25b488939b1ede42752458b": 3, "associ": [3, 5], "made": 3, "where": [3, 5, 11, 12], "you": [3, 8, 11], "control": 3, "rbac": 3, "name": [3, 4, 8, 10, 11, 12, 13], "addit": [3, 5, 10, 11], "entri": [3, 11], "attach": [3, 5], "each": [3, 8, 10, 11], "group": 3, "previou": [3, 11], "checksum": [3, 10], "checksum_typ": 3, "match": [3, 5, 12], "param": [3, 5, 8, 10], "correspond": 3, "suppli": 3, "need": [3, 7, 10, 11], "handl": [3, 10], "hash_typ": 3, "whether": [3, 4, 5, 8, 11, 12], "ssl": [3, 4, 5], "pattern": [3, 12], "against": [3, 7, 8, 11, 12], "kwarg": [3, 4, 5], "bodi": 3, "dictionari": [3, 4, 5, 7, 8], "format": [3, 5, 8, 11], "blank": 3, "add": [3, 5], "sinc": 3, "content": [3, 13], "immut": 3, "want": [3, 8], "chang": [3, 11], "That": 3, "field": [3, 5, 7, 11, 12], "old": 3, "ad": 3, "least": 3, "one": [3, 5, 10, 11], "suffici": 3, "good": 3, "idea": 3, "number": [3, 7, 8, 11, 12], "noth": [3, 6], "dist_resolut": 3, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "alia": [3, 5], "distribut": 3, "identifi": [3, 5, 9, 11], "boolean": 3, "we": [3, 11], "pagin": 3, "fals": [3, 5, 6, 10, 11], "start": [3, 4, 7, 8, 10], "prefix": 3, "rtype": 3, "has_vers": 3, "latest": 3, "exclud": 3, "without": [3, 5], "repres": [3, 5, 10], "info": [3, 11], "algorithm": 3, "count": 3, "valid": [3, 7], "file_st": 3, "like": [3, 5, 9, 11, 12], "signpost": 3, "healthi": [3, 4, 5], "rev": 3, "revis": 3, "simpl": 3, "consist": 3, "mechan": 3, "dispatch": 4, "job_nam": 4, "job_input": 4, "wait": 4, "output": [4, 5, 11, 12], "Will": [4, 7, 10], "sleep": 4, "linear": 4, "delai": 4, "until": [4, 10], "done": 4, "1": [4, 5, 8, 10, 11], "global": [4, 5], "input": [4, 10, 11], "job_id": 4, "previous": 4, "complet": [4, 11], "bool": [4, 5, 8, 10, 11, 12], "md": [5, 7, 10, 12], "admin_endpoint_suffix": 5, "admin": [5, 11], "alias": 5, "overwrit": 5, "what": 5, "end": [5, 10], "up": [5, 9], "blob": [5, 7], "asyncron": 5, "altern": [5, 11], "merg": 5, "AND": 5, "descript": 5, "metadata_list": 5, "specif": [5, 8, 11, 12], "expect": [5, 9, 11], "anoth": 5, "databas": 5, "return_full_metadata": 5, "10": [5, 7, 11], "offset": [5, 7], "0": [5, 7, 8, 11], "use_agg_md": 5, "off": 5, "logic": [5, 12], "filter": [5, 7], "b": [5, 11], "2": [5, 11], "have": [5, 11], "ar": [5, 7, 8, 9, 10, 11], "alwai": 5, "nest": 5, "c": [5, 11], "3": [5, 11], "same": [5, 11, 13], "whose": 5, "ani": [5, 10, 11], "But": 5, "differ": 5, "33": 5, "d": 5, "4": [5, 11], "e": [5, 10], "won": 5, "333": 5, "defin": [5, 8, 10], "api": [5, 8, 11], "just": [5, 11, 12], "max": 5, "num": 5, "OR": 5, "level": 6, "current": [6, 8, 10], "system": [6, 7, 8, 9], "elasticsearch": 7, "query_str": 7, "execut": [7, 8, 11], "graphql": [7, 8], "query_txt": [7, 8], "syntax": 7, "see": [7, 10, 11], "com": 7, "uc": 7, "cdi": 7, "guppi": 7, "master": 7, "doc": [7, 10], "data_typ": 7, "my_index": 7, "my_field": 7, "first": [7, 8], "filter_object": 7, "sort_object": 7, "verbos": [7, 8], "row": [7, 8, 11, 12], "sort": 7, "equal": 7, "respect": 7, "complex": 7, "appli": 7, "One": 7, "unaccess": 7, "regular": 7, "tier": 7, "mode": 7, "subject": [7, 8], "50": 7, "vital_statu": 7, "submitter_id": 7, "aliv": 7, "asc": 7, "sort_field": 7, "raw": [7, 11], "project_id": [7, 8], "my_program": 7, "my_project": 7, "submit": [8, 11], "export": [8, 10], "sheepdog": 8, "through": [8, 11], "peregrin": 8, "sub": 8, "program": [8, 11], "project": [8, 11], "dcf": 8, "node_nam": 8, "batch_siz": 8, "100": [8, 11], "node": 8, "how": [8, 11], "mani": [8, 11], "print": [8, 10], "progress": [8, 10], "log": [8, 10, 11, 12], "ccle": 8, "demograph": 8, "ordered_node_list": 8, "revers": 8, "graph": 8, "experi": 8, "empti": [8, 11], "uuid1": 8, "uuid2": 8, "node_typ": 8, "fileformat": 8, "which": [8, 10], "belong": 8, "either": 8, "tsv": [8, 11, 12], "screen": 8, "sampl": [8, 10], "ccle_sample_nod": 8, "d70b41b9": 8, "6f90": 8, "4714": 8, "8420": 8, "e043ab8b77b9": 8, "ccle_one_record": 8, "entir": 8, "schema": 8, "run": [8, 11], "introspect": 8, "regist": 8, "mark": 8, "open": [8, 10, 11], "mean": 8, "etc": 8, "max_tri": 8, "retri": 8, "fail": [8, 10], "chunk_siz": 8, "30": 8, "row_offset": 8, "spreadsheet": 8, "multipl": [8, 11], "csv": [8, 11, 12], "xlsx": 8, "worksheet": 8, "skip": 8, "data_spreadsheet": 8, "arrai": 8, "action": [9, 11], "These": 9, "broken": 9, "broad": 9, "categori": 9, "task": 9, "relat": 9, "persist": 9, "Such": 9, "mai": [9, 11], "involv": 9, "assign": 9, "everi": [9, 11], "alreadi": 9, "larg": 9, "most": 9, "util": 9, "capabl": 9, "make": [9, 11], "effici": 9, "main": 10, "datafil": 10, "manifest_1": 10, "my_common": 10, "org": 10, "gen3": [10, 11, 12], "my_credenti": 10, "howto": 10, "drsdownload": 10, "drs_download": 10, "download_list": 10, "show_progress": 10, "assist": 10, "minimum": 10, "interest": 10, "object_list": 10, "host": 10, "It": 10, "possibl": 10, "save_directori": 10, "unpack_packag": 10, "delete_unpacked_packag": 10, "directori": [10, 11], "call": [10, 13], "case": 10, "error": [10, 11, 12], "continu": 10, "save": 10, "show": 10, "bar": 10, "disabl": 10, "unpack": 10, "packag": 10, "after": 10, "them": [10, 11], "drs_hostnam": 10, "known": 10, "otherwis": 10, "success": 10, "resolv": 10, "permiss": 10, "wai": 10, "determin": [10, 11, 12], "reason": 10, "pend": 10, "objectid": 10, "datamanag": 10, "occur": 10, "attribut": [10, 11], "drsobjecttyp": 10, "unknown": 10, "factori": 10, "popul": [10, 12], "server": 10, "requir": 10, "byte": 10, "timestamp": 10, "last": 10, "g": 10, "child": 10, "bundl": 10, "self": 10, "indent": 10, "pretti": 10, "In": 10, "similar": 10, "linux": 10, "tree": 10, "command": [10, 11], "typic": 10, "discoveri": 10, "cach": 10, "files": 10, "static": 10, "bownload": 10, "successfulli": 10, "pars": [10, 11, 12, 13], "output_dir": 10, "conveni": 10, "infil": 10, "wrapper": 10, "cli": 10, "process": 11, "coroutin": 11, "asyncio": [11, 12], "librari": 11, "comma": 11, "header": 11, "download_manifest": 11, "per": [11, 12], "maximum": [11, 12], "desir": 11, "concurr": [11, 12], "across": 11, "thread": 11, "folder": 11, "place": 11, "temporari": 11, "write": 11, "safe": 11, "so": 11, "To": 11, "workaround": 11, "concat": 11, "output_filenam": [11, 12], "num_process": 11, "24": [11, 12], "input_manifest": 11, "python_subprocess_command": 11, "root": [11, 12], "domain": [11, 12], "live": [11, 12], "parallel": 11, "hit": 11, "total": 11, "help": 11, "everyth": 11, "attempt": 11, "By": 11, "someth": 11, "maco": 11, "x": 11, "python3": 11, "tab": 11, "session": 11, "column": [11, 12], "shown": 11, "below": 11, "255e396f": 11, "f1f8": 11, "11e9": 11, "9a07": 11, "0a80fada099c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "363455714": 11, "pdcdatastor": 11, "test1": 11, "0a80fada098c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "343434344": 11, "test2": 11, "0a80fada097c": 11, "473d83400bc1bc9dc635e334fadd433c": 11, "543434443": 11, "phs0001": 11, "phs0002": 11, "test3": 11, "0a80fada096c": 11, "test4": 11, "0a80fada010c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "test5": 11, "index_manifest": 11, "usag": 11, "py": 11, "giangb": 11, "planx": 11, "pla": 11, "net": 11, "manifest_fil": [11, 12], "path_to_manifest": 11, "replace_url": 11, "thread_num": 11, "api_kei": 11, "processed_fil": 11, "num_total_fil": 11, "synchron": 11, "warn": 11, "IF": 11, "know": 11, "THE": 11, "implic": 11, "manifest_file_delimit": [11, 12], "submit_additional_metadata_column": 11, "force_metadata_columns_even_if_empti": 11, "loop": 11, "password": [11, 12], "flag": 11, "replac": 11, "delimit": 11, "forc": 11, "creation": 11, "even": 11, "enabl": 11, "illustr": 11, "columna": 11, "columnb": 11, "columnc": 11, "dataa": 11, "datab": 11, "guid_exampl": 11, "md5_hash": 11, "fieldnam": 11, "There": 11, "those": 11, "abov": 11, "overrid": [11, 12], "assum": 11, "behavior": 11, "simpli": 11, "manifest_row_pars": [11, 12], "custom": 11, "import": 11, "verify_manifest": 11, "def": 11, "_get_authz_from_row": 11, "strip": 11, "parser": [11, 12], "verify_object_manifest": 11, "verif": 11, "error_nam": 11, "value_from_manifest": 11, "actual": 11, "value_from_indexd": 11, "93d9af72": 11, "b0f1": 11, "450c": 11, "a5c6": 11, "7d3d8d2083b4": 11, "dev": 11, "test": 11, "_get_acl_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "1680036087": [], "7316139": [], "indexd_field": [11, 12], "func_to_parse_row": [11, 12], "delimet": [11, 12], "metdata": 12, "ingest_manifest": 12, "NOT": 12, "guid_from_fil": 12, "_get_guid_for_row": 12, "indexed_file_object_guid": 12, "_query_for_associated_indexd_record_guid": 12, "guid_for_row": 12, "elsewher": 12, "metadata_sourc": 12, "1680036088": [], "0052626": [], "get_guid_from_fil": 12, "metadata_typ": 12, "dbgap": 12, "usual": 12, "fill": 12, "_guid_typ": 12, "is_indexed_file_object": 12, "lock": 12, "semaphor": 12, "semaphon": 12, "ammount": 12, "connect": 12, "wss": 13, "src_urlstr": 13, "dest_urlstr": 13, "appropri": 13, "src_w": 13, "src_wskei": 13, "dest_path": 13, "disk": 13, "w": 13, "wskei": 13, "ws_urlstr": 13, "form": 13, "src_path": 13, "dest_w": 13, "dest_wskei": 13, "1680299160": [], "900727": [], "1680299161": [], "180279": [], "1680538055": [], "6888537": [], "1680538056": [], "1099148": [], "1680538182": [11, 12], "0011957": 11, "2834604": 12}, "objects": {"gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "get_access_token_from_wts"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "download_single"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"], [1, 1, 1, "", "upload_file_to_guid"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_guids_prefix"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_valid_guids"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_create_aliases"], [5, 1, 1, "", "async_delete_alias"], [5, 1, 1, "", "async_delete_aliases"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_get_aliases"], [5, 1, 1, "", "async_update"], [5, 1, 1, "", "async_update_aliases"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_aliases"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_alias"], [5, 1, 1, "", "delete_aliases"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_aliases"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"], [5, 1, 1, "", "update_aliases"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3": [[9, 3, 0, "-", "tools"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_drs_object"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "titleterms": {"gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "auth": 0, "helper": 0, "file": 1, "class": [1, 3, 4, 5, 6, 7, 8], "welcom": 2, "sdk": 2, "": 2, "document": 2, "indic": 2, "tabl": 2, "index": [3, 11], "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "submiss": 8, "tool": [9, 10, 11, 12], "dr": 10, "download": [10, 11], "verifi": 11, "ingest": 12, "workspac": 13, "storag": 13}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 57}, "alltitles": {"Gen3 Auth Helper": [[0, "gen3-auth-helper"]], "Gen3 File Class": [[1, "gen3-file-class"]], "Welcome to Gen3 SDK\u2019s documentation!": [[2, "welcome-to-gen3-sdk-s-documentation"]], "Indices and tables": [[2, "indices-and-tables"]], "Gen3 Index Class": [[3, "gen3-index-class"]], "Gen3 Jobs Class": [[4, "gen3-jobs-class"]], "Gen3 Metadata Class": [[5, "gen3-metadata-class"]], "Gen3 Object Class": [[6, "gen3-object-class"]], "Gen3 Query Class": [[7, "gen3-query-class"]], "Gen3 Submission Class": [[8, "gen3-submission-class"]], "Gen3 Tools": [[9, "gen3-tools"]], "DRS Download Tools": [[10, "module-gen3.tools.download.drs_download"]], "Indexing Tools": [[11, "indexing-tools"]], "Download": [[11, "module-gen3.tools.indexing.download_manifest"]], "Index": [[11, "module-gen3.tools.indexing.index_manifest"]], "Verify": [[11, "module-gen3.tools.indexing.verify_manifest"]], "Metadata Tools": [[12, "metadata-tools"]], "Ingest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "Gen3 Workspace Storage": [[13, "gen3-workspace-storage"]]}, "indexentries": {"gen3auth (class in gen3.auth)": [[0, "gen3.auth.Gen3Auth"]], "curl() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.curl"]], "get_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token"]], "get_access_token_from_wts() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token_from_wts"]], "refresh_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.refresh_access_token"]], "gen3file (class in gen3.file)": [[1, "gen3.file.Gen3File"]], "delete_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file"]], "delete_file_locations() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file_locations"]], "download_single() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.download_single"]], "get_presigned_url() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.get_presigned_url"]], "upload_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file"]], "upload_file_to_guid() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file_to_guid"]], "gen3index (class in gen3.index)": [[3, "gen3.index.Gen3Index"]], "async_create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_create_record"]], "async_get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_record"]], "async_get_records_from_checksum() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_from_checksum"]], "async_get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_on_page"]], "async_get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_with_params"]], "async_query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_query_urls"]], "async_update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_update_record"]], "create_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_blank"]], "create_new_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_new_version"]], "create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_record"]], "delete_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.delete_record"]], "get() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get"]], "get_all_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_all_records"]], "get_guids_prefix() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_guids_prefix"]], "get_latest_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_latest_version"]], "get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record"]], "get_record_doc() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record_doc"]], "get_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records"]], "get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records_on_page"]], "get_stats() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_stats"]], "get_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_urls"]], "get_valid_guids() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_valid_guids"]], "get_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_version"]], "get_versions() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_versions"]], "get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_with_params"]], "is_healthy() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.is_healthy"]], "query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.query_urls"]], "update_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_blank"]], "update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_record"]], "gen3jobs (class in gen3.jobs)": [[4, "gen3.jobs.Gen3Jobs"]], "async_run_job_and_wait() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.async_run_job_and_wait"]], "create_job() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.create_job"]], "get_output() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_output"]], "get_status() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_status"]], "get_version() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_version"]], "is_healthy() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.is_healthy"]], "list_jobs() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.list_jobs"]], "gen3metadata (class in gen3.metadata)": [[5, "gen3.metadata.Gen3Metadata"]], "async_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create"]], "async_create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create_aliases"]], "async_delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_alias"]], "async_delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_aliases"]], "async_get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get"]], "async_get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get_aliases"]], "async_update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update"]], "async_update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update_aliases"]], "auth_provider (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.auth_provider"]], "batch_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.batch_create"]], "create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create"]], "create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_aliases"]], "create_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_index_key_path"]], "delete() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete"]], "delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_alias"]], "delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_aliases"]], "delete_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_index_key_path"]], "endpoint (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.endpoint"]], "get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get"]], "get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_aliases"]], "get_index_key_paths() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_index_key_paths"]], "get_version() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_version"]], "is_healthy() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.is_healthy"]], "query() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.query"]], "update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update"]], "update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update_aliases"]], "gen3object (class in gen3.object)": [[6, "gen3.object.Gen3Object"]], "delete_object() (gen3.object.gen3object method)": [[6, "gen3.object.Gen3Object.delete_object"]], "gen3query (class in gen3.query)": [[7, "gen3.query.Gen3Query"]], "graphql_query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.graphql_query"]], "query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.query"]], "raw_data_download() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.raw_data_download"]], "gen3submission (class in gen3.submission)": [[8, "gen3.submission.Gen3Submission"]], "create_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_program"]], "create_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_project"]], "delete_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_node"]], "delete_nodes() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_nodes"]], "delete_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_program"]], "delete_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_project"]], "delete_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_record"]], "delete_records() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_records"]], "export_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_node"]], "export_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_record"]], "get_dictionary_all() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_all"]], "get_dictionary_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_node"]], "get_graphql_schema() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_graphql_schema"]], "get_programs() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_programs"]], "get_project_dictionary() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_dictionary"]], "get_project_manifest() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_manifest"]], "get_projects() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_projects"]], "open_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.open_project"]], "query() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.query"]], "submit_file() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_file"]], "submit_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_record"]], "gen3.tools": [[9, "module-gen3.tools"]], "module": [[9, "module-gen3.tools"], [10, "module-gen3.tools.download.drs_download"], [11, "module-gen3.tools.indexing.download_manifest"], [11, "module-gen3.tools.indexing.index_manifest"], [11, "module-gen3.tools.indexing.verify_manifest"], [12, "module-gen3.tools.metadata.ingest_manifest"]], "downloadmanager (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadManager"]], "downloadstatus (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadStatus"]], "downloadable (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Downloadable"]], "manifest (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Manifest"]], "_manager (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable._manager"]], "access_methods (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.access_methods"]], "cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens"]], "children (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.children"]], "commons_url (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.commons_url"]], "create_object_list() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.create_object_list"]], "created_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.created_time"]], "download() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.download"]], "download() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.download"]], "download_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_drs_object"]], "download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_files_in_drs_manifest"]], "end_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.end_time"]], "file_name (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_name"]], "file_name (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_name"]], "file_size (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_size"]], "file_size (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_size"]], "gen3.tools.download.drs_download": [[10, "module-gen3.tools.download.drs_download"]], "get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.get_fresh_token"]], "hostname (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.hostname"]], "list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_access_in_drs_manifest"]], "list_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_drs_object"]], "list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_files_in_drs_manifest"]], "load() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load"]], "load_manifest() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load_manifest"]], "md5sum (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.md5sum"]], "object_id (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_id"]], "object_id (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.object_id"]], "object_type (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_type"]], "pprint() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.pprint"]], "resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.resolve_objects"]], "start_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.start_time"]], "status (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.status"]], "updated_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.updated_time"]], "user_access() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.user_access"]], "acls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ACLS"]], "authz (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.AUTHZ"]], "current_dir (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.CURRENT_DIR"]], "guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.GUID"]], "indexd_record_page_size (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE"]], "max_concurrent_requests (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS"]], "max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS"]], "md5 (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.MD5"]], "prev_guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.PREV_GUID"]], "size (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.SIZE"]], "tmp_folder (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.TMP_FOLDER"]], "threadcontrol (class in gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ThreadControl"]], "urls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.URLS"]], "async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.async_download_object_manifest"]], "async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.async_verify_object_manifest"]], "delete_all_guids() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.delete_all_guids"]], "gen3.tools.indexing.download_manifest": [[11, "module-gen3.tools.indexing.download_manifest"]], "gen3.tools.indexing.index_manifest": [[11, "module-gen3.tools.indexing.index_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, "module-gen3.tools.indexing.verify_manifest"]], "index_object_manifest() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.index_object_manifest"]], "column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID"]], "guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT"]], "guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"]], "max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS"]], "async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest"]], "async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd"]], "gen3.tools.metadata.ingest_manifest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "gen3wsstorage (class in gen3.wss)": [[13, "gen3.wss.Gen3WsStorage"]], "copy() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.copy"]], "download() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download"]], "download_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download_url"]], "ls() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls"]], "ls_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls_path"]], "rm() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm"]], "rm_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm_path"]], "upload() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload"]], "upload_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload_url"]]}}) \ No newline at end of file +Search.setIndex({"docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "terms": {"class": [0, 2, 10, 11, 13], "gen3auth": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "endpoint": [0, 1, 2, 3, 4, 5, 7, 8, 13], "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "idp": 0, "client_credenti": 0, "client_scop": 0, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "authbas": 0, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "request": [0, 1, 3, 5, 8, 11, 12], "implement": 0, "order": [0, 8], "support": [0, 1, 5, 8, 11], "jwt": 0, "authent": 0, "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "access": [0, 1, 3, 7, 10], "token": [0, 10], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "provid": [0, 1, 3, 5, 7, 8, 12], "refresh": [0, 10], "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "string": [0, 3, 5, 11, 13], "automat": 0, "when": [0, 3, 7, 10, 12], "thei": [0, 10], "expir": [0, 1], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "opt": 0, "The": [0, 1, 2, 3, 5, 8, 10, 11], "contain": [0, 2, 5, 8, 9, 10, 11, 12], "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "web": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "work": [0, 10], "workspac": [0, 2], "default": [0, 1, 3, 7, 8, 11, 12], "env": 0, "gen3_api_kei": 0, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "set": [0, 1, 5, 10], "includ": [0, 3], "search": [0, 2, 3], "path": [0, 1, 5, 10, 11, 13], "valu": [0, 1, 3, 5, 7, 10, 11], "doe": [0, 12], "interpret": 0, "wt": [0, 10], "an": [0, 3, 6, 8, 10, 11], "accesstoken": 0, "If": [0, 1, 7, 11, 12], "can": [0, 3, 4, 8, 11, 12], "specifi": [0, 1, 3, 11, 13], "local": [0, 13], "indic": [0, 11], "environ": 0, "fenc": [0, 1], "tupl": [0, 3, 11, 12], "client_id": 0, "client_secret": 0, "oidc": 0, "client": [0, 3], "ha": [0, 11], "grant": 0, "allow": [0, 6, 8, 10, 11, 12], "obtain": [0, 10], "space": [0, 11], "separ": [0, 11], "list": [0, 1, 3, 4, 5, 7, 8, 10, 11, 13], "scope": [0, 1], "user": [0, 10, 12], "data": [0, 1, 3, 5, 7, 8, 10, 11], "openid": 0, "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "instal": [0, 2, 11], "crdc": 0, "some": [0, 2], "arbitrari": 0, "kei": [0, 3, 5, 13], "variabl": [0, 7, 8], "rather": 0, "than": [0, 5], "pass": [0, 7, 8, 10], "argument": [0, 13], "constructor": 0, "id": [0, 1, 3, 5, 10, 11], "secret": 0, "note": [0, 3, 11, 12], "should": [0, 8, 11], "never": 0, "hardcod": 0, "http": [0, 7, 11, 12], "datacommon": 0, "o": 0, "gen3_oidc_client_creds_secret": 0, "initi": [0, 10], "follow": [0, 11], "curl": [0, 2], "given": [0, 3, 4, 5, 8, 10, 12, 13], "ex": [0, 11, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "respons": [0, 1, 3, 4, 5], "under": [0, 8, 13], "index": [0, 2, 5, 9], "authz": [0, 1, 2, 3, 9, 10, 11], "map": [0, 11], "get": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12, 13], "post": [0, 11], "put": 0, "delet": [0, 1, 2, 3, 5, 6, 8, 10, 11], "i": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12], "els": [0, 12], "filenam": [0, 8, 10, 11, 12], "get_access_token": [0, 2], "auto": [0, 2], "within": [0, 2, 9], "5": [0, 5], "minut": 0, "get_access_token_from_wt": [0, 2], "try": 0, "fetch": 0, "namespac": [0, 12], "refresh_access_token": [0, 2], "new": [0, 3], "gen3fil": [1, 2], "auth_provid": [1, 2, 3, 4, 5, 6, 7, 8, 13], "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "For": [1, 5, 6, 7, 8, 9, 11], "interact": [1, 3, 4, 5, 6, 8, 13], "manag": [1, 5, 10], "featur": [1, 6], "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "presign": 1, "url": [1, 2, 3, 9, 10, 11, 12, 13], "right": 1, "now": [1, 8], "instanc": [1, 3, 6, 7, 8, 9, 10], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "delete_fil": [1, 2], "guid": [1, 2, 3, 5, 6, 9, 11, 12], "method": [1, 7, 10], "deprec": 1, "delete_file_loc": [1, 2, 6], "instead": [1, 7, 11], "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "locat": [1, 6], "store": [1, 3, 10], "remov": [1, 6, 11, 13], "its": [1, 3], "record": [1, 3, 5, 7, 8, 11, 12], "indexd": [1, 3, 6, 10, 11, 12], "uuid": [1, 3, 8], "text": [1, 7, 8], "result": [1, 8, 10, 11], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "download_singl": [1, 2], "object_id": [1, 2, 9, 10], "singl": [1, 5, 8, 10], "": [1, 4, 8, 10, 11], "uniqu": [1, 5], "get_presigned_url": [1, 2], "protocol": 1, "retriev": [1, 8, 10, 12], "give": 1, "limit": [1, 3, 5, 12], "time": [1, 8, 10, 11], "pick": 1, "avail": [1, 2, 10, 11], "queri": [1, 2, 3, 5, 8, 11, 12], "upload_fil": [1, 2], "file_nam": [1, 2, 3, 9, 10, 11], "expires_in": 1, "bucket": [1, 6], "upload": [1, 2, 3, 8, 13], "author": 1, "storag": [1, 2, 6], "s3": [1, 10, 11], "az": 1, "isn": 1, "t": [1, 5, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "amount": [1, 9], "second": [1, 4], "sign": 1, "datetim": [1, 10], "utcnow": 1, "Be": 1, "sure": 1, "posit": [1, 7], "integ": [1, 3, 8], "also": 1, "treat": [1, 5], "max_presigned_url_ttl": 1, "configur": 1, "must": [1, 5], "allowed_data_upload_bucket": 1, "data_upload_bucket": 1, "represent": [1, 3], "document": [1, 3], "upload_file_to_guid": [1, 2], "exist": [1, 3, 5, 6, 9, 12], "readm": 2, "python": [2, 9, 11], "github": [2, 7], "repo": 2, "setup": 2, "inform": [2, 3, 10], "along": 2, "quickstart": 2, "script": 2, "more": [2, 5, 7, 9, 10], "detail": [2, 7, 10], "about": [2, 3], "variou": 2, "function": [2, 3, 4, 5, 9, 10, 11, 12], "mostli": 2, "docstr": 2, "code": [2, 8], "helper": 2, "gen3index": [2, 3], "async_create_record": [2, 3], "async_get_record": [2, 3], "async_get_records_from_checksum": [2, 3], "async_get_records_on_pag": [2, 3], "async_get_with_param": [2, 3], "async_query_url": [2, 3], "async_update_record": [2, 3], "create_blank": [2, 3], "create_new_vers": [2, 3], "create_record": [2, 3], "delete_record": [2, 3, 8], "get_all_record": [2, 3], "get_guids_prefix": [2, 3], "get_latest_vers": [2, 3], "get_record": [2, 3], "get_record_doc": [2, 3], "get_records_on_pag": [2, 3], "get_stat": [2, 3], "get_url": [2, 3], "get_valid_guid": [2, 3], "get_vers": [2, 3, 4, 5], "get_with_param": [2, 3], "is_healthi": [2, 3, 4, 5], "query_url": [2, 3], "update_blank": [2, 3], "update_record": [2, 3], "job": 2, "gen3job": [2, 4, 10], "async_run_job_and_wait": [2, 4], "create_job": [2, 4], "get_output": [2, 4], "get_statu": [2, 4], "list_job": [2, 4], "metadata": [2, 3, 6, 9, 11], "gen3metadata": [2, 5], "async_cr": [2, 5], "async_create_alias": [2, 5], "async_delete_alia": [2, 5], "async_delete_alias": [2, 5], "async_get": [2, 5], "async_get_alias": [2, 5], "async_upd": [2, 5], "async_update_alias": [2, 5], "batch_creat": [2, 5], "creat": [2, 3, 4, 5, 6, 8, 10, 11], "create_alias": [2, 5], "create_index_key_path": [2, 5], "delete_alia": [2, 5], "delete_alias": [2, 5], "delete_index_key_path": [2, 5], "get_alias": [2, 5], "get_index_key_path": [2, 5], "updat": [2, 3, 5, 10, 11], "update_alias": [2, 5], "gen3object": [2, 6], "delete_object": [2, 6], "gen3queri": [2, 7], "graphql_queri": [2, 7], "raw_data_download": [2, 7], "submiss": 2, "gen3submiss": [2, 8], "create_program": [2, 8], "create_project": [2, 8], "delete_nod": [2, 8], "delete_program": [2, 8], "delete_project": [2, 8], "export_nod": [2, 8], "export_record": [2, 8], "get_dictionary_al": [2, 8], "get_dictionary_nod": [2, 8], "get_graphql_schema": [2, 8], "get_program": [2, 8], "get_project_dictionari": [2, 8], "get_project_manifest": [2, 8], "get_project": [2, 8], "open_project": [2, 8], "submit_fil": [2, 8], "submit_record": [2, 8], "tool": 2, "dr": [2, 9], "downloadmanag": [2, 9, 10], "cache_hosts_wts_token": [2, 9, 10], "get_fresh_token": [2, 9, 10], "resolve_object": [2, 9, 10], "user_access": [2, 9, 10], "downloadstatu": [2, 9, 10], "statu": [2, 4, 9, 10], "start_tim": [2, 9, 10], "end_tim": [2, 9, 10], "object_typ": [2, 9, 10], "hostnam": [2, 9, 10], "file_s": [2, 9, 10, 11], "updated_tim": [2, 9, 10], "created_tim": [2, 9, 10], "access_method": [2, 9, 10], "children": [2, 9, 10], "_manag": [2, 9, 10], "pprint": [2, 9, 10], "manifest": [2, 8, 9, 10, 11, 12], "md5sum": [2, 9, 10], "commons_url": [2, 9, 10, 11, 12], "create_object_list": [2, 9, 10], "load": [2, 9, 10], "load_manifest": [2, 9, 10], "download_drs_object": [2, 9, 10], "download_files_in_drs_manifest": [2, 9, 10], "list_access_in_drs_manifest": [2, 9, 10], "list_drs_object": [2, 9, 10], "list_files_in_drs_manifest": [2, 9, 10], "current_dir": [2, 9, 11], "indexd_record_page_s": [2, 9, 11], "max_concurrent_request": [2, 9, 11, 12], "tmp_folder": [2, 9, 11], "async_download_object_manifest": [2, 9, 11], "size": [2, 3, 9, 10, 11], "md5": [2, 3, 9, 11], "acl": [2, 3, 9, 11], "prev_guid": [2, 9, 11], "threadcontrol": [2, 9, 11], "delete_all_guid": [2, 9, 11], "index_object_manifest": [2, 9, 11], "verifi": [2, 9], "async_verify_object_manifest": [2, 9, 11], "ingest": [2, 9], "column_to_use_as_guid": [2, 9, 12], "guid_type_for_indexed_file_object": [2, 9, 12], "guid_type_for_non_indexed_file_object": [2, 9, 12], "async_ingest_metadata_manifest": [2, 9, 12], "async_query_urls_from_indexd": [2, 9, 12], "gen3wsstorag": [2, 13], "copi": [2, 13], "download_url": [2, 13], "l": [2, 13], "ls_path": [2, 13], "rm": [2, 13], "rm_path": [2, 13], "upload_url": [2, 13], "modul": [2, 10, 11], "service_loc": [3, 4, 5], "public": [3, 5], "read": [3, 5, 11], "onli": [3, 5, 7, 8, 10, 11], "necessari": [3, 5], "basic": [3, 11, 12], "cred": 3, "async": [3, 4, 5, 9, 11, 12], "hash": [3, 11], "did": 3, "baseid": 3, "urls_metadata": 3, "version": [3, 4, 5], "_ssl": [3, 4, 5], "asynchron": [3, 4, 5], "dict": [3, 4, 5, 10, 11, 12], "eg": 3, "ab167e49d25b488939b1ede42752458b": 3, "associ": [3, 5], "made": 3, "where": [3, 5, 11, 12], "you": [3, 8, 11], "control": 3, "rbac": 3, "name": [3, 4, 8, 10, 11, 12, 13], "addit": [3, 5, 10, 11], "entri": [3, 11], "attach": [3, 5], "each": [3, 8, 10, 11], "group": 3, "previou": [3, 11], "checksum": [3, 10], "checksum_typ": 3, "match": [3, 5, 12], "param": [3, 5, 8, 10], "correspond": 3, "suppli": 3, "need": [3, 7, 10, 11], "handl": [3, 10], "hash_typ": 3, "whether": [3, 4, 5, 8, 11, 12], "ssl": [3, 4, 5], "pattern": [3, 12], "against": [3, 7, 8, 11, 12], "kwarg": [3, 4, 5], "bodi": 3, "dictionari": [3, 4, 5, 7, 8], "format": [3, 5, 8, 11], "blank": 3, "add": [3, 5], "sinc": 3, "content": [3, 13], "immut": 3, "want": [3, 8], "chang": [3, 11], "That": 3, "field": [3, 5, 7, 11, 12], "old": 3, "ad": 3, "least": 3, "one": [3, 5, 10, 11], "suffici": 3, "good": 3, "idea": 3, "number": [3, 7, 8, 11, 12], "noth": [3, 6], "dist_resolut": 3, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "alia": [3, 5], "distribut": 3, "identifi": [3, 5, 9, 11], "boolean": 3, "we": [3, 11], "pagin": 3, "fals": [3, 5, 6, 10, 11], "start": [3, 4, 7, 8, 10], "prefix": 3, "rtype": 3, "has_vers": 3, "latest": 3, "exclud": 3, "without": [3, 5], "repres": [3, 5, 10], "info": [3, 11], "algorithm": 3, "count": 3, "valid": [3, 7], "file_st": 3, "like": [3, 5, 9, 11, 12], "signpost": 3, "healthi": [3, 4, 5], "rev": 3, "revis": 3, "simpl": 3, "consist": 3, "mechan": 3, "dispatch": 4, "job_nam": 4, "job_input": 4, "wait": 4, "output": [4, 5, 11, 12], "Will": [4, 7, 10], "sleep": 4, "linear": 4, "delai": 4, "until": [4, 10], "done": 4, "1": [4, 5, 8, 10, 11], "global": [4, 5], "input": [4, 10, 11], "job_id": 4, "previous": 4, "complet": [4, 11], "bool": [4, 5, 8, 10, 11, 12], "md": [5, 7, 10, 12], "admin_endpoint_suffix": 5, "admin": [5, 11], "alias": 5, "overwrit": 5, "what": 5, "end": [5, 10], "up": [5, 9], "blob": [5, 7], "asyncron": 5, "altern": [5, 11], "merg": 5, "AND": 5, "descript": 5, "metadata_list": 5, "specif": [5, 8, 11, 12], "expect": [5, 9, 11], "anoth": 5, "databas": 5, "return_full_metadata": 5, "10": [5, 7, 11], "offset": [5, 7], "0": [5, 7, 8, 11], "use_agg_md": 5, "off": 5, "logic": [5, 12], "filter": [5, 7], "b": [5, 11], "2": [5, 11], "have": [5, 11], "ar": [5, 7, 8, 9, 10, 11], "alwai": 5, "nest": 5, "c": [5, 11], "3": [5, 11], "same": [5, 11, 13], "whose": 5, "ani": [5, 10, 11], "But": 5, "differ": 5, "33": 5, "d": 5, "4": [5, 11], "e": [5, 10], "won": 5, "333": 5, "defin": [5, 8, 10], "api": [5, 8, 11], "just": [5, 11, 12], "max": 5, "num": 5, "OR": 5, "level": 6, "current": [6, 8, 10], "system": [6, 7, 8, 9], "elasticsearch": 7, "query_str": 7, "execut": [7, 8, 11], "graphql": [7, 8], "query_txt": [7, 8], "syntax": 7, "see": [7, 10, 11], "com": 7, "uc": 7, "cdi": 7, "guppi": 7, "master": 7, "doc": [7, 10], "data_typ": 7, "my_index": 7, "my_field": 7, "first": [7, 8], "filter_object": 7, "sort_object": 7, "verbos": [7, 8], "row": [7, 8, 11, 12], "sort": 7, "equal": 7, "respect": 7, "complex": 7, "appli": 7, "One": 7, "unaccess": 7, "regular": 7, "tier": 7, "mode": 7, "subject": [7, 8], "50": 7, "vital_statu": 7, "submitter_id": 7, "aliv": 7, "asc": 7, "sort_field": 7, "raw": [7, 11], "project_id": [7, 8], "my_program": 7, "my_project": 7, "submit": [8, 11], "export": [8, 10], "sheepdog": 8, "through": [8, 11], "peregrin": 8, "sub": 8, "program": [8, 11], "project": [8, 11], "dcf": 8, "node_nam": 8, "batch_siz": 8, "100": [8, 11], "node": 8, "how": [8, 11], "mani": [8, 11], "print": [8, 10], "progress": [8, 10], "log": [8, 10, 11, 12], "ccle": 8, "demograph": 8, "ordered_node_list": 8, "revers": 8, "graph": 8, "experi": 8, "empti": [8, 11], "uuid1": 8, "uuid2": 8, "node_typ": 8, "fileformat": 8, "which": [8, 10], "belong": 8, "either": 8, "tsv": [8, 11, 12], "screen": 8, "sampl": [8, 10], "ccle_sample_nod": 8, "d70b41b9": 8, "6f90": 8, "4714": 8, "8420": 8, "e043ab8b77b9": 8, "ccle_one_record": 8, "entir": 8, "schema": 8, "run": [8, 11], "introspect": 8, "regist": 8, "mark": 8, "open": [8, 10, 11], "mean": 8, "etc": 8, "max_tri": 8, "retri": 8, "fail": [8, 10], "chunk_siz": 8, "30": 8, "row_offset": 8, "spreadsheet": 8, "multipl": [8, 11], "csv": [8, 11, 12], "xlsx": 8, "worksheet": 8, "skip": 8, "data_spreadsheet": 8, "arrai": 8, "action": [9, 11], "These": 9, "broken": 9, "broad": 9, "categori": 9, "task": 9, "relat": 9, "persist": 9, "Such": 9, "mai": [9, 11], "involv": 9, "assign": 9, "everi": [9, 11], "alreadi": 9, "larg": 9, "most": 9, "util": 9, "capabl": 9, "make": [9, 11], "effici": 9, "main": 10, "datafil": 10, "manifest_1": 10, "my_common": 10, "org": 10, "gen3": [10, 11, 12], "my_credenti": 10, "howto": 10, "drsdownload": 10, "drs_download": 10, "download_list": 10, "show_progress": 10, "assist": 10, "minimum": 10, "interest": 10, "object_list": 10, "host": 10, "It": 10, "possibl": 10, "save_directori": 10, "unpack_packag": 10, "delete_unpacked_packag": 10, "directori": [10, 11], "call": [10, 13], "case": 10, "error": [10, 11, 12], "continu": 10, "save": 10, "show": 10, "bar": 10, "disabl": 10, "unpack": 10, "packag": 10, "after": 10, "them": [10, 11], "drs_hostnam": 10, "known": 10, "otherwis": 10, "success": 10, "resolv": 10, "permiss": 10, "wai": 10, "determin": [10, 11, 12], "reason": 10, "pend": 10, "objectid": 10, "datamanag": 10, "occur": 10, "attribut": [10, 11], "drsobjecttyp": 10, "unknown": 10, "factori": 10, "popul": [10, 12], "server": 10, "requir": 10, "byte": 10, "timestamp": 10, "last": 10, "g": 10, "child": 10, "bundl": 10, "self": 10, "indent": 10, "pretti": 10, "In": 10, "similar": 10, "linux": 10, "tree": 10, "command": [10, 11], "typic": 10, "discoveri": 10, "cach": 10, "files": 10, "static": 10, "bownload": 10, "successfulli": 10, "pars": [10, 11, 12, 13], "output_dir": 10, "conveni": 10, "infil": 10, "wrapper": 10, "cli": 10, "process": 11, "coroutin": 11, "asyncio": [11, 12], "librari": 11, "comma": 11, "header": 11, "download_manifest": 11, "per": [11, 12], "maximum": [11, 12], "desir": 11, "concurr": [11, 12], "across": 11, "thread": 11, "folder": 11, "place": 11, "temporari": 11, "write": 11, "safe": 11, "so": 11, "To": 11, "workaround": 11, "concat": 11, "output_filenam": [11, 12], "num_process": 11, "24": [11, 12], "input_manifest": 11, "python_subprocess_command": 11, "root": [11, 12], "domain": [11, 12], "live": [11, 12], "parallel": 11, "hit": 11, "total": 11, "help": 11, "everyth": 11, "attempt": 11, "By": 11, "someth": 11, "maco": 11, "x": 11, "python3": 11, "tab": 11, "session": 11, "column": [11, 12], "shown": 11, "below": 11, "255e396f": 11, "f1f8": 11, "11e9": 11, "9a07": 11, "0a80fada099c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "363455714": 11, "pdcdatastor": 11, "test1": 11, "0a80fada098c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "343434344": 11, "test2": 11, "0a80fada097c": 11, "473d83400bc1bc9dc635e334fadd433c": 11, "543434443": 11, "phs0001": 11, "phs0002": 11, "test3": 11, "0a80fada096c": 11, "test4": 11, "0a80fada010c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "test5": 11, "index_manifest": 11, "usag": 11, "py": 11, "giangb": 11, "planx": 11, "pla": 11, "net": 11, "manifest_fil": [11, 12], "path_to_manifest": 11, "replace_url": 11, "thread_num": 11, "api_kei": 11, "processed_fil": 11, "num_total_fil": 11, "synchron": 11, "warn": 11, "IF": 11, "know": 11, "THE": 11, "implic": 11, "manifest_file_delimit": [11, 12], "submit_additional_metadata_column": 11, "force_metadata_columns_even_if_empti": 11, "loop": 11, "password": [11, 12], "flag": 11, "replac": 11, "delimit": 11, "forc": 11, "creation": 11, "even": 11, "enabl": 11, "illustr": 11, "columna": 11, "columnb": 11, "columnc": 11, "dataa": 11, "datab": 11, "guid_exampl": 11, "md5_hash": 11, "fieldnam": 11, "There": 11, "those": 11, "abov": 11, "overrid": [11, 12], "assum": 11, "behavior": 11, "simpli": 11, "manifest_row_pars": [11, 12], "custom": 11, "import": 11, "verify_manifest": 11, "def": 11, "_get_authz_from_row": 11, "strip": 11, "parser": [11, 12], "verify_object_manifest": 11, "verif": 11, "error_nam": 11, "value_from_manifest": 11, "actual": 11, "value_from_indexd": 11, "93d9af72": 11, "b0f1": 11, "450c": 11, "a5c6": 11, "7d3d8d2083b4": 11, "dev": 11, "test": 11, "_get_acl_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "1680036087": [], "7316139": [], "indexd_field": [11, 12], "func_to_parse_row": [11, 12], "delimet": [11, 12], "metdata": 12, "ingest_manifest": 12, "NOT": 12, "guid_from_fil": 12, "_get_guid_for_row": 12, "indexed_file_object_guid": 12, "_query_for_associated_indexd_record_guid": 12, "guid_for_row": 12, "elsewher": 12, "metadata_sourc": 12, "1680036088": [], "0052626": [], "get_guid_from_fil": 12, "metadata_typ": 12, "dbgap": 12, "usual": 12, "fill": 12, "_guid_typ": 12, "is_indexed_file_object": 12, "lock": 12, "semaphor": 12, "semaphon": 12, "ammount": 12, "connect": 12, "wss": 13, "src_urlstr": 13, "dest_urlstr": 13, "appropri": 13, "src_w": 13, "src_wskei": 13, "dest_path": 13, "disk": 13, "w": 13, "wskei": 13, "ws_urlstr": 13, "form": 13, "src_path": 13, "dest_w": 13, "dest_wskei": 13, "1680299160": [], "900727": [], "1680299161": [], "180279": [], "1680538055": [], "6888537": [], "1680538056": [], "1099148": [], "1680538182": [], "0011957": [], "2834604": [], "1682356864": [], "3120942": [], "5995133": [], "1682358633": [], "710197": [], "9979434": [], "1682523668": [], "0313597": [], "3855686": [], "1682526635": 11, "7836046": 11, "1682526636": 12, "077181": 12}, "objects": {"gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "get_access_token_from_wts"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "download_single"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"], [1, 1, 1, "", "upload_file_to_guid"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_guids_prefix"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_valid_guids"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_create_aliases"], [5, 1, 1, "", "async_delete_alias"], [5, 1, 1, "", "async_delete_aliases"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_get_aliases"], [5, 1, 1, "", "async_update"], [5, 1, 1, "", "async_update_aliases"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_aliases"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_alias"], [5, 1, 1, "", "delete_aliases"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_aliases"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"], [5, 1, 1, "", "update_aliases"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3": [[9, 3, 0, "-", "tools"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_drs_object"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "titleterms": {"gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "auth": 0, "helper": 0, "file": 1, "class": [1, 3, 4, 5, 6, 7, 8], "welcom": 2, "sdk": 2, "": 2, "document": 2, "indic": 2, "tabl": 2, "index": [3, 11], "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "submiss": 8, "tool": [9, 10, 11, 12], "dr": 10, "download": [10, 11], "verifi": 11, "ingest": 12, "workspac": 13, "storag": 13}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 57}, "alltitles": {"Gen3 Auth Helper": [[0, "gen3-auth-helper"]], "Gen3 File Class": [[1, "gen3-file-class"]], "Welcome to Gen3 SDK\u2019s documentation!": [[2, "welcome-to-gen3-sdk-s-documentation"]], "Indices and tables": [[2, "indices-and-tables"]], "Gen3 Index Class": [[3, "gen3-index-class"]], "Gen3 Jobs Class": [[4, "gen3-jobs-class"]], "Gen3 Metadata Class": [[5, "gen3-metadata-class"]], "Gen3 Object Class": [[6, "gen3-object-class"]], "Gen3 Query Class": [[7, "gen3-query-class"]], "Gen3 Submission Class": [[8, "gen3-submission-class"]], "Gen3 Tools": [[9, "gen3-tools"]], "DRS Download Tools": [[10, "module-gen3.tools.download.drs_download"]], "Indexing Tools": [[11, "indexing-tools"]], "Download": [[11, "module-gen3.tools.indexing.download_manifest"]], "Index": [[11, "module-gen3.tools.indexing.index_manifest"]], "Verify": [[11, "module-gen3.tools.indexing.verify_manifest"]], "Metadata Tools": [[12, "metadata-tools"]], "Ingest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "Gen3 Workspace Storage": [[13, "gen3-workspace-storage"]]}, "indexentries": {"gen3auth (class in gen3.auth)": [[0, "gen3.auth.Gen3Auth"]], "curl() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.curl"]], "get_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token"]], "get_access_token_from_wts() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token_from_wts"]], "refresh_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.refresh_access_token"]], "gen3file (class in gen3.file)": [[1, "gen3.file.Gen3File"]], "delete_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file"]], "delete_file_locations() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file_locations"]], "download_single() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.download_single"]], "get_presigned_url() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.get_presigned_url"]], "upload_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file"]], "upload_file_to_guid() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file_to_guid"]], "gen3index (class in gen3.index)": [[3, "gen3.index.Gen3Index"]], "async_create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_create_record"]], "async_get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_record"]], "async_get_records_from_checksum() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_from_checksum"]], "async_get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_on_page"]], "async_get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_with_params"]], "async_query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_query_urls"]], "async_update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_update_record"]], "create_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_blank"]], "create_new_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_new_version"]], "create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_record"]], "delete_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.delete_record"]], "get() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get"]], "get_all_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_all_records"]], "get_guids_prefix() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_guids_prefix"]], "get_latest_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_latest_version"]], "get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record"]], "get_record_doc() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record_doc"]], "get_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records"]], "get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records_on_page"]], "get_stats() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_stats"]], "get_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_urls"]], "get_valid_guids() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_valid_guids"]], "get_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_version"]], "get_versions() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_versions"]], "get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_with_params"]], "is_healthy() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.is_healthy"]], "query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.query_urls"]], "update_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_blank"]], "update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_record"]], "gen3jobs (class in gen3.jobs)": [[4, "gen3.jobs.Gen3Jobs"]], "async_run_job_and_wait() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.async_run_job_and_wait"]], "create_job() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.create_job"]], "get_output() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_output"]], "get_status() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_status"]], "get_version() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_version"]], "is_healthy() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.is_healthy"]], "list_jobs() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.list_jobs"]], "gen3metadata (class in gen3.metadata)": [[5, "gen3.metadata.Gen3Metadata"]], "async_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create"]], "async_create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create_aliases"]], "async_delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_alias"]], "async_delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_aliases"]], "async_get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get"]], "async_get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get_aliases"]], "async_update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update"]], "async_update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update_aliases"]], "auth_provider (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.auth_provider"]], "batch_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.batch_create"]], "create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create"]], "create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_aliases"]], "create_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_index_key_path"]], "delete() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete"]], "delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_alias"]], "delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_aliases"]], "delete_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_index_key_path"]], "endpoint (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.endpoint"]], "get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get"]], "get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_aliases"]], "get_index_key_paths() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_index_key_paths"]], "get_version() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_version"]], "is_healthy() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.is_healthy"]], "query() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.query"]], "update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update"]], "update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update_aliases"]], "gen3object (class in gen3.object)": [[6, "gen3.object.Gen3Object"]], "delete_object() (gen3.object.gen3object method)": [[6, "gen3.object.Gen3Object.delete_object"]], "gen3query (class in gen3.query)": [[7, "gen3.query.Gen3Query"]], "graphql_query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.graphql_query"]], "query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.query"]], "raw_data_download() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.raw_data_download"]], "gen3submission (class in gen3.submission)": [[8, "gen3.submission.Gen3Submission"]], "create_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_program"]], "create_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_project"]], "delete_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_node"]], "delete_nodes() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_nodes"]], "delete_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_program"]], "delete_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_project"]], "delete_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_record"]], "delete_records() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_records"]], "export_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_node"]], "export_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_record"]], "get_dictionary_all() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_all"]], "get_dictionary_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_node"]], "get_graphql_schema() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_graphql_schema"]], "get_programs() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_programs"]], "get_project_dictionary() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_dictionary"]], "get_project_manifest() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_manifest"]], "get_projects() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_projects"]], "open_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.open_project"]], "query() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.query"]], "submit_file() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_file"]], "submit_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_record"]], "gen3.tools": [[9, "module-gen3.tools"]], "module": [[9, "module-gen3.tools"], [10, "module-gen3.tools.download.drs_download"], [11, "module-gen3.tools.indexing.download_manifest"], [11, "module-gen3.tools.indexing.index_manifest"], [11, "module-gen3.tools.indexing.verify_manifest"], [12, "module-gen3.tools.metadata.ingest_manifest"]], "downloadmanager (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadManager"]], "downloadstatus (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadStatus"]], "downloadable (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Downloadable"]], "manifest (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Manifest"]], "_manager (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable._manager"]], "access_methods (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.access_methods"]], "cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens"]], "children (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.children"]], "commons_url (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.commons_url"]], "create_object_list() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.create_object_list"]], "created_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.created_time"]], "download() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.download"]], "download() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.download"]], "download_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_drs_object"]], "download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_files_in_drs_manifest"]], "end_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.end_time"]], "file_name (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_name"]], "file_name (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_name"]], "file_size (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_size"]], "file_size (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_size"]], "gen3.tools.download.drs_download": [[10, "module-gen3.tools.download.drs_download"]], "get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.get_fresh_token"]], "hostname (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.hostname"]], "list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_access_in_drs_manifest"]], "list_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_drs_object"]], "list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_files_in_drs_manifest"]], "load() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load"]], "load_manifest() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load_manifest"]], "md5sum (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.md5sum"]], "object_id (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_id"]], "object_id (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.object_id"]], "object_type (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_type"]], "pprint() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.pprint"]], "resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.resolve_objects"]], "start_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.start_time"]], "status (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.status"]], "updated_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.updated_time"]], "user_access() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.user_access"]], "acls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ACLS"]], "authz (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.AUTHZ"]], "current_dir (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.CURRENT_DIR"]], "guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.GUID"]], "indexd_record_page_size (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE"]], "max_concurrent_requests (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS"]], "max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS"]], "md5 (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.MD5"]], "prev_guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.PREV_GUID"]], "size (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.SIZE"]], "tmp_folder (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.TMP_FOLDER"]], "threadcontrol (class in gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ThreadControl"]], "urls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.URLS"]], "async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.async_download_object_manifest"]], "async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.async_verify_object_manifest"]], "delete_all_guids() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.delete_all_guids"]], "gen3.tools.indexing.download_manifest": [[11, "module-gen3.tools.indexing.download_manifest"]], "gen3.tools.indexing.index_manifest": [[11, "module-gen3.tools.indexing.index_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, "module-gen3.tools.indexing.verify_manifest"]], "index_object_manifest() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.index_object_manifest"]], "column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID"]], "guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT"]], "guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"]], "max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS"]], "async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest"]], "async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd"]], "gen3.tools.metadata.ingest_manifest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "gen3wsstorage (class in gen3.wss)": [[13, "gen3.wss.Gen3WsStorage"]], "copy() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.copy"]], "download() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download"]], "download_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download_url"]], "ls() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls"]], "ls_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls_path"]], "rm() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm"]], "rm_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm_path"]], "upload() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload"]], "upload_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload_url"]]}}) \ No newline at end of file diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html index f94b99b95..3738802d4 100644 --- a/docs/_build/html/tools/indexing.html +++ b/docs/_build/html/tools/indexing.html @@ -381,7 +381,7 @@

Indexing Tools
-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1680538182.0011957.log')[source]¶
+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1682526635.7836046.log')[source]¶

Verify all file object records into a manifest csv

Parameters:
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html index 076c9b35f..952cd42ad 100644 --- a/docs/_build/html/tools/metadata.html +++ b/docs/_build/html/tools/metadata.html @@ -102,7 +102,7 @@

Metadata Tools
-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1680538182.2834604.log', get_guid_from_file=True, metadata_type=None)[source]¶
+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1682526636.077181.log', get_guid_from_file=True, metadata_type=None)[source]¶

Ingest all metadata records into a manifest csv

Parameters:
diff --git a/docs/howto/discoveryMetadataTools.md b/docs/howto/discoveryMetadataTools.md index c89687d34..fa2dfecdb 100644 --- a/docs/howto/discoveryMetadataTools.md +++ b/docs/howto/discoveryMetadataTools.md @@ -1,8 +1,10 @@ ## Gen3 Discovery Page Metadata Tools -TOC +**Table of Contents** + - [Overview](#overview) -- [Combine Metadata with Current Discovery Metadata](#combine-metadata-with-current-discovery-metadata) +- [DOIs in Gen3](#dois-in-gen3-discovery-metadata-and-page-for-visualizing-public-doi-metadata) +- [dbGaP FHIR Metadata in Gen3 Discovery](#combine-dbgap-fhir-metadata-with-current-discovery-metadata) ### Overview @@ -18,7 +20,330 @@ So you can choose to use the CLI or write your own Python script and use the SDK functions yourself. Generally this provides the most flexibility, at less of a convenience. -### Combine Metadata with Current Discovery Metadata +### DOIs in Gen3: Discovery Metadata and Page for Visualizing Public DOI Metadata + +Gen3's SDK supports minting DOIs from DataCite, storing DOI metadata in a Gen3 instance, +and visualizing the DOI metadata in our Discovery Page to serve as a DOI "Landing Page". + +> **DOI?** A digital object identifier (DOI) is a persistent identifier or handle used to identify objects uniquely, standardized by the International Organization for Standardization (ISO). DOIs are in wide use mainly to identify academic, professional, and government information, such as journal articles, research reports, data sets, and official publications. However, they also have been used to identify other types of information resources, such as commercial videos. + +The general overview for how Gen3 supports DOIs is as follows: + +* Gen3 SDK/CLI used to gather Metadata from External Public Metadata Sources +* Gen3 SDK/CLI used to do any conversions to DOI Metadata +* Gen3 SDK/CLI communicates with DataCite API to mint DOI + * NOTE: the gathering of metadata, conversion to DOI fields, and final minting may or may not be a part of a regular data ingestion. It’s possible that this is used ad-hocly, as needed +* Gen3 SDK/CLI persists metadata in Gen3 +* Persisted metadata in Gen3 exposed via Discovery Page +* Discovery Page is used as the required DOI Landing Page + +> **What is DataCite?** In order to create a DOI, one must use a DOI registration service. In the US there are two: CrossRef and DataCite. We are focusing on DataCite, because that is what we were provided access to. + +#### Example Script: Manually Creating a Single DOI and Persisting the Metadata in Gen3 + +Prerequisites: + +* Environment variable `DATACITE_USERNAME` set as a valid DataCite username for interacting with their API +* Environment variable `DATACITE_PASSWORD` set as a valid DataCite password for interacting with their API + +This shows a full example of: + +* Setting up the necessary classes for interacting with Gen3 & Datacite +* Getting the DOI metadata (ideally from some external source like a file or another API, but here we've hard-coded it) +* Creating/Minting the DOI in DataCite +* Persisting the DOI metadata into a Gen3 Discovery record in the metadata service + +```python +import os +from requests.auth import HTTPBasicAuth + +from cdislogging import get_logger + +from gen3.doi import ( + DataCite, + DigitalObjectIdentifier, + DigitalObjectIdentifierCreator, + DigitalObjectIdentifierTitle, +) +from gen3.auth import Gen3Auth + +logging = get_logger("__name__", log_level="info") + +# This prefix should be provided by DataCite +PREFIX = "10.12345" +PUBLISHER = "Example" +COMMONS_DISCOVERY_PAGE = "https://example.com/discovery" + +DOI_DISCLAIMER = "" +DOI_ACCESS_INFORMATION = "You can find information about how to access this resource in the link below." +DOI_ACCESS_INFORMATION_LINK = "https://example.com/more/info" +DOI_CONTACT = "https://example.com/contact/" + + +def test_manual_single_doi(publish_dois=False): + # Setup + gen3_auth = Gen3Auth() + datacite = DataCite( + api=DataCite.TEST_URL, + auth_provider=HTTPBasicAuth( + os.environ.get("DATACITE_USERNAME"), + os.environ.get("DATACITE_PASSWORD"), + ), + ) + + gen3_metadata_guid = "Example-Study-01" + + # Get DOI metadata (ideally from some external source) + identifier = "10.82483/BDC-268Z-O151" + creators = [ + DigitalObjectIdentifierCreator( + name="Bar, Foo", + name_type=DigitalObjectIdentifierCreator.NAME_TYPE_PERSON, + ).as_dict() + ] + titles = [DigitalObjectIdentifierTitle("Some Example Study in Gen3").as_dict()] + publisher = "Example Gen3 Sponsor" + publication_year = 2023 + doi_type = "Dataset" + version = 1 + + doi_metadata = { + "identifier": identifier, + "creators": creators, + "titles": titles, + "publisher": publisher, + "publication_year": publication_year, + "doi_type": doi_type, + "version": version, + } + + # Create/Mint the DOI in DataCite + doi = DigitalObjectIdentifier(root_url=COMMONS_DISCOVERY_PAGE, **doi_metadata) + + if publish_dois: + logging.info(f"Publishing DOI `{identifier}`...") + doi.event = "publish" + + # works for only new DOIs + # You can use this for updates: `datacite.update_doi(doi)` + response = datacite.create_doi(doi) + doi = DigitalObjectIdentifier.from_datacite_create_doi_response(response) + + # Persist necessary DOI Metadata in Gen3 Discovery to support the landing page + metadata = datacite.persist_doi_metadata_in_gen3( + guid=gen3_metadata_guid, + doi=doi, + auth=gen3_auth, + additional_metadata={ + "disclaimer": DOI_DISCLAIMER, + "access_information": DOI_ACCESS_INFORMATION, + "access_information_link": DOI_ACCESS_INFORMATION_LINK, + "contact": DOI_CONTACT, + }, + prefix="doi_", + ) + logging.debug(f"Gen3 Metadata for GUID `{gen3_metadata_guid}`: {metadata}") + + +def main(): + test_manual_single_doi() + + +if __name__ == "__main__": + main() + +``` + +#### Example Partial Discovery Page Configuration for DOIs + +This is portion of the Gen3 Data Portal configuration that pertains to +the Discovery Page. The code provided shows an example of how to configure +the visualization of the DOI metadata. + +In order to be compliant with Landing Pages, the URL you provide during minting +needs to automatically display all this information. So if you have other tabs +of non-DOI information, they cannot be the first focused tab upon resolving the +DOI url. + +```json +"discoveryConfig": { + // ... + "detailView": { + // ... + "tabs": [ + { + "tabName": "DOI", + "groups": [ + { + "header": "Dataset Information", + "fields": [ + { + "type": "block", + "label": "", + "sourceField": "disclaimer", + "default": "" + }, + { + "type": "text", + "label": "Title:", + "sourceField": "doi_titles", + "default": "Not specified" + }, + { + "type": "link", + "label": "DOI:", + "sourceField": "doi_resolveable_link", + "default": "None" + }, + { + "type": "text", + "label": "Data available:", + "sourceField": "doi_is_available", + "default": "None" + }, + { + "type": "text", + "label": "Citation:", + "sourceField": "doi_citation", + "default": "Not specified" + }, + { + "type": "link", + "label": "Contact:", + "sourceField": "doi_contact", + "default": "Not specified" + } + ] + }, + { + "header": "How to Access the Data", + "fields": [ + { + "type": "block", + "label": "How to access the data:", + "sourceField": "doi_access_information", + "default": "Not specified" + }, + { + "type": "link", + "label": "Data and access information:", + "sourceField": "doi_access_information_link", + "default": "Not specified" + } + ] + }, + { + "header": "Additional Information", + "fields": [ + { + "type": "text", + "label": "Publisher:", + "sourceField": "doi_publisher", + "default": "Not specified" + }, + { + "type": "text", + "label": "Funded by:", + "sourceField": "doi_fundingReferences", + "default": "Not specified" + }, + { + "type": "text", + "label": "Publication Year:", + "sourceField": "doi_publication_year", + "default": "Not specified" + }, + { + "type": "text", + "label": "Resource Type:", + "sourceField": "doi_resource_type", + "default": "Not specified" + }, + { + "type": "text", + "label": "Version:", + "sourceField": "doi_version_information", + "default": "Not specified" + } + ] + }, + { + "header": "Description", + "fields": [ + { + "type": "block", + "label": "Description:", + "sourceField": "doi_descriptions", + "default": "Not specified" + } + ] + } + ] + }, + // ... +``` + +#### Work in Progress. Script to automate dbGaP scraping for updating datasets and minting DOIs + +- TODO: Push DOI from submitted to registered + +See below for a full example of DOI metadata gathering, minting, and persisting +into Gen3. + +```python +import os +from requests.auth import HTTPBasicAuth + +from cdislogging import get_logger + +from gen3.auth import Gen3Auth +from gen3.discovery_dois import mint_dois_for_dbgap_discovery_datasets +from gen3.utils import get_random_alphanumeric + +logging = get_logger("__name__", log_level="info") + +PREFIX = "10.12345" +PUBLISHER = "Example" +COMMONS_DISCOVERY_PAGE = "https://example.com/discovery" + +DOI_DISCLAIMER = "" +DOI_ACCESS_INFORMATION = "You can find information about how to access this resource in the link below." +DOI_ACCESS_INFORMATION_LINK = "https://example.com/more/info" +DOI_CONTACT = "https://example.com/contact/" + + +def get_doi_identifier(): + return ( + PREFIX + "/EXAMPLE-" + get_random_alphanumeric(4) + "-" + get_random_alphanumeric(4) + ) + + +def main(): + auth = Gen3Auth() + dbgap_phsid_field = "dbgap_accession" + + mint_dois_for_dbgap_discovery_datasets( + gen3_auth=auth, + datacite_auth=HTTPBasicAuth( + os.environ.get("DATACITE_USERNAME"), + os.environ.get("DATACITE_PASSWORD"), + ), + dbgap_phsid_field=dbgap_phsid_field, + get_doi_identifier_function=get_doi_identifier, + publisher=PUBLISHER, + commons_discovery_page=COMMONS_DISCOVERY_PAGE, + doi_disclaimer=DOI_DISCLAIMER, + doi_access_information=DOI_ACCESS_INFORMATION, + doi_access_information_link=DOI_ACCESS_INFORMATION_LINK, + doi_contact=DOI_CONTACT, + ) + +if __name__ == "__main__": + main() + +``` + +### Combine dbGaP FHIR Metadata with Current Discovery Metadata For CLI, see `gen3 discovery combine --help`. @@ -42,7 +367,7 @@ from gen3.tools.metadata.discovery import ( output_expanded_discovery_metadata, combine_discovery_metadata, ) -from gen3.nih import dbgapFHIR +from gen3.external.nih.dbgap_fhir import dbgapFHIR from gen3.utils import get_or_create_event_loop_for_thread diff --git a/gen3/cli/discovery.py b/gen3/cli/discovery.py index 748e31801..ea9a7f4fb 100644 --- a/gen3/cli/discovery.py +++ b/gen3/cli/discovery.py @@ -33,8 +33,18 @@ def discovery(): help="omit fields from empty columns if set", show_default=True, ) +@click.option( + "--guid-type", + "guid_type", + default="discovery_metadata", + help=( + "The value of this gets set as _guid_type in the root level metadata. " + "discovery_metadata is the default that enables the Gen3 Discovery Page to visualize the results." + ), + show_default=True, +) @click.pass_context -def discovery_publish(ctx, file, use_default_file, omit_empty): +def discovery_publish(ctx, file, use_default_file, omit_empty, guid_type): """ Run a discovery metadata ingestion on a given metadata TSV file with guid column. If [FILE] is omitted and --default-file not set, prompts for TSV file name. @@ -47,7 +57,11 @@ def discovery_publish(ctx, file, use_default_file, omit_empty): endpoint = ctx.obj.get("endpoint") loop.run_until_complete( publish_discovery_metadata( - auth, file, endpoint=endpoint, omit_empty_values=omit_empty + auth, + file, + endpoint=endpoint, + omit_empty_values=omit_empty, + guid_type=guid_type, ) ) diff --git a/gen3/doi.py b/gen3/doi.py new file mode 100644 index 000000000..fc6e9f647 --- /dev/null +++ b/gen3/doi.py @@ -0,0 +1,932 @@ +""" +This is a lower-level DOI class for interacting with Datacite and actually +minting the DOIs. + +For collecting DOI Metadata, other classes (outside of the ones in this module) +can interact with different APIs to gather the necessary metadata. +""" +import requests +import os + +from cdislogging import get_logger + +from gen3.tools.metadata.discovery import get_discovery_metadata +from gen3.metadata import Gen3Metadata +from gen3.utils import ( + raise_for_status_and_print_error, + is_status_code, +) + + +logging = get_logger("__name__") + + +class DataCiteDOIValidationError(Exception): + pass + + +class DataCite(object): + """ + For interacting with DataCite's API. + https://support.datacite.org/reference/introduction + + Example usage: + + ``` + import os + from requests.auth import HTTPBasicAuth + import sys + + from gen3.doi import DataCite, DigitalObjectIdentifer + + PREFIX = "10.12345" + + def main(): + datacite = DataCite( + auth_provider=HTTPBasicAuth( + os.environ["DATACITE_USERNAME"], os.environ["DATACITE_PASSWORD"] + ) + ) + + doi = DigitalObjectIdentifer( + prefix=PREFIX, + # identifier=f"{PREFIX}/test1", + creators=["test"], + titles=["test"], + publisher="test", + publication_year=2023, + doi_type="Dataset", + url="https://example.com", + version="0.1", + descriptions=[{"description": "this is a test resource"}], + foobar="test", + ) + + datacite.create_doi(doi) + + datacite.persist_doi_metadata_in_gen3(doi) + + if __name__ == "__main__": + main() + ``` + """ + + DEFAULT_DOI_ACCESS_INFORMATION = ( + "The data identified by this DOI is available through Gen3. Please refer " + "to other documentation for more details on access. Alternatively, use the contact " + "information provided on this page to request such information." + ) + DEFAULT_DOI_CONTACT_INFORMATION = ( + "Unavailable. Please refer to other links or " + "documentation on this site to determine an effective contact." + ) + DOI_RESOLVER = "https://doi.org" + PRODUCTION_URL = "https://api.datacite.org" + TEST_URL = "https://api.test.datacite.org" + + def __init__(self, api=PRODUCTION_URL, auth_provider=None): + self._auth_provider = auth_provider + self.api = api + + def create_doi(self, doi): + """ + Create DOI provided via DataCite's API + + Args: + doi (gen3.doi.DigitalObjectIdentifier): DOI to create + """ + payload = doi.as_dict() + headers = { + "Content-type": "application/vnd.api+json", + "Accept": "application/vnd.api+json", + } + + endpoint = self.api.rstrip("/") + "/dois" + logging.info(f"POST-ing to {endpoint}...") + logging.debug(f"Data payload to {endpoint}: {payload}") + response = requests.post( + endpoint, json=payload, auth=self._auth_provider, headers=headers + ) + logging.debug(f"Response: {response.text}") + raise_for_status_and_print_error(response) + + return response + + def read_doi(self, identifier): + """ + Get DOI provided by DataCite's API. + + Args: + identifier (str): DOI identifier to read + """ + headers = { + "Content-type": "application/vnd.api+json", + "Accept": "application/vnd.api+json", + } + + endpoint = self.api.rstrip("/") + f"/dois/{identifier}" + logging.info(f"GET-ing to {endpoint}...") + response = requests.get(endpoint, auth=self._auth_provider, headers=headers) + logging.debug(f"Response: {response.text}") + + # 404's should not raise exceptions, instead returning nothing + if is_status_code(response, "404"): + return None + + raise_for_status_and_print_error(response) + + return response + + def update_doi(self, doi): + """ + Update DOI provided via DataCite's API + + Args: + doi (gen3.doi.DigitalObjectIdentifier): DOI to update + """ + payload = doi.as_dict() + identifier = doi.identifier + headers = { + "Content-type": "application/vnd.api+json", + "Accept": "application/vnd.api+json", + } + + endpoint = self.api.rstrip("/") + f"/dois/{identifier}" + logging.info(f"PUT-ing to {endpoint}...") + logging.debug(f"Data payload to {endpoint}: {payload}") + response = requests.put( + endpoint, json=payload, auth=self._auth_provider, headers=headers + ) + logging.debug(f"Response: {response.text}") + raise_for_status_and_print_error(response) + + return response + + def delete_doi(self, identifier): + """ + Delete DOI provided via DataCite's API + + Args: + identifier (str): DOI identifier to delete + """ + headers = { + "Content-type": "application/vnd.api+json", + "Accept": "application/vnd.api+json", + } + + endpoint = self.api.rstrip("/") + f"/dois/{identifier}" + logging.info(f"DELETE-ing to {endpoint}...") + response = requests.delete(endpoint, auth=self._auth_provider, headers=headers) + logging.debug(f"Response: {response.text}") + raise_for_status_and_print_error(response) + + return response + + def persist_doi_metadata_in_gen3( + self, + guid, + doi, + auth, + additional_metadata=None, + prefix="", + ): + """ + Persist the DOI Metadata in Gen3. The default behavior is + to use the Gen3 Discovery feature set, where there exists an entry + for the DOI already in the Discovery Metadata under the `guid` provided. + + The DOI Metadata will be added under the existing `guid`'s `gen3_discovery` + metadata block so that it is available to the Gen3 Discovery Page. + + Args: + guid (str): globally unique ID for the metadata record + doi (DigitalObjectIdentifier): DOI object with required metadata + auth (TYPE): Description + additional_metadata (Dict, optional): Any additional metadata fields + to add to the DOI metadata into the Gen3 Metadata API. + This can include overrides to the default landing page info + for the following fields: + is_available (bool): True/False + disclaimer (str): Text to display at the top of the page + intended use is for tombstone pages, where the data is + no longer available and we need to note that + citation (str): By default: `Creator (PublicationYear). + Title. Version. Publisher. ResourceType. Identifier` + access_information (str): Text on how to access this data + access_information_link (str): HTTP link to additional info + about access + version_information (str): Text on any version information + contact (str): Contact information required for landing page + prefix (str): prefix for the metadata fields to prepend to all DOI + fields from `doi` AND all fields in `additional_metadata` + """ + metadata_service = Gen3Metadata(auth_provider=auth) + identifier = doi.identifier + + metadata = doi.as_gen3_metadata(prefix=prefix) + + additional_metadata = additional_metadata or {} + + is_available = additional_metadata.get("is_available", True) + is_available_text = "Yes" if is_available else "No" + metadata[prefix + "is_available"] = is_available_text + + if additional_metadata.get("is_available"): + del additional_metadata["is_available"] + + metadata[prefix + "disclaimer"] = additional_metadata.get("disclaimer", "") + if additional_metadata.get("disclaimer"): + del additional_metadata["disclaimer"] + + metadata[prefix + "citation"] = additional_metadata.get("citation") + if not metadata[prefix + "citation"]: + if not doi.optional_fields.get("version"): + logging.warning( + f"DOI {doi.identifier} is missing `version` in doi.optional_fields. " + f"Defaulting to `1` for citation." + ) + # `Creator (PublicationYear). Title. Version. Publisher. ResourceType. Identifier` + metadata[prefix + "citation"] = ( + f"{', '.join([creator.get('name') for creator in doi.creators])} " + f"({doi.publication_year}). " + f"{', '.join([titles.get('title') for titles in doi.titles])} " + f"{str(doi.optional_fields.get('version', '1'))}. " + f"{doi.publisher}. {doi.doi_type}. {doi.identifier}" + ) + if additional_metadata.get("citation"): + del additional_metadata["citation"] + + metadata[prefix + "version_information"] = additional_metadata.get( + "version_information" + ) + if not metadata[prefix + "version_information"]: + if not doi.optional_fields.get("version"): + logging.warning( + f"DOI {doi.identifier} is missing `version` in doi.optional_fields. " + f"Defaulting to `1`." + ) + metadata[ + prefix + "version_information" + ] = f"This is version {doi.optional_fields.get('version', '1')} of this {doi.doi_type}." + if additional_metadata.get("version_information"): + del additional_metadata["version_information"] + + metadata[prefix + "access_information"] = additional_metadata.get( + "access_information" + ) + if not metadata[prefix + "access_information"]: + logging.warning( + "No `access_information` provided in " + "`additional_metadata` in doi.persist_metadata_in_gen3() call. " + f"Consider providing this. Default will be used: {DataCite.DEFAULT_DOI_ACCESS_INFORMATION}" + ) + metadata[ + prefix + "access_information" + ] = DataCite.DEFAULT_DOI_ACCESS_INFORMATION + if additional_metadata.get("access_information"): + del additional_metadata["access_information"] + + metadata[prefix + "contact"] = additional_metadata.get("contact") + if not metadata[prefix + "contact"]: + logging.warning( + "No `contact` provided in " + "`additional_metadata` in doi.persist_metadata_in_gen3() call. " + f"Consider providing this. Default will be used: {DataCite.DEFAULT_DOI_CONTACT_INFORMATION}" + ) + metadata[prefix + "contact"] = DataCite.DEFAULT_DOI_CONTACT_INFORMATION + if additional_metadata.get("contact"): + del additional_metadata["contact"] + + # ensure prefix gets added to any additional metadata + additional_metadata_with_prefix = { + prefix + key.lstrip(prefix): value + for key, value in additional_metadata.items() + } + + metadata.update(additional_metadata_with_prefix) + logging.debug(f"Metadata for {guid}: {metadata}") + + try: + existing_record = metadata_service.get(guid=guid) + + # combine new metadata with existing metadata b/c MDS only supports + # shallow merge at the moment + existing_record.setdefault("gen3_discovery", {}).update(metadata) + metadata = existing_record + + metadata_service.update(guid=guid, metadata=metadata, aliases=[identifier]) + + logging.info(f"Updated existing record for {guid}.") + logging.debug( + f"Updated existing record metadata for {guid}: {existing_record}" + ) + except requests.exceptions.HTTPError as exc: + if is_status_code(exc, "404"): + logging.info( + f"Existing Gen3 Discovery metadata record with GUID " + f"`{guid}` not found. Creating..." + ) + metadata = get_discovery_metadata(metadata) + logging.debug( + f"Creating missing GUID `{guid}` with metadata: {metadata}." + ) + metadata_service.create( + guid=guid, metadata=metadata, aliases=[identifier] + ) + else: + # non 404s should actually escalate + raise + + return metadata + + +class DigitalObjectIdentifier(object): + """ + Object representation of a DOI (including it's metadata). + Convert to a DataCite-compatible payload by using `as_dict()` + + Example return: + { + "data": { + "id": "10.5438/0012", + "type": "dois", + "attributes": { + "event": "publish", + "doi": "10.5438/0012", + "creators": [{ + "name": "DataCite Metadata Working Group" + }], + "titles": [{ + "title": "DataCite Metadata Schema Documentation for the Publication and Citation of Research Data v4.0" + }], + "publisher": "DataCite e.V.", + "publicationYear": 2016, + "types": { + "resourceTypeGeneral": "Text" + }, + "url": "https://schema.datacite.org/meta/kernel-4.4/index.html", + "schemaVersion": "http://datacite.org/schema/kernel-4" + } + } + } + """ + + # to update to a later version, the classes that handle representing various + # objects will need to change. Also it's possible other parts of this implementation + # will need to change (including general types, how they should be representined in JSON, etc) + # As such, this will remain "hard coded" + SCHEMA_VERSION = "http://datacite.org/schema/kernel-4.4" + + RESOURCE_GENERAL_TYPES = [ + "Audiovisual", + "Book", + "BookChapter", + "Collection", + "ComputationalNotebook", + "ConferencePaper", + "ConferenceProceeding", + "DataPaper", + "Dataset", + "Dissertation", + "Event", + "Image", + "InteractiveResource", + "Journal", + "JournalArticle", + "Model", + "OutputManagementPlan", + "PeerReview", + "PhysicalObject", + "Preprint", + "Report", + "Service", + "Software", + "Sound", + "Standard", + "Text", + "Workflow", + "Other", + ] + + OPTIONAL_FIELDS = [ + "subjects", + "contributors", + "descriptions", + "dates", + "relatedIdentifiers", + "geoLocations", + "language", + "identifiers", + "sizes", + "formats", + "version", + "rightsList", + "fundingReferences", + ] + + DEFAULT_DOI_TYPE = "Dataset" + + def __init__( + self, + prefix=None, + identifier=None, + creators=None, + titles=None, + publisher=None, + publication_year=None, + doi_type=None, + url=None, + root_url=None, + event=None, + _type="dois", + **kwargs, + ): + """ + To create a DOI you must specify identifier OR prefix. + + Mandatory fields for registration are defaulted the init. + + Supply any additional reccomended/optional as keyword args + + Arg definitions summarized from the official schema for v4.4: + https://schema.datacite.org/meta/kernel-4.4/doc/DataCite-MetadataKernel_v4.4.pdf + + Args: + prefix (str, optional): DOI Prefix + identifier (str, optional): a unique string that identifies the resource (the DOI) + NOTE: `identifier` is NOT case sensitive but the `url` is. Different + Datacite API's tend to return the DOI either with the mixed cases + presented, fully lower, or fully upper (which adds to the confusion). + As such, anything provided/received will be forced UPPER_CASE + automatically to reduce confusion. + creators (List[Dict], optional): the main + researcher(s) involved in producing the data, + or the author(s) of the publication. + titles (List[Dict], optional): a name or titles by which a resource is known + Format [{"title": "some title"}, ...] + publisher (str, optional): The name of the entity that holds, archives, publishes + prints, distributes, releases, issues, or produces the resource. + This property will be used to formulate the citation, so consider the + prominence of the role. For software, use Publisher for the code + repository. If there is an entity other than a code repository, + that "holds, archives, publishes, prints, distributes, releases, + issues, or produces" the code, use the property + Contributor/contributorType/ hostingInstitution for the code repository. + publication_year (int, optional): the year the data or resource was or will be made + publicly available + doi_type (str, optional): a description of the resource (free-format text) + url (str, optional): the web address of the landing page for the resource + event (str, optional): what event to issue against the DOI + Event field logic per DataCite API: + publish - Triggers a state move from draft or registered to findable + register - Triggers a state move from draft to registered + hide - Triggers a state move from findable to registered + _type (str, optional): The type of thing to interact with in Datacite + """ + self.prefix = prefix + self.identifier = identifier or "" + self.creators = creators or [] + self.titles = titles or [] + self.publisher = publisher + self.publication_year = publication_year + self.doi_type = doi_type or DigitalObjectIdentifier.DEFAULT_DOI_TYPE + self.url = url + self.root_url = root_url + self.event = event + self._type = _type + + # See NOTE: in docstring under `identifier` + self.identifier = self.identifier.upper() + + # any additional kwargs get interpreted as optional fields + self.optional_fields = {} + + for key, value in kwargs.items(): + if key in DigitalObjectIdentifier.OPTIONAL_FIELDS: + self.optional_fields[key] = value + else: + logging.warning( + f"Skipping '{key}={value}' because '{key}' " + f"is not a supported optional DOI metadata field." + ) + logging.debug( + f"Supported optional DOI metadata fields: " + f"{DigitalObjectIdentifier.OPTIONAL_FIELDS}" + ) + + if not (self.identifier or self.prefix): + raise DataCiteDOIValidationError( + "You must either specify a) `identifier` which is a full DOI with prefix " + "or b) `prefix` (which will let Datacite autogenerate the rest of " + "the ID. You can't leave both blank." + ) + + if self.root_url and self.url: + logging.warning( + "You provided both `root_url` and `url`. " + f"Preferring `url` ({self.url}) and ignoring `root_url` ({self.root_url})." + ) + self.root_url = None + + if self.root_url and self.prefix and not self.identifier: + raise DataCiteDOIValidationError( + "To use `root_url`, you MUST specify `identifier` (which is a " + "full DOI with prefix). You CANNOT rely on `prefix` (which will " + "let Datacite autogenerate the rest of the ID)." + ) + + if not self.url: + if not self.root_url: + raise DataCiteDOIValidationError( + "You must either specify a) `url` which is a full landing page url " + "or b) `root_url` (which will automatically get appended with " + "the `identifier` to form final url). You can't leave both blank." + ) + + self.url = self._get_url_from_root() + + if doi_type and doi_type not in DigitalObjectIdentifier.RESOURCE_GENERAL_TYPES: + logging.error(f"{doi_type} is not an accepted resourceTypeGeneral") + raise DataCiteDOIValidationError( + f"{doi_type} is not an accepted resourceTypeGeneral" + ) + + def _get_url_from_root(self): + return self.root_url.rstrip("/") + f"/{self.identifier}" + + def as_gen3_metadata(self, prefix=""): + """ + Return a flat dictionary for ingestion into Gen3's Metadata Service + + Returns: + Dict: flat dictionary for ingestion into Gen3's Metadata Service + """ + data = {} + + if self.identifier: + data[prefix + "identifier"] = self.identifier + data[prefix + "resolveable_link"] = ( + DataCite.DOI_RESOLVER.rstrip("/") + "/" + self.identifier + ) + if self.creators: + data[prefix + "creators"] = "& ".join( + [item["name"] for item in self.creators] + ) + if self.titles: + data[prefix + "titles"] = ", ".join([item["title"] for item in self.titles]) + if self.publisher: + data[prefix + "publisher"] = self.publisher + if self.publication_year: + data[prefix + "publication_year"] = self.publication_year + if self.doi_type: + data[prefix + "resource_type"] = self.doi_type + if self.url: + data[prefix + "url"] = self.url + + for key, value in self.optional_fields.items(): + if key == "contributors": + data[prefix + key] = "& ".join([item["name"] for item in value]) + elif key == "descriptions": + data[prefix + key] = "\n\n".join( + [ + item["descriptionType"] + ": " + item["description"] + for item in value + ] + ) + elif key == "identifiers": + data[prefix + key] = ", ".join(value) + elif key == "fundingReferences": + # sometimes this is a list and sometimes is an object depending + # on which Datacite API you hit (PUT/POST/DELETE). Handle both + if type(value) == list: + value = ",".join([funder.get("funderName") for funder in value]) + else: + value = value.get("funderName") + + data[prefix + key] = value + else: + data[prefix + key] = value + + return data + + def as_dict(self): + data = { + "data": { + "id": self.identifier, + "type": self._type, + "attributes": { + "doi": self.identifier, + }, + } + } + + if self.creators: + data["data"]["attributes"]["creators"] = [item for item in self.creators] + if self.titles: + data["data"]["attributes"]["titles"] = self.titles + if self.publisher: + data["data"]["attributes"]["publisher"] = self.publisher + if self.publication_year: + data["data"]["attributes"]["publicationYear"] = self.publication_year + if self.doi_type: + data["data"]["attributes"]["types"] = {"resourceTypeGeneral": self.doi_type} + if self.url: + data["data"]["attributes"]["url"] = self.url + if self.event: + data["data"]["attributes"]["event"] = self.event + + data["data"]["attributes"]["schemaVersion"] = self.SCHEMA_VERSION + + for key, value in self.optional_fields.items(): + data["data"]["attributes"][key] = value + + # per DataCite API's spec, if you don't specify a full DOI you need + # to specify the prefix at least + if not self.identifier: + del data["data"]["attributes"]["doi"] + del data["data"]["id"] + data["data"]["attributes"]["prefix"] = self.prefix + + return data + + @staticmethod + def from_datacite_create_doi_response(response): + """ + Return a DigitalObjectIdentifier instance from the Response from a + Datacite Create DOI API call. + """ + raw_attributes = response.json().get("data", {}).get("attributes", {}) + attributes = {} + + # we only want the mandatory and optional fields we know how to handle, + # exclude all other fields returned by the API + for key, value in raw_attributes.items(): + if key in DigitalObjectIdentifier.OPTIONAL_FIELDS: + attributes[key] = value + + return DigitalObjectIdentifier( + prefix=raw_attributes.get("prefix"), + # Why .upper()? See NOTE: in docstring under `identifier` + identifier=raw_attributes.get("doi").upper(), + creators=raw_attributes.get("creators"), + titles=raw_attributes.get("titles"), + publisher=raw_attributes.get("publisher"), + publication_year=raw_attributes.get("publicationYear"), + url=raw_attributes.get("url"), + **attributes, + ) + + +class DigitalObjectIdentifierCreator(object): + """ + Class to help with representing a Creator object for the DataCite API + per their schema. + + NOTE: If you use anything other than the schema this was built for (v4.4), + this logic will need to be updated. + + From the DataCite Metadata Schema V 4.4: + + * 20.2 Creator 0-n + The institution or person + responsible for creating the + related resource + To supply multiple creators, + repeat this property. + + * 20.2.1 creatorName 1 + The full name of the related + item creator + + Examples: Charpy, Antoine; + Jemison, Mae; Foo Data + Center Note: The personal + name, format should be: + family, given. Non- roman + names may be transliterated + according to the ALA-LC + schemas12. + + * 20.2.1.a nameType 0-1 + The type of name Controlled List Values: + Organizational + Personal (default) + + * 20.2.2 givenName 0-1 + The personal or first name of + the creator + Examples based on the 20.11.1 names: Antoine; Mae + + * 20.2.3 familyName 0-1 + The surname or last name of + the creator + + Examples based on the 2.1 + names: Charpy; Jemison + """ + + NAME_TYPE_PERSON = "Personal" + NAME_TYPE_ORGANIZATION = "Organizational" + + def __init__( + self, + name, + name_type=NAME_TYPE_PERSON, + given_name=None, + family_name=None, + ): + self.name = name + + allowed_name_types = [ + DigitalObjectIdentifierCreator.NAME_TYPE_PERSON, + DigitalObjectIdentifierCreator.NAME_TYPE_ORGANIZATION, + ] + if name_type not in allowed_name_types: + logging.error( + f"Provided name_type '{name_type}' is NOT in {allowed_name_types}. " + f"Defaulting to '{DigitalObjectIdentifierCreator.NAME_TYPE_PERSON}'." + ) + name_type = DigitalObjectIdentifierCreator.NAME_TYPE_PERSON + + self.name_type = name_type + self.given_name = given_name + self.family_name = family_name + + def as_dict(self): + data = { + "name": self.name, + "nameType": self.name_type, + } + + if self.given_name: + data.update({"givenName": self.given_name}) + + if self.family_name: + data.update({"familyName": self.family_name}) + + if ( + self.given_name + and self.family_name + and self.name_type == DigitalObjectIdentifierCreator.NAME_TYPE_PERSON + ): + data.update({"name": f"{self.family_name}, {self.given_name}"}) + + return data + + +class DigitalObjectIdentifierFundingReference(object): + """ + This could theoretically support more options, but at the point of writing, + they weren't deemed necessary. Leaving this as an isolated class in case + we want to expand in the future. + + From the DataCite Metadata Schema V 4.4: + + * 19 FundingReference 0-n + Information about financial + support (funding) for the + resource being registered + It is a best practice to supply + funding information when + financial support has been + received. + + * 19.1 funderName 1 + Name of the funding provider Example: Gordon and Betty + Moore Foundation + + * 19.2 funderIdentifier 0-1 + Uniquely identifies a funding + entity, according to various + types. + Example: + https://doi.org/10.13039/100 + 000936 + + * 19.2.a funderIdentifierType 0-1 + The type of the funderIdentifier Controlled List Values: + GRID + ISNI + ROR + Other + """ + + def __init__( + self, + name, + ): + self.name = name + + def as_dict(self): + data = { + "funderName": f"{self.name}", + } + + return data + + +class DigitalObjectIdentifierTitle(object): + """ + This could theoretically support more options, but at the point of writing, + they weren't deemed necessary. Leaving this as an isolated class in case + we want to expand in the future. + + From the DataCite Metadata Schema V 4.4: + * 3 Title 1-n + A name or title by which a + resource is known. May be + the title of a dataset or the + name of a piece of software. + Free text. + + * 3.a titleType 0-1 + The type of Title (other than + the Main Title) + Controlled List Values: + AlternativeTitle + Subtitle + TranslatedTitle + Other + """ + + def __init__( + self, + title, + ): + self.title = title + + def as_dict(self): + data = { + "title": f"{self.title}", + } + + return data + + +class DigitalObjectIdentifierDescription(object): + """ + From the DataCite Metadata Schema V 4.4: + * 17 Description 0-n + All additional information that + does not fit in any of the other + categories. May be used for + technical information. + Free text + *** + It is a best practice to supply a + description. + + * 17.a descriptionType 1 + The type of the Description If Description is used, + descriptionType is mandatory. + + Controlled List Values: + Abstract + Methods + SeriesInformation + TableOfContents + TechnicalInfo + Other + + Note: SeriesInformation as a + container for series title, + volume, issue, page number, + and related fields, is now + superseded by the new + RelatedItem property + """ + + TYPE_ABSTRACT = "Abstract" + TYPE_METHODS = "Methods" + TYPE_SERIES_INFORMATION = "SeriesInformation" + TYPE_TABLE_OF_CONTENTS = "TableOfContents" + TYPE_TECHNICAL_INFO = "TechnicalInfo" + TYPE_OTHER = "Other" + + def __init__(self, description, description_type=TYPE_ABSTRACT): + self.description = description + + allowed_types = [ + DigitalObjectIdentifierDescription.TYPE_ABSTRACT, + DigitalObjectIdentifierDescription.TYPE_METHODS, + DigitalObjectIdentifierDescription.TYPE_SERIES_INFORMATION, + DigitalObjectIdentifierDescription.TYPE_TABLE_OF_CONTENTS, + DigitalObjectIdentifierDescription.TYPE_TECHNICAL_INFO, + DigitalObjectIdentifierDescription.TYPE_OTHER, + ] + if description_type not in allowed_types: + logging.error( + f"Provided description_type '{description_type}' is NOT in {allowed_types}. " + f"Defaulting to '{DigitalObjectIdentifierDescription.TYPE_ABSTRACT}'." + ) + description_type = DigitalObjectIdentifierDescription.NAME_TYPE_PERSON + + self.description_type = description_type + + def as_dict(self): + data = { + "description": f"{self.description}", + "descriptionType": f"{self.description_type}", + } + + return data diff --git a/gen3/tools/merge.py b/gen3/tools/merge.py index 1731d1e81..b80e12443 100644 --- a/gen3/tools/merge.py +++ b/gen3/tools/merge.py @@ -9,6 +9,8 @@ from datetime import datetime from cdislogging import get_logger +from gen3.utils import get_delimiter_from_extension + logging = get_logger("__name__") @@ -275,25 +277,6 @@ def merge_guids_into_metadata( logging.debug(f"output file:\n{os.path.abspath(output_filename)}") -def get_delimiter_from_extension(filename): - """ - Return the file delimter based on the extension. - - Args: - filename (str): file name with extension - - Returns: - str: delimeter character, either \t or , - """ - file_ext = os.path.splitext(filename) - if file_ext[-1].lower() == ".tsv": - file_delimiter = "\t" - else: - # default, assume CSV - file_delimiter = "," - return file_delimiter - - def write_header_to_file(filename, fieldnames, delimiter="\t"): """ Writes to a file in TSV format. diff --git a/gen3/tools/metadata/discovery.py b/gen3/tools/metadata/discovery.py index c75339d9e..c99bdf94b 100644 --- a/gen3/tools/metadata/discovery.py +++ b/gen3/tools/metadata/discovery.py @@ -375,7 +375,9 @@ async def publish_discovery_metadata( else: guid = discovery_metadata[guid_field] - # when publishing unregistered metadata, skip those who are already registered if both reset_unregistered_metadata and update_registered_metadata are set to false + # when publishing unregistered metadata, skip those who are already + # registered if both reset_unregistered_metadata and + # update_registered_metadata are set to false if ( is_unregistered_metadata and str(guid) in registered_metadata_guids @@ -408,7 +410,8 @@ async def publish_discovery_metadata( if reset_unregistered_metadata or ( str(guid) not in registered_metadata_guids ): - # only set GUID type to "unregistered_discovery_metadata" for unregistered metadata, or reset_unregistered_metadata is set + # only set GUID type to "unregistered_discovery_metadata" + # for unregistered metadata, or reset_unregistered_metadata is set new_guid_type = f"unregistered_{guid_type}" elif str(guid) in registered_metadata_guids: if update_registered_metadata: @@ -427,10 +430,9 @@ async def publish_discovery_metadata( logging.warning(f"{guid} is not already registered. Skipping.") continue - metadata = { - "_guid_type": new_guid_type, - "gen3_discovery": discovery_metadata, - } + metadata = get_discovery_metadata( + provided_metadata=discovery_metadata, guid_type=new_guid_type + ) pending_requests += [mds.async_create(guid, metadata, overwrite=True)] if len(pending_requests) == MAX_CONCURRENT_REQUESTS: @@ -440,6 +442,24 @@ async def publish_discovery_metadata( await asyncio.gather(*pending_requests) +def get_discovery_metadata( + provided_metadata, + guid_type="discovery_metadata", +): + """ + Return a metadata block representing Gen3 Discovery Metadata with the provided + metadata in the relevant block. + + Args: + provided_metadata (Dict): Metadata fields you want in Gen3 Discovery + guid_type (str, optional): value to override the default _guid_type + """ + return { + "_guid_type": guid_type, + "gen3_discovery": provided_metadata, + } + + def try_delete_discovery_guid(auth, guid): """ Deletes all discovery metadata under [guid] if it exists diff --git a/gen3/utils.py b/gen3/utils.py index 557724f4e..869969525 100644 --- a/gen3/utils.py +++ b/gen3/utils.py @@ -4,6 +4,8 @@ import sys import re import requests +import random +import string import os from urllib.parse import urlunsplit @@ -25,6 +27,12 @@ AUTHZ_FORMAT = r"^.*$" +def get_random_alphanumeric(length): + # end up with roughly the same amount of numbers as letters + letters = string.ascii_lowercase + "".join([str(item) for item in range(0, 10)]) * 3 + return "".join(random.choice(letters) for i in range(length)) + + def make_folders_for_filename(filename, current_directory=None): """ Make the directories up to the filename provided. Relative paths are supported. @@ -193,18 +201,48 @@ def log_backoff_giveup_except_on_no_retries(details): ) -def exception_do_not_retry(error): - def _is_status(code): - return ( - str(getattr(error, "code", None)) == code - or str(getattr(error, "status", None)) == code - or str(getattr(error, "status_code", None)) == code - or str(getattr(getattr(error, "response", {}), "code", "")) == code - or str(getattr(getattr(error, "response", {}), "status", "")) == code - or str(getattr(getattr(error, "response", {}), "status_code", "")) == code - ) +def get_delimiter_from_extension(filename): + """ + Return the file delimter based on the extension. + + Args: + filename (str): file name with extension + + Returns: + str: delimeter character, either \t or , + """ + file_ext = os.path.splitext(filename) + if file_ext[-1].lower() == ".tsv": + file_delimiter = "\t" + else: + # default, assume CSV + file_delimiter = "," + return file_delimiter - if _is_status("409") or _is_status("404"): + +def is_status_code(error, code): + """ + Args: + error (object): Ideally a requests.Response, this safely checks for + known attributes where the status code might be + code (str): The status code you want to check for in the error. ex: 404 + + Returns: + bool: Whether or not the error object contains the status code specified + """ + code = str(code) + return ( + str(getattr(error, "code", None)) == code + or str(getattr(error, "status", None)) == code + or str(getattr(error, "status_code", None)) == code + or str(getattr(getattr(error, "response", {}), "code", "")) == code + or str(getattr(getattr(error, "response", {}), "status", "")) == code + or str(getattr(getattr(error, "response", {}), "status_code", "")) == code + ) + + +def exception_do_not_retry(error): + if is_status_code(error, "409") or is_status_code(error, "404"): return True return False diff --git a/poetry.lock b/poetry.lock index 0793881cb..5aae11ee4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -196,22 +196,22 @@ files = [ [[package]] name = "attrs" -version = "22.2.0" +version = "23.1.0" description = "Classes Without Boilerplate" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, - {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, + {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, + {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, ] [package.extras] -cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]"] -docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"] -tests = ["attrs[tests-no-zope]", "zope.interface"] -tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] [[package]] name = "authlib" @@ -522,63 +522,63 @@ files = [ [[package]] name = "coverage" -version = "7.2.2" +version = "7.2.3" description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "coverage-7.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c90e73bdecb7b0d1cea65a08cb41e9d672ac6d7995603d6465ed4914b98b9ad7"}, - {file = "coverage-7.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e2926b8abedf750c2ecf5035c07515770944acf02e1c46ab08f6348d24c5f94d"}, - {file = "coverage-7.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57b77b9099f172804e695a40ebaa374f79e4fb8b92f3e167f66facbf92e8e7f5"}, - {file = "coverage-7.2.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:efe1c0adad110bf0ad7fb59f833880e489a61e39d699d37249bdf42f80590169"}, - {file = "coverage-7.2.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2199988e0bc8325d941b209f4fd1c6fa007024b1442c5576f1a32ca2e48941e6"}, - {file = "coverage-7.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:81f63e0fb74effd5be736cfe07d710307cc0a3ccb8f4741f7f053c057615a137"}, - {file = "coverage-7.2.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:186e0fc9cf497365036d51d4d2ab76113fb74f729bd25da0975daab2e107fd90"}, - {file = "coverage-7.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:420f94a35e3e00a2b43ad5740f935358e24478354ce41c99407cddd283be00d2"}, - {file = "coverage-7.2.2-cp310-cp310-win32.whl", hash = "sha256:38004671848b5745bb05d4d621526fca30cee164db42a1f185615f39dc997292"}, - {file = "coverage-7.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:0ce383d5f56d0729d2dd40e53fe3afeb8f2237244b0975e1427bfb2cf0d32bab"}, - {file = "coverage-7.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3eb55b7b26389dd4f8ae911ba9bc8c027411163839dea4c8b8be54c4ee9ae10b"}, - {file = "coverage-7.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d2b96123a453a2d7f3995ddb9f28d01fd112319a7a4d5ca99796a7ff43f02af5"}, - {file = "coverage-7.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:299bc75cb2a41e6741b5e470b8c9fb78d931edbd0cd009c58e5c84de57c06731"}, - {file = "coverage-7.2.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5e1df45c23d4230e3d56d04414f9057eba501f78db60d4eeecfcb940501b08fd"}, - {file = "coverage-7.2.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:006ed5582e9cbc8115d2e22d6d2144a0725db542f654d9d4fda86793832f873d"}, - {file = "coverage-7.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d683d230b5774816e7d784d7ed8444f2a40e7a450e5720d58af593cb0b94a212"}, - {file = "coverage-7.2.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8efb48fa743d1c1a65ee8787b5b552681610f06c40a40b7ef94a5b517d885c54"}, - {file = "coverage-7.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4c752d5264053a7cf2fe81c9e14f8a4fb261370a7bb344c2a011836a96fb3f57"}, - {file = "coverage-7.2.2-cp311-cp311-win32.whl", hash = "sha256:55272f33da9a5d7cccd3774aeca7a01e500a614eaea2a77091e9be000ecd401d"}, - {file = "coverage-7.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:92ebc1619650409da324d001b3a36f14f63644c7f0a588e331f3b0f67491f512"}, - {file = "coverage-7.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5afdad4cc4cc199fdf3e18088812edcf8f4c5a3c8e6cb69127513ad4cb7471a9"}, - {file = "coverage-7.2.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0484d9dd1e6f481b24070c87561c8d7151bdd8b044c93ac99faafd01f695c78e"}, - {file = "coverage-7.2.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d530191aa9c66ab4f190be8ac8cc7cfd8f4f3217da379606f3dd4e3d83feba69"}, - {file = "coverage-7.2.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ac0f522c3b6109c4b764ffec71bf04ebc0523e926ca7cbe6c5ac88f84faced0"}, - {file = "coverage-7.2.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ba279aae162b20444881fc3ed4e4f934c1cf8620f3dab3b531480cf602c76b7f"}, - {file = "coverage-7.2.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:53d0fd4c17175aded9c633e319360d41a1f3c6e352ba94edcb0fa5167e2bad67"}, - {file = "coverage-7.2.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c99cb7c26a3039a8a4ee3ca1efdde471e61b4837108847fb7d5be7789ed8fd9"}, - {file = "coverage-7.2.2-cp37-cp37m-win32.whl", hash = "sha256:5cc0783844c84af2522e3a99b9b761a979a3ef10fb87fc4048d1ee174e18a7d8"}, - {file = "coverage-7.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:817295f06eacdc8623dc4df7d8b49cea65925030d4e1e2a7c7218380c0072c25"}, - {file = "coverage-7.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6146910231ece63facfc5984234ad1b06a36cecc9fd0c028e59ac7c9b18c38c6"}, - {file = "coverage-7.2.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:387fb46cb8e53ba7304d80aadca5dca84a2fbf6fe3faf6951d8cf2d46485d1e5"}, - {file = "coverage-7.2.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:046936ab032a2810dcaafd39cc4ef6dd295df1a7cbead08fe996d4765fca9fe4"}, - {file = "coverage-7.2.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e627dee428a176ffb13697a2c4318d3f60b2ccdde3acdc9b3f304206ec130ccd"}, - {file = "coverage-7.2.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fa54fb483decc45f94011898727802309a109d89446a3c76387d016057d2c84"}, - {file = "coverage-7.2.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3668291b50b69a0c1ef9f462c7df2c235da3c4073f49543b01e7eb1dee7dd540"}, - {file = "coverage-7.2.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7c20b731211261dc9739bbe080c579a1835b0c2d9b274e5fcd903c3a7821cf88"}, - {file = "coverage-7.2.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5764e1f7471cb8f64b8cda0554f3d4c4085ae4b417bfeab236799863703e5de2"}, - {file = "coverage-7.2.2-cp38-cp38-win32.whl", hash = "sha256:4f01911c010122f49a3e9bdc730eccc66f9b72bd410a3a9d3cb8448bb50d65d3"}, - {file = "coverage-7.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:c448b5c9e3df5448a362208b8d4b9ed85305528313fca1b479f14f9fe0d873b8"}, - {file = "coverage-7.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bfe7085783cda55e53510482fa7b5efc761fad1abe4d653b32710eb548ebdd2d"}, - {file = "coverage-7.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9d22e94e6dc86de981b1b684b342bec5e331401599ce652900ec59db52940005"}, - {file = "coverage-7.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:507e4720791977934bba016101579b8c500fb21c5fa3cd4cf256477331ddd988"}, - {file = "coverage-7.2.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc4803779f0e4b06a2361f666e76f5c2e3715e8e379889d02251ec911befd149"}, - {file = "coverage-7.2.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db8c2c5ace167fd25ab5dd732714c51d4633f58bac21fb0ff63b0349f62755a8"}, - {file = "coverage-7.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4f68ee32d7c4164f1e2c8797535a6d0a3733355f5861e0f667e37df2d4b07140"}, - {file = "coverage-7.2.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d52f0a114b6a58305b11a5cdecd42b2e7f1ec77eb20e2b33969d702feafdd016"}, - {file = "coverage-7.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:797aad79e7b6182cb49c08cc5d2f7aa7b2128133b0926060d0a8889ac43843be"}, - {file = "coverage-7.2.2-cp39-cp39-win32.whl", hash = "sha256:db45eec1dfccdadb179b0f9ca616872c6f700d23945ecc8f21bb105d74b1c5fc"}, - {file = "coverage-7.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:8dbe2647bf58d2c5a6c5bcc685f23b5f371909a5624e9f5cd51436d6a9f6c6ef"}, - {file = "coverage-7.2.2-pp37.pp38.pp39-none-any.whl", hash = "sha256:872d6ce1f5be73f05bea4df498c140b9e7ee5418bfa2cc8204e7f9b817caa968"}, - {file = "coverage-7.2.2.tar.gz", hash = "sha256:36dd42da34fe94ed98c39887b86db9d06777b1c8f860520e21126a75507024f2"}, + {file = "coverage-7.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e58c0d41d336569d63d1b113bd573db8363bc4146f39444125b7f8060e4e04f5"}, + {file = "coverage-7.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:344e714bd0fe921fc72d97404ebbdbf9127bac0ca1ff66d7b79efc143cf7c0c4"}, + {file = "coverage-7.2.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:974bc90d6f6c1e59ceb1516ab00cf1cdfbb2e555795d49fa9571d611f449bcb2"}, + {file = "coverage-7.2.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0743b0035d4b0e32bc1df5de70fba3059662ace5b9a2a86a9f894cfe66569013"}, + {file = "coverage-7.2.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d0391fb4cfc171ce40437f67eb050a340fdbd0f9f49d6353a387f1b7f9dd4fa"}, + {file = "coverage-7.2.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4a42e1eff0ca9a7cb7dc9ecda41dfc7cbc17cb1d02117214be0561bd1134772b"}, + {file = "coverage-7.2.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:be19931a8dcbe6ab464f3339966856996b12a00f9fe53f346ab3be872d03e257"}, + {file = "coverage-7.2.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:72fcae5bcac3333a4cf3b8f34eec99cea1187acd55af723bcbd559adfdcb5535"}, + {file = "coverage-7.2.3-cp310-cp310-win32.whl", hash = "sha256:aeae2aa38395b18106e552833f2a50c27ea0000122bde421c31d11ed7e6f9c91"}, + {file = "coverage-7.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:83957d349838a636e768251c7e9979e899a569794b44c3728eaebd11d848e58e"}, + {file = "coverage-7.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:dfd393094cd82ceb9b40df4c77976015a314b267d498268a076e940fe7be6b79"}, + {file = "coverage-7.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:182eb9ac3f2b4874a1f41b78b87db20b66da6b9cdc32737fbbf4fea0c35b23fc"}, + {file = "coverage-7.2.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bb1e77a9a311346294621be905ea8a2c30d3ad371fc15bb72e98bfcfae532df"}, + {file = "coverage-7.2.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca0f34363e2634deffd390a0fef1aa99168ae9ed2af01af4a1f5865e362f8623"}, + {file = "coverage-7.2.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55416d7385774285b6e2a5feca0af9652f7f444a4fa3d29d8ab052fafef9d00d"}, + {file = "coverage-7.2.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:06ddd9c0249a0546997fdda5a30fbcb40f23926df0a874a60a8a185bc3a87d93"}, + {file = "coverage-7.2.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:fff5aaa6becf2c6a1699ae6a39e2e6fb0672c2d42eca8eb0cafa91cf2e9bd312"}, + {file = "coverage-7.2.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ea53151d87c52e98133eb8ac78f1206498c015849662ca8dc246255265d9c3c4"}, + {file = "coverage-7.2.3-cp311-cp311-win32.whl", hash = "sha256:8f6c930fd70d91ddee53194e93029e3ef2aabe26725aa3c2753df057e296b925"}, + {file = "coverage-7.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:fa546d66639d69aa967bf08156eb8c9d0cd6f6de84be9e8c9819f52ad499c910"}, + {file = "coverage-7.2.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b2317d5ed777bf5a033e83d4f1389fd4ef045763141d8f10eb09a7035cee774c"}, + {file = "coverage-7.2.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be9824c1c874b73b96288c6d3de793bf7f3a597770205068c6163ea1f326e8b9"}, + {file = "coverage-7.2.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2c3b2803e730dc2797a017335827e9da6da0e84c745ce0f552e66400abdfb9a1"}, + {file = "coverage-7.2.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f69770f5ca1994cb32c38965e95f57504d3aea96b6c024624fdd5bb1aa494a1"}, + {file = "coverage-7.2.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1127b16220f7bfb3f1049ed4a62d26d81970a723544e8252db0efde853268e21"}, + {file = "coverage-7.2.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:aa784405f0c640940595fa0f14064d8e84aff0b0f762fa18393e2760a2cf5841"}, + {file = "coverage-7.2.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:3146b8e16fa60427e03884301bf8209221f5761ac754ee6b267642a2fd354c48"}, + {file = "coverage-7.2.3-cp37-cp37m-win32.whl", hash = "sha256:1fd78b911aea9cec3b7e1e2622c8018d51c0d2bbcf8faaf53c2497eb114911c1"}, + {file = "coverage-7.2.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0f3736a5d34e091b0a611964c6262fd68ca4363df56185902528f0b75dbb9c1f"}, + {file = "coverage-7.2.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:981b4df72c93e3bc04478153df516d385317628bd9c10be699c93c26ddcca8ab"}, + {file = "coverage-7.2.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c0045f8f23a5fb30b2eb3b8a83664d8dc4fb58faddf8155d7109166adb9f2040"}, + {file = "coverage-7.2.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f760073fcf8f3d6933178d67754f4f2d4e924e321f4bb0dcef0424ca0215eba1"}, + {file = "coverage-7.2.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c86bd45d1659b1ae3d0ba1909326b03598affbc9ed71520e0ff8c31a993ad911"}, + {file = "coverage-7.2.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:172db976ae6327ed4728e2507daf8a4de73c7cc89796483e0a9198fd2e47b462"}, + {file = "coverage-7.2.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d2a3a6146fe9319926e1d477842ca2a63fe99af5ae690b1f5c11e6af074a6b5c"}, + {file = "coverage-7.2.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:f649dd53833b495c3ebd04d6eec58479454a1784987af8afb77540d6c1767abd"}, + {file = "coverage-7.2.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:7c4ed4e9f3b123aa403ab424430b426a1992e6f4c8fd3cb56ea520446e04d152"}, + {file = "coverage-7.2.3-cp38-cp38-win32.whl", hash = "sha256:eb0edc3ce9760d2f21637766c3aa04822030e7451981ce569a1b3456b7053f22"}, + {file = "coverage-7.2.3-cp38-cp38-win_amd64.whl", hash = "sha256:63cdeaac4ae85a179a8d6bc09b77b564c096250d759eed343a89d91bce8b6367"}, + {file = "coverage-7.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:20d1a2a76bb4eb00e4d36b9699f9b7aba93271c9c29220ad4c6a9581a0320235"}, + {file = "coverage-7.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ea748802cc0de4de92ef8244dd84ffd793bd2e7be784cd8394d557a3c751e21"}, + {file = "coverage-7.2.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21b154aba06df42e4b96fc915512ab39595105f6c483991287021ed95776d934"}, + {file = "coverage-7.2.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd214917cabdd6f673a29d708574e9fbdb892cb77eb426d0eae3490d95ca7859"}, + {file = "coverage-7.2.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c2e58e45fe53fab81f85474e5d4d226eeab0f27b45aa062856c89389da2f0d9"}, + {file = "coverage-7.2.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:87ecc7c9a1a9f912e306997ffee020297ccb5ea388421fe62a2a02747e4d5539"}, + {file = "coverage-7.2.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:387065e420aed3c71b61af7e82c7b6bc1c592f7e3c7a66e9f78dd178699da4fe"}, + {file = "coverage-7.2.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ea3f5bc91d7d457da7d48c7a732beaf79d0c8131df3ab278e6bba6297e23c6c4"}, + {file = "coverage-7.2.3-cp39-cp39-win32.whl", hash = "sha256:ae7863a1d8db6a014b6f2ff9c1582ab1aad55a6d25bac19710a8df68921b6e30"}, + {file = "coverage-7.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:3f04becd4fcda03c0160d0da9c8f0c246bc78f2f7af0feea1ec0930e7c93fa4a"}, + {file = "coverage-7.2.3-pp37.pp38.pp39-none-any.whl", hash = "sha256:965ee3e782c7892befc25575fa171b521d33798132692df428a09efacaffe8d0"}, + {file = "coverage-7.2.3.tar.gz", hash = "sha256:d298c2815fa4891edd9abe5ad6e6cb4207104c7dd9fd13aea3fdebf6f9b91259"}, ] [package.dependencies] @@ -589,31 +589,31 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "40.0.1" +version = "40.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "dev" optional = false python-versions = ">=3.6" files = [ - {file = "cryptography-40.0.1-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:918cb89086c7d98b1b86b9fdb70c712e5a9325ba6f7d7cfb509e784e0cfc6917"}, - {file = "cryptography-40.0.1-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9618a87212cb5200500e304e43691111570e1f10ec3f35569fdfcd17e28fd797"}, - {file = "cryptography-40.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a4805a4ca729d65570a1b7cac84eac1e431085d40387b7d3bbaa47e39890b88"}, - {file = "cryptography-40.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63dac2d25c47f12a7b8aa60e528bfb3c51c5a6c5a9f7c86987909c6c79765554"}, - {file = "cryptography-40.0.1-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0a4e3406cfed6b1f6d6e87ed243363652b2586b2d917b0609ca4f97072994405"}, - {file = "cryptography-40.0.1-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1e0af458515d5e4028aad75f3bb3fe7a31e46ad920648cd59b64d3da842e4356"}, - {file = "cryptography-40.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:d8aa3609d337ad85e4eb9bb0f8bcf6e4409bfb86e706efa9a027912169e89122"}, - {file = "cryptography-40.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cf91e428c51ef692b82ce786583e214f58392399cf65c341bc7301d096fa3ba2"}, - {file = "cryptography-40.0.1-cp36-abi3-win32.whl", hash = "sha256:650883cc064297ef3676b1db1b7b1df6081794c4ada96fa457253c4cc40f97db"}, - {file = "cryptography-40.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:a805a7bce4a77d51696410005b3e85ae2839bad9aa38894afc0aa99d8e0c3160"}, - {file = "cryptography-40.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cd033d74067d8928ef00a6b1327c8ea0452523967ca4463666eeba65ca350d4c"}, - {file = "cryptography-40.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d36bbeb99704aabefdca5aee4eba04455d7a27ceabd16f3b3ba9bdcc31da86c4"}, - {file = "cryptography-40.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:32057d3d0ab7d4453778367ca43e99ddb711770477c4f072a51b3ca69602780a"}, - {file = "cryptography-40.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:f5d7b79fa56bc29580faafc2ff736ce05ba31feaa9d4735048b0de7d9ceb2b94"}, - {file = "cryptography-40.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7c872413353c70e0263a9368c4993710070e70ab3e5318d85510cc91cce77e7c"}, - {file = "cryptography-40.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:28d63d75bf7ae4045b10de5413fb1d6338616e79015999ad9cf6fc538f772d41"}, - {file = "cryptography-40.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6f2bbd72f717ce33100e6467572abaedc61f1acb87b8d546001328d7f466b778"}, - {file = "cryptography-40.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:cc3a621076d824d75ab1e1e530e66e7e8564e357dd723f2533225d40fe35c60c"}, - {file = "cryptography-40.0.1.tar.gz", hash = "sha256:2803f2f8b1e95f614419926c7e6f55d828afc614ca5ed61543877ae668cc3472"}, + {file = "cryptography-40.0.2-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:8f79b5ff5ad9d3218afb1e7e20ea74da5f76943ee5edb7f76e56ec5161ec782b"}, + {file = "cryptography-40.0.2-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:05dc219433b14046c476f6f09d7636b92a1c3e5808b9a6536adf4932b3b2c440"}, + {file = "cryptography-40.0.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4df2af28d7bedc84fe45bd49bc35d710aede676e2a4cb7fc6d103a2adc8afe4d"}, + {file = "cryptography-40.0.2-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dcca15d3a19a66e63662dc8d30f8036b07be851a8680eda92d079868f106288"}, + {file = "cryptography-40.0.2-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a04386fb7bc85fab9cd51b6308633a3c271e3d0d3eae917eebab2fac6219b6d2"}, + {file = "cryptography-40.0.2-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:adc0d980fd2760c9e5de537c28935cc32b9353baaf28e0814df417619c6c8c3b"}, + {file = "cryptography-40.0.2-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:d5a1bd0e9e2031465761dfa920c16b0065ad77321d8a8c1f5ee331021fda65e9"}, + {file = "cryptography-40.0.2-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:a95f4802d49faa6a674242e25bfeea6fc2acd915b5e5e29ac90a32b1139cae1c"}, + {file = "cryptography-40.0.2-cp36-abi3-win32.whl", hash = "sha256:aecbb1592b0188e030cb01f82d12556cf72e218280f621deed7d806afd2113f9"}, + {file = "cryptography-40.0.2-cp36-abi3-win_amd64.whl", hash = "sha256:b12794f01d4cacfbd3177b9042198f3af1c856eedd0a98f10f141385c809a14b"}, + {file = "cryptography-40.0.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:142bae539ef28a1c76794cca7f49729e7c54423f615cfd9b0b1fa90ebe53244b"}, + {file = "cryptography-40.0.2-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:956ba8701b4ffe91ba59665ed170a2ebbdc6fc0e40de5f6059195d9f2b33ca0e"}, + {file = "cryptography-40.0.2-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4f01c9863da784558165f5d4d916093737a75203a5c5286fde60e503e4276c7a"}, + {file = "cryptography-40.0.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:3daf9b114213f8ba460b829a02896789751626a2a4e7a43a28ee77c04b5e4958"}, + {file = "cryptography-40.0.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48f388d0d153350f378c7f7b41497a54ff1513c816bcbbcafe5b829e59b9ce5b"}, + {file = "cryptography-40.0.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c0764e72b36a3dc065c155e5b22f93df465da9c39af65516fe04ed3c68c92636"}, + {file = "cryptography-40.0.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:cbaba590180cba88cb99a5f76f90808a624f18b169b90a4abb40c1fd8c19420e"}, + {file = "cryptography-40.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7a38250f433cd41df7fcb763caa3ee9362777fdb4dc642b9a349721d2bf47404"}, + {file = "cryptography-40.0.2.tar.gz", hash = "sha256:c33c0d32b8594fa647d2e01dbccc303478e16fdd7cf98652d5b3ed11aa5e5c99"}, ] [package.dependencies] @@ -1346,52 +1346,52 @@ files = [ [[package]] name = "numpy" -version = "1.24.2" +version = "1.24.3" description = "Fundamental package for array computing in Python" category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "numpy-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d"}, - {file = "numpy-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5"}, - {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253"}, - {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978"}, - {file = "numpy-1.24.2-cp310-cp310-win32.whl", hash = "sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9"}, - {file = "numpy-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0"}, - {file = "numpy-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a"}, - {file = "numpy-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0"}, - {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281"}, - {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910"}, - {file = "numpy-1.24.2-cp311-cp311-win32.whl", hash = "sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95"}, - {file = "numpy-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04"}, - {file = "numpy-1.24.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2"}, - {file = "numpy-1.24.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5"}, - {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a"}, - {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96"}, - {file = "numpy-1.24.2-cp38-cp38-win32.whl", hash = "sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d"}, - {file = "numpy-1.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756"}, - {file = "numpy-1.24.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a"}, - {file = "numpy-1.24.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f"}, - {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb"}, - {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780"}, - {file = "numpy-1.24.2-cp39-cp39-win32.whl", hash = "sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468"}, - {file = "numpy-1.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f"}, - {file = "numpy-1.24.2.tar.gz", hash = "sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22"}, + {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"}, + {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"}, + {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"}, + {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"}, + {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"}, + {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"}, + {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"}, + {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"}, + {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"}, + {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"}, + {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"}, + {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"}, + {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"}, + {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"}, + {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"}, + {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"}, + {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"}, + {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"}, + {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"}, + {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"}, + {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"}, + {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"}, + {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"}, + {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"}, + {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"}, ] [[package]] name = "packaging" -version = "23.0" +version = "23.1" description = "Core utilities for Python packages" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, - {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] [[package]] @@ -1461,25 +1461,25 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "psycopg2" -version = "2.9.5" +version = "2.9.6" description = "psycopg2 - Python-PostgreSQL Database Adapter" category = "dev" optional = false python-versions = ">=3.6" files = [ - {file = "psycopg2-2.9.5-cp310-cp310-win32.whl", hash = "sha256:d3ef67e630b0de0779c42912fe2cbae3805ebaba30cda27fea2a3de650a9414f"}, - {file = "psycopg2-2.9.5-cp310-cp310-win_amd64.whl", hash = "sha256:4cb9936316d88bfab614666eb9e32995e794ed0f8f6b3b718666c22819c1d7ee"}, - {file = "psycopg2-2.9.5-cp311-cp311-win32.whl", hash = "sha256:093e3894d2d3c592ab0945d9eba9d139c139664dcf83a1c440b8a7aa9bb21955"}, - {file = "psycopg2-2.9.5-cp311-cp311-win_amd64.whl", hash = "sha256:920bf418000dd17669d2904472efeab2b20546efd0548139618f8fa305d1d7ad"}, - {file = "psycopg2-2.9.5-cp36-cp36m-win32.whl", hash = "sha256:b9ac1b0d8ecc49e05e4e182694f418d27f3aedcfca854ebd6c05bb1cffa10d6d"}, - {file = "psycopg2-2.9.5-cp36-cp36m-win_amd64.whl", hash = "sha256:fc04dd5189b90d825509caa510f20d1d504761e78b8dfb95a0ede180f71d50e5"}, - {file = "psycopg2-2.9.5-cp37-cp37m-win32.whl", hash = "sha256:922cc5f0b98a5f2b1ff481f5551b95cd04580fd6f0c72d9b22e6c0145a4840e0"}, - {file = "psycopg2-2.9.5-cp37-cp37m-win_amd64.whl", hash = "sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a"}, - {file = "psycopg2-2.9.5-cp38-cp38-win32.whl", hash = "sha256:f5b6320dbc3cf6cfb9f25308286f9f7ab464e65cfb105b64cc9c52831748ced2"}, - {file = "psycopg2-2.9.5-cp38-cp38-win_amd64.whl", hash = "sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e"}, - {file = "psycopg2-2.9.5-cp39-cp39-win32.whl", hash = "sha256:322fd5fca0b1113677089d4ebd5222c964b1760e361f151cbb2706c4912112c5"}, - {file = "psycopg2-2.9.5-cp39-cp39-win_amd64.whl", hash = "sha256:190d51e8c1b25a47484e52a79638a8182451d6f6dff99f26ad9bd81e5359a0fa"}, - {file = "psycopg2-2.9.5.tar.gz", hash = "sha256:a5246d2e683a972e2187a8714b5c2cf8156c064629f9a9b1a873c1730d9e245a"}, + {file = "psycopg2-2.9.6-cp310-cp310-win32.whl", hash = "sha256:f7a7a5ee78ba7dc74265ba69e010ae89dae635eea0e97b055fb641a01a31d2b1"}, + {file = "psycopg2-2.9.6-cp310-cp310-win_amd64.whl", hash = "sha256:f75001a1cbbe523e00b0ef896a5a1ada2da93ccd752b7636db5a99bc57c44494"}, + {file = "psycopg2-2.9.6-cp311-cp311-win32.whl", hash = "sha256:53f4ad0a3988f983e9b49a5d9765d663bbe84f508ed655affdb810af9d0972ad"}, + {file = "psycopg2-2.9.6-cp311-cp311-win_amd64.whl", hash = "sha256:b81fcb9ecfc584f661b71c889edeae70bae30d3ef74fa0ca388ecda50b1222b7"}, + {file = "psycopg2-2.9.6-cp36-cp36m-win32.whl", hash = "sha256:11aca705ec888e4f4cea97289a0bf0f22a067a32614f6ef64fcf7b8bfbc53744"}, + {file = "psycopg2-2.9.6-cp36-cp36m-win_amd64.whl", hash = "sha256:36c941a767341d11549c0fbdbb2bf5be2eda4caf87f65dfcd7d146828bd27f39"}, + {file = "psycopg2-2.9.6-cp37-cp37m-win32.whl", hash = "sha256:869776630c04f335d4124f120b7fb377fe44b0a7645ab3c34b4ba42516951889"}, + {file = "psycopg2-2.9.6-cp37-cp37m-win_amd64.whl", hash = "sha256:a8ad4a47f42aa6aec8d061fdae21eaed8d864d4bb0f0cade5ad32ca16fcd6258"}, + {file = "psycopg2-2.9.6-cp38-cp38-win32.whl", hash = "sha256:2362ee4d07ac85ff0ad93e22c693d0f37ff63e28f0615a16b6635a645f4b9214"}, + {file = "psycopg2-2.9.6-cp38-cp38-win_amd64.whl", hash = "sha256:d24ead3716a7d093b90b27b3d73459fe8cd90fd7065cf43b3c40966221d8c394"}, + {file = "psycopg2-2.9.6-cp39-cp39-win32.whl", hash = "sha256:1861a53a6a0fd248e42ea37c957d36950da00266378746588eab4f4b5649e95f"}, + {file = "psycopg2-2.9.6-cp39-cp39-win_amd64.whl", hash = "sha256:ded2faa2e6dfb430af7713d87ab4abbfc764d8d7fb73eafe96a24155f906ebf5"}, + {file = "psycopg2-2.9.6.tar.gz", hash = "sha256:f15158418fd826831b28585e2ab48ed8df2d0d98f502a2b4fe619e7d5ca29011"}, ] [[package]] @@ -1779,14 +1779,14 @@ idna2008 = ["idna"] [[package]] name = "setuptools" -version = "67.6.1" +version = "67.7.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "setuptools-67.6.1-py3-none-any.whl", hash = "sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078"}, - {file = "setuptools-67.6.1.tar.gz", hash = "sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a"}, + {file = "setuptools-67.7.2-py3-none-any.whl", hash = "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b"}, + {file = "setuptools-67.7.2.tar.gz", hash = "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"}, ] [package.extras] @@ -2016,7 +2016,7 @@ watchdog = ["watchdog"] name = "xmltodict" version = "0.13.0" description = "Makes working with XML feel like you are working with JSON" -category = "dev" +category = "main" optional = false python-versions = ">=3.4" files = [ @@ -2026,86 +2026,86 @@ files = [ [[package]] name = "yarl" -version = "1.8.2" +version = "1.9.1" description = "Yet another URL library" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"}, - {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"}, - {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58a3c13d1c3005dbbac5c9f0d3210b60220a65a999b1833aa46bd6677c69b08e"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10b08293cda921157f1e7c2790999d903b3fd28cd5c208cf8826b3b508026996"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de986979bbd87272fe557e0a8fcb66fd40ae2ddfe28a8b1ce4eae22681728fef"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c4fcfa71e2c6a3cb568cf81aadc12768b9995323186a10827beccf5fa23d4f8"}, - {file = "yarl-1.8.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae4d7ff1049f36accde9e1ef7301912a751e5bae0a9d142459646114c70ecba6"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bf071f797aec5b96abfc735ab97da9fd8f8768b43ce2abd85356a3127909d146"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:74dece2bfc60f0f70907c34b857ee98f2c6dd0f75185db133770cd67300d505f"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:df60a94d332158b444301c7f569659c926168e4d4aad2cfbf4bce0e8fb8be826"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:63243b21c6e28ec2375f932a10ce7eda65139b5b854c0f6b82ed945ba526bff3"}, - {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cfa2bbca929aa742b5084fd4663dd4b87c191c844326fcb21c3afd2d11497f80"}, - {file = "yarl-1.8.2-cp310-cp310-win32.whl", hash = "sha256:b05df9ea7496df11b710081bd90ecc3a3db6adb4fee36f6a411e7bc91a18aa42"}, - {file = "yarl-1.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:24ad1d10c9db1953291f56b5fe76203977f1ed05f82d09ec97acb623a7976574"}, - {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2a1fca9588f360036242f379bfea2b8b44cae2721859b1c56d033adfd5893634"}, - {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f37db05c6051eff17bc832914fe46869f8849de5b92dc4a3466cd63095d23dfd"}, - {file = "yarl-1.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:77e913b846a6b9c5f767b14dc1e759e5aff05502fe73079f6f4176359d832581"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0978f29222e649c351b173da2b9b4665ad1feb8d1daa9d971eb90df08702668a"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:388a45dc77198b2460eac0aca1efd6a7c09e976ee768b0d5109173e521a19daf"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2305517e332a862ef75be8fad3606ea10108662bc6fe08509d5ca99503ac2aee"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42430ff511571940d51e75cf42f1e4dbdded477e71c1b7a17f4da76c1da8ea76"}, - {file = "yarl-1.8.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3150078118f62371375e1e69b13b48288e44f6691c1069340081c3fd12c94d5b"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c15163b6125db87c8f53c98baa5e785782078fbd2dbeaa04c6141935eb6dab7a"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4d04acba75c72e6eb90745447d69f84e6c9056390f7a9724605ca9c56b4afcc6"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e7fd20d6576c10306dea2d6a5765f46f0ac5d6f53436217913e952d19237efc4"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:75c16b2a900b3536dfc7014905a128a2bea8fb01f9ee26d2d7d8db0a08e7cb2c"}, - {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6d88056a04860a98341a0cf53e950e3ac9f4e51d1b6f61a53b0609df342cc8b2"}, - {file = "yarl-1.8.2-cp311-cp311-win32.whl", hash = "sha256:fb742dcdd5eec9f26b61224c23baea46c9055cf16f62475e11b9b15dfd5c117b"}, - {file = "yarl-1.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:8c46d3d89902c393a1d1e243ac847e0442d0196bbd81aecc94fcebbc2fd5857c"}, - {file = "yarl-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ceff9722e0df2e0a9e8a79c610842004fa54e5b309fe6d218e47cd52f791d7ef"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f6b4aca43b602ba0f1459de647af954769919c4714706be36af670a5f44c9c1"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1684a9bd9077e922300ecd48003ddae7a7474e0412bea38d4631443a91d61077"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ebb78745273e51b9832ef90c0898501006670d6e059f2cdb0e999494eb1450c2"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3adeef150d528ded2a8e734ebf9ae2e658f4c49bf413f5f157a470e17a4a2e89"}, - {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57a7c87927a468e5a1dc60c17caf9597161d66457a34273ab1760219953f7f4c"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:efff27bd8cbe1f9bd127e7894942ccc20c857aa8b5a0327874f30201e5ce83d0"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a783cd344113cb88c5ff7ca32f1f16532a6f2142185147822187913eb989f739"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:705227dccbe96ab02c7cb2c43e1228e2826e7ead880bb19ec94ef279e9555b5b"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:34c09b43bd538bf6c4b891ecce94b6fa4f1f10663a8d4ca589a079a5018f6ed7"}, - {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a48f4f7fea9a51098b02209d90297ac324241bf37ff6be6d2b0149ab2bd51b37"}, - {file = "yarl-1.8.2-cp37-cp37m-win32.whl", hash = "sha256:0414fd91ce0b763d4eadb4456795b307a71524dbacd015c657bb2a39db2eab89"}, - {file = "yarl-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:d881d152ae0007809c2c02e22aa534e702f12071e6b285e90945aa3c376463c5"}, - {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5df5e3d04101c1e5c3b1d69710b0574171cc02fddc4b23d1b2813e75f35a30b1"}, - {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7a66c506ec67eb3159eea5096acd05f5e788ceec7b96087d30c7d2865a243918"}, - {file = "yarl-1.8.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2b4fa2606adf392051d990c3b3877d768771adc3faf2e117b9de7eb977741229"}, - {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e21fb44e1eff06dd6ef971d4bdc611807d6bd3691223d9c01a18cec3677939e"}, - {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93202666046d9edadfe9f2e7bf5e0782ea0d497b6d63da322e541665d65a044e"}, - {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc77086ce244453e074e445104f0ecb27530d6fd3a46698e33f6c38951d5a0f1"}, - {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dd68a92cab699a233641f5929a40f02a4ede8c009068ca8aa1fe87b8c20ae3"}, - {file = "yarl-1.8.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b372aad2b5f81db66ee7ec085cbad72c4da660d994e8e590c997e9b01e44901"}, - {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e6f3515aafe0209dd17fb9bdd3b4e892963370b3de781f53e1746a521fb39fc0"}, - {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:dfef7350ee369197106805e193d420b75467b6cceac646ea5ed3049fcc950a05"}, - {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:728be34f70a190566d20aa13dc1f01dc44b6aa74580e10a3fb159691bc76909d"}, - {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ff205b58dc2929191f68162633d5e10e8044398d7a45265f90a0f1d51f85f72c"}, - {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:baf211dcad448a87a0d9047dc8282d7de59473ade7d7fdf22150b1d23859f946"}, - {file = "yarl-1.8.2-cp38-cp38-win32.whl", hash = "sha256:272b4f1599f1b621bf2aabe4e5b54f39a933971f4e7c9aa311d6d7dc06965165"}, - {file = "yarl-1.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:326dd1d3caf910cd26a26ccbfb84c03b608ba32499b5d6eeb09252c920bcbe4f"}, - {file = "yarl-1.8.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f8ca8ad414c85bbc50f49c0a106f951613dfa5f948ab69c10ce9b128d368baf8"}, - {file = "yarl-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:418857f837347e8aaef682679f41e36c24250097f9e2f315d39bae3a99a34cbf"}, - {file = "yarl-1.8.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ae0eec05ab49e91a78700761777f284c2df119376e391db42c38ab46fd662b77"}, - {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:009a028127e0a1755c38b03244c0bea9d5565630db9c4cf9572496e947137a87"}, - {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3edac5d74bb3209c418805bda77f973117836e1de7c000e9755e572c1f7850d0"}, - {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da65c3f263729e47351261351b8679c6429151ef9649bba08ef2528ff2c423b2"}, - {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ef8fb25e52663a1c85d608f6dd72e19bd390e2ecaf29c17fb08f730226e3a08"}, - {file = "yarl-1.8.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bcd7bb1e5c45274af9a1dd7494d3c52b2be5e6bd8d7e49c612705fd45420b12d"}, - {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:44ceac0450e648de86da8e42674f9b7077d763ea80c8ceb9d1c3e41f0f0a9951"}, - {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:97209cc91189b48e7cfe777237c04af8e7cc51eb369004e061809bcdf4e55220"}, - {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:48dd18adcf98ea9cd721a25313aef49d70d413a999d7d89df44f469edfb38a06"}, - {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e59399dda559688461762800d7fb34d9e8a6a7444fd76ec33220a926c8be1516"}, - {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d617c241c8c3ad5c4e78a08429fa49e4b04bedfc507b34b4d8dceb83b4af3588"}, - {file = "yarl-1.8.2-cp39-cp39-win32.whl", hash = "sha256:cb6d48d80a41f68de41212f3dfd1a9d9898d7841c8f7ce6696cf2fd9cb57ef83"}, - {file = "yarl-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778"}, - {file = "yarl-1.8.2.tar.gz", hash = "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562"}, + {file = "yarl-1.9.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e124b283a04cc06d22443cae536f93d86cd55108fa369f22b8fe1f2288b2fe1c"}, + {file = "yarl-1.9.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:56956b13ec275de31fe4fb991510b735c4fb3e1b01600528c952b9ac90464430"}, + {file = "yarl-1.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ecaa5755a39f6f26079bf13f336c67af589c222d76b53cd3824d3b684b84d1f1"}, + {file = "yarl-1.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92a101f6d5a9464e86092adc36cd40ef23d18a25bfb1eb32eaeb62edc22776bb"}, + {file = "yarl-1.9.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92e37999e36f9f3ded78e9d839face6baa2abdf9344ea8ed2735f495736159de"}, + {file = "yarl-1.9.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef7e2f6c47c41e234600a02e1356b799761485834fe35d4706b0094cb3a587ee"}, + {file = "yarl-1.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d7a0075a55380b19aa43b9e8056e128b058460d71d75018a4f9d60ace01e78c"}, + {file = "yarl-1.9.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2f01351b7809182822b21061d2a4728b7b9e08f4585ba90ee4c5c4d3faa0812"}, + {file = "yarl-1.9.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6cf47fe9df9b1ededc77e492581cdb6890a975ad96b4172e1834f1b8ba0fc3ba"}, + {file = "yarl-1.9.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:098bdc06ffb4db39c73883325b8c738610199f5f12e85339afedf07e912a39af"}, + {file = "yarl-1.9.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:6cdb47cbbacae8e1d7941b0d504d0235d686090eef5212ca2450525905e9cf02"}, + {file = "yarl-1.9.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:73a4b46689f2d59c8ec6b71c9a0cdced4e7863dd6eb98a8c30ea610e191f9e1c"}, + {file = "yarl-1.9.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:65d952e464df950eed32bb5dcbc1b4443c7c2de4d7abd7265b45b1b3b27f5fa2"}, + {file = "yarl-1.9.1-cp310-cp310-win32.whl", hash = "sha256:39a7a9108e9fc633ae381562f8f0355bb4ba00355218b5fb19cf5263fcdbfa68"}, + {file = "yarl-1.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:b63d41e0eecf3e3070d44f97456cf351fff7cb960e97ecb60a936b877ff0b4f6"}, + {file = "yarl-1.9.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4295790981630c4dab9d6de7b0f555a4c8defe3ed7704a8e9e595a321e59a0f5"}, + {file = "yarl-1.9.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b2b2382d59dec0f1fdca18ea429c4c4cee280d5e0dbc841180abb82e188cf6e9"}, + {file = "yarl-1.9.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:575975d28795a61e82c85f114c02333ca54cbd325fd4e4b27598c9832aa732e7"}, + {file = "yarl-1.9.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9bb794882818fae20ff65348985fdf143ea6dfaf6413814db1848120db8be33e"}, + {file = "yarl-1.9.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89da1fd6068553e3a333011cc17ad91c414b2100c32579ddb51517edc768b49c"}, + {file = "yarl-1.9.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d817593d345fefda2fae877accc8a0d9f47ada57086da6125fa02a62f6d1a94"}, + {file = "yarl-1.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85aa6fd779e194901386709e0eedd45710b68af2709f82a84839c44314b68c10"}, + {file = "yarl-1.9.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eed9827033b7f67ad12cb70bd0cb59d36029144a7906694317c2dbf5c9eb5ddd"}, + {file = "yarl-1.9.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:df747104ef27ab1aa9a1145064fa9ea26ad8cf24bfcbdba7db7abf0f8b3676b9"}, + {file = "yarl-1.9.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:efec77851231410125cb5be04ec96fa4a075ca637f415a1f2d2c900b09032a8a"}, + {file = "yarl-1.9.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:d5c407e530cf2979ea383885516ae79cc4f3c3530623acf5e42daf521f5c2564"}, + {file = "yarl-1.9.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:f76edb386178a54ea7ceffa798cb830c3c22ab50ea10dfb25dc952b04848295f"}, + {file = "yarl-1.9.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:75676110bce59944dd48fd18d0449bd37eaeb311b38a0c768f7670864b5f8b68"}, + {file = "yarl-1.9.1-cp311-cp311-win32.whl", hash = "sha256:9ba5a18c4fbd408fe49dc5da85478a76bc75c1ce912d7fd7b43ed5297c4403e1"}, + {file = "yarl-1.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:b20a5ddc4e243cbaa54886bfe9af6ffc4ba4ef58f17f1bb691e973eb65bba84d"}, + {file = "yarl-1.9.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:791357d537a09a194f92b834f28c98d074e7297bac0a8f1d5b458a906cafa17c"}, + {file = "yarl-1.9.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89099c887338608da935ba8bee027564a94f852ac40e472de15d8309517ad5fe"}, + {file = "yarl-1.9.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:395ea180257a3742d09dcc5071739682a95f7874270ebe3982d6696caec75be0"}, + {file = "yarl-1.9.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:90ebaf448b5f048352ec7c76cb8d452df30c27cb6b8627dfaa9cf742a14f141a"}, + {file = "yarl-1.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f878a78ed2ccfbd973cab46dd0933ecd704787724db23979e5731674d76eb36f"}, + {file = "yarl-1.9.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:74390c2318d066962500045aa145f5412169bce842e734b8c3e6e3750ad5b817"}, + {file = "yarl-1.9.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f8e73f526140c1c32f5fca4cd0bc3b511a1abcd948f45b2a38a95e4edb76ca72"}, + {file = "yarl-1.9.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ac8e593df1fbea820da7676929f821a0c7c2cecb8477d010254ce8ed54328ea8"}, + {file = "yarl-1.9.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:01cf88cb80411978a14aa49980968c1aeb7c18a90ac978c778250dd234d8e0ba"}, + {file = "yarl-1.9.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:97d76a3128f48fa1c721ef8a50e2c2f549296b2402dc8a8cde12ff60ed922f53"}, + {file = "yarl-1.9.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:01a073c9175481dfed6b40704a1b67af5a9435fc4a58a27d35fd6b303469b0c7"}, + {file = "yarl-1.9.1-cp37-cp37m-win32.whl", hash = "sha256:ecad20c3ef57c513dce22f58256361d10550a89e8eaa81d5082f36f8af305375"}, + {file = "yarl-1.9.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f5bcb80006efe9bf9f49ae89711253dd06df8053ff814622112a9219346566a7"}, + {file = "yarl-1.9.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e7ddebeabf384099814353a2956ed3ab5dbaa6830cc7005f985fcb03b5338f05"}, + {file = "yarl-1.9.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:13a1ad1f35839b3bb5226f59816b71e243d95d623f5b392efaf8820ddb2b3cd5"}, + {file = "yarl-1.9.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f0cd87949d619157a0482c6c14e5011f8bf2bc0b91cb5087414d9331f4ef02dd"}, + {file = "yarl-1.9.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d21887cbcf6a3cc5951662d8222bc9c04e1b1d98eebe3bb659c3a04ed49b0eec"}, + {file = "yarl-1.9.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4764114e261fe49d5df9b316b3221493d177247825c735b2aae77bc2e340d800"}, + {file = "yarl-1.9.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3abe37fd89a93ebe0010417ca671f422fa6fcffec54698f623b09f46b4d4a512"}, + {file = "yarl-1.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9fe3a1c073ab80a28a06f41d2b623723046709ed29faf2c56bea41848597d86"}, + {file = "yarl-1.9.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3b5f8da07a21f2e57551f88a6709c2d340866146cf7351e5207623cfe8aad16"}, + {file = "yarl-1.9.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:88f6413ff5edfb9609e2769e32ce87a62353e66e75d264bf0eaad26fb9daa8f2"}, + {file = "yarl-1.9.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b5d5fb6c94b620a7066a3adb7c246c87970f453813979818e4707ac32ce4d7bd"}, + {file = "yarl-1.9.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f206adb89424dca4a4d0b31981869700e44cd62742527e26d6b15a510dd410a2"}, + {file = "yarl-1.9.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:44fa6158e6b4b8ccfa2872c3900a226b29e8ce543ce3e48aadc99816afa8874d"}, + {file = "yarl-1.9.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:08c8599d6aa8a24425f8635f6c06fa8726afe3be01c8e53e236f519bcfa5db5b"}, + {file = "yarl-1.9.1-cp38-cp38-win32.whl", hash = "sha256:6b09cce412386ea9b4dda965d8e78d04ac5b5792b2fa9cced3258ec69c7d1c16"}, + {file = "yarl-1.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:09c56a32c26e24ef98d5757c5064e252836f621f9a8b42737773aa92936b8e08"}, + {file = "yarl-1.9.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b86e98c3021b7e2740d8719bf074301361bf2f51221ca2765b7a58afbfbd9042"}, + {file = "yarl-1.9.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5faf3ec98747318cb980aaf9addf769da68a66431fc203a373d95d7ee9c1fbb4"}, + {file = "yarl-1.9.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a21789bdf28549d4eb1de6910cabc762c9f6ae3eef85efc1958197c1c6ef853b"}, + {file = "yarl-1.9.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8b8d4b478a9862447daef4cafc89d87ea4ed958672f1d11db7732b77ead49cc"}, + {file = "yarl-1.9.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:307a782736ebf994e7600dcaeea3b3113083584da567272f2075f1540919d6b3"}, + {file = "yarl-1.9.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:46c4010de941e2e1365c07fb4418ddca10fcff56305a6067f5ae857f8c98f3a7"}, + {file = "yarl-1.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bab67d041c78e305ff3eef5e549304d843bd9b603c8855b68484ee663374ce15"}, + {file = "yarl-1.9.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1baf8cdaaab65d9ccedbf8748d626ad648b74b0a4d033e356a2f3024709fb82f"}, + {file = "yarl-1.9.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:27efc2e324f72df02818cd72d7674b1f28b80ab49f33a94f37c6473c8166ce49"}, + {file = "yarl-1.9.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ca14b84091700ae7c1fcd3a6000bd4ec1a3035009b8bcb94f246741ca840bb22"}, + {file = "yarl-1.9.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c3ca8d71b23bdf164b36d06df2298ec8a5bd3de42b17bf3e0e8e6a7489195f2c"}, + {file = "yarl-1.9.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:8c72a1dc7e2ea882cd3df0417c808ad3b69e559acdc43f3b096d67f2fb801ada"}, + {file = "yarl-1.9.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d966cd59df9a4b218480562e8daab39e87e746b78a96add51a3ab01636fc4291"}, + {file = "yarl-1.9.1-cp39-cp39-win32.whl", hash = "sha256:518a92a34c741836a315150460b5c1c71ae782d569eabd7acf53372e437709f7"}, + {file = "yarl-1.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:78755ce43b6e827e65ec0c68be832f86d059fcf05d4b33562745ebcfa91b26b1"}, + {file = "yarl-1.9.1.tar.gz", hash = "sha256:5ce0bcab7ec759062c818d73837644cde567ab8aa1e0d6c45db38dfb7c284441"}, ] [package.dependencies] @@ -2118,4 +2118,4 @@ fhir = ["fhirclient"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "7fe5d05e0cc53f260af45a159cbd80f131dab403c93ae0cb313164a760b4c75a" +content-hash = "a021962a44bf6c04178cd9e89d58704035c718a0860bcf0625809524b12208e7" diff --git a/pyproject.toml b/pyproject.toml index 1ed4a2b54..5758143ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "gen3" homepage = "https://gen3.org/" -version = "4.17.2" +version = "4.18.0" description = "Gen3 CLI and Python SDK" authors = ["Center for Translational Data Science at the University of Chicago "] license = "Apache-2.0" @@ -35,6 +35,7 @@ python-dateutil = "*" aiofiles = "^0.8.0" pandas = "^1.4.2" httpx = "*" +xmltodict = "^0.13.0" # A list of all of the optional dependencies, some of which are included in the # below `extras`. They can be opted into by apps. diff --git a/tests/test_doi.py b/tests/test_doi.py new file mode 100644 index 000000000..3a41d6328 --- /dev/null +++ b/tests/test_doi.py @@ -0,0 +1,459 @@ +""" +Tests gen3.doi +""" +import json +import os +import pytest +import sys + +import requests +from requests.auth import HTTPBasicAuth +from unittest.mock import MagicMock, patch + +import gen3 +from gen3.doi import ( + DataCite, + DigitalObjectIdentifier, + DataCiteDOIValidationError, + DigitalObjectIdentifierCreator, + DigitalObjectIdentifierTitle, +) + + +@pytest.mark.parametrize( + "expect_failure,doi_type,prefix,identifier,creators,titles,publisher,publication_year,descriptions", + [ + # everything normal + ( + False, + "Dataset", + "10.12345", + "ID1234", + ["test"], + ["test title"], + "test", + 2000, + ["this is a test description"], + ), + # providing prefix but not ID + ( + False, + "Dataset", + "10.12345", + "ID1234", + ["test"], + ["test title"], + "test", + 2000, + ["this is a test description"], + ), + # providing ID but not prefix + ( + False, + "Dataset", + "10.12345", + "ID1234", + ["test"], + ["test title"], + "test", + 2000, + ["this is a test description"], + ), + # using non valid DOI Type + ( + True, + "Not a valid doi type", + "10.12345", + "ID1234", + ["test"], + ["test title"], + "test", + 2000, + ["this is a test description"], + ), + # not providing prefix or ID + ( + True, + "Dataset", + None, + None, + ["test"], + ["test title"], + "test", + 2000, + ["this is a test description"], + ), + # special chars and spaces in creators + ( + False, + "Dataset", + "10.12345", + "ID1234", + ["test", "!@#$% ^&*|\\O({@[< >?;:]})", "Name. MIDDLE Last"], + ["test title"], + "test", + 2000, + ["this is a test description"], + ), + # special chars and spaces in titles + ( + False, + "Dataset", + "10.12345", + "ID1234", + ["test"], + ["!@#$% ^&*|\\O({@[< >?;:]})"], + "test", + 2000, + ["this is a test description"], + ), + # special chars and spaces in publisher + ( + False, + "Dataset", + "10.12345", + "ID1234", + ["test"], + ["test"], + "Some publisher with Numbers 123 and Symbols !@#$%^&*|\\O({@[<>?;:]})", + 2000, + ["this is a test description"], + ), + # large publication_year + ( + False, + "Dataset", + "10.12345", + "ID1234", + ["test"], + ["test title"], + "!@#$%^&*|\\O({@[<>?;:]})", + 5000, + ["this is a test description"], + ), + # special chars and spaces in description + ( + False, + "Dataset", + "10.12345", + "ID1234", + ["test"], + ["test title"], + "test", + 2000, + ["this is a test description !@#$%^&*|\\O({@[<>?;:]})"], + ), + # multiple creators + ( + False, + "Dataset", + "10.12345", + "ID1234", + ["test", "test2", "test3"], + ["test title"], + "test", + 2000, + ["this is a test description !@#$%^&*|\\O({@[<>?;:]})"], + ), + # multiple titles + ( + False, + "Dataset", + "10.12345", + "ID1234", + ["test"], + ["test title", "test title2", "test title 3"], + "test", + 2000, + ["this is a test description !@#$%^&*|\\O({@[<>?;:]})"], + ), + # multiple descriptions + ( + False, + "Dataset", + "10.12345", + "ID1234", + ["test"], + ["test title"], + "test", + 2000, + ["test description", "test description2", "test description 3"], + ), + ], +) +@patch("gen3.doi.requests.post") +def test_create_doi( + requests_mock, + expect_failure, + prefix, + identifier, + doi_type, + creators, + titles, + publisher, + publication_year, + descriptions, +): + """ + Test DOI creation under various inputs, ensure it still works even when + special characters are used. Ensure it doesn't work when invalid input is + provided (like not specify prefix OR id). + """ + datacite = DataCite( + auth_provider=HTTPBasicAuth("DATACITE_USERNAME", "DATACITE_PASSWORD") + ) + if expect_failure: + with pytest.raises(DataCiteDOIValidationError): + doi = DigitalObjectIdentifier( + prefix=prefix, + identifier=identifier, + creators=creators, + titles=titles, + publisher=publisher, + publication_year=publication_year, + doi_type=doi_type, + url="https://example.com", + version="0.1", + descriptions=descriptions, + foobar="test", + ) + else: + doi = DigitalObjectIdentifier( + prefix=prefix, + identifier=identifier, + creators=creators, + titles=titles, + publisher=publisher, + publication_year=publication_year, + doi_type=doi_type, + url="https://example.com", + version="0.1", + descriptions=descriptions, + foobar="test", + ) + + def _mock_request(url, **kwargs): + assert url.endswith("/dois") + + mocked_response = MagicMock(requests.Response) + mocked_response.status_code = 200 + mocked_response.json.return_value = { + "status": "OK", + } + mocked_response.raise_for_status.side_effect = lambda *args: None + + return mocked_response + + requests_mock.side_effect = _mock_request + + if not expect_failure: + response = datacite.create_doi(doi) + response.raise_for_status() + assert requests_mock.call_count == 1 + + data_in_request = ( + requests_mock.call_args.kwargs.get("json", {}) + .get("data", {}) + .get("attributes", {}) + ) + + for creator in creators: + assert creator in data_in_request.get("creators") + for title in titles: + assert title in data_in_request.get("titles") + assert data_in_request.get("publisher") == publisher + assert data_in_request.get("publicationYear") == publication_year + assert data_in_request.get("types", {}).get("resourceTypeGeneral") == doi_type + for description in descriptions: + assert description in data_in_request.get("descriptions") + + +@pytest.mark.parametrize( + "doi_metadata_already_exists, existing_metadata", + [ + # non Discovery, top-level metadata + (True, {"alreadyexists": "stuff"}), + # Discovery, non DOI metadata + ( + True, + { + "_guid_type": "discovery_metadata", + "gen3_discovery": {"otherstuff": "foobar"}, + }, + ), + # Discovery, DOI metadata + ( + True, + { + "_guid_type": "discovery_metadata", + "gen3_discovery": {"doi_version": 0}, + }, + ), + (False, {}), + ], +) +@patch("gen3.doi.requests.get") +@patch("gen3.doi.requests.post") +@patch("gen3.doi.requests.put") +def test_doi_metadata_persist( + doi_requests_put_mock, + doi_requests_post_mock, + doi_requests_get_mock, + gen3_auth, + doi_metadata_already_exists, + existing_metadata, +): + """ + Test the DOI metadata persistance into Gen3's metadata service with and + without existing metadata existing. + """ + # Setup + datacite = DataCite( + api=DataCite.TEST_URL, + auth_provider=HTTPBasicAuth( + os.environ.get("DATACITE_USERNAME"), + os.environ.get("DATACITE_PASSWORD"), + ), + ) + + gen3_metadata_guid = "Example-Study-01" + + # Get DOI metadata (ideally from some external source) + identifier = "10.12345/268Z-O151" + creators = [ + DigitalObjectIdentifierCreator( + name="Bar, Foo", + name_type=DigitalObjectIdentifierCreator.NAME_TYPE_PERSON, + ).as_dict() + ] + titles = [DigitalObjectIdentifierTitle("Some Example Study in Gen3").as_dict()] + publisher = "Example Gen3 Sponsor" + publication_year = 2023 + doi_type = "Dataset" + version = 1 + + doi_metadata = { + "identifier": identifier, + "creators": creators, + "titles": titles, + "publisher": publisher, + "publication_year": publication_year, + "doi_type": doi_type, + "version": version, + } + + # Create/Mint the DOI in DataCite + doi = DigitalObjectIdentifier(root_url="foobar", **doi_metadata) + + def _mock_metadata_create_request(url, **kwargs): + assert f"/metadata/{gen3_metadata_guid}" in url + + mocked_response = MagicMock(requests.Response) + mocked_response.status_code = 200 + mocked_response.json.return_value = { + "status": "OK", + } + mocked_response.raise_for_status.side_effect = lambda *args: None + + return mocked_response + + def _mock_metadata_get_request(url, **kwargs): + assert f"/metadata/{gen3_metadata_guid}" in url + + mocked_response = MagicMock(requests.Response) + mocked_response.status_code = 200 + + nonlocal existing_metadata + mocked_response.json.return_value = existing_metadata + + mocked_response.raise_for_status.side_effect = lambda *args: None + + if f"/metadata/{gen3_metadata_guid}/aliases" in url: + mocked_response.json.return_value = {"aliases": []} + + return mocked_response + + def _mock_metadata_get_request_doesnt_exist(url, **kwargs): + assert f"/metadata/{gen3_metadata_guid}" in url + + mocked_response = MagicMock(requests.Response) + mocked_response.status_code = 404 + mocked_response.json.return_value = {} + + def _raise(): + error = requests.exceptions.HTTPError(mocked_response) + error.status_code = 404 + raise error + + mocked_response.raise_for_status.side_effect = _raise + + if f"/metadata/{gen3_metadata_guid}/aliases" in url: + mocked_response.json.return_value = {"aliases": []} + + return mocked_response + + doi_requests_put_mock.side_effect = _mock_metadata_create_request + doi_requests_post_mock.side_effect = _mock_metadata_create_request + + if doi_metadata_already_exists: + doi_requests_get_mock.side_effect = _mock_metadata_get_request + else: + doi_requests_get_mock.side_effect = _mock_metadata_get_request_doesnt_exist + + # Persist necessary DOI Metadata in Gen3 Discovery to support the landing page + metadata = datacite.persist_doi_metadata_in_gen3( + guid=gen3_metadata_guid, + doi=doi, + auth=gen3_auth, + additional_metadata={ + "disclaimer": "disclaimer 1", + "access_information": "access_information 1", + "access_information_link": "access_information_link 1", + "contact": "contact 1", + }, + prefix="doi_", + ) + assert doi_requests_get_mock.call_count == 1 + + expected_output = { + "_guid_type": "discovery_metadata", + "gen3_discovery": { + "doi_identifier": identifier, + "doi_resolveable_link": f"https://doi.org/{identifier}", + "doi_creators": "Bar, Foo", + "doi_titles": "Some Example Study in Gen3", + "doi_publisher": "Example Gen3 Sponsor", + "doi_publication_year": 2023, + "doi_resource_type": "Dataset", + "doi_url": f"foobar/{identifier}", + "doi_version": 1, + "doi_is_available": "Yes", + "doi_disclaimer": "disclaimer 1", + "doi_citation": f"Bar, Foo (2023). Some Example Study in Gen3 1. Example Gen3 Sponsor. Dataset. {identifier}", + "doi_version_information": "This is version 1 of this Dataset.", + "doi_access_information": "access_information 1", + "doi_contact": "contact 1", + "doi_access_information_link": "access_information_link 1", + }, + } + + if doi_metadata_already_exists: + # metadata created should prefer the new DOI information over existing + expected_updated_output = existing_metadata + expected_updated_output.update(expected_output) + assert doi_requests_put_mock.call_count == 2 + + metadata_call = doi_requests_put_mock.call_args_list[0] + alias_call = doi_requests_put_mock.call_args_list[1] + + assert metadata_call.kwargs.get("json") == expected_updated_output + assert f"{identifier}" in alias_call.kwargs.get("json", {}).get("aliases", []) + else: + # 1 for metadata, 1 for alias + assert doi_requests_post_mock.call_count == 2 + + metadata_call = doi_requests_post_mock.call_args_list[0] + alias_call = doi_requests_post_mock.call_args_list[1] + + assert metadata_call.kwargs.get("json") == expected_output + assert f"{identifier}" in alias_call.kwargs.get("json", {}).get("aliases", [])