diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml new file mode 100644 index 000000000..3a27e8466 --- /dev/null +++ b/.github/workflows/pypi.yaml @@ -0,0 +1,16 @@ +name: PyPI +on: + push: + tags: + - '*' +jobs: + PyPIPoetryPublish: + name: PyPI Poetry Publish + uses: uc-cdis/.github/.github/workflows/python_package_index_publish.yaml@master + with: + PYTHON_VERSION: '3.9' + # This will attempt push to test PyPI first and only push to prod if it works + DO_TEST_PUBLISH_FIRST: true + secrets: + PYPI_TEST_API_TOKEN: ${{ secrets.PYPI_TEST_API_TOKEN }} + PYPI_PROD_API_TOKEN: ${{ secrets.PYPI_PROD_API_TOKEN }} \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 20d9257b6..000000000 --- a/.travis.yml +++ /dev/null @@ -1,48 +0,0 @@ -dist: jammy -language: python -python: -- 3.8 -- 3.9 -- 3.10 -jobs: - allow_failures: - - python: 3.8 - - python: 3.10 -before_install: -- pip install --upgrade pip -- curl -sSL https://install.python-poetry.org | python - -- which poetry -- poetry --version -install: -- poetry install -vv --all-extras --no-interaction --no-cache -- poetry show -vv -script: -- echo Unit testing is now done in GH Actions, this just pushing to pypi as necessary -before_deploy: -- poetry config repositories.testpypi https://test.pypi.org/legacy/ -- poetry config pypi-token.testpypi $PYPI_API_TOKEN -- poetry config pypi-token.pypi $PYPI_PROD_API_TOKEN -- poetry build -deploy: -# test pypi deployment on branches that are not master -- provider: script - skip_cleanup: true - script: poetry publish -n -vv -r testpypi || true - on: - python: 3.9 - repo: uc-cdis/gen3sdk-python - tags: false - all_branches: true - condition: "$TRAVIS_BRANCH != master" -# real pypi deployment on tags on master -- provider: script - skip_cleanup: true - script: poetry publish -n -vv - on: - python: 3.9 - repo: uc-cdis/gen3sdk-python - tags: true -env: - global: - - secure: jCslhsRF+cpozrC75nmYkBizmdBL/01StonnarwaWmCFBEzX22i6XklpUyROYRpvHTQ7SHJlFh+kmvptT4UmRxFCqKq/RNFq2We0rRhnfWNRsQOok5ENU5MDMgD0kl3RUeEElpVk36aQ8cUyybNyuel0ZFC1F+C5GOUKaCtiDrcX9RVu5jYbydi1W1RfsCNkmyJND8efdqBXD81BueXqP7xYxdyqz+f3du/1qIx5GR/X+jv6WE2Oum6EMq5tbiWf3qc6OnxNdNd0p/O7seBIZyvOZvFvX8Cx3PsCfSbWyeX0hcZCDIhnG5YTah944MplvVR70lOx0GvzmxHZCpxffUfliXLJCrBQy9RZbomXcXI6c1h88DMt6odYNDlZJpcV12sQypbWBr7FJb1En93FcSqpdoPKEU9eR+WzQ0/GNFL5KibZ95yQdwT31gnlnz7NLQz8ubdkrKMpZTbDM1HCAgLN326jMwYpn/RxRtXFcrxvA6sTxpsCQhWssIBRHv5sY0c5vlDkLADNAeA1s9+SOKWAKHJh5NzFby+vt59vi5QSkdWRsKwZsk01NNrBUFcJuDxsp9wTEO3NigzoYrnZXk2ews9+WRaiTmsEWa4/LSfY0afQA4ezbGVT4yY/tje2XFhsaYCT/JsJpHBgyNDadX6FaAraKw4oqsQDu76/KjU= - - secure: cFv/73omvtKB7/3OYuIodzNwSLUlqJuGgtyTf0W6pN5oJSpcE9U4Ti/ub2aUABaH15e0Fdp+MxaL+j4x5/TmzGwrmpP8Ts2nueJNh1XeC2aO2S+YevRuV+8L4rwbFvZzYe/rL9hbKx9Ghiatc/fk147j4Svf/ZNWPBHgBloZaXWGR2kRxrRrjnogXceTR6K2mPUMUr0+azYDH0Zwuu3Y5HF/CT5DpxWGs657VjM6EiGsZ6UUq4sqj4XZKKjTtp/I8zPH1z6NliBDqVtASX3UsZewbiDaa1DIs4NBN5QDSR3/vQfVoouPFy7M/oIqIpvDFOWfHahA8ac3/CdlMAT64KknRodpm/s2uUxJOtrDOSBMtdREEc+yMuBPf85+s+Vk6B8upkIYnpI2d7vDaHSFBIlQk/IbaoUsRUVeacQ6b20JhPyYMrKnvG9/aW7KfYJr23SpRwjwvh6D4ejbmFtePrU/uk1Js81lCUkYHBBqDvRXW/BNMKxyXTyMlCL4om4gIlQpBmru+vpDGoHHLzO+I9axmBBKWq1HgR5nF89Nd6AMoSTzAwt+7xmMsgaDaMewZczBnLyeGiKNKATkQ66qdhqUB+6dmKpTyDCFOwRaDUOaH71f3wVamOQpoHUdCPNnIh1B6RuZaXylGyO2qtsr65bC2uZCHVw9dk3eeqL2TUU= diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index 567a9e0da..c9a3049f3 100644 Binary files a/docs/_build/doctrees/environment.pickle and b/docs/_build/doctrees/environment.pickle differ diff --git a/docs/_build/doctrees/tools/indexing.doctree b/docs/_build/doctrees/tools/indexing.doctree index 82f969911..b6ce88608 100644 Binary files a/docs/_build/doctrees/tools/indexing.doctree and b/docs/_build/doctrees/tools/indexing.doctree differ diff --git a/docs/_build/doctrees/tools/metadata.doctree b/docs/_build/doctrees/tools/metadata.doctree index 7164fa686..1fbaf51b0 100644 Binary files a/docs/_build/doctrees/tools/metadata.doctree and b/docs/_build/doctrees/tools/metadata.doctree differ diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index 3c5b95d6a..4a3f953f0 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "terms": {"class": [0, 2, 10, 11, 13], "gen3auth": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "endpoint": [0, 1, 2, 3, 4, 5, 7, 8, 13], "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "idp": 0, "client_credenti": 0, "client_scop": 0, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "authbas": 0, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "request": [0, 1, 3, 5, 8, 11, 12], "implement": 0, "order": [0, 8], "support": [0, 1, 5, 8, 11], "jwt": 0, "authent": 0, "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "access": [0, 1, 3, 7, 10], "token": [0, 10], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "provid": [0, 1, 3, 5, 7, 8, 12], "refresh": [0, 10], "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "string": [0, 3, 5, 11, 13], "automat": 0, "when": [0, 3, 7, 10, 12], "thei": [0, 10], "expir": [0, 1], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "opt": 0, "The": [0, 1, 2, 3, 5, 8, 10, 11], "contain": [0, 2, 5, 8, 9, 10, 11, 12], "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "web": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "work": [0, 10], "workspac": [0, 2], "default": [0, 1, 3, 7, 8, 11, 12], "env": 0, "gen3_api_kei": 0, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "set": [0, 1, 5, 10], "includ": [0, 3], "search": [0, 2, 3], "path": [0, 1, 5, 10, 11, 13], "valu": [0, 1, 3, 5, 7, 10, 11], "doe": [0, 12], "interpret": 0, "wt": [0, 10], "an": [0, 3, 6, 8, 10, 11], "accesstoken": 0, "If": [0, 1, 7, 11, 12], "can": [0, 3, 4, 8, 11, 12], "specifi": [0, 1, 3, 11, 13], "local": [0, 13], "indic": [0, 11], "environ": 0, "fenc": [0, 1], "tupl": [0, 3, 11, 12], "client_id": 0, "client_secret": 0, "oidc": 0, "client": [0, 3], "ha": [0, 11], "grant": 0, "allow": [0, 6, 8, 10, 11, 12], "obtain": [0, 10], "space": [0, 11], "separ": [0, 11], "list": [0, 1, 3, 4, 5, 7, 8, 10, 11, 13], "scope": [0, 1], "user": [0, 10, 12], "data": [0, 1, 3, 5, 7, 8, 10, 11], "openid": 0, "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "instal": [0, 2, 11], "crdc": 0, "some": [0, 2], "arbitrari": 0, "kei": [0, 3, 5, 13], "variabl": [0, 7, 8], "rather": 0, "than": [0, 5], "pass": [0, 7, 8, 10], "argument": [0, 13], "constructor": 0, "id": [0, 1, 3, 5, 10, 11], "secret": 0, "note": [0, 3, 11, 12], "should": [0, 8, 11], "never": 0, "hardcod": 0, "http": [0, 7, 11, 12], "datacommon": 0, "o": 0, "gen3_oidc_client_creds_secret": 0, "initi": [0, 10], "follow": [0, 11], "curl": [0, 2], "given": [0, 3, 4, 5, 8, 10, 12, 13], "ex": [0, 11, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "respons": [0, 1, 3, 4, 5], "under": [0, 8, 13], "index": [0, 2, 5, 9], "authz": [0, 1, 2, 3, 9, 10, 11], "map": [0, 11], "get": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12, 13], "post": [0, 11], "put": 0, "delet": [0, 1, 2, 3, 5, 6, 8, 10, 11], "i": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12], "els": [0, 12], "filenam": [0, 8, 10, 11, 12], "get_access_token": [0, 2], "auto": [0, 2], "within": [0, 2, 9], "5": [0, 5], "minut": 0, "get_access_token_from_wt": [0, 2], "try": 0, "fetch": 0, "namespac": [0, 12], "refresh_access_token": [0, 2], "new": [0, 3], "gen3fil": [1, 2], "auth_provid": [1, 2, 3, 4, 5, 6, 7, 8, 13], "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "For": [1, 5, 6, 7, 8, 9, 11], "interact": [1, 3, 4, 5, 6, 8, 13], "manag": [1, 5, 10], "featur": [1, 6], "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "presign": 1, "url": [1, 2, 3, 9, 10, 11, 12, 13], "right": 1, "now": [1, 8], "instanc": [1, 3, 6, 7, 8, 9, 10], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "delete_fil": [1, 2], "guid": [1, 2, 3, 5, 6, 9, 11, 12], "method": [1, 7, 10], "deprec": 1, "delete_file_loc": [1, 2, 6], "instead": [1, 7, 11], "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "locat": [1, 6], "store": [1, 3, 10], "remov": [1, 6, 11, 13], "its": [1, 3], "record": [1, 3, 5, 7, 8, 11, 12], "indexd": [1, 3, 6, 10, 11, 12], "uuid": [1, 3, 8], "text": [1, 7, 8], "result": [1, 8, 10, 11], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "download_singl": [1, 2], "object_id": [1, 2, 9, 10], "singl": [1, 5, 8], "": [1, 4, 8, 10, 11], "uniqu": [1, 5], "get_presigned_url": [1, 2], "protocol": 1, "retriev": [1, 8, 10, 12], "give": 1, "limit": [1, 3, 5, 12], "time": [1, 3, 8, 10, 11], "pick": 1, "avail": [1, 2, 10, 11], "queri": [1, 2, 3, 5, 8, 11, 12], "upload_fil": [1, 2], "file_nam": [1, 2, 3, 9, 10, 11], "expires_in": 1, "bucket": [1, 6], "upload": [1, 2, 3, 8, 13], "author": 1, "storag": [1, 2, 6], "s3": [1, 10, 11], "az": 1, "isn": 1, "t": [1, 5, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "amount": [1, 9], "second": [1, 4], "sign": 1, "datetim": [1, 3, 10], "utcnow": 1, "Be": 1, "sure": 1, "posit": [1, 7], "integ": [1, 3, 8], "also": 1, "treat": [1, 5], "max_presigned_url_ttl": 1, "configur": 1, "must": [1, 5], "allowed_data_upload_bucket": 1, "data_upload_bucket": 1, "represent": [1, 3], "document": [1, 3], "upload_file_to_guid": [1, 2], "exist": [1, 3, 5, 6, 9, 12], "readm": 2, "python": [2, 9, 11], "github": [2, 7], "repo": 2, "setup": 2, "inform": [2, 3, 10], "along": 2, "quickstart": 2, "script": 2, "more": [2, 5, 7, 9, 10], "detail": [2, 7, 10], "about": [2, 3], "variou": 2, "function": [2, 3, 4, 5, 9, 10, 11, 12], "mostli": 2, "docstr": 2, "code": [2, 8], "helper": 2, "gen3index": [2, 3], "async_create_record": [2, 3], "async_get_record": [2, 3], "async_get_records_from_checksum": [2, 3], "async_get_records_on_pag": [2, 3], "async_get_with_param": [2, 3], "async_query_url": [2, 3], "async_update_record": [2, 3], "create_blank": [2, 3], "create_new_vers": [2, 3], "create_record": [2, 3], "delete_record": [2, 3, 8], "get_all_record": [2, 3], "get_guids_prefix": [2, 3], "get_latest_vers": [2, 3], "get_record": [2, 3], "get_record_doc": [2, 3], "get_records_on_pag": [2, 3], "get_stat": [2, 3], "get_url": [2, 3], "get_valid_guid": [2, 3], "get_vers": [2, 3, 4, 5], "get_with_param": [2, 3], "is_healthi": [2, 3, 4, 5], "query_url": [2, 3], "update_blank": [2, 3], "update_record": [2, 3], "job": 2, "gen3job": [2, 4, 10], "async_run_job_and_wait": [2, 4], "create_job": [2, 4], "get_output": [2, 4], "get_statu": [2, 4], "list_job": [2, 4], "metadata": [2, 3, 6, 9, 11], "gen3metadata": [2, 5], "async_cr": [2, 5], "async_create_alias": [2, 5], "async_delete_alia": [2, 5], "async_delete_alias": [2, 5], "async_get": [2, 5], "async_get_alias": [2, 5], "async_upd": [2, 5], "async_update_alias": [2, 5], "batch_creat": [2, 5], "creat": [2, 3, 4, 5, 6, 8, 10, 11], "create_alias": [2, 5], "create_index_key_path": [2, 5], "delete_alia": [2, 5], "delete_alias": [2, 5], "delete_index_key_path": [2, 5], "get_alias": [2, 5], "get_index_key_path": [2, 5], "updat": [2, 3, 5, 10, 11], "update_alias": [2, 5], "gen3object": [2, 6], "delete_object": [2, 6], "gen3queri": [2, 7], "graphql_queri": [2, 7], "raw_data_download": [2, 7], "submiss": 2, "gen3submiss": [2, 8], "create_program": [2, 8], "create_project": [2, 8], "delete_nod": [2, 8], "delete_program": [2, 8], "delete_project": [2, 8], "export_nod": [2, 8], "export_record": [2, 8], "get_dictionary_al": [2, 8], "get_dictionary_nod": [2, 8], "get_graphql_schema": [2, 8], "get_program": [2, 8], "get_project_dictionari": [2, 8], "get_project_manifest": [2, 8], "get_project": [2, 8], "open_project": [2, 8], "submit_fil": [2, 8], "submit_record": [2, 8], "tool": 2, "dr": [2, 9], "downloadmanag": [2, 9, 10], "cache_hosts_wts_token": [2, 9, 10], "get_fresh_token": [2, 9, 10], "resolve_object": [2, 9, 10], "user_access": [2, 9, 10], "downloadstatu": [2, 9, 10], "statu": [2, 4, 9, 10], "start_tim": [2, 9, 10], "end_tim": [2, 9, 10], "object_typ": [2, 9, 10], "hostnam": [2, 9, 10], "file_s": [2, 9, 10, 11], "updated_tim": [2, 9, 10], "created_tim": [2, 9, 10], "access_method": [2, 9, 10], "children": [2, 9, 10], "_manag": [2, 9, 10], "pprint": [2, 9, 10], "manifest": [2, 8, 9, 10, 11, 12], "md5sum": [2, 9, 10], "commons_url": [2, 9, 10, 11, 12], "create_object_list": [2, 9, 10], "load": [2, 9, 10], "load_manifest": [2, 9, 10], "download_drs_object": [], "download_files_in_drs_manifest": [2, 9, 10], "list_access_in_drs_manifest": [2, 9, 10], "list_drs_object": [2, 9, 10], "list_files_in_drs_manifest": [2, 9, 10], "current_dir": [2, 9, 11], "indexd_record_page_s": [2, 9, 11], "max_concurrent_request": [2, 9, 11, 12], "tmp_folder": [2, 9, 11], "async_download_object_manifest": [2, 9, 11], "size": [2, 3, 9, 10, 11], "md5": [2, 3, 9, 11], "acl": [2, 3, 9, 11], "prev_guid": [2, 9, 11], "threadcontrol": [2, 9, 11], "delete_all_guid": [2, 9, 11], "index_object_manifest": [2, 9, 11], "verifi": [2, 9], "async_verify_object_manifest": [2, 9, 11], "ingest": [2, 9], "column_to_use_as_guid": [2, 9, 12], "guid_type_for_indexed_file_object": [2, 9, 12], "guid_type_for_non_indexed_file_object": [2, 9, 12], "async_ingest_metadata_manifest": [2, 9, 12], "async_query_urls_from_indexd": [2, 9, 12], "gen3wsstorag": [2, 13], "copi": [2, 13], "download_url": [2, 13], "l": [2, 13], "ls_path": [2, 13], "rm": [2, 13], "rm_path": [2, 13], "upload_url": [2, 13], "modul": [2, 10, 11], "service_loc": [3, 4, 5], "public": [3, 5], "read": [3, 5, 11], "onli": [3, 5, 7, 8, 10, 11], "necessari": [3, 5], "basic": [3, 11, 12], "cred": 3, "async": [3, 4, 5, 9, 11, 12], "hash": [3, 11], "did": 3, "baseid": 3, "urls_metadata": 3, "version": [3, 4, 5], "_ssl": [3, 4, 5], "asynchron": [3, 4, 5], "dict": [3, 4, 5, 10, 11, 12], "eg": 3, "ab167e49d25b488939b1ede42752458b": 3, "associ": [3, 5], "made": 3, "where": [3, 5, 11, 12], "you": [3, 8, 11], "control": 3, "rbac": 3, "name": [3, 4, 8, 10, 11, 12, 13], "addit": [3, 5, 10, 11], "entri": [3, 11], "attach": [3, 5], "each": [3, 8, 10, 11], "group": 3, "previou": [3, 11], "checksum": [3, 10], "checksum_typ": 3, "match": [3, 5, 12], "param": [3, 5, 8, 10], "correspond": 3, "suppli": 3, "need": [3, 7, 10, 11], "handl": [3, 10], "hash_typ": 3, "whether": [3, 4, 5, 8, 11, 12], "ssl": [3, 4, 5], "pattern": [3, 12], "against": [3, 7, 8, 11, 12], "kwarg": [3, 4, 5], "bodi": 3, "dictionari": [3, 4, 5, 7, 8], "format": [3, 5, 8, 11], "blank": 3, "add": [3, 5], "sinc": 3, "content": [3, 13], "immut": 3, "want": [3, 8], "chang": [3, 11], "That": 3, "field": [3, 5, 7, 11, 12], "old": 3, "ad": 3, "least": 3, "one": [3, 5, 10, 11], "suffici": 3, "good": 3, "idea": 3, "number": [3, 7, 8, 11, 12], "noth": [3, 6], "dist_resolut": 3, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "alia": [3, 5], "distribut": 3, "identifi": [3, 5, 9, 11], "boolean": 3, "we": [3, 11], "pagin": 3, "fals": [3, 5, 6, 10, 11], "start": [3, 4, 7, 8, 10], "prefix": 3, "rtype": 3, "has_vers": 3, "latest": 3, "exclud": 3, "without": [3, 5], "repres": [3, 5, 10], "info": [3, 11], "algorithm": 3, "count": 3, "valid": [3, 7], "file_st": 3, "like": [3, 5, 9, 11, 12], "signpost": 3, "healthi": [3, 4, 5], "rev": 3, "revis": 3, "simpl": 3, "consist": 3, "mechan": 3, "dispatch": 4, "job_nam": 4, "job_input": 4, "wait": 4, "output": [4, 5, 11, 12], "Will": [4, 7, 10], "sleep": 4, "linear": 4, "delai": 4, "until": [4, 10], "done": 4, "1": [4, 5, 8, 10, 11], "global": [4, 5], "input": [4, 10, 11], "job_id": 4, "previous": 4, "complet": [4, 11], "bool": [4, 5, 8, 10, 11, 12], "md": [5, 7, 10, 12], "admin_endpoint_suffix": 5, "admin": [5, 11], "alias": 5, "overwrit": 5, "what": 5, "end": [5, 10], "up": [5, 9], "blob": [5, 7], "asyncron": 5, "altern": [5, 11], "merg": 5, "AND": 5, "descript": [3, 5], "metadata_list": 5, "specif": [5, 8, 11, 12], "expect": [5, 9, 11], "anoth": 5, "databas": 5, "return_full_metadata": 5, "10": [5, 7, 11], "offset": [5, 7], "0": [5, 7, 8, 11], "use_agg_md": 5, "off": 5, "logic": [5, 12], "filter": [5, 7], "b": [5, 11], "2": [5, 11], "have": [5, 11], "ar": [5, 7, 8, 9, 10, 11], "alwai": 5, "nest": 5, "c": [5, 11], "3": [5, 11], "same": [5, 11, 13], "whose": 5, "ani": [5, 10, 11], "But": 5, "differ": 5, "33": 5, "d": 5, "4": [5, 11], "e": [5, 10], "won": 5, "333": 5, "defin": [5, 8, 10], "api": [5, 8, 11], "just": [5, 11, 12], "max": 5, "num": 5, "OR": 5, "level": 6, "current": [6, 8, 10], "system": [6, 7, 8, 9], "elasticsearch": 7, "query_str": 7, "execut": [7, 8, 11], "graphql": [7, 8], "query_txt": [7, 8], "syntax": 7, "see": [7, 10, 11], "com": 7, "uc": 7, "cdi": 7, "guppi": 7, "master": 7, "doc": [7, 10], "data_typ": 7, "my_index": 7, "my_field": 7, "first": [7, 8], "filter_object": 7, "sort_object": 7, "verbos": [7, 8], "row": [7, 8, 11, 12], "sort": 7, "equal": 7, "respect": 7, "complex": 7, "appli": 7, "One": 7, "unaccess": 7, "regular": 7, "tier": 7, "mode": 7, "subject": [7, 8], "50": 7, "vital_statu": 7, "submitter_id": 7, "aliv": 7, "asc": 7, "sort_field": 7, "raw": [7, 11], "project_id": [7, 8], "my_program": 7, "my_project": 7, "submit": [8, 11], "export": [8, 10], "sheepdog": 8, "through": [8, 11], "peregrin": 8, "sub": 8, "program": [8, 11], "project": [8, 11], "dcf": 8, "node_nam": 8, "batch_siz": 8, "100": [8, 11], "node": 8, "how": [8, 11], "mani": [8, 11], "print": [8, 10], "progress": [8, 10], "log": [8, 10, 11, 12], "ccle": 8, "demograph": 8, "ordered_node_list": 8, "revers": 8, "graph": 8, "experi": 8, "empti": [8, 11], "uuid1": 8, "uuid2": 8, "node_typ": 8, "fileformat": 8, "which": [8, 10], "belong": 8, "either": 8, "tsv": [8, 11, 12], "screen": 8, "sampl": [8, 10], "ccle_sample_nod": 8, "d70b41b9": 8, "6f90": 8, "4714": 8, "8420": 8, "e043ab8b77b9": 8, "ccle_one_record": 8, "entir": 8, "schema": 8, "run": [8, 11], "introspect": 8, "regist": 8, "mark": 8, "open": [8, 10, 11], "mean": 8, "etc": 8, "max_tri": 8, "retri": 8, "fail": [8, 10], "chunk_siz": 8, "30": 8, "row_offset": 8, "spreadsheet": 8, "multipl": [8, 11], "csv": [8, 11, 12], "xlsx": 8, "worksheet": 8, "skip": 8, "data_spreadsheet": 8, "arrai": 8, "action": [9, 11], "These": 9, "broken": 9, "broad": 9, "categori": 9, "task": 9, "relat": 9, "persist": 9, "Such": 9, "mai": [9, 11], "involv": 9, "assign": 9, "everi": [9, 11], "alreadi": 9, "larg": 9, "most": 9, "util": 9, "capabl": 9, "make": [9, 11], "effici": 9, "main": 10, "datafil": 10, "manifest_1": 10, "my_common": 10, "org": 10, "gen3": [10, 11, 12], "my_credenti": 10, "howto": 10, "drsdownload": 10, "drs_download": 10, "download_list": 10, "show_progress": 10, "assist": 10, "minimum": 10, "interest": 10, "object_list": 10, "host": 10, "It": 10, "possibl": 10, "save_directori": 10, "unpack_packag": 10, "delete_unpacked_packag": 10, "directori": [10, 11], "call": [10, 13], "case": 10, "error": [10, 11, 12], "continu": 10, "save": 10, "show": 10, "bar": 10, "disabl": 10, "unpack": 10, "packag": 10, "after": 10, "them": [10, 11], "drs_hostnam": 10, "known": 10, "otherwis": 10, "success": 10, "resolv": 10, "permiss": 10, "wai": 10, "determin": [10, 11, 12], "reason": 10, "pend": 10, "objectid": 10, "datamanag": 10, "occur": 10, "attribut": [10, 11], "drsobjecttyp": 10, "unknown": 10, "factori": 10, "popul": [10, 12], "server": 10, "requir": 10, "byte": 10, "timestamp": 10, "last": 10, "g": 10, "child": 10, "bundl": 10, "self": 10, "indent": 10, "pretti": 10, "In": 10, "similar": 10, "linux": 10, "tree": 10, "command": [10, 11], "typic": 10, "discoveri": 10, "cach": 10, "files": 10, "static": 10, "bownload": 10, "successfulli": 10, "pars": [10, 11, 12, 13], "output_dir": 10, "conveni": 10, "infil": 10, "wrapper": 10, "cli": 10, "process": 11, "coroutin": 11, "asyncio": [11, 12], "librari": 11, "comma": 11, "header": 11, "download_manifest": 11, "per": [11, 12], "maximum": [11, 12], "desir": 11, "concurr": [11, 12], "across": 11, "thread": 11, "folder": 11, "place": 11, "temporari": 11, "write": 11, "safe": 11, "so": 11, "To": 11, "workaround": 11, "concat": 11, "output_filenam": [11, 12], "num_process": 11, "24": [11, 12], "input_manifest": 11, "python_subprocess_command": 11, "root": [11, 12], "domain": [11, 12], "live": [11, 12], "parallel": 11, "hit": 11, "total": 11, "help": 11, "everyth": 11, "attempt": 11, "By": 11, "someth": 11, "maco": 11, "x": 11, "python3": 11, "tab": 11, "session": 11, "column": [11, 12], "shown": 11, "below": 11, "255e396f": 11, "f1f8": 11, "11e9": 11, "9a07": 11, "0a80fada099c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "363455714": 11, "pdcdatastor": 11, "test1": 11, "0a80fada098c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "343434344": 11, "test2": 11, "0a80fada097c": 11, "473d83400bc1bc9dc635e334fadd433c": 11, "543434443": 11, "phs0001": 11, "phs0002": 11, "test3": 11, "0a80fada096c": 11, "test4": 11, "0a80fada010c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "test5": 11, "index_manifest": 11, "usag": 11, "py": 11, "giangb": 11, "planx": 11, "pla": 11, "net": 11, "manifest_fil": [11, 12], "path_to_manifest": 11, "replace_url": 11, "thread_num": 11, "api_kei": 11, "processed_fil": 11, "num_total_fil": 11, "synchron": 11, "warn": 11, "IF": 11, "know": 11, "THE": 11, "implic": 11, "manifest_file_delimit": [11, 12], "submit_additional_metadata_column": 11, "force_metadata_columns_even_if_empti": 11, "loop": 11, "password": [11, 12], "flag": 11, "replac": 11, "delimit": 11, "forc": 11, "creation": [3, 11], "even": 11, "enabl": 11, "illustr": 11, "columna": 11, "columnb": 11, "columnc": 11, "dataa": 11, "datab": 11, "guid_exampl": 11, "md5_hash": 11, "fieldnam": 11, "There": 11, "those": 11, "abov": 11, "overrid": [11, 12], "assum": 11, "behavior": 11, "simpli": 11, "manifest_row_pars": [11, 12], "custom": 11, "import": 11, "verify_manifest": 11, "def": 11, "_get_authz_from_row": 11, "strip": 11, "parser": [11, 12], "verify_object_manifest": 11, "verif": 11, "error_nam": 11, "value_from_manifest": 11, "actual": 11, "value_from_indexd": 11, "93d9af72": 11, "b0f1": 11, "450c": 11, "a5c6": 11, "7d3d8d2083b4": 11, "dev": 11, "test": 11, "_get_acl_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "1692675811": [], "9771066": [], "indexd_field": [11, 12], "func_to_parse_row": [11, 12], "delimet": [11, 12], "metdata": 12, "ingest_manifest": 12, "NOT": 12, "guid_from_fil": 12, "_get_guid_for_row": 12, "indexed_file_object_guid": 12, "_query_for_associated_indexd_record_guid": 12, "guid_for_row": 12, "elsewher": 12, "metadata_sourc": 12, "1692675812": [], "3144128": [], "get_guid_from_fil": 12, "metadata_typ": 12, "dbgap": 12, "usual": 12, "fill": 12, "_guid_typ": 12, "is_indexed_file_object": 12, "lock": 12, "semaphor": 12, "semaphon": 12, "ammount": 12, "connect": 12, "wss": 13, "src_urlstr": 13, "dest_urlstr": 13, "appropri": 13, "src_w": 13, "src_wskei": 13, "dest_path": 13, "disk": 13, "w": 13, "wskei": 13, "ws_urlstr": 13, "form": 13, "src_path": 13, "dest_w": 13, "dest_wskei": 13, "1692676927": [], "2291234": [], "6143494": [], "1692716275": [], "1734753": [], "577429": [], "1692716527": [], "4222114": [], "9511294": [], "1692717039": [], "3338752": [], "744264": [], "1692741377": [], "0982177": [], "4559627": [], "1692743380": [], "4060478": [], "7860734": [], "1692743691": [], "983341": [], "1692743692": [], "4543839": [], "1692825679": [], "249731": [], "5510318": [], "1692886743": [], "381985": [], "8074749": [], "1693592991": [], "493957": [], "9576306": [], "1693593902": [], "1950202": [], "507454": [], "1696860505": [], "0205865": [], "4065857": [], "1696863250": [], "490177": [], "780603": [], "1696863654": [], "490772": [], "8764913": [], "1696866574": [], "146243": [], "4969604": [], "1696867172": [], "15485": [], "4488156": [], "1696870997": [], "41439": [], "7891934": [], "1696874363": [], "5903769": [], "1696874364": [], "0760994": [], "1696874786": [], "0779042": [], "4368396": [], "1696875823": [], "7846825": [], "1696875824": [], "2559378": [], "1696877990": [], "9968846": [], "1696877991": [], "3616166": [], "1696878184": [], "6719804": [], "1696878185": [], "1095488": [], "1696878605": [], "0427673": [], "4048421": [], "1696885653": [], "7007155": [], "1696885654": [], "1520922": [], "1697467909": [], "0874708": [], "51657": [], "1697484656": [], "68711": [], "1697484657": [], "1308327": [], "1700001291": [], "6339526": [], "9630976": [], "1700002025": [], "294231": [], "5915642": [], "1700004267": [], "4130003": [], "8317993": [], "1700005111": [], "074543": [], "4001298": [], "1700005642": [], "8724742": [], "1700005643": [], "1597712": [], "1700499325": [], "4715705": [], "7661629": [], "1700591014": [], "4958894": [], "8457959": [], "1700592493": [], "8351357": [], "1700592494": [], "1116993": [], "1700678708": [], "9883742": [], "1700678709": [], "2699912": [], "1700678999": [], "9725351": [], "1700679000": [], "272037": [], "1701839488": [], "6993823": [], "1701839489": [], "0258799": [], "1701900364": [], "6706986": [], "9645383": [], "1701992223": [], "1760724": [], "50045": [], "1703187722": [], "008161": [], "3511112": [], "1707152739": [], "444163": [], "7294014": [], "1707153921": [], "9775102": [], "1707153922": [], "2934358": [], "1707755283": [], "0218701": [], "3274062": [], "1707755397": [], "8761852": [], "1707755398": [], "2227244": [], "1707775436": [], "8775291": [], "1707775437": [], "2080507": [], "1708924717": [], "6038976": [], "9058018": [], "1708924776": [], "1099105": [], "3872406": [], "content_created_d": 3, "content_updated_d": 3, "date": 3, "being": 3, "1708971147": [11, 12], "0555665": 11, "3980484": 12}, "objects": {"gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "get_access_token_from_wts"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "download_single"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"], [1, 1, 1, "", "upload_file_to_guid"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_guids_prefix"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_valid_guids"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_create_aliases"], [5, 1, 1, "", "async_delete_alias"], [5, 1, 1, "", "async_delete_aliases"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_get_aliases"], [5, 1, 1, "", "async_update"], [5, 1, 1, "", "async_update_aliases"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_aliases"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_alias"], [5, 1, 1, "", "delete_aliases"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_aliases"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"], [5, 1, 1, "", "update_aliases"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3": [[9, 3, 0, "-", "tools"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "titleterms": {"gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "auth": 0, "helper": 0, "file": 1, "class": [1, 3, 4, 5, 6, 7, 8], "welcom": 2, "sdk": 2, "": 2, "document": 2, "indic": 2, "tabl": 2, "index": [3, 11], "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "submiss": 8, "tool": [9, 10, 11, 12], "dr": 10, "download": [10, 11], "verifi": 11, "ingest": 12, "workspac": 13, "storag": 13}, "envversion": {"sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 60}, "alltitles": {"Gen3 Auth Helper": [[0, "gen3-auth-helper"]], "Gen3 File Class": [[1, "gen3-file-class"]], "Welcome to Gen3 SDK\u2019s documentation!": [[2, "welcome-to-gen3-sdk-s-documentation"]], "Indices and tables": [[2, "indices-and-tables"]], "Gen3 Index Class": [[3, "gen3-index-class"]], "Gen3 Jobs Class": [[4, "gen3-jobs-class"]], "Gen3 Metadata Class": [[5, "gen3-metadata-class"]], "Gen3 Object Class": [[6, "gen3-object-class"]], "Gen3 Query Class": [[7, "gen3-query-class"]], "Gen3 Submission Class": [[8, "gen3-submission-class"]], "Gen3 Tools": [[9, "gen3-tools"]], "DRS Download Tools": [[10, "module-gen3.tools.download.drs_download"]], "Indexing Tools": [[11, "indexing-tools"]], "Download": [[11, "module-gen3.tools.indexing.download_manifest"]], "Index": [[11, "module-gen3.tools.indexing.index_manifest"]], "Verify": [[11, "module-gen3.tools.indexing.verify_manifest"]], "Metadata Tools": [[12, "metadata-tools"]], "Ingest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "Gen3 Workspace Storage": [[13, "gen3-workspace-storage"]]}, "indexentries": {"gen3auth (class in gen3.auth)": [[0, "gen3.auth.Gen3Auth"]], "curl() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.curl"]], "get_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token"]], "get_access_token_from_wts() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token_from_wts"]], "refresh_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.refresh_access_token"]], "gen3file (class in gen3.file)": [[1, "gen3.file.Gen3File"]], "delete_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file"]], "delete_file_locations() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file_locations"]], "download_single() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.download_single"]], "get_presigned_url() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.get_presigned_url"]], "upload_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file"]], "upload_file_to_guid() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file_to_guid"]], "gen3index (class in gen3.index)": [[3, "gen3.index.Gen3Index"]], "async_create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_create_record"]], "async_get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_record"]], "async_get_records_from_checksum() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_from_checksum"]], "async_get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_on_page"]], "async_get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_with_params"]], "async_query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_query_urls"]], "async_update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_update_record"]], "create_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_blank"]], "create_new_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_new_version"]], "create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_record"]], "delete_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.delete_record"]], "get() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get"]], "get_all_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_all_records"]], "get_guids_prefix() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_guids_prefix"]], "get_latest_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_latest_version"]], "get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record"]], "get_record_doc() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record_doc"]], "get_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records"]], "get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records_on_page"]], "get_stats() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_stats"]], "get_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_urls"]], "get_valid_guids() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_valid_guids"]], "get_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_version"]], "get_versions() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_versions"]], "get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_with_params"]], "is_healthy() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.is_healthy"]], "query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.query_urls"]], "update_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_blank"]], "update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_record"]], "gen3jobs (class in gen3.jobs)": [[4, "gen3.jobs.Gen3Jobs"]], "async_run_job_and_wait() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.async_run_job_and_wait"]], "create_job() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.create_job"]], "get_output() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_output"]], "get_status() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_status"]], "get_version() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_version"]], "is_healthy() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.is_healthy"]], "list_jobs() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.list_jobs"]], "gen3metadata (class in gen3.metadata)": [[5, "gen3.metadata.Gen3Metadata"]], "async_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create"]], "async_create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create_aliases"]], "async_delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_alias"]], "async_delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_aliases"]], "async_get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get"]], "async_get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get_aliases"]], "async_update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update"]], "async_update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update_aliases"]], "auth_provider (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.auth_provider"]], "batch_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.batch_create"]], "create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create"]], "create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_aliases"]], "create_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_index_key_path"]], "delete() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete"]], "delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_alias"]], "delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_aliases"]], "delete_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_index_key_path"]], "endpoint (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.endpoint"]], "get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get"]], "get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_aliases"]], "get_index_key_paths() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_index_key_paths"]], "get_version() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_version"]], "is_healthy() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.is_healthy"]], "query() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.query"]], "update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update"]], "update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update_aliases"]], "gen3object (class in gen3.object)": [[6, "gen3.object.Gen3Object"]], "delete_object() (gen3.object.gen3object method)": [[6, "gen3.object.Gen3Object.delete_object"]], "gen3query (class in gen3.query)": [[7, "gen3.query.Gen3Query"]], "graphql_query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.graphql_query"]], "query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.query"]], "raw_data_download() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.raw_data_download"]], "gen3submission (class in gen3.submission)": [[8, "gen3.submission.Gen3Submission"]], "create_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_program"]], "create_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_project"]], "delete_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_node"]], "delete_nodes() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_nodes"]], "delete_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_program"]], "delete_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_project"]], "delete_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_record"]], "delete_records() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_records"]], "export_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_node"]], "export_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_record"]], "get_dictionary_all() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_all"]], "get_dictionary_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_node"]], "get_graphql_schema() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_graphql_schema"]], "get_programs() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_programs"]], "get_project_dictionary() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_dictionary"]], "get_project_manifest() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_manifest"]], "get_projects() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_projects"]], "open_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.open_project"]], "query() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.query"]], "submit_file() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_file"]], "submit_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_record"]], "gen3.tools": [[9, "module-gen3.tools"]], "module": [[9, "module-gen3.tools"], [10, "module-gen3.tools.download.drs_download"], [11, "module-gen3.tools.indexing.download_manifest"], [11, "module-gen3.tools.indexing.index_manifest"], [11, "module-gen3.tools.indexing.verify_manifest"], [12, "module-gen3.tools.metadata.ingest_manifest"]], "downloadmanager (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadManager"]], "downloadstatus (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadStatus"]], "downloadable (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Downloadable"]], "manifest (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Manifest"]], "_manager (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable._manager"]], "access_methods (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.access_methods"]], "cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens"]], "children (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.children"]], "commons_url (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.commons_url"]], "create_object_list() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.create_object_list"]], "created_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.created_time"]], "download() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.download"]], "download() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.download"]], "download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_files_in_drs_manifest"]], "end_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.end_time"]], "file_name (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_name"]], "file_name (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_name"]], "file_size (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_size"]], "file_size (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_size"]], "gen3.tools.download.drs_download": [[10, "module-gen3.tools.download.drs_download"]], "get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.get_fresh_token"]], "hostname (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.hostname"]], "list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_access_in_drs_manifest"]], "list_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_drs_object"]], "list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_files_in_drs_manifest"]], "load() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load"]], "load_manifest() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load_manifest"]], "md5sum (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.md5sum"]], "object_id (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_id"]], "object_id (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.object_id"]], "object_type (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_type"]], "pprint() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.pprint"]], "resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.resolve_objects"]], "start_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.start_time"]], "status (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.status"]], "updated_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.updated_time"]], "user_access() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.user_access"]], "acls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ACLS"]], "authz (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.AUTHZ"]], "current_dir (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.CURRENT_DIR"]], "guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.GUID"]], "indexd_record_page_size (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE"]], "max_concurrent_requests (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS"]], "max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS"]], "md5 (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.MD5"]], "prev_guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.PREV_GUID"]], "size (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.SIZE"]], "tmp_folder (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.TMP_FOLDER"]], "threadcontrol (class in gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ThreadControl"]], "urls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.URLS"]], "async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.async_download_object_manifest"]], "async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.async_verify_object_manifest"]], "delete_all_guids() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.delete_all_guids"]], "gen3.tools.indexing.download_manifest": [[11, "module-gen3.tools.indexing.download_manifest"]], "gen3.tools.indexing.index_manifest": [[11, "module-gen3.tools.indexing.index_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, "module-gen3.tools.indexing.verify_manifest"]], "index_object_manifest() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.index_object_manifest"]], "column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID"]], "guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT"]], "guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"]], "max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS"]], "async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest"]], "async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd"]], "gen3.tools.metadata.ingest_manifest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "gen3wsstorage (class in gen3.wss)": [[13, "gen3.wss.Gen3WsStorage"]], "copy() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.copy"]], "download() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download"]], "download_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download_url"]], "ls() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls"]], "ls_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls_path"]], "rm() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm"]], "rm_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm_path"]], "upload() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload"]], "upload_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload_url"]]}}) \ No newline at end of file +Search.setIndex({"docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "terms": {"class": [0, 2, 10, 11, 13], "gen3auth": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "endpoint": [0, 1, 2, 3, 4, 5, 7, 8, 13], "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "idp": 0, "client_credenti": 0, "client_scop": 0, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "authbas": 0, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "request": [0, 1, 3, 5, 8, 11, 12], "implement": 0, "order": [0, 8], "support": [0, 1, 5, 8, 11], "jwt": 0, "authent": 0, "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "access": [0, 1, 3, 7, 10], "token": [0, 10], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "provid": [0, 1, 3, 5, 7, 8, 12], "refresh": [0, 10], "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "string": [0, 3, 5, 11, 13], "automat": 0, "when": [0, 3, 7, 10, 12], "thei": [0, 10], "expir": [0, 1], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "opt": 0, "The": [0, 1, 2, 3, 5, 8, 10, 11], "contain": [0, 2, 5, 8, 9, 10, 11, 12], "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "web": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "work": [0, 10], "workspac": [0, 2], "default": [0, 1, 3, 7, 8, 11, 12], "env": 0, "gen3_api_kei": 0, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "set": [0, 1, 5, 10], "includ": [0, 3], "search": [0, 2, 3], "path": [0, 1, 5, 10, 11, 13], "valu": [0, 1, 3, 5, 7, 10, 11], "doe": [0, 12], "interpret": 0, "wt": [0, 10], "an": [0, 3, 6, 8, 10, 11], "accesstoken": 0, "If": [0, 1, 7, 11, 12], "can": [0, 3, 4, 8, 11, 12], "specifi": [0, 1, 3, 11, 13], "local": [0, 13], "indic": [0, 11], "environ": 0, "fenc": [0, 1], "tupl": [0, 3, 11, 12], "client_id": 0, "client_secret": 0, "oidc": 0, "client": [0, 3], "ha": [0, 11], "grant": 0, "allow": [0, 6, 8, 10, 11, 12], "obtain": [0, 10], "space": [0, 11], "separ": [0, 11], "list": [0, 1, 3, 4, 5, 7, 8, 10, 11, 13], "scope": [0, 1], "user": [0, 10, 12], "data": [0, 1, 3, 5, 7, 8, 10, 11], "openid": 0, "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "instal": [0, 2, 11], "crdc": 0, "some": [0, 2], "arbitrari": 0, "kei": [0, 3, 5, 13], "variabl": [0, 7, 8], "rather": 0, "than": [0, 5], "pass": [0, 7, 8, 10], "argument": [0, 13], "constructor": 0, "id": [0, 1, 3, 5, 10, 11], "secret": 0, "note": [0, 3, 11, 12], "should": [0, 8, 11], "never": 0, "hardcod": 0, "http": [0, 7, 11, 12], "datacommon": 0, "o": 0, "gen3_oidc_client_creds_secret": 0, "initi": [0, 10], "follow": [0, 11], "curl": [0, 2], "given": [0, 3, 4, 5, 8, 10, 12, 13], "ex": [0, 11, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "respons": [0, 1, 3, 4, 5], "under": [0, 8, 13], "index": [0, 2, 5, 9], "authz": [0, 1, 2, 3, 9, 10, 11], "map": [0, 11], "get": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12, 13], "post": [0, 11], "put": 0, "delet": [0, 1, 2, 3, 5, 6, 8, 10, 11], "i": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12], "els": [0, 12], "filenam": [0, 8, 10, 11, 12], "get_access_token": [0, 2], "auto": [0, 2], "within": [0, 2, 9], "5": [0, 5], "minut": 0, "get_access_token_from_wt": [0, 2], "try": 0, "fetch": 0, "namespac": [0, 12], "refresh_access_token": [0, 2], "new": [0, 3], "gen3fil": [1, 2], "auth_provid": [1, 2, 3, 4, 5, 6, 7, 8, 13], "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "For": [1, 5, 6, 7, 8, 9, 11], "interact": [1, 3, 4, 5, 6, 8, 13], "manag": [1, 5, 10], "featur": [1, 6], "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "presign": 1, "url": [1, 2, 3, 9, 10, 11, 12, 13], "right": 1, "now": [1, 8], "instanc": [1, 3, 6, 7, 8, 9, 10], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "delete_fil": [1, 2], "guid": [1, 2, 3, 5, 6, 9, 11, 12], "method": [1, 7, 10], "deprec": 1, "delete_file_loc": [1, 2, 6], "instead": [1, 7, 11], "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "locat": [1, 6], "store": [1, 3, 10], "remov": [1, 6, 11, 13], "its": [1, 3], "record": [1, 3, 5, 7, 8, 11, 12], "indexd": [1, 3, 6, 10, 11, 12], "uuid": [1, 3, 8], "text": [1, 7, 8], "result": [1, 8, 10, 11], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "download_singl": [1, 2], "object_id": [1, 2, 9, 10], "singl": [1, 5, 8], "": [1, 4, 8, 10, 11], "uniqu": [1, 5], "get_presigned_url": [1, 2], "protocol": 1, "retriev": [1, 8, 10, 12], "give": 1, "limit": [1, 3, 5, 12], "time": [1, 3, 8, 10, 11], "pick": 1, "avail": [1, 2, 10, 11], "queri": [1, 2, 3, 5, 8, 11, 12], "upload_fil": [1, 2], "file_nam": [1, 2, 3, 9, 10, 11], "expires_in": 1, "bucket": [1, 6], "upload": [1, 2, 3, 8, 13], "author": 1, "storag": [1, 2, 6], "s3": [1, 10, 11], "az": 1, "isn": 1, "t": [1, 5, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "amount": [1, 9], "second": [1, 4], "sign": 1, "datetim": [1, 3, 10], "utcnow": 1, "Be": 1, "sure": 1, "posit": [1, 7], "integ": [1, 3, 8], "also": 1, "treat": [1, 5], "max_presigned_url_ttl": 1, "configur": 1, "must": [1, 5], "allowed_data_upload_bucket": 1, "data_upload_bucket": 1, "represent": [1, 3], "document": [1, 3], "upload_file_to_guid": [1, 2], "exist": [1, 3, 5, 6, 9, 12], "readm": 2, "python": [2, 9, 11], "github": [2, 7], "repo": 2, "setup": 2, "inform": [2, 3, 10], "along": 2, "quickstart": 2, "script": 2, "more": [2, 5, 7, 9, 10], "detail": [2, 7, 10], "about": [2, 3], "variou": 2, "function": [2, 3, 4, 5, 9, 10, 11, 12], "mostli": 2, "docstr": 2, "code": [2, 8], "helper": 2, "gen3index": [2, 3], "async_create_record": [2, 3], "async_get_record": [2, 3], "async_get_records_from_checksum": [2, 3], "async_get_records_on_pag": [2, 3], "async_get_with_param": [2, 3], "async_query_url": [2, 3], "async_update_record": [2, 3], "create_blank": [2, 3], "create_new_vers": [2, 3], "create_record": [2, 3], "delete_record": [2, 3, 8], "get_all_record": [2, 3], "get_guids_prefix": [2, 3], "get_latest_vers": [2, 3], "get_record": [2, 3], "get_record_doc": [2, 3], "get_records_on_pag": [2, 3], "get_stat": [2, 3], "get_url": [2, 3], "get_valid_guid": [2, 3], "get_vers": [2, 3, 4, 5], "get_with_param": [2, 3], "is_healthi": [2, 3, 4, 5], "query_url": [2, 3], "update_blank": [2, 3], "update_record": [2, 3], "job": 2, "gen3job": [2, 4, 10], "async_run_job_and_wait": [2, 4], "create_job": [2, 4], "get_output": [2, 4], "get_statu": [2, 4], "list_job": [2, 4], "metadata": [2, 3, 6, 9, 11], "gen3metadata": [2, 5], "async_cr": [2, 5], "async_create_alias": [2, 5], "async_delete_alia": [2, 5], "async_delete_alias": [2, 5], "async_get": [2, 5], "async_get_alias": [2, 5], "async_upd": [2, 5], "async_update_alias": [2, 5], "batch_creat": [2, 5], "creat": [2, 3, 4, 5, 6, 8, 10, 11], "create_alias": [2, 5], "create_index_key_path": [2, 5], "delete_alia": [2, 5], "delete_alias": [2, 5], "delete_index_key_path": [2, 5], "get_alias": [2, 5], "get_index_key_path": [2, 5], "updat": [2, 3, 5, 10, 11], "update_alias": [2, 5], "gen3object": [2, 6], "delete_object": [2, 6], "gen3queri": [2, 7], "graphql_queri": [2, 7], "raw_data_download": [2, 7], "submiss": 2, "gen3submiss": [2, 8], "create_program": [2, 8], "create_project": [2, 8], "delete_nod": [2, 8], "delete_program": [2, 8], "delete_project": [2, 8], "export_nod": [2, 8], "export_record": [2, 8], "get_dictionary_al": [2, 8], "get_dictionary_nod": [2, 8], "get_graphql_schema": [2, 8], "get_program": [2, 8], "get_project_dictionari": [2, 8], "get_project_manifest": [2, 8], "get_project": [2, 8], "open_project": [2, 8], "submit_fil": [2, 8], "submit_record": [2, 8], "tool": 2, "dr": [2, 9], "downloadmanag": [2, 9, 10], "cache_hosts_wts_token": [2, 9, 10], "get_fresh_token": [2, 9, 10], "resolve_object": [2, 9, 10], "user_access": [2, 9, 10], "downloadstatu": [2, 9, 10], "statu": [2, 4, 9, 10], "start_tim": [2, 9, 10], "end_tim": [2, 9, 10], "object_typ": [2, 9, 10], "hostnam": [2, 9, 10], "file_s": [2, 9, 10, 11], "updated_tim": [2, 9, 10], "created_tim": [2, 9, 10], "access_method": [2, 9, 10], "children": [2, 9, 10], "_manag": [2, 9, 10], "pprint": [2, 9, 10], "manifest": [2, 8, 9, 10, 11, 12], "md5sum": [2, 9, 10], "commons_url": [2, 9, 10, 11, 12], "create_object_list": [2, 9, 10], "load": [2, 9, 10], "load_manifest": [2, 9, 10], "download_drs_object": [], "download_files_in_drs_manifest": [2, 9, 10], "list_access_in_drs_manifest": [2, 9, 10], "list_drs_object": [2, 9, 10], "list_files_in_drs_manifest": [2, 9, 10], "current_dir": [2, 9, 11], "indexd_record_page_s": [2, 9, 11], "max_concurrent_request": [2, 9, 11, 12], "tmp_folder": [2, 9, 11], "async_download_object_manifest": [2, 9, 11], "size": [2, 3, 9, 10, 11], "md5": [2, 3, 9, 11], "acl": [2, 3, 9, 11], "prev_guid": [2, 9, 11], "threadcontrol": [2, 9, 11], "delete_all_guid": [2, 9, 11], "index_object_manifest": [2, 9, 11], "verifi": [2, 9], "async_verify_object_manifest": [2, 9, 11], "ingest": [2, 9], "column_to_use_as_guid": [2, 9, 12], "guid_type_for_indexed_file_object": [2, 9, 12], "guid_type_for_non_indexed_file_object": [2, 9, 12], "async_ingest_metadata_manifest": [2, 9, 12], "async_query_urls_from_indexd": [2, 9, 12], "gen3wsstorag": [2, 13], "copi": [2, 13], "download_url": [2, 13], "l": [2, 13], "ls_path": [2, 13], "rm": [2, 13], "rm_path": [2, 13], "upload_url": [2, 13], "modul": [2, 10, 11], "service_loc": [3, 4, 5], "public": [3, 5], "read": [3, 5, 11], "onli": [3, 5, 7, 8, 10, 11], "necessari": [3, 5], "basic": [3, 11, 12], "cred": 3, "async": [3, 4, 5, 9, 11, 12], "hash": [3, 11], "did": 3, "baseid": 3, "urls_metadata": 3, "version": [3, 4, 5], "_ssl": [3, 4, 5], "asynchron": [3, 4, 5], "dict": [3, 4, 5, 10, 11, 12], "eg": 3, "ab167e49d25b488939b1ede42752458b": 3, "associ": [3, 5], "made": 3, "where": [3, 5, 11, 12], "you": [3, 8, 11], "control": 3, "rbac": 3, "name": [3, 4, 8, 10, 11, 12, 13], "addit": [3, 5, 10, 11], "entri": [3, 11], "attach": [3, 5], "each": [3, 8, 10, 11], "group": 3, "previou": [3, 11], "checksum": [3, 10], "checksum_typ": 3, "match": [3, 5, 12], "param": [3, 5, 8, 10], "correspond": 3, "suppli": 3, "need": [3, 7, 10, 11], "handl": [3, 10], "hash_typ": 3, "whether": [3, 4, 5, 8, 11, 12], "ssl": [3, 4, 5], "pattern": [3, 12], "against": [3, 7, 8, 11, 12], "kwarg": [3, 4, 5], "bodi": 3, "dictionari": [3, 4, 5, 7, 8], "format": [3, 5, 8, 11], "blank": 3, "add": [3, 5], "sinc": 3, "content": [3, 13], "immut": 3, "want": [3, 8], "chang": [3, 11], "That": 3, "field": [3, 5, 7, 11, 12], "old": 3, "ad": 3, "least": 3, "one": [3, 5, 10, 11], "suffici": 3, "good": 3, "idea": 3, "number": [3, 7, 8, 11, 12], "noth": [3, 6], "dist_resolut": 3, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "alia": [3, 5], "distribut": 3, "identifi": [3, 5, 9, 11], "boolean": 3, "we": [3, 11], "pagin": 3, "fals": [3, 5, 6, 10, 11], "start": [3, 4, 7, 8, 10], "prefix": 3, "rtype": 3, "has_vers": 3, "latest": 3, "exclud": 3, "without": [3, 5], "repres": [3, 5, 10], "info": [3, 11], "algorithm": 3, "count": 3, "valid": [3, 7], "file_st": 3, "like": [3, 5, 9, 11, 12], "signpost": 3, "healthi": [3, 4, 5], "rev": 3, "revis": 3, "simpl": 3, "consist": 3, "mechan": 3, "dispatch": 4, "job_nam": 4, "job_input": 4, "wait": 4, "output": [4, 5, 11, 12], "Will": [4, 7, 10], "sleep": 4, "linear": 4, "delai": 4, "until": [4, 10], "done": 4, "1": [4, 5, 8, 10, 11], "global": [4, 5], "input": [4, 10, 11], "job_id": 4, "previous": 4, "complet": [4, 11], "bool": [4, 5, 8, 10, 11, 12], "md": [5, 7, 10, 12], "admin_endpoint_suffix": 5, "admin": [5, 11], "alias": 5, "overwrit": 5, "what": 5, "end": [5, 10], "up": [5, 9], "blob": [5, 7], "asyncron": 5, "altern": [5, 11], "merg": 5, "AND": 5, "descript": [3, 5], "metadata_list": 5, "specif": [5, 8, 11, 12], "expect": [5, 9, 11], "anoth": 5, "databas": 5, "return_full_metadata": 5, "10": [5, 7, 11], "offset": [5, 7], "0": [5, 7, 8, 11], "use_agg_md": 5, "off": 5, "logic": [5, 12], "filter": [5, 7], "b": [5, 11], "2": [5, 11], "have": [5, 11], "ar": [5, 7, 8, 9, 10, 11], "alwai": 5, "nest": 5, "c": [5, 11], "3": [5, 11], "same": [5, 11, 13], "whose": 5, "ani": [5, 10, 11], "But": 5, "differ": 5, "33": 5, "d": 5, "4": [5, 11], "e": [5, 10], "won": 5, "333": 5, "defin": [5, 8, 10], "api": [5, 8, 11], "just": [5, 11, 12], "max": 5, "num": 5, "OR": 5, "level": 6, "current": [6, 8, 10], "system": [6, 7, 8, 9], "elasticsearch": 7, "query_str": 7, "execut": [7, 8, 11], "graphql": [7, 8], "query_txt": [7, 8], "syntax": 7, "see": [7, 10, 11], "com": 7, "uc": 7, "cdi": 7, "guppi": 7, "master": 7, "doc": [7, 10], "data_typ": 7, "my_index": 7, "my_field": 7, "first": [7, 8], "filter_object": 7, "sort_object": 7, "verbos": [7, 8], "row": [7, 8, 11, 12], "sort": 7, "equal": 7, "respect": 7, "complex": 7, "appli": 7, "One": 7, "unaccess": 7, "regular": 7, "tier": 7, "mode": 7, "subject": [7, 8], "50": 7, "vital_statu": 7, "submitter_id": 7, "aliv": 7, "asc": 7, "sort_field": 7, "raw": [7, 11], "project_id": [7, 8], "my_program": 7, "my_project": 7, "submit": [8, 11], "export": [8, 10], "sheepdog": 8, "through": [8, 11], "peregrin": 8, "sub": 8, "program": [8, 11], "project": [8, 11], "dcf": 8, "node_nam": 8, "batch_siz": 8, "100": [8, 11], "node": 8, "how": [8, 11], "mani": [8, 11], "print": [8, 10], "progress": [8, 10], "log": [8, 10, 11, 12], "ccle": 8, "demograph": 8, "ordered_node_list": 8, "revers": 8, "graph": 8, "experi": 8, "empti": [8, 11], "uuid1": 8, "uuid2": 8, "node_typ": 8, "fileformat": 8, "which": [8, 10], "belong": 8, "either": 8, "tsv": [8, 11, 12], "screen": 8, "sampl": [8, 10], "ccle_sample_nod": 8, "d70b41b9": 8, "6f90": 8, "4714": 8, "8420": 8, "e043ab8b77b9": 8, "ccle_one_record": 8, "entir": 8, "schema": 8, "run": [8, 11], "introspect": 8, "regist": 8, "mark": 8, "open": [8, 10, 11], "mean": 8, "etc": 8, "max_tri": 8, "retri": 8, "fail": [8, 10], "chunk_siz": 8, "30": 8, "row_offset": 8, "spreadsheet": 8, "multipl": [8, 11], "csv": [8, 11, 12], "xlsx": 8, "worksheet": 8, "skip": 8, "data_spreadsheet": 8, "arrai": 8, "action": [9, 11], "These": 9, "broken": 9, "broad": 9, "categori": 9, "task": 9, "relat": 9, "persist": 9, "Such": 9, "mai": [9, 11], "involv": 9, "assign": 9, "everi": [9, 11], "alreadi": 9, "larg": 9, "most": 9, "util": 9, "capabl": 9, "make": [9, 11], "effici": 9, "main": 10, "datafil": 10, "manifest_1": 10, "my_common": 10, "org": 10, "gen3": [10, 11, 12], "my_credenti": 10, "howto": 10, "drsdownload": 10, "drs_download": 10, "download_list": 10, "show_progress": 10, "assist": 10, "minimum": 10, "interest": 10, "object_list": 10, "host": 10, "It": 10, "possibl": 10, "save_directori": 10, "unpack_packag": 10, "delete_unpacked_packag": 10, "directori": [10, 11], "call": [10, 13], "case": 10, "error": [10, 11, 12], "continu": 10, "save": 10, "show": 10, "bar": 10, "disabl": 10, "unpack": 10, "packag": 10, "after": 10, "them": [10, 11], "drs_hostnam": 10, "known": 10, "otherwis": 10, "success": 10, "resolv": 10, "permiss": 10, "wai": 10, "determin": [10, 11, 12], "reason": 10, "pend": 10, "objectid": 10, "datamanag": 10, "occur": 10, "attribut": [10, 11], "drsobjecttyp": 10, "unknown": 10, "factori": 10, "popul": [10, 12], "server": 10, "requir": 10, "byte": 10, "timestamp": 10, "last": 10, "g": 10, "child": 10, "bundl": 10, "self": 10, "indent": 10, "pretti": 10, "In": 10, "similar": 10, "linux": 10, "tree": 10, "command": [10, 11], "typic": 10, "discoveri": 10, "cach": 10, "files": 10, "static": 10, "bownload": 10, "successfulli": 10, "pars": [10, 11, 12, 13], "output_dir": 10, "conveni": 10, "infil": 10, "wrapper": 10, "cli": 10, "process": 11, "coroutin": 11, "asyncio": [11, 12], "librari": 11, "comma": 11, "header": 11, "download_manifest": 11, "per": [11, 12], "maximum": [11, 12], "desir": 11, "concurr": [11, 12], "across": 11, "thread": 11, "folder": 11, "place": 11, "temporari": 11, "write": 11, "safe": 11, "so": 11, "To": 11, "workaround": 11, "concat": 11, "output_filenam": [11, 12], "num_process": 11, "24": [11, 12], "input_manifest": 11, "python_subprocess_command": 11, "root": [11, 12], "domain": [11, 12], "live": [11, 12], "parallel": 11, "hit": 11, "total": 11, "help": 11, "everyth": 11, "attempt": 11, "By": 11, "someth": 11, "maco": 11, "x": 11, "python3": 11, "tab": 11, "session": 11, "column": [11, 12], "shown": 11, "below": 11, "255e396f": 11, "f1f8": 11, "11e9": 11, "9a07": 11, "0a80fada099c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "363455714": 11, "pdcdatastor": 11, "test1": 11, "0a80fada098c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "343434344": 11, "test2": 11, "0a80fada097c": 11, "473d83400bc1bc9dc635e334fadd433c": 11, "543434443": 11, "phs0001": 11, "phs0002": 11, "test3": 11, "0a80fada096c": 11, "test4": 11, "0a80fada010c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "test5": 11, "index_manifest": 11, "usag": 11, "py": 11, "giangb": 11, "planx": 11, "pla": 11, "net": 11, "manifest_fil": [11, 12], "path_to_manifest": 11, "replace_url": 11, "thread_num": 11, "api_kei": 11, "processed_fil": 11, "num_total_fil": 11, "synchron": 11, "warn": 11, "IF": 11, "know": 11, "THE": 11, "implic": 11, "manifest_file_delimit": [11, 12], "submit_additional_metadata_column": 11, "force_metadata_columns_even_if_empti": 11, "loop": 11, "password": [11, 12], "flag": 11, "replac": 11, "delimit": 11, "forc": 11, "creation": [3, 11], "even": 11, "enabl": 11, "illustr": 11, "columna": 11, "columnb": 11, "columnc": 11, "dataa": 11, "datab": 11, "guid_exampl": 11, "md5_hash": 11, "fieldnam": 11, "There": 11, "those": 11, "abov": 11, "overrid": [11, 12], "assum": 11, "behavior": 11, "simpli": 11, "manifest_row_pars": [11, 12], "custom": 11, "import": 11, "verify_manifest": 11, "def": 11, "_get_authz_from_row": 11, "strip": 11, "parser": [11, 12], "verify_object_manifest": 11, "verif": 11, "error_nam": 11, "value_from_manifest": 11, "actual": 11, "value_from_indexd": 11, "93d9af72": 11, "b0f1": 11, "450c": 11, "a5c6": 11, "7d3d8d2083b4": 11, "dev": 11, "test": 11, "_get_acl_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "1692675811": [], "9771066": [], "indexd_field": [11, 12], "func_to_parse_row": [11, 12], "delimet": [11, 12], "metdata": 12, "ingest_manifest": 12, "NOT": 12, "guid_from_fil": 12, "_get_guid_for_row": 12, "indexed_file_object_guid": 12, "_query_for_associated_indexd_record_guid": 12, "guid_for_row": 12, "elsewher": 12, "metadata_sourc": 12, "1692675812": [], "3144128": [], "get_guid_from_fil": 12, "metadata_typ": 12, "dbgap": 12, "usual": 12, "fill": 12, "_guid_typ": 12, "is_indexed_file_object": 12, "lock": 12, "semaphor": 12, "semaphon": 12, "ammount": 12, "connect": 12, "wss": 13, "src_urlstr": 13, "dest_urlstr": 13, "appropri": 13, "src_w": 13, "src_wskei": 13, "dest_path": 13, "disk": 13, "w": 13, "wskei": 13, "ws_urlstr": 13, "form": 13, "src_path": 13, "dest_w": 13, "dest_wskei": 13, "1692676927": [], "2291234": [], "6143494": [], "1692716275": [], "1734753": [], "577429": [], "1692716527": [], "4222114": [], "9511294": [], "1692717039": [], "3338752": [], "744264": [], "1692741377": [], "0982177": [], "4559627": [], "1692743380": [], "4060478": [], "7860734": [], "1692743691": [], "983341": [], "1692743692": [], "4543839": [], "1692825679": [], "249731": [], "5510318": [], "1692886743": [], "381985": [], "8074749": [], "1693592991": [], "493957": [], "9576306": [], "1693593902": [], "1950202": [], "507454": [], "1696860505": [], "0205865": [], "4065857": [], "1696863250": [], "490177": [], "780603": [], "1696863654": [], "490772": [], "8764913": [], "1696866574": [], "146243": [], "4969604": [], "1696867172": [], "15485": [], "4488156": [], "1696870997": [], "41439": [], "7891934": [], "1696874363": [], "5903769": [], "1696874364": [], "0760994": [], "1696874786": [], "0779042": [], "4368396": [], "1696875823": [], "7846825": [], "1696875824": [], "2559378": [], "1696877990": [], "9968846": [], "1696877991": [], "3616166": [], "1696878184": [], "6719804": [], "1696878185": [], "1095488": [], "1696878605": [], "0427673": [], "4048421": [], "1696885653": [], "7007155": [], "1696885654": [], "1520922": [], "1697467909": [], "0874708": [], "51657": [], "1697484656": [], "68711": [], "1697484657": [], "1308327": [], "1700001291": [], "6339526": [], "9630976": [], "1700002025": [], "294231": [], "5915642": [], "1700004267": [], "4130003": [], "8317993": [], "1700005111": [], "074543": [], "4001298": [], "1700005642": [], "8724742": [], "1700005643": [], "1597712": [], "1700499325": [], "4715705": [], "7661629": [], "1700591014": [], "4958894": [], "8457959": [], "1700592493": [], "8351357": [], "1700592494": [], "1116993": [], "1700678708": [], "9883742": [], "1700678709": [], "2699912": [], "1700678999": [], "9725351": [], "1700679000": [], "272037": [], "1701839488": [], "6993823": [], "1701839489": [], "0258799": [], "1701900364": [], "6706986": [], "9645383": [], "1701992223": [], "1760724": [], "50045": [], "1703187722": [], "008161": [], "3511112": [], "1707152739": [], "444163": [], "7294014": [], "1707153921": [], "9775102": [], "1707153922": [], "2934358": [], "1707755283": [], "0218701": [], "3274062": [], "1707755397": [], "8761852": [], "1707755398": [], "2227244": [], "1707775436": [], "8775291": [], "1707775437": [], "2080507": [], "1708924717": [], "6038976": [], "9058018": [], "1708924776": [], "1099105": [], "3872406": [], "content_created_d": 3, "content_updated_d": 3, "date": 3, "being": 3, "1708971147": [], "0555665": [], "3980484": [], "1709071889": [], "1303432": [], "4402523": [], "1709222307": [11, 12], "153123": 11, "5054574": 12}, "objects": {"gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "get_access_token_from_wts"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "download_single"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"], [1, 1, 1, "", "upload_file_to_guid"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_guids_prefix"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_valid_guids"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_create_aliases"], [5, 1, 1, "", "async_delete_alias"], [5, 1, 1, "", "async_delete_aliases"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_get_aliases"], [5, 1, 1, "", "async_update"], [5, 1, 1, "", "async_update_aliases"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_aliases"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_alias"], [5, 1, 1, "", "delete_aliases"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_aliases"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"], [5, 1, 1, "", "update_aliases"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3": [[9, 3, 0, "-", "tools"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "titleterms": {"gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "auth": 0, "helper": 0, "file": 1, "class": [1, 3, 4, 5, 6, 7, 8], "welcom": 2, "sdk": 2, "": 2, "document": 2, "indic": 2, "tabl": 2, "index": [3, 11], "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "submiss": 8, "tool": [9, 10, 11, 12], "dr": 10, "download": [10, 11], "verifi": 11, "ingest": 12, "workspac": 13, "storag": 13}, "envversion": {"sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 60}, "alltitles": {"Gen3 Auth Helper": [[0, "gen3-auth-helper"]], "Gen3 File Class": [[1, "gen3-file-class"]], "Welcome to Gen3 SDK\u2019s documentation!": [[2, "welcome-to-gen3-sdk-s-documentation"]], "Indices and tables": [[2, "indices-and-tables"]], "Gen3 Index Class": [[3, "gen3-index-class"]], "Gen3 Jobs Class": [[4, "gen3-jobs-class"]], "Gen3 Metadata Class": [[5, "gen3-metadata-class"]], "Gen3 Object Class": [[6, "gen3-object-class"]], "Gen3 Query Class": [[7, "gen3-query-class"]], "Gen3 Submission Class": [[8, "gen3-submission-class"]], "Gen3 Tools": [[9, "gen3-tools"]], "DRS Download Tools": [[10, "module-gen3.tools.download.drs_download"]], "Indexing Tools": [[11, "indexing-tools"]], "Download": [[11, "module-gen3.tools.indexing.download_manifest"]], "Index": [[11, "module-gen3.tools.indexing.index_manifest"]], "Verify": [[11, "module-gen3.tools.indexing.verify_manifest"]], "Metadata Tools": [[12, "metadata-tools"]], "Ingest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "Gen3 Workspace Storage": [[13, "gen3-workspace-storage"]]}, "indexentries": {"gen3auth (class in gen3.auth)": [[0, "gen3.auth.Gen3Auth"]], "curl() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.curl"]], "get_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token"]], "get_access_token_from_wts() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token_from_wts"]], "refresh_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.refresh_access_token"]], "gen3file (class in gen3.file)": [[1, "gen3.file.Gen3File"]], "delete_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file"]], "delete_file_locations() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file_locations"]], "download_single() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.download_single"]], "get_presigned_url() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.get_presigned_url"]], "upload_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file"]], "upload_file_to_guid() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file_to_guid"]], "gen3index (class in gen3.index)": [[3, "gen3.index.Gen3Index"]], "async_create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_create_record"]], "async_get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_record"]], "async_get_records_from_checksum() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_from_checksum"]], "async_get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_on_page"]], "async_get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_with_params"]], "async_query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_query_urls"]], "async_update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_update_record"]], "create_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_blank"]], "create_new_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_new_version"]], "create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_record"]], "delete_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.delete_record"]], "get() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get"]], "get_all_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_all_records"]], "get_guids_prefix() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_guids_prefix"]], "get_latest_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_latest_version"]], "get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record"]], "get_record_doc() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record_doc"]], "get_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records"]], "get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records_on_page"]], "get_stats() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_stats"]], "get_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_urls"]], "get_valid_guids() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_valid_guids"]], "get_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_version"]], "get_versions() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_versions"]], "get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_with_params"]], "is_healthy() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.is_healthy"]], "query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.query_urls"]], "update_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_blank"]], "update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_record"]], "gen3jobs (class in gen3.jobs)": [[4, "gen3.jobs.Gen3Jobs"]], "async_run_job_and_wait() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.async_run_job_and_wait"]], "create_job() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.create_job"]], "get_output() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_output"]], "get_status() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_status"]], "get_version() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_version"]], "is_healthy() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.is_healthy"]], "list_jobs() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.list_jobs"]], "gen3metadata (class in gen3.metadata)": [[5, "gen3.metadata.Gen3Metadata"]], "async_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create"]], "async_create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create_aliases"]], "async_delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_alias"]], "async_delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_aliases"]], "async_get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get"]], "async_get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get_aliases"]], "async_update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update"]], "async_update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update_aliases"]], "auth_provider (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.auth_provider"]], "batch_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.batch_create"]], "create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create"]], "create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_aliases"]], "create_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_index_key_path"]], "delete() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete"]], "delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_alias"]], "delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_aliases"]], "delete_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_index_key_path"]], "endpoint (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.endpoint"]], "get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get"]], "get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_aliases"]], "get_index_key_paths() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_index_key_paths"]], "get_version() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_version"]], "is_healthy() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.is_healthy"]], "query() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.query"]], "update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update"]], "update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update_aliases"]], "gen3object (class in gen3.object)": [[6, "gen3.object.Gen3Object"]], "delete_object() (gen3.object.gen3object method)": [[6, "gen3.object.Gen3Object.delete_object"]], "gen3query (class in gen3.query)": [[7, "gen3.query.Gen3Query"]], "graphql_query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.graphql_query"]], "query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.query"]], "raw_data_download() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.raw_data_download"]], "gen3submission (class in gen3.submission)": [[8, "gen3.submission.Gen3Submission"]], "create_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_program"]], "create_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_project"]], "delete_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_node"]], "delete_nodes() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_nodes"]], "delete_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_program"]], "delete_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_project"]], "delete_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_record"]], "delete_records() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_records"]], "export_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_node"]], "export_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_record"]], "get_dictionary_all() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_all"]], "get_dictionary_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_node"]], "get_graphql_schema() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_graphql_schema"]], "get_programs() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_programs"]], "get_project_dictionary() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_dictionary"]], "get_project_manifest() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_manifest"]], "get_projects() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_projects"]], "open_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.open_project"]], "query() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.query"]], "submit_file() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_file"]], "submit_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_record"]], "gen3.tools": [[9, "module-gen3.tools"]], "module": [[9, "module-gen3.tools"], [10, "module-gen3.tools.download.drs_download"], [11, "module-gen3.tools.indexing.download_manifest"], [11, "module-gen3.tools.indexing.index_manifest"], [11, "module-gen3.tools.indexing.verify_manifest"], [12, "module-gen3.tools.metadata.ingest_manifest"]], "downloadmanager (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadManager"]], "downloadstatus (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadStatus"]], "downloadable (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Downloadable"]], "manifest (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Manifest"]], "_manager (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable._manager"]], "access_methods (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.access_methods"]], "cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens"]], "children (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.children"]], "commons_url (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.commons_url"]], "create_object_list() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.create_object_list"]], "created_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.created_time"]], "download() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.download"]], "download() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.download"]], "download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_files_in_drs_manifest"]], "end_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.end_time"]], "file_name (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_name"]], "file_name (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_name"]], "file_size (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_size"]], "file_size (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_size"]], "gen3.tools.download.drs_download": [[10, "module-gen3.tools.download.drs_download"]], "get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.get_fresh_token"]], "hostname (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.hostname"]], "list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_access_in_drs_manifest"]], "list_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_drs_object"]], "list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_files_in_drs_manifest"]], "load() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load"]], "load_manifest() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load_manifest"]], "md5sum (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.md5sum"]], "object_id (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_id"]], "object_id (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.object_id"]], "object_type (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_type"]], "pprint() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.pprint"]], "resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.resolve_objects"]], "start_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.start_time"]], "status (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.status"]], "updated_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.updated_time"]], "user_access() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.user_access"]], "acls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ACLS"]], "authz (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.AUTHZ"]], "current_dir (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.CURRENT_DIR"]], "current_dir (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.CURRENT_DIR"]], "guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.GUID"]], "indexd_record_page_size (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE"]], "max_concurrent_requests (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS"]], "max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS"]], "md5 (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.MD5"]], "prev_guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.PREV_GUID"]], "size (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.SIZE"]], "tmp_folder (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.TMP_FOLDER"]], "threadcontrol (class in gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ThreadControl"]], "urls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.URLS"]], "async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.async_download_object_manifest"]], "async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.async_verify_object_manifest"]], "delete_all_guids() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.delete_all_guids"]], "gen3.tools.indexing.download_manifest": [[11, "module-gen3.tools.indexing.download_manifest"]], "gen3.tools.indexing.index_manifest": [[11, "module-gen3.tools.indexing.index_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, "module-gen3.tools.indexing.verify_manifest"]], "index_object_manifest() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.index_object_manifest"]], "column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID"]], "guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT"]], "guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"]], "max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS"]], "async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest"]], "async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd"]], "gen3.tools.metadata.ingest_manifest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "gen3wsstorage (class in gen3.wss)": [[13, "gen3.wss.Gen3WsStorage"]], "copy() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.copy"]], "download() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download"]], "download_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download_url"]], "ls() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls"]], "ls_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls_path"]], "rm() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm"]], "rm_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm_path"]], "upload() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload"]], "upload_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload_url"]]}}) \ No newline at end of file diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html index bef41f01e..458a819ff 100644 --- a/docs/_build/html/tools/indexing.html +++ b/docs/_build/html/tools/indexing.html @@ -381,7 +381,7 @@

Indexing Tools
-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1708971147.0555665.log')[source]
+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1709222307.153123.log')[source]

Verify all file object records into a manifest csv

Parameters:
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html index 7d7c41e0b..b35d74eb1 100644 --- a/docs/_build/html/tools/metadata.html +++ b/docs/_build/html/tools/metadata.html @@ -102,7 +102,7 @@

Metadata Tools
-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1708971147.3980484.log', get_guid_from_file=True, metadata_type=None)[source]
+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1709222307.5054574.log', get_guid_from_file=True, metadata_type=None)[source]

Ingest all metadata records into a manifest csv

Parameters:
diff --git a/pyproject.toml b/pyproject.toml index 30c59bd6c..14b9f73bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "gen3" homepage = "https://gen3.org/" -version = "4.23.0" +version = "4.23.1" description = "Gen3 CLI and Python SDK" authors = ["Center for Translational Data Science at the University of Chicago "] license = "Apache-2.0"