From d0369a976cd8cfe1bf7b84de72e9f73f82c7b7d5 Mon Sep 17 00:00:00 2001 From: Kyle Burton Date: Thu, 31 Aug 2023 22:23:00 -0500 Subject: [PATCH 1/8] Convert date strings to datetime objects when adding new version --- .secrets.baseline | 16 +++++++-------- indexd/index/drivers/alchemy.py | 35 +++++++++++++++++++++++---------- pyproject.toml | 4 ++-- tests/test_client.py | 2 ++ 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index d2fc203d..8e939c75 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -349,49 +349,49 @@ "filename": "tests/test_client.py", "hashed_secret": "ff9c79b737b3ea7386618cc9437d3fb0a772182b", "is_verified": false, - "line_number": 2406 + "line_number": 2408 }, { "type": "Hex High Entropy String", "filename": "tests/test_client.py", "hashed_secret": "c8176f1e75e62e15dabaa4087fb7194451c8f6d2", "is_verified": false, - "line_number": 2409 + "line_number": 2411 }, { "type": "Hex High Entropy String", "filename": "tests/test_client.py", "hashed_secret": "d5198f8eddb1cbeb437899cd99e5ee97ab8531b4", "is_verified": false, - "line_number": 2409 + "line_number": 2411 }, { "type": "Hex High Entropy String", "filename": "tests/test_client.py", "hashed_secret": "02dc196562514eaa3e2feac1f441ccf6ad81e09d", "is_verified": false, - "line_number": 2413 + "line_number": 2415 }, { "type": "Hex High Entropy String", "filename": "tests/test_client.py", "hashed_secret": "f1cb2d91a95165a2ab909eadd9f7b65f312c7e2d", "is_verified": false, - "line_number": 2414 + "line_number": 2416 }, { "type": "Hex High Entropy String", "filename": "tests/test_client.py", "hashed_secret": "58db546de03270b55a4c889a5c5e6296b29fef25", "is_verified": false, - "line_number": 2415 + "line_number": 2417 }, { "type": "Hex High Entropy String", "filename": "tests/test_client.py", "hashed_secret": "b6c0bd08fde409c18760f32bef8705191840c402", "is_verified": false, - "line_number": 2416 + "line_number": 2418 } ], "tests/test_deprecated_aliases_endpoints.py": [ @@ -413,5 +413,5 @@ } ] }, - "generated_at": "2023-04-20T22:58:41Z" + "generated_at": "2023-09-01T03:22:57Z" } diff --git a/indexd/index/drivers/alchemy.py b/indexd/index/drivers/alchemy.py index 780d5796..54bd865e 100644 --- a/indexd/index/drivers/alchemy.py +++ b/indexd/index/drivers/alchemy.py @@ -676,6 +676,20 @@ def get_urls(self, size=None, hashes=None, ids=None, start=0, limit=100): for r in query ] + def _validate_and_format_content_dates( + self, record, content_created_date, content_updated_date + ): + if content_created_date is not None: + record.content_created_date = datetime.datetime.fromisoformat( + content_created_date + ) + # Users cannot set content_updated_date without a content_created_date + record.content_updated_date = ( + datetime.datetime.fromisoformat(content_updated_date) + if content_updated_date is not None + else record.content_created_date # Set updated to created if no updated is provided + ) + def add( self, form, @@ -755,16 +769,11 @@ def add( record.description = description - if content_created_date is not None: - record.content_created_date = datetime.datetime.fromisoformat( - content_created_date - ) - # Users cannot set content_updated_date without a content_created_date - record.content_updated_date = ( - datetime.datetime.fromisoformat(content_updated_date) - if content_updated_date is not None - else record.content_created_date # Set updated to created if no updated is provided - ) + self._validate_and_format_content_dates( + record=record, + content_created_date=content_created_date, + content_updated_date=content_updated_date, + ) session.merge(base_version) @@ -1384,6 +1393,12 @@ def add_version( for m_key, m_value in metadata.items() ] + self._validate_and_format_content_dates( + record=record, + content_created_date=content_created_date, + content_updated_date=content_updated_date, + ) + try: session.add(record) create_urls_metadata(urls_metadata, record, session) diff --git a/pyproject.toml b/pyproject.toml index 49bc2005..45acbbde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "indexd" -version = "5.0.0" +version = "5.0.1" description = "Gen3 Indexing Service" authors = ["CTDS UChicago "] license = "Apache-2.0" @@ -10,7 +10,7 @@ include = [ ] [tool.poetry.dependencies] -python = "3.9.*" +python = "^3.9" alembic = "^1.9.4" authutils = "^6.0.0" cdislogging = "^1.0.0" diff --git a/tests/test_client.py b/tests/test_client.py index eadc38f4..93cf119c 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -2115,6 +2115,8 @@ def test_create_index_version(client, user): "urls": ["s3://endpointurl/bucket2/key"], "hashes": {"md5": "8b9942cf415384b27cadf1f4d2d981f5"}, "acl": ["a"], + "content_updated_date": "2023-03-14T17:02:54", + "content_created_date": "2023-03-13T17:02:54", } res_2 = client.post("/index/" + rec["did"], json=dataNew, headers=user) From 693f0076f8efee93b1733ba3197f8715cc45cae4 Mon Sep 17 00:00:00 2001 From: Kyle Burton Date: Thu, 31 Aug 2023 22:32:22 -0500 Subject: [PATCH 2/8] Update indexd version --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 45acbbde..1ee236a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "indexd" -version = "5.0.1" +version = "5.0.2" description = "Gen3 Indexing Service" authors = ["CTDS UChicago "] license = "Apache-2.0" @@ -10,7 +10,7 @@ include = [ ] [tool.poetry.dependencies] -python = "^3.9" +python = "3.9.*" alembic = "^1.9.4" authutils = "^6.0.0" cdislogging = "^1.0.0" From 810605ee426ae6e4830a5e5e0e82afde52da5630 Mon Sep 17 00:00:00 2001 From: Kyle Burton Date: Fri, 1 Sep 2023 11:05:23 -0500 Subject: [PATCH 3/8] Travis jammy and postgresql13 update --- .secrets.baseline | 4 ++-- .travis.yml | 20 +++++++++++++++++++- travis/pg_hba.conf | 10 ++++++++++ travis/postgresql.conf | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 travis/pg_hba.conf create mode 100644 travis/postgresql.conf diff --git a/.secrets.baseline b/.secrets.baseline index 8e939c75..bf18f840 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -100,7 +100,7 @@ "filename": ".travis.yml", "hashed_secret": "8c0fdc72bf4daf2a2338287334d0cbd221fa9ef4", "is_verified": false, - "line_number": 27 + "line_number": 45 } ], "README.md": [ @@ -413,5 +413,5 @@ } ] }, - "generated_at": "2023-09-01T03:22:57Z" + "generated_at": "2023-09-01T16:05:20Z" } diff --git a/.travis.yml b/.travis.yml index 2f41e460..15253974 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: xenial +dist: jammy language: python python: @@ -9,6 +9,24 @@ sudo: false services: - postgresql +addons: + postgresql: "13" + apt: + sources: + - sourceline: deb http://apt.postgresql.org/pub/repos/apt/ jammy-pgdg main + 13 + key_url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + packages: + - postgresql-13 + +before_install: + # Copy custom configs from the repo because PG-13 isn't set up to run like + # it normally does on Travis out of the box. + # Source: https://github.com/NCI-GDC/psqlgraph/blob/94f315db2c039217752cba85d9c63988f2059317/.travis.yml + - sudo cp travis/postgresql.conf /etc/postgresql/13/main/postgresql.conf + - sudo cp travis/pg_hba.conf /etc/postgresql/13/main/pg_hba.conf + - sudo pg_ctlcluster 13 main restart + jdk: - oraclejdk8 diff --git a/travis/pg_hba.conf b/travis/pg_hba.conf new file mode 100644 index 00000000..e080219f --- /dev/null +++ b/travis/pg_hba.conf @@ -0,0 +1,10 @@ +# This config file will be used for the Travis test run. +# +# The new PostgreSQL 13 changes some settings from what they originally were +# in Travis, so we'll set them back. In particular we want to enable +# passwordless authentication for connections to PostgreSQL. +# Source: https://github.com/NCI-GDC/psqlgraph/blob/94f315db2c039217752cba85d9c63988f2059317/travis/pg_hba.conf +local all postgres trust +local all all trust +host all all 127.0.0.1/32 trust +host all all ::1/128 trust diff --git a/travis/postgresql.conf b/travis/postgresql.conf new file mode 100644 index 00000000..d3959e56 --- /dev/null +++ b/travis/postgresql.conf @@ -0,0 +1,32 @@ +# This config file will be used for PostgreSQL 13 because Travis doesn't +# have configurations set up for it yet. The most important part will be the +# ramfs storage location change. It also defaults to port 5433 so we need to +# change that back, too. +# Copied from https://github.com/NCI-GDC/psqlgraph/blob/94f315db2c039217752cba85d9c63988f2059317/travis/postgresql.conf +data_directory = '/var/ramfs/postgresql/13/main' +hba_file = '/etc/postgresql/13/main/pg_hba.conf' +ident_file = '/etc/postgresql/13/main/pg_ident.conf' +external_pid_file = '/var/run/postgresql/13-main.pid' +port = 5432 +max_connections = 255 +unix_socket_directories = '/var/run/postgresql' +ssl = on +ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem' +ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key' +shared_buffers = 128MB +dynamic_shared_memory_type = posix +max_wal_size = 256MB +min_wal_size = 80MB +log_line_prefix = '%t ' +log_timezone = 'UTC' +cluster_name = '13/main' +stats_temp_directory = '/var/run/postgresql/13-main.pg_stat_tmp' +datestyle = 'iso, mdy' +timezone = 'UTC' +lc_messages = 'en_US.UTF-8' +lc_monetary = 'en_US.UTF-8' +lc_numeric = 'en_US.UTF-8' +lc_time = 'en_US.UTF-8' +default_text_search_config = 'pg_catalog.english' +include_dir = 'conf.d' +fsync = false From f48fc26948ed9e23b2c204fa92b35b48641b1283 Mon Sep 17 00:00:00 2001 From: Kyle Burton Date: Tue, 26 Sep 2023 19:47:49 -0500 Subject: [PATCH 4/8] Use cdislogger over flask app logging --- indexd/app.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/indexd/app.py b/indexd/app.py index e0bf52a3..73f4c6f8 100644 --- a/indexd/app.py +++ b/indexd/app.py @@ -17,17 +17,18 @@ from .blueprint import blueprint as cross_blueprint from indexd.urls.blueprint import blueprint as index_urls_blueprint +logger = cdislogging.get_logger(__name__) + def app_init(app, settings=None): app.url_map.strict_slashes = False - app.logger.addHandler(cdislogging.get_stream_handler()) if not settings: from .default_settings import settings app.config.update(settings["config"]) if settings.get("AUTO_MIGRATE", True): engine_name = settings["config"]["INDEX"]["driver"].engine.dialect.name - app.logger.info(f"Auto migrating. Engine name: {engine_name}") + logger.info(f"Auto migrating. Engine name: {engine_name}") if engine_name == "sqlite": IndexBase.metadata.create_all() AliasBase.metadata.create_all() @@ -37,7 +38,7 @@ def app_init(app, settings=None): else: alembic_main(["--raiseerr", "upgrade", "head"]) else: - app.logger.info("Auto migrations are disabled") + logger.info("Auto migrations are disabled") app.auth = settings["auth"] app.hostname = os.environ.get("HOSTNAME") or "http://example.io" From c94fe9eca73d9da9a30688c0e204418e7a15bfcf Mon Sep 17 00:00:00 2001 From: Kyle Burton Date: Wed, 27 Sep 2023 18:03:59 -0500 Subject: [PATCH 5/8] Remove travis, add GH Action CI for Unit Tests --- .github/workflows/ci.yaml | 21 ++++++++++++++++++ .secrets.baseline | 12 +++++------ .travis.yml | 45 --------------------------------------- travis/pg_hba.conf | 10 --------- travis/postgresql.conf | 32 ---------------------------- 5 files changed, 27 insertions(+), 93 deletions(-) create mode 100644 .github/workflows/ci.yaml delete mode 100644 .travis.yml delete mode 100644 travis/pg_hba.conf delete mode 100644 travis/postgresql.conf diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..c3c495d4 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,21 @@ +name: CI +on: + push: + pull_request: + types: [opened, reopened] +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true +jobs: + Security: + name: Security Pipeline + uses: uc-cdis/.github/.github/workflows/securitypipeline.yaml@master + with: + python-poetry: 'false' + secrets: inherit + + UnitTest: + name: Python Unit Test with Postgres + uses: uc-cdis/.github/.github/workflows/python_unit_test.yaml@master + with: + python-version: '3.9' diff --git a/.secrets.baseline b/.secrets.baseline index bf18f840..9aefa253 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -94,13 +94,13 @@ } ], "results": { - ".travis.yml": [ + ".github/workflows/ci.yaml": [ { - "type": "Base64 High Entropy String", - "filename": ".travis.yml", - "hashed_secret": "8c0fdc72bf4daf2a2338287334d0cbd221fa9ef4", + "type": "Secret Keyword", + "filename": ".github/workflows/ci.yaml", + "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0", "is_verified": false, - "line_number": 45 + "line_number": 15 } ], "README.md": [ @@ -413,5 +413,5 @@ } ] }, - "generated_at": "2023-09-01T16:05:20Z" + "generated_at": "2023-09-27T23:03:38Z" } diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 15253974..00000000 --- a/.travis.yml +++ /dev/null @@ -1,45 +0,0 @@ -dist: jammy -language: python - -python: - - "3.9" - -sudo: false - -services: - - postgresql - -addons: - postgresql: "13" - apt: - sources: - - sourceline: deb http://apt.postgresql.org/pub/repos/apt/ jammy-pgdg main - 13 - key_url: https://www.postgresql.org/media/keys/ACCC4CF8.asc - packages: - - postgresql-13 - -before_install: - # Copy custom configs from the repo because PG-13 isn't set up to run like - # it normally does on Travis out of the box. - # Source: https://github.com/NCI-GDC/psqlgraph/blob/94f315db2c039217752cba85d9c63988f2059317/.travis.yml - - sudo cp travis/postgresql.conf /etc/postgresql/13/main/postgresql.conf - - sudo cp travis/pg_hba.conf /etc/postgresql/13/main/pg_hba.conf - - sudo pg_ctlcluster 13 main restart - -jdk: - - oraclejdk8 - -install: - - pip install poetry - - poetry install -vv - - -script: pytest -vv --cov=indexd --cov-report xml tests - -after_script: - - COVERALLS_REPO_TOKEN=$COVERALLS_TOKEN coveralls - -env: - global: - secure: h7ZQM1IMz67u8V+g/EqOXGH7SngyTZkaTRHDnnxlwql5vJ+gF8dlPgAzSN9xBn9SpSS0o5jqcZZ+6tN7FZEnPlZjCU15hN0G450r75zsKMFpWKl/8Xz3SwBJHx529ZOe9R9nrwOVmnedHxqbiQqgCxWwTDWsvmce6dF3GwigOfKNuW2UD2wdRX3quaMDAILZ0WyO6vGx6SVnr2sdhN4sJqCugXxjN72o/JLHnRBg5Cx2U4BdPguqaaEV6KjloxgTTtZEp0IZkrnv0WrxCgNuDoPYkfPEAD2hzrAgUkj0c+6WQg/aOV/k0MAKNLX4et+3haRAFeDlzAJBGPgUJN3zvPXGkPixTBiD5zDyplzjvF24qCSmXeH5SMKu//6lGeOuNgqJria3VD8xt+lAdGg4Fgzp+zDLQ17CSmJx5KRyQroZsp/YqD0ezT0mQZMRLuTNuw+UWQquk1ufotOD4o5KGssvcu03XD7SVc5CnsWrmlkN8MH5g19ddwNOSKKYu7Ran4//XOWOfSfzLCgYPgQbYebZXdpbxNNWK57V9sPHKRzrQCo26HNXWgWebbzYI1z5aoq6bY5dB7Un1YFUxQlBdNzZnx/soErPL+omH8nw9Usfv20uhB4lQo1plkoctpqLITuVDe8ZtSbUEZB7cZGcrfb/c5E9TLC9EqZu/e60qN8= diff --git a/travis/pg_hba.conf b/travis/pg_hba.conf deleted file mode 100644 index e080219f..00000000 --- a/travis/pg_hba.conf +++ /dev/null @@ -1,10 +0,0 @@ -# This config file will be used for the Travis test run. -# -# The new PostgreSQL 13 changes some settings from what they originally were -# in Travis, so we'll set them back. In particular we want to enable -# passwordless authentication for connections to PostgreSQL. -# Source: https://github.com/NCI-GDC/psqlgraph/blob/94f315db2c039217752cba85d9c63988f2059317/travis/pg_hba.conf -local all postgres trust -local all all trust -host all all 127.0.0.1/32 trust -host all all ::1/128 trust diff --git a/travis/postgresql.conf b/travis/postgresql.conf deleted file mode 100644 index d3959e56..00000000 --- a/travis/postgresql.conf +++ /dev/null @@ -1,32 +0,0 @@ -# This config file will be used for PostgreSQL 13 because Travis doesn't -# have configurations set up for it yet. The most important part will be the -# ramfs storage location change. It also defaults to port 5433 so we need to -# change that back, too. -# Copied from https://github.com/NCI-GDC/psqlgraph/blob/94f315db2c039217752cba85d9c63988f2059317/travis/postgresql.conf -data_directory = '/var/ramfs/postgresql/13/main' -hba_file = '/etc/postgresql/13/main/pg_hba.conf' -ident_file = '/etc/postgresql/13/main/pg_ident.conf' -external_pid_file = '/var/run/postgresql/13-main.pid' -port = 5432 -max_connections = 255 -unix_socket_directories = '/var/run/postgresql' -ssl = on -ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem' -ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key' -shared_buffers = 128MB -dynamic_shared_memory_type = posix -max_wal_size = 256MB -min_wal_size = 80MB -log_line_prefix = '%t ' -log_timezone = 'UTC' -cluster_name = '13/main' -stats_temp_directory = '/var/run/postgresql/13-main.pg_stat_tmp' -datestyle = 'iso, mdy' -timezone = 'UTC' -lc_messages = 'en_US.UTF-8' -lc_monetary = 'en_US.UTF-8' -lc_numeric = 'en_US.UTF-8' -lc_time = 'en_US.UTF-8' -default_text_search_config = 'pg_catalog.english' -include_dir = 'conf.d' -fsync = false From 41666a7f19668b4de43192d7dfdb7a2e1b9cbb59 Mon Sep 17 00:00:00 2001 From: burtonk <117617405+k-burt-uch@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:59:02 -0500 Subject: [PATCH 6/8] Update .github/workflows/ci.yaml Co-authored-by: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c3c495d4..e9f4b7dc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,8 +1,8 @@ name: CI on: push: - pull_request: - types: [opened, reopened] + pull_request: + types: [opened, reopened] concurrency: group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' cancel-in-progress: true From c32122e8230f23bbbca282516b6ad89d057a6528 Mon Sep 17 00:00:00 2001 From: Kyle Burton Date: Thu, 28 Sep 2023 17:06:29 -0500 Subject: [PATCH 7/8] Testing coveralls --- .github/workflows/ci.yaml | 6 ++++++ tests/ci_commands_script.sh | 3 +++ 2 files changed, 9 insertions(+) create mode 100644 tests/ci_commands_script.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e9f4b7dc..19da4ca3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,3 +19,9 @@ jobs: uses: uc-cdis/.github/.github/workflows/python_unit_test.yaml@master with: python-version: '3.9' + test-script: 'tests/ci_commands_script.sh' + Coveralls: + name: Publish to coveralls.io + uses: coverallsapp/github-action@v1.1.2 + with: + github-token: ${{ github.token }} diff --git a/tests/ci_commands_script.sh b/tests/ci_commands_script.sh new file mode 100644 index 00000000..e82905b1 --- /dev/null +++ b/tests/ci_commands_script.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +poetry run pytest -vv --cov=indexd --cov-report xml tests From 797fdcf44f0439bdf9925f6d1cce6df1b0de98aa Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Fri, 29 Sep 2023 11:00:12 -0500 Subject: [PATCH 8/8] Add coveralls (#368) --- .github/workflows/ci.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 19da4ca3..3c787408 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,8 +20,4 @@ jobs: with: python-version: '3.9' test-script: 'tests/ci_commands_script.sh' - Coveralls: - name: Publish to coveralls.io - uses: coverallsapp/github-action@v1.1.2 - with: - github-token: ${{ github.token }} + run-coveralls: true