From 7fd2a24e72316cc75e954eeb11074d0faf09eee3 Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Mon, 16 Sep 2024 20:56:05 +0000 Subject: [PATCH 01/14] chore: init action to publish json schemas as release artifacts --- .github/workflows/build.yml | 13 +++++++++ .../chord/management/commands/schemas.py | 27 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 chord_metadata_service/chord/management/commands/schemas.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b2c9bf1ae..4915471ef 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,3 +33,16 @@ jobs: image-name: ghcr.io/bento-platform/katsu development-dockerfile: bento.dev.Dockerfile dockerfile: bento.Dockerfile + + # TODO: test, maybe just add to a protected build directory? + - name: Make release artifacts + run: | + ./manage.py schema phenopacket >> ./dist/schemas/phenopacket_schema.json + ./manage.py schema experiment >> ./dist/schemas/experiment_schema.json + ./manage.py schema discovery >> ./dist/schemas/discovery.json + + - name: Upload release artifacts + uses: actions/upload-artifact@v4 + with: + name: JSON-schemas + path: ./dist/schemas diff --git a/chord_metadata_service/chord/management/commands/schemas.py b/chord_metadata_service/chord/management/commands/schemas.py new file mode 100644 index 000000000..9914987dd --- /dev/null +++ b/chord_metadata_service/chord/management/commands/schemas.py @@ -0,0 +1,27 @@ +import json +from typing import Any +from django.core.management.base import BaseCommand, CommandParser + +from chord_metadata_service.chord.data_types import DATA_TYPE_EXPERIMENT, DATA_TYPE_PHENOPACKET +from chord_metadata_service.discovery.schemas import DISCOVERY_SCHEMA +from chord_metadata_service.experiments.schemas import EXPERIMENT_SCHEMA +from chord_metadata_service.phenopackets.schemas import PHENOPACKET_SCHEMA + +NAME_TO_SCHEMA: dict[str, object] = { + DATA_TYPE_PHENOPACKET: PHENOPACKET_SCHEMA, + DATA_TYPE_EXPERIMENT: EXPERIMENT_SCHEMA, + "discovery": DISCOVERY_SCHEMA, +} + +class Command(BaseCommand): + help = """ + Compiles and returns a JSON-schema in a single JSON file for artifact. + Use in GitHub Actions in order to publish usable schemas on releases. + """ + + def add_arguments(self, parser: CommandParser) -> None: + parser.add_argument("schema", action="store", type=str, choices=NAME_TO_SCHEMA.keys()) + + def handle(self, *args: Any, **options: Any) -> str | None: + schema = NAME_TO_SCHEMA[options["schema"]] + self.stdout.write(json.dumps(schema)) From 3ecf74aab68bcb8ca4af04f34896bd88e49e6b57 Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Mon, 16 Sep 2024 21:03:09 +0000 Subject: [PATCH 02/14] lint --- .../chord/management/commands/schemas.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/chord_metadata_service/chord/management/commands/schemas.py b/chord_metadata_service/chord/management/commands/schemas.py index 9914987dd..8a13c5a08 100644 --- a/chord_metadata_service/chord/management/commands/schemas.py +++ b/chord_metadata_service/chord/management/commands/schemas.py @@ -13,15 +13,16 @@ "discovery": DISCOVERY_SCHEMA, } + class Command(BaseCommand): - help = """ - Compiles and returns a JSON-schema in a single JSON file for artifact. - Use in GitHub Actions in order to publish usable schemas on releases. - """ + help = """ + Compiles and returns a JSON-schema in a single JSON file for artifact. + Use in GitHub Actions in order to publish usable schemas on releases. + """ - def add_arguments(self, parser: CommandParser) -> None: - parser.add_argument("schema", action="store", type=str, choices=NAME_TO_SCHEMA.keys()) + def add_arguments(self, parser: CommandParser) -> None: + parser.add_argument("schema", action="store", type=str, choices=NAME_TO_SCHEMA.keys()) - def handle(self, *args: Any, **options: Any) -> str | None: - schema = NAME_TO_SCHEMA[options["schema"]] - self.stdout.write(json.dumps(schema)) + def handle(self, *args: Any, **options: Any) -> str | None: + schema = NAME_TO_SCHEMA[options["schema"]] + self.stdout.write(json.dumps(schema)) From a40ebdca19bfee4aa0e43f063ec059e8d9087dd1 Mon Sep 17 00:00:00 2001 From: Victor Rocheleau Date: Mon, 16 Sep 2024 21:04:19 +0000 Subject: [PATCH 03/14] init artifact dir --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4915471ef..bd9b7f9f3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,6 +37,7 @@ jobs: # TODO: test, maybe just add to a protected build directory? - name: Make release artifacts run: | + mkdir -p ./dist/schemas ./manage.py schema phenopacket >> ./dist/schemas/phenopacket_schema.json ./manage.py schema experiment >> ./dist/schemas/experiment_schema.json ./manage.py schema discovery >> ./dist/schemas/discovery.json From fe02f362e9c005d74d71842ddc9f3972e973bd2a Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 17 Sep 2024 13:39:47 -0400 Subject: [PATCH 04/14] chore: release gh actions workflow test --- .github/workflows/build.yml | 14 --------- .github/workflows/release.yml | 59 +++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd9b7f9f3..b2c9bf1ae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,17 +33,3 @@ jobs: image-name: ghcr.io/bento-platform/katsu development-dockerfile: bento.dev.Dockerfile dockerfile: bento.Dockerfile - - # TODO: test, maybe just add to a protected build directory? - - name: Make release artifacts - run: | - mkdir -p ./dist/schemas - ./manage.py schema phenopacket >> ./dist/schemas/phenopacket_schema.json - ./manage.py schema experiment >> ./dist/schemas/experiment_schema.json - ./manage.py schema discovery >> ./dist/schemas/discovery.json - - - name: Upload release artifacts - uses: actions/upload-artifact@v4 - with: - name: JSON-schemas - path: ./dist/schemas diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..d15d0f1f6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,59 @@ +name: Create and upload release artifacts +on: + release: + types: [published, prereleased] + # temporary PR trigger for test + pull_request: + branches: + - develop + - 'features/**' + +jobs: + release-artifacts: + runs-on: ubuntu-latest + + permissions: + contents: read + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install Poetry + run: python -m pip install poetry + - name: Install dependencies + run: poetry install + - name: Make release artifacts + run: | + mkdir -p ./dist/schemas + ./manage.py schema phenopacket >> ./dist/schemas/phenopacket_schema.json + ./manage.py schema experiment >> ./dist/schemas/experiment_schema.json + ./manage.py schema discovery >> ./dist/schemas/discovery.json + - name: Upload release artifacts + uses: actions/github-scripts@v7 + with: + script: | + const fs = require(fs); + + # temporary hardcoding for test + const tag = "v8.0.2-test" + # const tag = context.ref.replace("refs/tags/", ""); + console.log("tag = ", tag); + + const release = await github.rest.repos.getReleaseByTag({ + owner: context.repo.owner, + repo: context.repo.repo, + tag + }); + await github.rest.repos.uploadReleaseAsset({ + owner: context.repo.owner, + repo: context.repo.repo, + release_id: release.data.id, + name: "JSON Schemas", + data: await fs.readdirSync("./dist/schemas") + }); From cebe42784a888d6dc3adad353db8801600a46147 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 17 Sep 2024 13:41:19 -0400 Subject: [PATCH 05/14] fix: release actions typo --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d15d0f1f6..c5fa82e52 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,7 @@ jobs: ./manage.py schema experiment >> ./dist/schemas/experiment_schema.json ./manage.py schema discovery >> ./dist/schemas/discovery.json - name: Upload release artifacts - uses: actions/github-scripts@v7 + uses: actions/github-script@v7 with: script: | const fs = require(fs); From 8bda0ff1f03c59fda126ace8f4aeb757daa6fef9 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 17 Sep 2024 13:43:28 -0400 Subject: [PATCH 06/14] fix: release action uses poetry to create schemas --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c5fa82e52..1bc00a039 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,9 +31,9 @@ jobs: - name: Make release artifacts run: | mkdir -p ./dist/schemas - ./manage.py schema phenopacket >> ./dist/schemas/phenopacket_schema.json - ./manage.py schema experiment >> ./dist/schemas/experiment_schema.json - ./manage.py schema discovery >> ./dist/schemas/discovery.json + poetry run ./manage.py schema phenopacket >> ./dist/schemas/phenopacket_schema.json + poetry run ./manage.py schema experiment >> ./dist/schemas/experiment_schema.json + poetry run ./manage.py schema discovery >> ./dist/schemas/discovery.json - name: Upload release artifacts uses: actions/github-script@v7 with: From 7789a08d68300a64ed61c6427a0d222c875566c0 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 17 Sep 2024 13:45:47 -0400 Subject: [PATCH 07/14] fix: release workflow schemas typo --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1bc00a039..52965ce84 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,9 +31,9 @@ jobs: - name: Make release artifacts run: | mkdir -p ./dist/schemas - poetry run ./manage.py schema phenopacket >> ./dist/schemas/phenopacket_schema.json - poetry run ./manage.py schema experiment >> ./dist/schemas/experiment_schema.json - poetry run ./manage.py schema discovery >> ./dist/schemas/discovery.json + poetry run ./manage.py schemas phenopacket >> ./dist/schemas/phenopacket_schema.json + poetry run ./manage.py schemas experiment >> ./dist/schemas/experiment_schema.json + poetry run ./manage.py schemas discovery >> ./dist/schemas/discovery.json - name: Upload release artifacts uses: actions/github-script@v7 with: From 7b2b0cc45c6adb2efe26211d9472498a5fae7245 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 17 Sep 2024 13:55:12 -0400 Subject: [PATCH 08/14] debug get release --- .github/workflows/release.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 52965ce84..8e910c4cb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,20 +40,21 @@ jobs: script: | const fs = require(fs); - # temporary hardcoding for test - const tag = "v8.0.2-test" - # const tag = context.ref.replace("refs/tags/", ""); + // temporary hardcoding for test + const tag = "v8.0.2-test"; + // const tag = context.ref.replace("refs/tags/", ""); console.log("tag = ", tag); const release = await github.rest.repos.getReleaseByTag({ owner: context.repo.owner, repo: context.repo.repo, - tag + tag: tag, }); + console.log("release id = ", release.data.id) await github.rest.repos.uploadReleaseAsset({ owner: context.repo.owner, repo: context.repo.repo, release_id: release.data.id, name: "JSON Schemas", - data: await fs.readdirSync("./dist/schemas") + data: await fs.readdirSync("./dist/schemas"), }); From f171f733791f75770571cd2f8432928902a12657 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 17 Sep 2024 13:56:56 -0400 Subject: [PATCH 09/14] fix: release action script fs import --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8e910c4cb..360d0de03 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,7 +38,7 @@ jobs: uses: actions/github-script@v7 with: script: | - const fs = require(fs); + const fs = require("fs"); // temporary hardcoding for test const tag = "v8.0.2-test"; From 04d84cee94403f46a5b41eff339e7ad4d2623229 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 17 Sep 2024 14:03:26 -0400 Subject: [PATCH 10/14] fix: release workflow permissions --- .github/workflows/release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 360d0de03..186cd38d3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,8 @@ jobs: runs-on: ubuntu-latest permissions: - contents: read + contents: write + repository-projects: write steps: - name: Checkout From 178077575a0795fc71a53c9183a7494c9702be10 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 17 Sep 2024 14:19:27 -0400 Subject: [PATCH 11/14] upload schemas as zip --- .github/workflows/release.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 186cd38d3..13f5aeb26 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,6 +35,7 @@ jobs: poetry run ./manage.py schemas phenopacket >> ./dist/schemas/phenopacket_schema.json poetry run ./manage.py schemas experiment >> ./dist/schemas/experiment_schema.json poetry run ./manage.py schemas discovery >> ./dist/schemas/discovery.json + zip -r json-schemas.zip ./dist/schemas - name: Upload release artifacts uses: actions/github-script@v7 with: @@ -56,6 +57,6 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, release_id: release.data.id, - name: "JSON Schemas", - data: await fs.readdirSync("./dist/schemas"), + name: "json-schemas.zip", + data: await fs.readFile("./json-schemas.zip"), }); From c618ff36b79870cf16ecbcb958fc3e08b7d14f8f Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 17 Sep 2024 14:23:38 -0400 Subject: [PATCH 12/14] fix: release action read file no cb --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 13f5aeb26..225df504b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,5 +58,5 @@ jobs: repo: context.repo.repo, release_id: release.data.id, name: "json-schemas.zip", - data: await fs.readFile("./json-schemas.zip"), + data: await fs.readFileSync("./json-schemas.zip"), }); From d0819aaececa8204e5cd44a3b6b6ceebbb9ccc6e Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 17 Sep 2024 14:42:00 -0400 Subject: [PATCH 13/14] fix: pretty JSON artifact --- .github/workflows/release.yml | 5 +++-- chord_metadata_service/chord/management/commands/schemas.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 225df504b..a3bd30a56 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,8 @@ jobs: poetry run ./manage.py schemas phenopacket >> ./dist/schemas/phenopacket_schema.json poetry run ./manage.py schemas experiment >> ./dist/schemas/experiment_schema.json poetry run ./manage.py schemas discovery >> ./dist/schemas/discovery.json - zip -r json-schemas.zip ./dist/schemas + pushd ./dist/schemas + zip -r ../../json-schemas.zip * - name: Upload release artifacts uses: actions/github-script@v7 with: @@ -57,6 +58,6 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, release_id: release.data.id, - name: "json-schemas.zip", + name: "json-schemas-clean.zip", data: await fs.readFileSync("./json-schemas.zip"), }); diff --git a/chord_metadata_service/chord/management/commands/schemas.py b/chord_metadata_service/chord/management/commands/schemas.py index 8a13c5a08..c20e1a6ef 100644 --- a/chord_metadata_service/chord/management/commands/schemas.py +++ b/chord_metadata_service/chord/management/commands/schemas.py @@ -25,4 +25,4 @@ def add_arguments(self, parser: CommandParser) -> None: def handle(self, *args: Any, **options: Any) -> str | None: schema = NAME_TO_SCHEMA[options["schema"]] - self.stdout.write(json.dumps(schema)) + self.stdout.write(json.dumps(schema, indent=2)) From 4dac6510540586619df58d38f1a1437a9888bc31 Mon Sep 17 00:00:00 2001 From: v-rocheleau Date: Tue, 17 Sep 2024 15:02:29 -0400 Subject: [PATCH 14/14] chore: release workflow trigger on published and prereleased only --- .github/workflows/release.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a3bd30a56..5c5f2f5aa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,11 +2,6 @@ name: Create and upload release artifacts on: release: types: [published, prereleased] - # temporary PR trigger for test - pull_request: - branches: - - develop - - 'features/**' jobs: release-artifacts: @@ -43,21 +38,19 @@ jobs: script: | const fs = require("fs"); - // temporary hardcoding for test - const tag = "v8.0.2-test"; - // const tag = context.ref.replace("refs/tags/", ""); - console.log("tag = ", tag); + // get tag + const tag = context.ref.replace("refs/tags/", ""); + // get release from tag const release = await github.rest.repos.getReleaseByTag({ owner: context.repo.owner, repo: context.repo.repo, tag: tag, }); - console.log("release id = ", release.data.id) await github.rest.repos.uploadReleaseAsset({ owner: context.repo.owner, repo: context.repo.repo, release_id: release.data.id, - name: "json-schemas-clean.zip", + name: "json-schemas.zip", data: await fs.readFileSync("./json-schemas.zip"), });