From f94114d76b75d0b81537ae44f73e4df40cc787a0 Mon Sep 17 00:00:00 2001 From: sulhicader Date: Sun, 29 Oct 2023 22:50:58 +0530 Subject: [PATCH 01/24] packaging: Initial workflow files --- .../workflows/helm-release-university.yaml | 29 ++++++ .github/workflows/wave-bundle-university.yaml | 48 +++++++++ .../workflows/wave-publish-university.yaml | 29 ++++++ .github/workflows/wave-university.yaml | 31 ++++++ ...2o.wave.university.1.0.0.bundle.Dockerfile | 4 + ...o.wave.university.1.0.0.runtime.Dockerfile | 33 +++++++ .../helm/charts/university/Chart.yaml | 7 ++ .../charts/university/templates/_helpers.tpl | 42 ++++++++ .../university/templates/configmap.yaml | 9 ++ .../helm/charts/university/templates/job.yaml | 86 ++++++++++++++++ .../charts/university/templates/secret.yaml | 23 +++++ .../helm/charts/university/values.yaml | 98 +++++++++++++++++++ 12 files changed, 439 insertions(+) create mode 100644 .github/workflows/helm-release-university.yaml create mode 100644 .github/workflows/wave-bundle-university.yaml create mode 100644 .github/workflows/wave-publish-university.yaml create mode 100644 .github/workflows/wave-university.yaml create mode 100644 university/bundle_generated_files/ai.h2o.wave.university.1.0.0.bundle.Dockerfile create mode 100644 university/bundle_generated_files/ai.h2o.wave.university.1.0.0.runtime.Dockerfile create mode 100644 university/bundle_generated_files/helm/charts/university/Chart.yaml create mode 100644 university/bundle_generated_files/helm/charts/university/templates/_helpers.tpl create mode 100644 university/bundle_generated_files/helm/charts/university/templates/configmap.yaml create mode 100644 university/bundle_generated_files/helm/charts/university/templates/job.yaml create mode 100644 university/bundle_generated_files/helm/charts/university/templates/secret.yaml create mode 100644 university/bundle_generated_files/helm/charts/university/values.yaml diff --git a/.github/workflows/helm-release-university.yaml b/.github/workflows/helm-release-university.yaml new file mode 100644 index 0000000000..3d16e137c1 --- /dev/null +++ b/.github/workflows/helm-release-university.yaml @@ -0,0 +1,29 @@ +name: Release Generated Helm Chart + +on: + workflow_call: + +jobs: + retrieve-metadata: + runs-on: ubuntu-latest + outputs: + build-version: ${{ steps.get-build-version.outputs.VERSION }} + + steps: + - uses: actions/checkout@v3 + + - name: Get App Version + id: get-build-version + run: | + echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT" + + release-helm-chart: + needs: retrieve-metadata + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + + uses: h2oai/workflows-library/.github/workflows/wave-bundle-helm-release.yaml@main + with: + build-version: ${{ needs.retrieve-metadata.outputs.build-version }} + helm-chart-artifact: wave-bundle-helm diff --git a/.github/workflows/wave-bundle-university.yaml b/.github/workflows/wave-bundle-university.yaml new file mode 100644 index 0000000000..a826852f13 --- /dev/null +++ b/.github/workflows/wave-bundle-university.yaml @@ -0,0 +1,48 @@ +name: Wave Bundle + +on: + workflow_call: + +jobs: + airgapped: + name: Create Wave Bundle + runs-on: ubuntu-latest + outputs: + build-version: ${{ steps.get-build-version.outputs.VERSION }} + + steps: + - uses: actions/checkout@v3 + + - name: Download H2O CLI + run: mkdir -p .bin && curl -o .bin/h2o https://h2oai-cloud-release.s3.amazonaws.com/releases/ai/h2o/h2o-cloud/v0.28.2/cli/linux-amd64/h2o + + - name: Change permissions + run: chmod +x .bin/h2o + + - name: Get App Version + id: get-build-version + run: | + echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT" + + - name: Make air-gapped bundle + run: | + .bin/h2o bundle \ + --docker-base-image 524466471676.dkr.ecr.us-east-1.amazonaws.com/q8s/launcher:v0.23.0-38 \ + --docker-include static \ + --docker-use-buildkit \ + --generate-helm-charts \ + --helm-chart-version ${{ steps.get-build-version.outputs.VERSION }} \ + --helm-chart-name ${{ github.event.repository.name }} \ + --generate-dockerfile + + - uses: actions/upload-artifact@v3 + with: + name: wave-bundle + path: | + ./*.Dockerfile + ./*.wave + + - uses: actions/upload-artifact@v3 + with: + name: wave-bundle-helm + path: helm/ diff --git a/.github/workflows/wave-publish-university.yaml b/.github/workflows/wave-publish-university.yaml new file mode 100644 index 0000000000..58ce20b841 --- /dev/null +++ b/.github/workflows/wave-publish-university.yaml @@ -0,0 +1,29 @@ +name: Publish Wave Bundle + +on: + workflow_call: + +jobs: + retrieve-metadata: + runs-on: ubuntu-latest + outputs: + build-version: ${{ steps.get-build-version.outputs.VERSION }} + + steps: + - uses: actions/checkout@v3 + + - name: Get App Version + id: get-build-version + run: | + echo "VERSION=$(make version)" >> "$GITHUB_OUTPUT" + + build-and-publish: + needs: retrieve-metadata + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + + uses: h2oai/workflows-library/.github/workflows/wave-bundle-docker-build-publish.yaml@main + with: + build-version: ${{ needs.retrieve-metadata.outputs.build-version }} + bundle-artifact: wave-bundle diff --git a/.github/workflows/wave-university.yaml b/.github/workflows/wave-university.yaml new file mode 100644 index 0000000000..777d4696b5 --- /dev/null +++ b/.github/workflows/wave-university.yaml @@ -0,0 +1,31 @@ +name: Wave Bundle + +on: + workflow_dispatch: + inputs: + version: + description: 'Release Version' + required: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: "${{ github.event.inputs.version }}" + TAG: "v${{ github.event.inputs.version }}" + +jobs: + bundle: + name: Create Wave Bundle + working-directory: ./university + uses: ./.github/workflows/wave-bundle-university.yaml + + publish: + needs: bundle + name: Build and Publish + working-directory: ./university + uses: ./.github/workflows/wave-publish-university.yaml + + helm-publish: + needs: bundle + name: Build and Publish Helm Chart + working-directory: ./university + uses: ./.github/workflows/helm-release-university.yaml diff --git a/university/bundle_generated_files/ai.h2o.wave.university.1.0.0.bundle.Dockerfile b/university/bundle_generated_files/ai.h2o.wave.university.1.0.0.bundle.Dockerfile new file mode 100644 index 0000000000..da9fa85d8b --- /dev/null +++ b/university/bundle_generated_files/ai.h2o.wave.university.1.0.0.bundle.Dockerfile @@ -0,0 +1,4 @@ +FROM alpine:3.18 + +COPY ../ai.h2o.wave.university.1.0.0.wave /app/ai.h2o.wave.university.1.0.0.wave +ENV WAVE_BUNDLE_FILE /app/ai.h2o.wave.university.1.0.0.wave diff --git a/university/bundle_generated_files/ai.h2o.wave.university.1.0.0.runtime.Dockerfile b/university/bundle_generated_files/ai.h2o.wave.university.1.0.0.runtime.Dockerfile new file mode 100644 index 0000000000..c91b4e8154 --- /dev/null +++ b/university/bundle_generated_files/ai.h2o.wave.university.1.0.0.runtime.Dockerfile @@ -0,0 +1,33 @@ +FROM 524466471676.dkr.ecr.us-east-1.amazonaws.com/q8s/launcher:v0.23.0-gpu38 + +WORKDIR /app/ + +# Create a virtual environment +RUN python3 -m venv /app/venv + +LABEL \ + # Version of the bundle cmd used to generate this Dockerfile + ai.h2o.appstore.bundle.version = 0.0.1 \ + # App name as defined in the app.toml + ai.h2o.appstore.app.name = "ai.h2o.wave.university" \ + # App version as defined in the app.toml + ai.h2o.appstore.app.version = "1.0.0" + +# Unpack .wave bundle +RUN --mount=type=bind,src=ai.h2o.wave.university.1.0.0.wave,target=/app/ai.h2o.wave.university.1.0.0.wave \ + unzip ai.h2o.wave.university.1.0.0.wave || UNZIP_EXIT_CODE=$? && \ + # Ignore exit code 2, seems to be a false positive + if [ $UNZIP_EXIT_CODE != 0 ] && [ $UNZIP_EXIT_CODE != 2 ]; then echo "unzip failed with exit code $UNZIP_EXIT_CODE"; exit $UNZIP_EXIT_CODE; fi + +# Install Python dependencies +RUN --mount=type=cache,target=/home/.cache \ + --mount=type=cache,target=/root/.cache \ + --mount=type=cache,target=/.cache \ + /app/venv/bin/pip3 install -r requirements.txt + +# Path to the venv directory +ENV H2O_CLOUD_VENV_PATH /app/venv +# Path to the app directory +ENV H2O_CLOUD_APP_ROOT /app +# App main module +ENV H2O_CLOUD_PY_MODULE=h2o_wave_university.university diff --git a/university/bundle_generated_files/helm/charts/university/Chart.yaml b/university/bundle_generated_files/helm/charts/university/Chart.yaml new file mode 100644 index 0000000000..82fde127af --- /dev/null +++ b/university/bundle_generated_files/helm/charts/university/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +appVersion: 1.0.0 +name: university +version: 0.1.0 +description: | + Interactive Wave learning experience. +type: application diff --git a/university/bundle_generated_files/helm/charts/university/templates/_helpers.tpl b/university/bundle_generated_files/helm/charts/university/templates/_helpers.tpl new file mode 100644 index 0000000000..6b5305508e --- /dev/null +++ b/university/bundle_generated_files/helm/charts/university/templates/_helpers.tpl @@ -0,0 +1,42 @@ +{{- define "university.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "university.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "university.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "university.labels" -}} +helm.sh/chart: {{ include "university.chart" . }} +{{ include "university.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{- define "university.selectorLabels" -}} +app.kubernetes.io/name: {{ include "university.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{- define "university.import.auth.oidc.usernamePassword.secretName" -}} +{{- if .Values.import.auth.oidc.usernamePassword.existingSecret }} +{{- .Values.import.auth.oidc.usernamePassword.existingSecret }} +{{- else }} +{{- printf "%s-import-creds" (include "university.fullname" .) }} +{{- end }} +{{- end }} diff --git a/university/bundle_generated_files/helm/charts/university/templates/configmap.yaml b/university/bundle_generated_files/helm/charts/university/templates/configmap.yaml new file mode 100644 index 0000000000..ecc8009e75 --- /dev/null +++ b/university/bundle_generated_files/helm/charts/university/templates/configmap.yaml @@ -0,0 +1,9 @@ +{{- if .Values.caCertificates }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "university.fullname" . }}-ca-certificates +data: + root-ca-bundle.crt: | + {{ .Values.caCertificates | nindent 4 | trim }} +{{- end }} diff --git a/university/bundle_generated_files/helm/charts/university/templates/job.yaml b/university/bundle_generated_files/helm/charts/university/templates/job.yaml new file mode 100644 index 0000000000..b0c8bd2315 --- /dev/null +++ b/university/bundle_generated_files/helm/charts/university/templates/job.yaml @@ -0,0 +1,86 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "university.fullname" . }}-import + labels: + {{- include "university.labels" . | nindent 4 }} + annotations: + # marks this as a hook - https://helm.sh/docs/topics/charts_hooks/ + # this resource will be automatically removed after the install/upgrade is complete, given + # the job completes successfully + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + backoffLimit: 10 + template: + metadata: + name: {{ include "university.fullname" . }}-import + labels: + {{- include "university.labels" . | nindent 8 }} + spec: + restartPolicy: Never + {{- with coalesce .Values.import.image.pullSecrets .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + initContainers: + - name: cp-bundle + imagePullPolicy: {{ .Values.import.image.pullPolicy | default .Values.image.pullPolicy | quote }} + image: "{{ required "'university.bundleImage.repository' is required!" .Values.university.bundleImage.repository }}:{{ required "'university.bundleImage.tag' is required!" .Values.university.bundleImage.tag }}" + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + command: ["/bin/sh"] + args: ["-c", "cp ${WAVE_BUNDLE_FILE} /workspace/app-bundle.wave"] + volumeMounts: + - mountPath: /workspace + name: "wave-bundle-volume" + containers: + - name: import-app + imagePullPolicy: {{ .Values.import.image.pullPolicy | default .Values.image.pullPolicy | quote }} + image: "{{ required "'import.image.repository' is required!" .Values.import.image.repository }}:{{ required "'import.image.tag' is required!" .Values.import.image.tag }}" + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.import.auth.oidc.usernamePassword.enabled }} + env: + - name: H2O_IMPORT_APP_USERNAME + valueFrom: + secretKeyRef: + name: "{{ include "university.import.auth.oidc.usernamePassword.secretName" . }}" + key: "{{ .Values.import.auth.oidc.usernamePassword.usernameKey }}" + - name: H2O_IMPORT_APP_PASSWORD + valueFrom: + secretKeyRef: + name: "{{ include "university.import.auth.oidc.usernamePassword.secretName" . }}" + key: "{{ .Values.import.auth.oidc.usernamePassword.passwordKey }}" + {{- end }} + command: ["/app/bin/hac-import-app"] + args: [ + "import", + "--app-store-endpoint", "{{ .Values.import.appStore.endpoint }}", + "--oidc-provider", "{{ .Values.import.auth.oidc.provider }}", + "--oidc-client-id", "{{ .Values.import.auth.oidc.clientID }}", + "--image", "{{ required "'university.runtimeImage.repository' is required!" .Values.university.runtimeImage.repository }}:{{ required "'university.runtimeImage.tag' is required!" .Values.university.runtimeImage.tag }}", + "--bundle-path", "/workspace/app-bundle.wave", + ] + volumeMounts: + - mountPath: /workspace + name: "wave-bundle-volume" + readOnly: true + {{- if .Values.caCertificates }} + - name: ca-certificates + mountPath: /etc/ssl/certs/root-ca-bundle.crt + subPath: root-ca-bundle.crt + {{- end }} + volumes: + - name: "wave-bundle-volume" + emptyDir: {} + {{- if .Values.caCertificates }} + - name: ca-certificates + configMap: + name: {{ include "university.fullname" . }}-ca-certificates + {{- end }} diff --git a/university/bundle_generated_files/helm/charts/university/templates/secret.yaml b/university/bundle_generated_files/helm/charts/university/templates/secret.yaml new file mode 100644 index 0000000000..85c223fffc --- /dev/null +++ b/university/bundle_generated_files/helm/charts/university/templates/secret.yaml @@ -0,0 +1,23 @@ +{{ if and .Values.import.auth.oidc.usernamePassword.enabled (not .Values.import.auth.oidc.usernamePassword.existingSecret) -}} + +{{- $secretName := include "university.import.auth.oidc.usernamePassword.secretName" . }} +{{- $sec := lookup "v1" "Secret" .Release.Namespace $secretName }} + +{{- $username := get (coalesce $sec.data dict) .Values.import.auth.oidc.usernamePassword.usernameKey | b64dec }} +{{- $username = .Values.import.auth.oidc.usernamePassword.username | default $username }} + +{{- $password := get (coalesce $sec.data dict) .Values.import.auth.oidc.usernamePassword.passwordKey | b64dec }} +{{- $password = .Values.import.auth.oidc.usernamePassword.password | default $password }} + +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName | quote }} + labels: + {{- include "university.labels" . | nindent 4 }} +type: Opaque +data: + {{ .Values.import.auth.oidc.usernamePassword.usernameKey }}: {{ $username | b64enc | quote }} + {{ .Values.import.auth.oidc.usernamePassword.passwordKey }}: {{ $password | b64enc | quote }} + +{{- end -}} diff --git a/university/bundle_generated_files/helm/charts/university/values.yaml b/university/bundle_generated_files/helm/charts/university/values.yaml new file mode 100644 index 0000000000..072d8fd78c --- /dev/null +++ b/university/bundle_generated_files/helm/charts/university/values.yaml @@ -0,0 +1,98 @@ +## -- Partially overrides name of the deployed resources. Release name is maintained. +nameOverride: "" +## -- Fully overrides name of the deployed resources. +fullnameOverride: "" + +## -- Private certificate +caCertificates: | + -----BEGIN CERTIFICATE----- + MIIC+jCCAeICCQC/TlM20nWRcDANBgkqhkiG9w0BAQsFADA/MQswCQYDVQQGEwJY + WDEKMAgGA1UECAwBTjETMBEGA1UECwwKUmVwbGljYXRlZDEPMA0GA1UEAwwGUm9v + dENBMB4XDTIzMDMwNjA5NTg0NFoXDTMzMDExMjA5NTg0NFowPzELMAkGA1UEBhMC + WFgxCjAIBgNVBAgMAU4xEzARBgNVBAsMClJlcGxpY2F0ZWQxDzANBgNVBAMMBlJv + b3RDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMW9YCzq+lnkc2TC + qUl2ptGlX6+ZPa/gv5kRAo/YtBmAo661XzgUoR526ms5xPscOTzG8sZ2zC6hWtvK + Obl825AMpCBqemc5fl43LWlrBENpxenNgJWQFkdMPuI6IaWPt+KpTkxwfFAB0k4g + tCQLu/haaNppym9L5C46KoG4a44j4n66VjyZ8CK3xMn3iSIsRQUUTrEKql+8zdsK + KS8GdhUkEC4xjLbJ1QLPgAU8EMikUVv/cQ3R7QUV8pw5VUSeBCYHe9Xn+V/BaNyh + Iu8UtA+Mhk7uyCUwE88uhawavyUmpI94f2t1+XpbedK1inGsJAmh01HJweOgS0fz + /I2wi9cCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAfbs7UmepgO2v79V/HzKk4Yp3 + Umv3pBqFT2dI90HSwQzbTOM5IhBiOZnehrT89qqd/glqBQt/huI2eYFHrUQkGAjv + Wez5atS80A2OQoYsUpHaiCs0j9wYo4L9Ovpgw66MnqNZBe3+EtzEtBDe6aITqkbS + k/KiGs4iEfimVFERz4PsJVm8Ah/pSizygpu1ZU3H+Mokj02cx2oqKe0V4Aarr/su + +IGLp5Hy3uSeoDBclVAtJnUFN3sKDzHG4YARwuEAhrrkzz7/no2ukB0yjVw7TS3g + XjZHwSK2Zf1CkF42PAxg5fUvev2m3ySySeLIZeKxbK++XnM5rxRIwbtyqgLykA== + -----END CERTIFICATE----- + +image: + # -- Default imagePullPolicy for pods. + pullPolicy: IfNotPresent + # -- Optional default imagePullSecrets for pods. + pullSecrets: + - name: "haic-h2oaicloud-registry" + +# -- Default Pod Security Context for the pods. +securityContext: + runAsNonRoot: true + runAsUser: 65532 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + +university: + bundleImage: + # -- Image repository that contains the university App bundle. + repository: "10.96.1.85/haic/wave_university" + # -- Image tag that contains the university App bundle. + tag: "bundle_0.0.1" + # -- Overrides the image pull policy used for the university App bundle image. + pullPolicy: + # -- Overrides the image pull secrets used for the university App bundle image. + pullSecrets: + - name: "haic-h2oaicloud-registry" + runtimeImage: + # -- Image repository that contains the university App runtime. + repository: "10.96.1.85/haic/wave_university" + # -- Image tag that contains the university App runtime. + tag: "runtime_0.0.1" + +import: + image: + # -- Image repository used for the App import Job pod. + repository: "10.96.1.85/h2oaicloud/import-app" + # -- Image tag used for the App import Job pod. + tag: "1.0.0-snapshot.1" + # -- Overrides the image pull policy used for the App import Job pod. + pullPolicy: + # -- Overrides the image pull secrets used for the App import Job pod. + pullSecrets: + - name: "haic-h2oaicloud-registry" + appStore: + # -- App Store URL used for the App import. + endpoint: "https://18.116.49.137.nip.io" + auth: + # -- OIDC configuration. + oidc: + # -- The issuer URL for the OIDC provider. + provider: "https://auth.18.116.49.137.nip.io/auth/realms/hac" + # -- The client ID for the OIDC provider. + clientID: "hac-appstore-public" + # -- Configures the username/password used to obtain the bearer token. Will be + # replaced by service accounts in the future. + usernamePassword: + # -- If enabled, the username/password is used to obtain the bearer token. + enabled: true + # -- Name of the existing secret containing the username/password. Created if it does not exist. + existingSecret: + # -- Key in the secret containing the username. + usernameKey: "username" + # -- If set, given value is used and persisted in the secret. If empty, the secret must + # exist and the value is read from there. + username: test + # -- Key in the secret containing the password. + passwordKey: "password" + # -- If set, given value is used and persisted in the secret. If empty, the secret must + # exist and the value is read from there. + password: test From 33c3f71bb27b35ac1b433d264fc7b81226ca19bd Mon Sep 17 00:00:00 2001 From: sulhicader Date: Sun, 29 Oct 2023 22:53:10 +0530 Subject: [PATCH 02/24] packaging: changed the workflow tigger on #2163 --- .github/workflows/wave-university.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/wave-university.yaml b/.github/workflows/wave-university.yaml index 777d4696b5..5bd90b119e 100644 --- a/.github/workflows/wave-university.yaml +++ b/.github/workflows/wave-university.yaml @@ -1,16 +1,11 @@ name: Wave Bundle -on: - workflow_dispatch: - inputs: - version: - description: 'Release Version' - required: true +on: [push] env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: "${{ github.event.inputs.version }}" - TAG: "v${{ github.event.inputs.version }}" + VERSION: "0.0.1" + TAG: "0.0.1" jobs: bundle: From f0c6a4feeb372071a1aa6198ec0ba9854ab2f98a Mon Sep 17 00:00:00 2001 From: sulhicader Date: Sun, 29 Oct 2023 23:00:51 +0530 Subject: [PATCH 03/24] packaging: changed the workflow file names on #2163 --- .../{helm-release-university.yaml => helm-release.yaml} | 0 .../{wave-bundle-university.yaml => wave-bundle.yaml} | 0 .../{wave-publish-university.yaml => wave-publish.yaml} | 0 .github/workflows/{wave-university.yaml => wave.yaml} | 6 +++--- 4 files changed, 3 insertions(+), 3 deletions(-) rename .github/workflows/{helm-release-university.yaml => helm-release.yaml} (100%) rename .github/workflows/{wave-bundle-university.yaml => wave-bundle.yaml} (100%) rename .github/workflows/{wave-publish-university.yaml => wave-publish.yaml} (100%) rename .github/workflows/{wave-university.yaml => wave.yaml} (69%) diff --git a/.github/workflows/helm-release-university.yaml b/.github/workflows/helm-release.yaml similarity index 100% rename from .github/workflows/helm-release-university.yaml rename to .github/workflows/helm-release.yaml diff --git a/.github/workflows/wave-bundle-university.yaml b/.github/workflows/wave-bundle.yaml similarity index 100% rename from .github/workflows/wave-bundle-university.yaml rename to .github/workflows/wave-bundle.yaml diff --git a/.github/workflows/wave-publish-university.yaml b/.github/workflows/wave-publish.yaml similarity index 100% rename from .github/workflows/wave-publish-university.yaml rename to .github/workflows/wave-publish.yaml diff --git a/.github/workflows/wave-university.yaml b/.github/workflows/wave.yaml similarity index 69% rename from .github/workflows/wave-university.yaml rename to .github/workflows/wave.yaml index 5bd90b119e..8be76fe39e 100644 --- a/.github/workflows/wave-university.yaml +++ b/.github/workflows/wave.yaml @@ -11,16 +11,16 @@ jobs: bundle: name: Create Wave Bundle working-directory: ./university - uses: ./.github/workflows/wave-bundle-university.yaml + uses: ./.github/workflows/wave-bundle.yaml publish: needs: bundle name: Build and Publish working-directory: ./university - uses: ./.github/workflows/wave-publish-university.yaml + uses: ./.github/workflows/wave-publish.yaml helm-publish: needs: bundle name: Build and Publish Helm Chart working-directory: ./university - uses: ./.github/workflows/helm-release-university.yaml + uses: ./.github/workflows/helm-release.yaml From 3a002541c734a2d71ca2605686fdb7086094e903 Mon Sep 17 00:00:00 2001 From: sulhicader Date: Thu, 16 Nov 2023 19:47:39 +0530 Subject: [PATCH 04/24] packaging: Added to set version in app.toml on #2163 --- .github/workflows/wave-bundle.yaml | 3 +++ .github/workflows/wave.yaml | 13 ++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wave-bundle.yaml b/.github/workflows/wave-bundle.yaml index a826852f13..551f0e70ae 100644 --- a/.github/workflows/wave-bundle.yaml +++ b/.github/workflows/wave-bundle.yaml @@ -19,6 +19,9 @@ jobs: - name: Change permissions run: chmod +x .bin/h2o + - name: Set version + run: sed -i -r -e "s/\{\{VERSION\}\}/$(env.VERSION)/g" app.toml + - name: Get App Version id: get-build-version run: | diff --git a/.github/workflows/wave.yaml b/.github/workflows/wave.yaml index 8be76fe39e..d8b5854b77 100644 --- a/.github/workflows/wave.yaml +++ b/.github/workflows/wave.yaml @@ -1,11 +1,16 @@ name: Wave Bundle -on: [push] +on: + workflow_dispatch: + inputs: + version: + description: 'Release Version' + required: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: "0.0.1" - TAG: "0.0.1" + VERSION: "${{ github.event.inputs.version }}" + TAG: "v${{ github.event.inputs.version }}" jobs: bundle: @@ -16,11 +21,9 @@ jobs: publish: needs: bundle name: Build and Publish - working-directory: ./university uses: ./.github/workflows/wave-publish.yaml helm-publish: needs: bundle name: Build and Publish Helm Chart - working-directory: ./university uses: ./.github/workflows/helm-release.yaml From 5b64c3f7059b99e4cd9b40ad6fcc9cab82ff00ab Mon Sep 17 00:00:00 2001 From: sulhicader Date: Thu, 16 Nov 2023 19:49:59 +0530 Subject: [PATCH 05/24] packaging: Added workflows from workflow-library on #2163 --- .github/workflows/helm-release.yaml | 2 +- .../wave-bundle-docker-build-publish.yaml | 110 ++++++++++++++++++ .../workflows/wave-bundle-helm-release.yaml | 88 ++++++++++++++ .github/workflows/wave-publish.yaml | 2 +- 4 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/wave-bundle-docker-build-publish.yaml create mode 100644 .github/workflows/wave-bundle-helm-release.yaml diff --git a/.github/workflows/helm-release.yaml b/.github/workflows/helm-release.yaml index 3d16e137c1..6863b3770b 100644 --- a/.github/workflows/helm-release.yaml +++ b/.github/workflows/helm-release.yaml @@ -23,7 +23,7 @@ jobs: id-token: write # This is required for requesting the JWT contents: read # This is required for actions/checkout - uses: h2oai/workflows-library/.github/workflows/wave-bundle-helm-release.yaml@main + uses: ./.github/workflows/wave-bundle-helm-release.yaml with: build-version: ${{ needs.retrieve-metadata.outputs.build-version }} helm-chart-artifact: wave-bundle-helm diff --git a/.github/workflows/wave-bundle-docker-build-publish.yaml b/.github/workflows/wave-bundle-docker-build-publish.yaml new file mode 100644 index 0000000000..b3c10449f6 --- /dev/null +++ b/.github/workflows/wave-bundle-docker-build-publish.yaml @@ -0,0 +1,110 @@ +name: Build and Publish Docker image from generated Dockerfile + +on: + workflow_call: + inputs: + build-version: + type: string + description: The version of the application/image to be pushed + required: true + bundle-artifact: + type: string + description: The name of the artifact containing the generated Dockerfiles and the wave bundle + required: true + working-directory: + type: string + description: Path to the working directory, where docker build will be executed + default: . + +jobs: + docker: + name: Build and Publish App Docker Image + runs-on: ubuntu-latest + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + + steps: + - uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Download Wave bundle and Dockerfiles + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.bundle-artifact }} + path: ./ + + - name: Rename Dockerfiles + run: | + mv ./*bundle.Dockerfile ./generated.bundle.Dockerfile + mv ./*runtime.Dockerfile ./generated.runtime.Dockerfile + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::524466471676:role/workflows-library-wave-bundling-github-actions + role-session-name: GitHub_to_AWS_via_FederatedOIDC + aws-region: us-east-1 + + - name: "Login to Amazon ECR" + uses: aws-actions/amazon-ecr-login@v1 + + - name: "Ensure ECR Repository for the bundle" + # Tries to describe the repository and if it fails, creates it + run: | + aws ecr describe-repositories \ + --repository-names h2oai/${{ github.event.repository.name }}-bundle 2>/dev/null || + aws ecr create-repository \ + --repository-name h2oai/${{ github.event.repository.name }}-bundle \ + --image-tag-mutability IMMUTABLE \ + --tag \ + Key=GithubRepo,Value=github.com/h2oai/${{ github.event.repository.name }} \ + Key=ManagedBy,Value=GitHubActions \ + Key=CreatedByWorkflow,Value=${{ github.workflow_ref }} + + - name: "Ensure ECR Repository for the runtime" + # Tries to describe the repository and if it fails, creates it + run: | + aws ecr describe-repositories \ + --repository-names h2oai/${{ github.event.repository.name }} 2>/dev/null || + aws ecr create-repository \ + --repository-name h2oai/${{ github.event.repository.name }} \ + --image-tag-mutability IMMUTABLE \ + --tag \ + Key=GithubRepo,Value=github.com/h2oai/${{ github.event.repository.name }} \ + Key=ManagedBy,Value=GitHubActions \ + Key=CreatedByWorkflow,Value=${{ github.workflow_ref }} + + - name: Build and Export Bundle Image + uses: docker/build-push-action@v4 + id: bundle-build + with: + push: true + context: ${{ inputs.working-directory }} + file: ./generated.bundle.Dockerfile + platforms: linux/amd64 + provenance: false + tags: | + 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ github.event.repository.name }}-bundle:${{ inputs.build-version }} + + - name: Build and Export Runtime Image + uses: docker/build-push-action@v4 + id: runtime-build + with: + push: true + context: ${{ inputs.working-directory }} + file: ./generated.runtime.Dockerfile + platforms: linux/amd64 + provenance: false + tags: | + 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ github.event.repository.name }}:${{ inputs.build-version }} + + - name: Published Images Summary + run: | + echo "#### Image Tags" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "docker image push 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ github.event.repository.name }}-bundle:${{ inputs.build-version }}" >> $GITHUB_STEP_SUMMARY + echo "docker image push 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ github.event.repository.name }}:${{ inputs.build-version }}" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/wave-bundle-helm-release.yaml b/.github/workflows/wave-bundle-helm-release.yaml new file mode 100644 index 0000000000..684d969f42 --- /dev/null +++ b/.github/workflows/wave-bundle-helm-release.yaml @@ -0,0 +1,88 @@ +name: Release Helm Chart + +on: + workflow_call: + inputs: + build-version: + type: string + description: The version of the application + required: true + helm-version: + type: string + description: The version of the Helm Chart to be pushed. If not provided `build-version` will be used. + required: false + helm-chart-artifact: + type: string + description: The name of the artifact containing the generated Helm chart + required: true + +jobs: + helm: + name: Helm Release + runs-on: ubuntu-latest + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + + steps: + - uses: actions/checkout@v3 + + - name: Download Wave bundle and Dockerfile + uses: actions/download-artifact@v3 + id: helm-chart-download + with: + name: ${{ inputs.helm-chart-artifact }} + path: helm/ + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::524466471676:role/workflows-library-wave-bundling-github-actions + role-session-name: GitHub_to_AWS_via_FederatedOIDC + aws-region: us-east-1 + + - name: "Ensure ECR Repository" + # Tires to describe the repository and if it fails, creates it + run: | + aws ecr describe-repositories \ + --repository-names charts/${{ github.event.repository.name }} 2>/dev/null || + aws ecr create-repository \ + --repository-name charts/${{ github.event.repository.name }} \ + --image-tag-mutability IMMUTABLE \ + --tag \ + Key=GithubRepo,Value=github.com/h2oai/${{ github.event.repository.name }} \ + Key=ManagedBy,Value=GitHubActions \ + Key=CreatedByWorkflow,Value=${{ github.workflow_ref }} + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + token: ${{ github.token }} + + - name: Helm Login to ECR + run: | + aws ecr get-login-password --region us-east-1 \ + | helm registry login --username AWS --password-stdin \ + 524466471676.dkr.ecr.us-east-1.amazonaws.com + + - name: Locate Chart.yaml + id: locate-chart-dir + run: | + echo "PATH=$(dirname $(find ${{ steps.helm-chart-download.outputs.download-path }} -name Chart.yaml))" >> "$GITHUB_OUTPUT" + + - name: Package Helm Chart + run: | + helm package \ + ${{ steps.locate-chart-dir.outputs.PATH }} \ + --version ${{ inputs.helm-version || inputs.build-version }} \ + --app-version ${{ inputs.build-version }} \ + --dependency-update + + - name: Locate Helm package + id: locate-helm-package + run: | + echo "PATH=$(find . -name *-${{ inputs.build-version }}.tgz -type f -exec basename {} \; | tr -d '[:space:]')" >> "$GITHUB_OUTPUT" + + - name: Push Helm Chart to ECR + run: | + helm push ${{ steps.locate-helm-package.outputs.PATH }} oci://524466471676.dkr.ecr.us-east-1.amazonaws.com/charts diff --git a/.github/workflows/wave-publish.yaml b/.github/workflows/wave-publish.yaml index 58ce20b841..aaaa162a08 100644 --- a/.github/workflows/wave-publish.yaml +++ b/.github/workflows/wave-publish.yaml @@ -23,7 +23,7 @@ jobs: id-token: write # This is required for requesting the JWT contents: read # This is required for actions/checkout - uses: h2oai/workflows-library/.github/workflows/wave-bundle-docker-build-publish.yaml@main + uses: ./.github/workflows/wave-bundle-docker-build-publish.yaml with: build-version: ${{ needs.retrieve-metadata.outputs.build-version }} bundle-artifact: wave-bundle From c817498fe1e78b41a0f93731838a81d8e03e4490 Mon Sep 17 00:00:00 2001 From: sulhicader Date: Thu, 16 Nov 2023 20:12:46 +0530 Subject: [PATCH 06/24] packaging: Changed workflow trigger mode to push on #2163 --- .github/workflows/wave.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/wave.yaml b/.github/workflows/wave.yaml index d8b5854b77..dd2b71bea2 100644 --- a/.github/workflows/wave.yaml +++ b/.github/workflows/wave.yaml @@ -1,16 +1,12 @@ name: Wave Bundle on: - workflow_dispatch: - inputs: - version: - description: 'Release Version' - required: true + push: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: "${{ github.event.inputs.version }}" - TAG: "v${{ github.event.inputs.version }}" + VERSION: "0.0.1" + TAG: "v0.0.1" jobs: bundle: From cd63621bc2f2756f9b9b694b84d52af1ff3a02c5 Mon Sep 17 00:00:00 2001 From: sulhicader Date: Thu, 16 Nov 2023 20:15:29 +0530 Subject: [PATCH 07/24] packaging: Added workflow directory to each step on #2163 --- .github/workflows/wave-bundle.yaml | 6 ++++++ .github/workflows/wave.yaml | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wave-bundle.yaml b/.github/workflows/wave-bundle.yaml index 551f0e70ae..117d0b299f 100644 --- a/.github/workflows/wave-bundle.yaml +++ b/.github/workflows/wave-bundle.yaml @@ -14,12 +14,15 @@ jobs: - uses: actions/checkout@v3 - name: Download H2O CLI + working-directory: ./university run: mkdir -p .bin && curl -o .bin/h2o https://h2oai-cloud-release.s3.amazonaws.com/releases/ai/h2o/h2o-cloud/v0.28.2/cli/linux-amd64/h2o - name: Change permissions + working-directory: ./university run: chmod +x .bin/h2o - name: Set version + working-directory: ./university run: sed -i -r -e "s/\{\{VERSION\}\}/$(env.VERSION)/g" app.toml - name: Get App Version @@ -28,6 +31,7 @@ jobs: echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT" - name: Make air-gapped bundle + working-directory: ./university run: | .bin/h2o bundle \ --docker-base-image 524466471676.dkr.ecr.us-east-1.amazonaws.com/q8s/launcher:v0.23.0-38 \ @@ -39,6 +43,7 @@ jobs: --generate-dockerfile - uses: actions/upload-artifact@v3 + working-directory: ./university with: name: wave-bundle path: | @@ -46,6 +51,7 @@ jobs: ./*.wave - uses: actions/upload-artifact@v3 + working-directory: ./university with: name: wave-bundle-helm path: helm/ diff --git a/.github/workflows/wave.yaml b/.github/workflows/wave.yaml index dd2b71bea2..37e7974dd0 100644 --- a/.github/workflows/wave.yaml +++ b/.github/workflows/wave.yaml @@ -11,7 +11,6 @@ env: jobs: bundle: name: Create Wave Bundle - working-directory: ./university uses: ./.github/workflows/wave-bundle.yaml publish: From 2c6f3615bc427bac1e805e6678b3b4eae61a5b62 Mon Sep 17 00:00:00 2001 From: sulhicader Date: Thu, 16 Nov 2023 20:26:40 +0530 Subject: [PATCH 08/24] packaging: Added workflow directory to each step on #2163 --- .github/workflows/wave-bundle.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wave-bundle.yaml b/.github/workflows/wave-bundle.yaml index 117d0b299f..0365547cfc 100644 --- a/.github/workflows/wave-bundle.yaml +++ b/.github/workflows/wave-bundle.yaml @@ -40,18 +40,18 @@ jobs: --generate-helm-charts \ --helm-chart-version ${{ steps.get-build-version.outputs.VERSION }} \ --helm-chart-name ${{ github.event.repository.name }} \ + --helm-app-bundle-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ github.event.repository.name }}-bundle \ + --helm-app-runtime-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ github.event.repository.name }} \ --generate-dockerfile - uses: actions/upload-artifact@v3 - working-directory: ./university with: name: wave-bundle path: | - ./*.Dockerfile - ./*.wave + ./university/*.Dockerfile + ./university/*.wave - uses: actions/upload-artifact@v3 - working-directory: ./university with: name: wave-bundle-helm - path: helm/ + path: university/helm/ From bc9991d332c3480c2d49444fffe2d0f1a07c4fd3 Mon Sep 17 00:00:00 2001 From: sulhicader Date: Thu, 16 Nov 2023 20:47:42 +0530 Subject: [PATCH 09/24] packaging: Added workflow directory to each step on #2163 --- .github/workflows/snyk-scan.yml | 1 - .github/workflows/wave-bundle.yaml | 7 ++++++- .github/workflows/wave.yaml | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/snyk-scan.yml b/.github/workflows/snyk-scan.yml index 15e8559e8f..bc577370c2 100644 --- a/.github/workflows/snyk-scan.yml +++ b/.github/workflows/snyk-scan.yml @@ -2,7 +2,6 @@ name: Snyk Security Vulnerability Scan on: workflow_dispatch: - pull_request: push: tags: - "v[0-9]+.[0-9]+.[0-9]+" diff --git a/.github/workflows/wave-bundle.yaml b/.github/workflows/wave-bundle.yaml index 0365547cfc..f5ce889139 100644 --- a/.github/workflows/wave-bundle.yaml +++ b/.github/workflows/wave-bundle.yaml @@ -2,6 +2,11 @@ name: Wave Bundle on: workflow_call: + inputs: + build-version: + type: string + description: The version of the application/image to be pushed + required: true jobs: airgapped: @@ -28,7 +33,7 @@ jobs: - name: Get App Version id: get-build-version run: | - echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT" + echo "VERSION=${{ build-version }}" >> "$GITHUB_OUTPUT" - name: Make air-gapped bundle working-directory: ./university diff --git a/.github/workflows/wave.yaml b/.github/workflows/wave.yaml index 37e7974dd0..22784ca340 100644 --- a/.github/workflows/wave.yaml +++ b/.github/workflows/wave.yaml @@ -12,6 +12,8 @@ jobs: bundle: name: Create Wave Bundle uses: ./.github/workflows/wave-bundle.yaml + with: + build-version: ${{ VERSION }} publish: needs: bundle From 5ee8dbfc15d94dd1bda31b36cd8aa52db5e5f61f Mon Sep 17 00:00:00 2001 From: sulhicader Date: Thu, 16 Nov 2023 20:48:33 +0530 Subject: [PATCH 10/24] packaging: Added workflow directory to each step on #2163 --- .github/workflows/wave.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yaml b/.github/workflows/wave.yaml index 22784ca340..e5d11f0cf5 100644 --- a/.github/workflows/wave.yaml +++ b/.github/workflows/wave.yaml @@ -13,7 +13,7 @@ jobs: name: Create Wave Bundle uses: ./.github/workflows/wave-bundle.yaml with: - build-version: ${{ VERSION }} + build-version: ${{ env.VERSION }} publish: needs: bundle From e6b32d212eb3c60cd38943d92786cd4c9f397db3 Mon Sep 17 00:00:00 2001 From: sulhicader Date: Thu, 16 Nov 2023 20:52:10 +0530 Subject: [PATCH 11/24] packaging: Added workflow directory to each step on #2163 --- .github/workflows/wave.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wave.yaml b/.github/workflows/wave.yaml index e5d11f0cf5..e04b9c009a 100644 --- a/.github/workflows/wave.yaml +++ b/.github/workflows/wave.yaml @@ -13,7 +13,7 @@ jobs: name: Create Wave Bundle uses: ./.github/workflows/wave-bundle.yaml with: - build-version: ${{ env.VERSION }} + build-version: 0.0.1 publish: needs: bundle From fd3610a8a4d30a63493cbd40d5098d7c6a4743c7 Mon Sep 17 00:00:00 2001 From: sulhicader Date: Fri, 17 Nov 2023 11:04:18 +0530 Subject: [PATCH 12/24] packaging: Added workflow directory to each step on #2163 --- .github/workflows/wave-bundle.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wave-bundle.yaml b/.github/workflows/wave-bundle.yaml index f5ce889139..a9077b5eb7 100644 --- a/.github/workflows/wave-bundle.yaml +++ b/.github/workflows/wave-bundle.yaml @@ -28,12 +28,12 @@ jobs: - name: Set version working-directory: ./university - run: sed -i -r -e "s/\{\{VERSION\}\}/$(env.VERSION)/g" app.toml + run: sed -i -r -e "s/\{\{VERSION\}\}/0.0.1/g" app.toml - name: Get App Version id: get-build-version run: | - echo "VERSION=${{ build-version }}" >> "$GITHUB_OUTPUT" + echo "VERSION=0.0.1" >> "$GITHUB_OUTPUT" - name: Make air-gapped bundle working-directory: ./university @@ -43,8 +43,8 @@ jobs: --docker-include static \ --docker-use-buildkit \ --generate-helm-charts \ - --helm-chart-version ${{ steps.get-build-version.outputs.VERSION }} \ - --helm-chart-name ${{ github.event.repository.name }} \ + --helm-chart-version 0.0.1 \ + --helm-chart-name university \ --helm-app-bundle-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ github.event.repository.name }}-bundle \ --helm-app-runtime-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ github.event.repository.name }} \ --generate-dockerfile From acad0439e06c0c8bcf6096fd8be0243eb7d8061c Mon Sep 17 00:00:00 2001 From: sulhicader Date: Fri, 17 Nov 2023 11:10:33 +0530 Subject: [PATCH 13/24] packaging: Added workflow directory to each step on #2163 --- .github/workflows/helm-release.yaml | 2 +- .github/workflows/wave-publish.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/helm-release.yaml b/.github/workflows/helm-release.yaml index 6863b3770b..72025782a9 100644 --- a/.github/workflows/helm-release.yaml +++ b/.github/workflows/helm-release.yaml @@ -15,7 +15,7 @@ jobs: - name: Get App Version id: get-build-version run: | - echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT" + echo "VERSION=0.0.1" >> "$GITHUB_OUTPUT" release-helm-chart: needs: retrieve-metadata diff --git a/.github/workflows/wave-publish.yaml b/.github/workflows/wave-publish.yaml index aaaa162a08..0d7e8ae7d0 100644 --- a/.github/workflows/wave-publish.yaml +++ b/.github/workflows/wave-publish.yaml @@ -15,7 +15,7 @@ jobs: - name: Get App Version id: get-build-version run: | - echo "VERSION=$(make version)" >> "$GITHUB_OUTPUT" + echo "VERSION=0.0.1" >> "$GITHUB_OUTPUT" build-and-publish: needs: retrieve-metadata From 51a91b63422f7b0067242b26b4364af662d57acc Mon Sep 17 00:00:00 2001 From: sulhicader Date: Fri, 17 Nov 2023 14:25:32 +0530 Subject: [PATCH 14/24] packaging: Added workflow directory to each step on #2163 --- .github/workflows/helm-release.yaml | 1 + .../wave-bundle-docker-build-publish.yaml | 24 +++++++++++-------- .../workflows/wave-bundle-helm-release.yaml | 10 +++++--- .github/workflows/wave-bundle.yaml | 7 +++--- .github/workflows/wave-publish.yaml | 1 + 5 files changed, 26 insertions(+), 17 deletions(-) diff --git a/.github/workflows/helm-release.yaml b/.github/workflows/helm-release.yaml index 72025782a9..6aab5970d7 100644 --- a/.github/workflows/helm-release.yaml +++ b/.github/workflows/helm-release.yaml @@ -27,3 +27,4 @@ jobs: with: build-version: ${{ needs.retrieve-metadata.outputs.build-version }} helm-chart-artifact: wave-bundle-helm + wave-app-name: university diff --git a/.github/workflows/wave-bundle-docker-build-publish.yaml b/.github/workflows/wave-bundle-docker-build-publish.yaml index b3c10449f6..a4b23a29e1 100644 --- a/.github/workflows/wave-bundle-docker-build-publish.yaml +++ b/.github/workflows/wave-bundle-docker-build-publish.yaml @@ -15,6 +15,10 @@ on: type: string description: Path to the working directory, where docker build will be executed default: . + wave-app-name: + type: string + description: The name of the wave app + required: true jobs: docker: @@ -55,12 +59,12 @@ jobs: # Tries to describe the repository and if it fails, creates it run: | aws ecr describe-repositories \ - --repository-names h2oai/${{ github.event.repository.name }}-bundle 2>/dev/null || + --repository-names h2oai/${{ inputs.wave-app-name }}-bundle 2>/dev/null || aws ecr create-repository \ - --repository-name h2oai/${{ github.event.repository.name }}-bundle \ + --repository-name h2oai/${{ inputs.wave-app-name }}-bundle \ --image-tag-mutability IMMUTABLE \ --tag \ - Key=GithubRepo,Value=github.com/h2oai/${{ github.event.repository.name }} \ + Key=GithubRepo,Value=github.com/h2oai/${{ inputs.wave-app-name }} \ Key=ManagedBy,Value=GitHubActions \ Key=CreatedByWorkflow,Value=${{ github.workflow_ref }} @@ -68,12 +72,12 @@ jobs: # Tries to describe the repository and if it fails, creates it run: | aws ecr describe-repositories \ - --repository-names h2oai/${{ github.event.repository.name }} 2>/dev/null || + --repository-names h2oai/${{ inputs.wave-app-name }} 2>/dev/null || aws ecr create-repository \ - --repository-name h2oai/${{ github.event.repository.name }} \ + --repository-name h2oai/${{ inputs.wave-app-name }} \ --image-tag-mutability IMMUTABLE \ --tag \ - Key=GithubRepo,Value=github.com/h2oai/${{ github.event.repository.name }} \ + Key=GithubRepo,Value=github.com/h2oai/${{ inputs.wave-app-name }} \ Key=ManagedBy,Value=GitHubActions \ Key=CreatedByWorkflow,Value=${{ github.workflow_ref }} @@ -87,7 +91,7 @@ jobs: platforms: linux/amd64 provenance: false tags: | - 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ github.event.repository.name }}-bundle:${{ inputs.build-version }} + 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ inputs.wave-app-name }}-bundle:${{ inputs.build-version }} - name: Build and Export Runtime Image uses: docker/build-push-action@v4 @@ -99,12 +103,12 @@ jobs: platforms: linux/amd64 provenance: false tags: | - 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ github.event.repository.name }}:${{ inputs.build-version }} + 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ inputs.wave-app-name }}:${{ inputs.build-version }} - name: Published Images Summary run: | echo "#### Image Tags" >> $GITHUB_STEP_SUMMARY echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - echo "docker image push 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ github.event.repository.name }}-bundle:${{ inputs.build-version }}" >> $GITHUB_STEP_SUMMARY - echo "docker image push 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ github.event.repository.name }}:${{ inputs.build-version }}" >> $GITHUB_STEP_SUMMARY + echo "docker image push 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ inputs.wave-app-name }}-bundle:${{ inputs.build-version }}" >> $GITHUB_STEP_SUMMARY + echo "docker image push 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ inputs.wave-app-name }}:${{ inputs.build-version }}" >> $GITHUB_STEP_SUMMARY echo "\`\`\`" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/wave-bundle-helm-release.yaml b/.github/workflows/wave-bundle-helm-release.yaml index 684d969f42..99155152e7 100644 --- a/.github/workflows/wave-bundle-helm-release.yaml +++ b/.github/workflows/wave-bundle-helm-release.yaml @@ -15,6 +15,10 @@ on: type: string description: The name of the artifact containing the generated Helm chart required: true + wave-app-name: + type: string + description: The name of the wave app + required: true jobs: helm: @@ -45,12 +49,12 @@ jobs: # Tires to describe the repository and if it fails, creates it run: | aws ecr describe-repositories \ - --repository-names charts/${{ github.event.repository.name }} 2>/dev/null || + --repository-names charts/${{ inputs.wave-app-name }} 2>/dev/null || aws ecr create-repository \ - --repository-name charts/${{ github.event.repository.name }} \ + --repository-name charts/${{ inputs.wave-app-name }} \ --image-tag-mutability IMMUTABLE \ --tag \ - Key=GithubRepo,Value=github.com/h2oai/${{ github.event.repository.name }} \ + Key=GithubRepo,Value=github.com/h2oai/${{ inputs.wave-app-name }} \ Key=ManagedBy,Value=GitHubActions \ Key=CreatedByWorkflow,Value=${{ github.workflow_ref }} diff --git a/.github/workflows/wave-bundle.yaml b/.github/workflows/wave-bundle.yaml index a9077b5eb7..34d46005b8 100644 --- a/.github/workflows/wave-bundle.yaml +++ b/.github/workflows/wave-bundle.yaml @@ -20,7 +20,7 @@ jobs: - name: Download H2O CLI working-directory: ./university - run: mkdir -p .bin && curl -o .bin/h2o https://h2oai-cloud-release.s3.amazonaws.com/releases/ai/h2o/h2o-cloud/v0.28.2/cli/linux-amd64/h2o + run: mkdir -p .bin && curl -o .bin/h2o https://h2oai-cloud-release.s3.amazonaws.com/releases/ai/h2o/h2o-cloud/latest/cli/linux-amd64/h2o - name: Change permissions working-directory: ./university @@ -40,13 +40,12 @@ jobs: run: | .bin/h2o bundle \ --docker-base-image 524466471676.dkr.ecr.us-east-1.amazonaws.com/q8s/launcher:v0.23.0-38 \ - --docker-include static \ --docker-use-buildkit \ --generate-helm-charts \ --helm-chart-version 0.0.1 \ --helm-chart-name university \ - --helm-app-bundle-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ github.event.repository.name }}-bundle \ - --helm-app-runtime-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/${{ github.event.repository.name }} \ + --helm-app-bundle-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/university-bundle \ + --helm-app-runtime-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/university \ --generate-dockerfile - uses: actions/upload-artifact@v3 diff --git a/.github/workflows/wave-publish.yaml b/.github/workflows/wave-publish.yaml index 0d7e8ae7d0..050df02e25 100644 --- a/.github/workflows/wave-publish.yaml +++ b/.github/workflows/wave-publish.yaml @@ -27,3 +27,4 @@ jobs: with: build-version: ${{ needs.retrieve-metadata.outputs.build-version }} bundle-artifact: wave-bundle + wave-app-name: university From cdc11bc6fe56a14c4ba238ca84b61d7a3ff9eb5e Mon Sep 17 00:00:00 2001 From: sulhicader Date: Mon, 20 Nov 2023 06:06:44 +0530 Subject: [PATCH 15/24] Attached new flows to publish-university --- ...ease.yaml => helm-release-university.yaml} | 11 ++++--- .github/workflows/publish-university.yml | 31 +++++-------------- ...undle.yaml => wave-bundle-university.yaml} | 8 ++--- ...lish.yaml => wave-publish-university.yaml} | 11 ++++--- .github/workflows/wave-university.yaml | 30 ++++++++++++++++++ .github/workflows/wave.yaml | 26 ---------------- 6 files changed, 55 insertions(+), 62 deletions(-) rename .github/workflows/{helm-release.yaml => helm-release-university.yaml} (68%) rename .github/workflows/{wave-bundle.yaml => wave-bundle-university.yaml} (87%) rename .github/workflows/{wave-publish.yaml => wave-publish-university.yaml} (67%) create mode 100644 .github/workflows/wave-university.yaml delete mode 100644 .github/workflows/wave.yaml diff --git a/.github/workflows/helm-release.yaml b/.github/workflows/helm-release-university.yaml similarity index 68% rename from .github/workflows/helm-release.yaml rename to .github/workflows/helm-release-university.yaml index 6aab5970d7..46978c21fc 100644 --- a/.github/workflows/helm-release.yaml +++ b/.github/workflows/helm-release-university.yaml @@ -2,12 +2,15 @@ name: Release Generated Helm Chart on: workflow_call: + inputs: + build-version: + type: string + description: The version of the application/image to be pushed + required: true jobs: retrieve-metadata: runs-on: ubuntu-latest - outputs: - build-version: ${{ steps.get-build-version.outputs.VERSION }} steps: - uses: actions/checkout@v3 @@ -15,7 +18,7 @@ jobs: - name: Get App Version id: get-build-version run: | - echo "VERSION=0.0.1" >> "$GITHUB_OUTPUT" + echo "VERSION=${{ inputs.build-version }}" >> "$GITHUB_OUTPUT" release-helm-chart: needs: retrieve-metadata @@ -25,6 +28,6 @@ jobs: uses: ./.github/workflows/wave-bundle-helm-release.yaml with: - build-version: ${{ needs.retrieve-metadata.outputs.build-version }} + build-version: ${{ inputs.build-version }} helm-chart-artifact: wave-bundle-helm wave-app-name: university diff --git a/.github/workflows/publish-university.yml b/.github/workflows/publish-university.yml index 20df752b07..2f1978a55e 100644 --- a/.github/workflows/publish-university.yml +++ b/.github/workflows/publish-university.yml @@ -1,30 +1,15 @@ name: Publish Wave University on: - workflow_dispatch: - inputs: - version: - description: 'Release Version' - required: true + push: + -env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: "${{ github.event.inputs.version }}" jobs: - publish: - name: Publish Wave University - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - with: - token: ${{ secrets.GIT_TOKEN }} - - name: Build university - run: make publish-university - - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@master - with: - packages_dir: university/dist - password: ${{ secrets.PYPI_UNIVERSITY_TOKEN }} + bundle_generator: + name: Bundle and Publish + runs-on: ubuntu-20.04 + uses: ./.github/workflows/wave-university.yaml + with: + build-version: 1.0.0 diff --git a/.github/workflows/wave-bundle.yaml b/.github/workflows/wave-bundle-university.yaml similarity index 87% rename from .github/workflows/wave-bundle.yaml rename to .github/workflows/wave-bundle-university.yaml index 34d46005b8..111e9a204e 100644 --- a/.github/workflows/wave-bundle.yaml +++ b/.github/workflows/wave-bundle-university.yaml @@ -12,8 +12,6 @@ jobs: airgapped: name: Create Wave Bundle runs-on: ubuntu-latest - outputs: - build-version: ${{ steps.get-build-version.outputs.VERSION }} steps: - uses: actions/checkout@v3 @@ -28,12 +26,12 @@ jobs: - name: Set version working-directory: ./university - run: sed -i -r -e "s/\{\{VERSION\}\}/0.0.1/g" app.toml + run: sed -i -r -e "s/\{\{VERSION\}\}/${{ inputs.build-version }}/g" app.toml - name: Get App Version id: get-build-version run: | - echo "VERSION=0.0.1" >> "$GITHUB_OUTPUT" + echo "VERSION=${{ inputs.build-version }}" >> "$GITHUB_OUTPUT" - name: Make air-gapped bundle working-directory: ./university @@ -42,7 +40,7 @@ jobs: --docker-base-image 524466471676.dkr.ecr.us-east-1.amazonaws.com/q8s/launcher:v0.23.0-38 \ --docker-use-buildkit \ --generate-helm-charts \ - --helm-chart-version 0.0.1 \ + --helm-chart-version ${{ inputs.build-version }} \ --helm-chart-name university \ --helm-app-bundle-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/university-bundle \ --helm-app-runtime-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/university \ diff --git a/.github/workflows/wave-publish.yaml b/.github/workflows/wave-publish-university.yaml similarity index 67% rename from .github/workflows/wave-publish.yaml rename to .github/workflows/wave-publish-university.yaml index 050df02e25..65c091a744 100644 --- a/.github/workflows/wave-publish.yaml +++ b/.github/workflows/wave-publish-university.yaml @@ -2,12 +2,15 @@ name: Publish Wave Bundle on: workflow_call: + inputs: + build-version: + type: string + description: The version of the application/image to be pushed + required: true jobs: retrieve-metadata: runs-on: ubuntu-latest - outputs: - build-version: ${{ steps.get-build-version.outputs.VERSION }} steps: - uses: actions/checkout@v3 @@ -15,7 +18,7 @@ jobs: - name: Get App Version id: get-build-version run: | - echo "VERSION=0.0.1" >> "$GITHUB_OUTPUT" + echo "VERSION=${{ inputs.build-version }}" >> "$GITHUB_OUTPUT" build-and-publish: needs: retrieve-metadata @@ -25,6 +28,6 @@ jobs: uses: ./.github/workflows/wave-bundle-docker-build-publish.yaml with: - build-version: ${{ needs.retrieve-metadata.outputs.build-version }} + build-version: ${{ inputs.build-version }} bundle-artifact: wave-bundle wave-app-name: university diff --git a/.github/workflows/wave-university.yaml b/.github/workflows/wave-university.yaml new file mode 100644 index 0000000000..89dab03308 --- /dev/null +++ b/.github/workflows/wave-university.yaml @@ -0,0 +1,30 @@ +name: Wave Bundle + +on: + workflow_call: + inputs: + build-version: + type: string + description: The version of the application/image to be pushed + required: true + +jobs: + bundle: + name: Create Wave Bundle + uses: ./.github/workflows/wave-bundle-university.yaml + with: + build-version: ${{ inputs.build-version }} + + publish: + needs: bundle + name: Build and Publish + uses: ./.github/workflows/wave-publish-university.yaml + with: + build-version: ${{ inputs.build-version }} + + helm-publish: + needs: bundle + name: Build and Publish Helm Chart + uses: ./.github/workflows/helm-release-university.yaml + with: + build-version: ${{ inputs.build-version }} diff --git a/.github/workflows/wave.yaml b/.github/workflows/wave.yaml deleted file mode 100644 index e04b9c009a..0000000000 --- a/.github/workflows/wave.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: Wave Bundle - -on: - push: - -env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: "0.0.1" - TAG: "v0.0.1" - -jobs: - bundle: - name: Create Wave Bundle - uses: ./.github/workflows/wave-bundle.yaml - with: - build-version: 0.0.1 - - publish: - needs: bundle - name: Build and Publish - uses: ./.github/workflows/wave-publish.yaml - - helm-publish: - needs: bundle - name: Build and Publish Helm Chart - uses: ./.github/workflows/helm-release.yaml From 66dbde37d502c5798e4c470604cd3e2420e062cf Mon Sep 17 00:00:00 2001 From: sulhicader Date: Mon, 20 Nov 2023 06:09:03 +0530 Subject: [PATCH 16/24] packaging: Attached new flows to publish-university on #2163 --- .github/workflows/publish-university.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/publish-university.yml b/.github/workflows/publish-university.yml index 2f1978a55e..639ff03780 100644 --- a/.github/workflows/publish-university.yml +++ b/.github/workflows/publish-university.yml @@ -9,7 +9,6 @@ jobs: bundle_generator: name: Bundle and Publish - runs-on: ubuntu-20.04 uses: ./.github/workflows/wave-university.yaml with: build-version: 1.0.0 From ca86f56a379265a1376468214fee1b25efa4dbc0 Mon Sep 17 00:00:00 2001 From: sulhicader Date: Mon, 20 Nov 2023 06:27:19 +0530 Subject: [PATCH 17/24] packaging: Reverted some changes on #2163 --- .github/workflows/publish-university.yml | 28 +++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-university.yml b/.github/workflows/publish-university.yml index 639ff03780..ef3e4a1b50 100644 --- a/.github/workflows/publish-university.yml +++ b/.github/workflows/publish-university.yml @@ -1,14 +1,36 @@ name: Publish Wave University on: - push: - + workflow_dispatch: + inputs: + version: + description: 'Release Version' + required: true +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: "${{ github.event.inputs.version }}" jobs: + publish: + name: Publish Wave University + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + token: ${{ secrets.GIT_TOKEN }} + + - name: Build university + run: make publish-university + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + packages_dir: university/dist + password: ${{ secrets.PYPI_UNIVERSITY_TOKEN }} bundle_generator: name: Bundle and Publish uses: ./.github/workflows/wave-university.yaml with: - build-version: 1.0.0 + build-version: ${{ env.VERSION }} From dc9a1c20571ec493d8a49cd3006846897f82fb3a Mon Sep 17 00:00:00 2001 From: sulhicader Date: Mon, 20 Nov 2023 08:59:24 +0530 Subject: [PATCH 18/24] packaging: Reverted some changes on #2163 --- .github/workflows/snyk-scan.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/snyk-scan.yml b/.github/workflows/snyk-scan.yml index bc577370c2..15e8559e8f 100644 --- a/.github/workflows/snyk-scan.yml +++ b/.github/workflows/snyk-scan.yml @@ -2,6 +2,7 @@ name: Snyk Security Vulnerability Scan on: workflow_dispatch: + pull_request: push: tags: - "v[0-9]+.[0-9]+.[0-9]+" From 4ad597c09fdfbb9d0a07afbefffd7fc21b6f2eb5 Mon Sep 17 00:00:00 2001 From: sulhicader Date: Mon, 20 Nov 2023 09:01:51 +0530 Subject: [PATCH 19/24] packaging: Removed generated files on #2163 --- ...2o.wave.university.1.0.0.bundle.Dockerfile | 4 - ...o.wave.university.1.0.0.runtime.Dockerfile | 33 ------- .../helm/charts/university/Chart.yaml | 7 -- .../charts/university/templates/_helpers.tpl | 42 -------- .../university/templates/configmap.yaml | 9 -- .../helm/charts/university/templates/job.yaml | 86 ---------------- .../charts/university/templates/secret.yaml | 23 ----- .../helm/charts/university/values.yaml | 98 ------------------- 8 files changed, 302 deletions(-) delete mode 100644 university/bundle_generated_files/ai.h2o.wave.university.1.0.0.bundle.Dockerfile delete mode 100644 university/bundle_generated_files/ai.h2o.wave.university.1.0.0.runtime.Dockerfile delete mode 100644 university/bundle_generated_files/helm/charts/university/Chart.yaml delete mode 100644 university/bundle_generated_files/helm/charts/university/templates/_helpers.tpl delete mode 100644 university/bundle_generated_files/helm/charts/university/templates/configmap.yaml delete mode 100644 university/bundle_generated_files/helm/charts/university/templates/job.yaml delete mode 100644 university/bundle_generated_files/helm/charts/university/templates/secret.yaml delete mode 100644 university/bundle_generated_files/helm/charts/university/values.yaml diff --git a/university/bundle_generated_files/ai.h2o.wave.university.1.0.0.bundle.Dockerfile b/university/bundle_generated_files/ai.h2o.wave.university.1.0.0.bundle.Dockerfile deleted file mode 100644 index da9fa85d8b..0000000000 --- a/university/bundle_generated_files/ai.h2o.wave.university.1.0.0.bundle.Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM alpine:3.18 - -COPY ../ai.h2o.wave.university.1.0.0.wave /app/ai.h2o.wave.university.1.0.0.wave -ENV WAVE_BUNDLE_FILE /app/ai.h2o.wave.university.1.0.0.wave diff --git a/university/bundle_generated_files/ai.h2o.wave.university.1.0.0.runtime.Dockerfile b/university/bundle_generated_files/ai.h2o.wave.university.1.0.0.runtime.Dockerfile deleted file mode 100644 index c91b4e8154..0000000000 --- a/university/bundle_generated_files/ai.h2o.wave.university.1.0.0.runtime.Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM 524466471676.dkr.ecr.us-east-1.amazonaws.com/q8s/launcher:v0.23.0-gpu38 - -WORKDIR /app/ - -# Create a virtual environment -RUN python3 -m venv /app/venv - -LABEL \ - # Version of the bundle cmd used to generate this Dockerfile - ai.h2o.appstore.bundle.version = 0.0.1 \ - # App name as defined in the app.toml - ai.h2o.appstore.app.name = "ai.h2o.wave.university" \ - # App version as defined in the app.toml - ai.h2o.appstore.app.version = "1.0.0" - -# Unpack .wave bundle -RUN --mount=type=bind,src=ai.h2o.wave.university.1.0.0.wave,target=/app/ai.h2o.wave.university.1.0.0.wave \ - unzip ai.h2o.wave.university.1.0.0.wave || UNZIP_EXIT_CODE=$? && \ - # Ignore exit code 2, seems to be a false positive - if [ $UNZIP_EXIT_CODE != 0 ] && [ $UNZIP_EXIT_CODE != 2 ]; then echo "unzip failed with exit code $UNZIP_EXIT_CODE"; exit $UNZIP_EXIT_CODE; fi - -# Install Python dependencies -RUN --mount=type=cache,target=/home/.cache \ - --mount=type=cache,target=/root/.cache \ - --mount=type=cache,target=/.cache \ - /app/venv/bin/pip3 install -r requirements.txt - -# Path to the venv directory -ENV H2O_CLOUD_VENV_PATH /app/venv -# Path to the app directory -ENV H2O_CLOUD_APP_ROOT /app -# App main module -ENV H2O_CLOUD_PY_MODULE=h2o_wave_university.university diff --git a/university/bundle_generated_files/helm/charts/university/Chart.yaml b/university/bundle_generated_files/helm/charts/university/Chart.yaml deleted file mode 100644 index 82fde127af..0000000000 --- a/university/bundle_generated_files/helm/charts/university/Chart.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v2 -appVersion: 1.0.0 -name: university -version: 0.1.0 -description: | - Interactive Wave learning experience. -type: application diff --git a/university/bundle_generated_files/helm/charts/university/templates/_helpers.tpl b/university/bundle_generated_files/helm/charts/university/templates/_helpers.tpl deleted file mode 100644 index 6b5305508e..0000000000 --- a/university/bundle_generated_files/helm/charts/university/templates/_helpers.tpl +++ /dev/null @@ -1,42 +0,0 @@ -{{- define "university.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{- define "university.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{- define "university.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{- define "university.labels" -}} -helm.sh/chart: {{ include "university.chart" . }} -{{ include "university.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{- define "university.selectorLabels" -}} -app.kubernetes.io/name: {{ include "university.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{- define "university.import.auth.oidc.usernamePassword.secretName" -}} -{{- if .Values.import.auth.oidc.usernamePassword.existingSecret }} -{{- .Values.import.auth.oidc.usernamePassword.existingSecret }} -{{- else }} -{{- printf "%s-import-creds" (include "university.fullname" .) }} -{{- end }} -{{- end }} diff --git a/university/bundle_generated_files/helm/charts/university/templates/configmap.yaml b/university/bundle_generated_files/helm/charts/university/templates/configmap.yaml deleted file mode 100644 index ecc8009e75..0000000000 --- a/university/bundle_generated_files/helm/charts/university/templates/configmap.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- if .Values.caCertificates }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "university.fullname" . }}-ca-certificates -data: - root-ca-bundle.crt: | - {{ .Values.caCertificates | nindent 4 | trim }} -{{- end }} diff --git a/university/bundle_generated_files/helm/charts/university/templates/job.yaml b/university/bundle_generated_files/helm/charts/university/templates/job.yaml deleted file mode 100644 index b0c8bd2315..0000000000 --- a/university/bundle_generated_files/helm/charts/university/templates/job.yaml +++ /dev/null @@ -1,86 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "university.fullname" . }}-import - labels: - {{- include "university.labels" . | nindent 4 }} - annotations: - # marks this as a hook - https://helm.sh/docs/topics/charts_hooks/ - # this resource will be automatically removed after the install/upgrade is complete, given - # the job completes successfully - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-weight": "0" - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -spec: - backoffLimit: 10 - template: - metadata: - name: {{ include "university.fullname" . }}-import - labels: - {{- include "university.labels" . | nindent 8 }} - spec: - restartPolicy: Never - {{- with coalesce .Values.import.image.pullSecrets .Values.image.pullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - initContainers: - - name: cp-bundle - imagePullPolicy: {{ .Values.import.image.pullPolicy | default .Values.image.pullPolicy | quote }} - image: "{{ required "'university.bundleImage.repository' is required!" .Values.university.bundleImage.repository }}:{{ required "'university.bundleImage.tag' is required!" .Values.university.bundleImage.tag }}" - {{- with .Values.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - command: ["/bin/sh"] - args: ["-c", "cp ${WAVE_BUNDLE_FILE} /workspace/app-bundle.wave"] - volumeMounts: - - mountPath: /workspace - name: "wave-bundle-volume" - containers: - - name: import-app - imagePullPolicy: {{ .Values.import.image.pullPolicy | default .Values.image.pullPolicy | quote }} - image: "{{ required "'import.image.repository' is required!" .Values.import.image.repository }}:{{ required "'import.image.tag' is required!" .Values.import.image.tag }}" - {{- with .Values.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- if .Values.import.auth.oidc.usernamePassword.enabled }} - env: - - name: H2O_IMPORT_APP_USERNAME - valueFrom: - secretKeyRef: - name: "{{ include "university.import.auth.oidc.usernamePassword.secretName" . }}" - key: "{{ .Values.import.auth.oidc.usernamePassword.usernameKey }}" - - name: H2O_IMPORT_APP_PASSWORD - valueFrom: - secretKeyRef: - name: "{{ include "university.import.auth.oidc.usernamePassword.secretName" . }}" - key: "{{ .Values.import.auth.oidc.usernamePassword.passwordKey }}" - {{- end }} - command: ["/app/bin/hac-import-app"] - args: [ - "import", - "--app-store-endpoint", "{{ .Values.import.appStore.endpoint }}", - "--oidc-provider", "{{ .Values.import.auth.oidc.provider }}", - "--oidc-client-id", "{{ .Values.import.auth.oidc.clientID }}", - "--image", "{{ required "'university.runtimeImage.repository' is required!" .Values.university.runtimeImage.repository }}:{{ required "'university.runtimeImage.tag' is required!" .Values.university.runtimeImage.tag }}", - "--bundle-path", "/workspace/app-bundle.wave", - ] - volumeMounts: - - mountPath: /workspace - name: "wave-bundle-volume" - readOnly: true - {{- if .Values.caCertificates }} - - name: ca-certificates - mountPath: /etc/ssl/certs/root-ca-bundle.crt - subPath: root-ca-bundle.crt - {{- end }} - volumes: - - name: "wave-bundle-volume" - emptyDir: {} - {{- if .Values.caCertificates }} - - name: ca-certificates - configMap: - name: {{ include "university.fullname" . }}-ca-certificates - {{- end }} diff --git a/university/bundle_generated_files/helm/charts/university/templates/secret.yaml b/university/bundle_generated_files/helm/charts/university/templates/secret.yaml deleted file mode 100644 index 85c223fffc..0000000000 --- a/university/bundle_generated_files/helm/charts/university/templates/secret.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{ if and .Values.import.auth.oidc.usernamePassword.enabled (not .Values.import.auth.oidc.usernamePassword.existingSecret) -}} - -{{- $secretName := include "university.import.auth.oidc.usernamePassword.secretName" . }} -{{- $sec := lookup "v1" "Secret" .Release.Namespace $secretName }} - -{{- $username := get (coalesce $sec.data dict) .Values.import.auth.oidc.usernamePassword.usernameKey | b64dec }} -{{- $username = .Values.import.auth.oidc.usernamePassword.username | default $username }} - -{{- $password := get (coalesce $sec.data dict) .Values.import.auth.oidc.usernamePassword.passwordKey | b64dec }} -{{- $password = .Values.import.auth.oidc.usernamePassword.password | default $password }} - -apiVersion: v1 -kind: Secret -metadata: - name: {{ $secretName | quote }} - labels: - {{- include "university.labels" . | nindent 4 }} -type: Opaque -data: - {{ .Values.import.auth.oidc.usernamePassword.usernameKey }}: {{ $username | b64enc | quote }} - {{ .Values.import.auth.oidc.usernamePassword.passwordKey }}: {{ $password | b64enc | quote }} - -{{- end -}} diff --git a/university/bundle_generated_files/helm/charts/university/values.yaml b/university/bundle_generated_files/helm/charts/university/values.yaml deleted file mode 100644 index 072d8fd78c..0000000000 --- a/university/bundle_generated_files/helm/charts/university/values.yaml +++ /dev/null @@ -1,98 +0,0 @@ -## -- Partially overrides name of the deployed resources. Release name is maintained. -nameOverride: "" -## -- Fully overrides name of the deployed resources. -fullnameOverride: "" - -## -- Private certificate -caCertificates: | - -----BEGIN CERTIFICATE----- - MIIC+jCCAeICCQC/TlM20nWRcDANBgkqhkiG9w0BAQsFADA/MQswCQYDVQQGEwJY - WDEKMAgGA1UECAwBTjETMBEGA1UECwwKUmVwbGljYXRlZDEPMA0GA1UEAwwGUm9v - dENBMB4XDTIzMDMwNjA5NTg0NFoXDTMzMDExMjA5NTg0NFowPzELMAkGA1UEBhMC - WFgxCjAIBgNVBAgMAU4xEzARBgNVBAsMClJlcGxpY2F0ZWQxDzANBgNVBAMMBlJv - b3RDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMW9YCzq+lnkc2TC - qUl2ptGlX6+ZPa/gv5kRAo/YtBmAo661XzgUoR526ms5xPscOTzG8sZ2zC6hWtvK - Obl825AMpCBqemc5fl43LWlrBENpxenNgJWQFkdMPuI6IaWPt+KpTkxwfFAB0k4g - tCQLu/haaNppym9L5C46KoG4a44j4n66VjyZ8CK3xMn3iSIsRQUUTrEKql+8zdsK - KS8GdhUkEC4xjLbJ1QLPgAU8EMikUVv/cQ3R7QUV8pw5VUSeBCYHe9Xn+V/BaNyh - Iu8UtA+Mhk7uyCUwE88uhawavyUmpI94f2t1+XpbedK1inGsJAmh01HJweOgS0fz - /I2wi9cCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAfbs7UmepgO2v79V/HzKk4Yp3 - Umv3pBqFT2dI90HSwQzbTOM5IhBiOZnehrT89qqd/glqBQt/huI2eYFHrUQkGAjv - Wez5atS80A2OQoYsUpHaiCs0j9wYo4L9Ovpgw66MnqNZBe3+EtzEtBDe6aITqkbS - k/KiGs4iEfimVFERz4PsJVm8Ah/pSizygpu1ZU3H+Mokj02cx2oqKe0V4Aarr/su - +IGLp5Hy3uSeoDBclVAtJnUFN3sKDzHG4YARwuEAhrrkzz7/no2ukB0yjVw7TS3g - XjZHwSK2Zf1CkF42PAxg5fUvev2m3ySySeLIZeKxbK++XnM5rxRIwbtyqgLykA== - -----END CERTIFICATE----- - -image: - # -- Default imagePullPolicy for pods. - pullPolicy: IfNotPresent - # -- Optional default imagePullSecrets for pods. - pullSecrets: - - name: "haic-h2oaicloud-registry" - -# -- Default Pod Security Context for the pods. -securityContext: - runAsNonRoot: true - runAsUser: 65532 - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - -university: - bundleImage: - # -- Image repository that contains the university App bundle. - repository: "10.96.1.85/haic/wave_university" - # -- Image tag that contains the university App bundle. - tag: "bundle_0.0.1" - # -- Overrides the image pull policy used for the university App bundle image. - pullPolicy: - # -- Overrides the image pull secrets used for the university App bundle image. - pullSecrets: - - name: "haic-h2oaicloud-registry" - runtimeImage: - # -- Image repository that contains the university App runtime. - repository: "10.96.1.85/haic/wave_university" - # -- Image tag that contains the university App runtime. - tag: "runtime_0.0.1" - -import: - image: - # -- Image repository used for the App import Job pod. - repository: "10.96.1.85/h2oaicloud/import-app" - # -- Image tag used for the App import Job pod. - tag: "1.0.0-snapshot.1" - # -- Overrides the image pull policy used for the App import Job pod. - pullPolicy: - # -- Overrides the image pull secrets used for the App import Job pod. - pullSecrets: - - name: "haic-h2oaicloud-registry" - appStore: - # -- App Store URL used for the App import. - endpoint: "https://18.116.49.137.nip.io" - auth: - # -- OIDC configuration. - oidc: - # -- The issuer URL for the OIDC provider. - provider: "https://auth.18.116.49.137.nip.io/auth/realms/hac" - # -- The client ID for the OIDC provider. - clientID: "hac-appstore-public" - # -- Configures the username/password used to obtain the bearer token. Will be - # replaced by service accounts in the future. - usernamePassword: - # -- If enabled, the username/password is used to obtain the bearer token. - enabled: true - # -- Name of the existing secret containing the username/password. Created if it does not exist. - existingSecret: - # -- Key in the secret containing the username. - usernameKey: "username" - # -- If set, given value is used and persisted in the secret. If empty, the secret must - # exist and the value is read from there. - username: test - # -- Key in the secret containing the password. - passwordKey: "password" - # -- If set, given value is used and persisted in the secret. If empty, the secret must - # exist and the value is read from there. - password: test From b43ef51128262f3b962ef01c9ba8eda39126e74c Mon Sep 17 00:00:00 2001 From: sulhicader Date: Wed, 31 Jan 2024 12:09:43 +0530 Subject: [PATCH 20/24] Handled review comments #2163 --- .../workflows/helm-release-university.yaml | 33 ------- .github/workflows/publish-university.yml | 97 ++++++++++++++----- .../workflows/wave-bundle-helm-release.yaml | 92 ------------------ 3 files changed, 72 insertions(+), 150 deletions(-) delete mode 100644 .github/workflows/helm-release-university.yaml delete mode 100644 .github/workflows/wave-bundle-helm-release.yaml diff --git a/.github/workflows/helm-release-university.yaml b/.github/workflows/helm-release-university.yaml deleted file mode 100644 index 46978c21fc..0000000000 --- a/.github/workflows/helm-release-university.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: Release Generated Helm Chart - -on: - workflow_call: - inputs: - build-version: - type: string - description: The version of the application/image to be pushed - required: true - -jobs: - retrieve-metadata: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Get App Version - id: get-build-version - run: | - echo "VERSION=${{ inputs.build-version }}" >> "$GITHUB_OUTPUT" - - release-helm-chart: - needs: retrieve-metadata - permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout - - uses: ./.github/workflows/wave-bundle-helm-release.yaml - with: - build-version: ${{ inputs.build-version }} - helm-chart-artifact: wave-bundle-helm - wave-app-name: university diff --git a/.github/workflows/publish-university.yml b/.github/workflows/publish-university.yml index ef3e4a1b50..d22f173e7b 100644 --- a/.github/workflows/publish-university.yml +++ b/.github/workflows/publish-university.yml @@ -1,36 +1,83 @@ name: Publish Wave University on: - workflow_dispatch: - inputs: - version: - description: 'Release Version' - required: true + push: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: "${{ github.event.inputs.version }}" + VERSION: "1.0.1" jobs: - publish: - name: Publish Wave University - runs-on: ubuntu-20.04 + + + airgapped: + name: Create Wave Bundle + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Download H2O CLI + working-directory: ./university + run: mkdir -p .bin && curl -o .bin/h2o https://h2oai-cloud-release.s3.amazonaws.com/releases/ai/h2o/h2o-cloud/latest/cli/linux-amd64/h2o + + - name: Change permissions + working-directory: ./university + run: chmod +x .bin/h2o + + - name: Set version + working-directory: ./university + run: sed -i -r -e "s/\{\{VERSION\}\}/${{ env.VERSION }}/g" app.toml + + - name: Get App Version + id: get-build-version + run: | + echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT" + + - name: Make air-gapped bundle + working-directory: ./university + run: | + .bin/h2o bundle \ + --docker-base-image 524466471676.dkr.ecr.us-east-1.amazonaws.com/q8s/launcher:v0.23.0-38 \ + --docker-use-buildkit \ + --generate-helm-charts \ + --helm-chart-version ${{ env.VERSION }} \ + --helm-chart-name university \ + --helm-app-bundle-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/university-bundle \ + --helm-app-runtime-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/university \ + --generate-dockerfile + + - uses: actions/upload-artifact@v3 + with: + name: wave-bundle + path: | + ./university/*.Dockerfile + ./university/*.wave + + - uses: actions/upload-artifact@v3 + with: + name: wave-bundle-helm + path: university/helm/ + + retrieve-metadata: + runs-on: ubuntu-latest + steps: - - uses: actions/checkout@v2 - with: - token: ${{ secrets.GIT_TOKEN }} - - - name: Build university - run: make publish-university - - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@master - with: - packages_dir: university/dist - password: ${{ secrets.PYPI_UNIVERSITY_TOKEN }} - - bundle_generator: - name: Bundle and Publish - uses: ./.github/workflows/wave-university.yaml + - uses: actions/checkout@v3 + + - name: Get App Version + id: get-build-version + run: | + echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT" + + build-and-publish: + needs: retrieve-metadata + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + + uses: ./.github/workflows/wave-bundle-docker-build-publish.yaml with: build-version: ${{ env.VERSION }} + bundle-artifact: wave-bundle + wave-app-name: university diff --git a/.github/workflows/wave-bundle-helm-release.yaml b/.github/workflows/wave-bundle-helm-release.yaml deleted file mode 100644 index 99155152e7..0000000000 --- a/.github/workflows/wave-bundle-helm-release.yaml +++ /dev/null @@ -1,92 +0,0 @@ -name: Release Helm Chart - -on: - workflow_call: - inputs: - build-version: - type: string - description: The version of the application - required: true - helm-version: - type: string - description: The version of the Helm Chart to be pushed. If not provided `build-version` will be used. - required: false - helm-chart-artifact: - type: string - description: The name of the artifact containing the generated Helm chart - required: true - wave-app-name: - type: string - description: The name of the wave app - required: true - -jobs: - helm: - name: Helm Release - runs-on: ubuntu-latest - permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout - - steps: - - uses: actions/checkout@v3 - - - name: Download Wave bundle and Dockerfile - uses: actions/download-artifact@v3 - id: helm-chart-download - with: - name: ${{ inputs.helm-chart-artifact }} - path: helm/ - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: arn:aws:iam::524466471676:role/workflows-library-wave-bundling-github-actions - role-session-name: GitHub_to_AWS_via_FederatedOIDC - aws-region: us-east-1 - - - name: "Ensure ECR Repository" - # Tires to describe the repository and if it fails, creates it - run: | - aws ecr describe-repositories \ - --repository-names charts/${{ inputs.wave-app-name }} 2>/dev/null || - aws ecr create-repository \ - --repository-name charts/${{ inputs.wave-app-name }} \ - --image-tag-mutability IMMUTABLE \ - --tag \ - Key=GithubRepo,Value=github.com/h2oai/${{ inputs.wave-app-name }} \ - Key=ManagedBy,Value=GitHubActions \ - Key=CreatedByWorkflow,Value=${{ github.workflow_ref }} - - - name: Set up Helm - uses: azure/setup-helm@v3 - with: - token: ${{ github.token }} - - - name: Helm Login to ECR - run: | - aws ecr get-login-password --region us-east-1 \ - | helm registry login --username AWS --password-stdin \ - 524466471676.dkr.ecr.us-east-1.amazonaws.com - - - name: Locate Chart.yaml - id: locate-chart-dir - run: | - echo "PATH=$(dirname $(find ${{ steps.helm-chart-download.outputs.download-path }} -name Chart.yaml))" >> "$GITHUB_OUTPUT" - - - name: Package Helm Chart - run: | - helm package \ - ${{ steps.locate-chart-dir.outputs.PATH }} \ - --version ${{ inputs.helm-version || inputs.build-version }} \ - --app-version ${{ inputs.build-version }} \ - --dependency-update - - - name: Locate Helm package - id: locate-helm-package - run: | - echo "PATH=$(find . -name *-${{ inputs.build-version }}.tgz -type f -exec basename {} \; | tr -d '[:space:]')" >> "$GITHUB_OUTPUT" - - - name: Push Helm Chart to ECR - run: | - helm push ${{ steps.locate-helm-package.outputs.PATH }} oci://524466471676.dkr.ecr.us-east-1.amazonaws.com/charts From 72a07bd46d4faec24c1b79fd56b7e451c5d5a3fb Mon Sep 17 00:00:00 2001 From: sulhicader Date: Wed, 31 Jan 2024 12:22:37 +0530 Subject: [PATCH 21/24] Handled review comments #2163 --- .github/workflows/publish-university.yml | 96 ++++++++++-------------- 1 file changed, 41 insertions(+), 55 deletions(-) diff --git a/.github/workflows/publish-university.yml b/.github/workflows/publish-university.yml index d22f173e7b..1934e2a116 100644 --- a/.github/workflows/publish-university.yml +++ b/.github/workflows/publish-university.yml @@ -8,56 +8,54 @@ env: VERSION: "1.0.1" jobs: - - airgapped: - name: Create Wave Bundle - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Download H2O CLI - working-directory: ./university - run: mkdir -p .bin && curl -o .bin/h2o https://h2oai-cloud-release.s3.amazonaws.com/releases/ai/h2o/h2o-cloud/latest/cli/linux-amd64/h2o + name: Create Wave Bundle + runs-on: ubuntu-latest - - name: Change permissions - working-directory: ./university - run: chmod +x .bin/h2o + steps: + - uses: actions/checkout@v3 - - name: Set version - working-directory: ./university - run: sed -i -r -e "s/\{\{VERSION\}\}/${{ env.VERSION }}/g" app.toml + - name: Download H2O CLI + working-directory: ./university + run: mkdir -p .bin && curl -o .bin/h2o https://h2oai-cloud-release.s3.amazonaws.com/releases/ai/h2o/h2o-cloud/latest/cli/linux-amd64/h2o - - name: Get App Version - id: get-build-version - run: | - echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT" + - name: Change permissions + working-directory: ./university + run: chmod +x .bin/h2o - - name: Make air-gapped bundle - working-directory: ./university - run: | - .bin/h2o bundle \ - --docker-base-image 524466471676.dkr.ecr.us-east-1.amazonaws.com/q8s/launcher:v0.23.0-38 \ - --docker-use-buildkit \ - --generate-helm-charts \ - --helm-chart-version ${{ env.VERSION }} \ - --helm-chart-name university \ - --helm-app-bundle-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/university-bundle \ - --helm-app-runtime-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/university \ - --generate-dockerfile + - name: Set version + working-directory: ./university + run: sed -i -r -e "s/\{\{VERSION\}\}/${{ env.VERSION }}/g" app.toml - - uses: actions/upload-artifact@v3 - with: - name: wave-bundle - path: | - ./university/*.Dockerfile - ./university/*.wave + - name: Get App Version + id: get-build-version + run: | + echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT" - - uses: actions/upload-artifact@v3 - with: - name: wave-bundle-helm - path: university/helm/ + - name: Make air-gapped bundle + working-directory: ./university + run: | + .bin/h2o bundle \ + --docker-base-image 524466471676.dkr.ecr.us-east-1.amazonaws.com/q8s/launcher:v0.23.0-38 \ + --docker-use-buildkit \ + --generate-helm-charts \ + --helm-chart-version ${{ env.VERSION }} \ + --helm-chart-name university \ + --helm-app-bundle-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/university-bundle \ + --helm-app-runtime-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/university \ + --generate-dockerfile + + - uses: actions/upload-artifact@v3 + with: + name: wave-bundle + path: | + ./university/*.Dockerfile + ./university/*.wave + + - uses: actions/upload-artifact@v3 + with: + name: wave-bundle-helm + path: university/helm/ retrieve-metadata: runs-on: ubuntu-latest @@ -69,15 +67,3 @@ jobs: id: get-build-version run: | echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT" - - build-and-publish: - needs: retrieve-metadata - permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout - - uses: ./.github/workflows/wave-bundle-docker-build-publish.yaml - with: - build-version: ${{ env.VERSION }} - bundle-artifact: wave-bundle - wave-app-name: university From 41bf0f734d5c313a5ac00d111fa99fccc816b11a Mon Sep 17 00:00:00 2001 From: sulhicader Date: Wed, 31 Jan 2024 12:36:14 +0530 Subject: [PATCH 22/24] Handled review comments #2163 --- .github/workflows/publish-university.yml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publish-university.yml b/.github/workflows/publish-university.yml index 1934e2a116..97fa710297 100644 --- a/.github/workflows/publish-university.yml +++ b/.github/workflows/publish-university.yml @@ -8,10 +8,13 @@ env: VERSION: "1.0.1" jobs: - airgapped: + create-bundle: name: Create Wave Bundle runs-on: ubuntu-latest + outputs: + build-version: ${{ env.VERSION }} + steps: - uses: actions/checkout@v3 @@ -57,13 +60,14 @@ jobs: name: wave-bundle-helm path: university/helm/ - retrieve-metadata: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 + build-and-publish: + needs: create-bundle + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout - - name: Get App Version - id: get-build-version - run: | - echo "VERSION=${{ env.VERSION }}" >> "$GITHUB_OUTPUT" + uses: ./.github/workflows/wave-bundle-docker-build-publish.yaml + with: + build-version: ${{ needs.create-bundle.outputs.build-version }} + bundle-artifact: wave-bundle + wave-app-name: university From 1ad2d02dfbc64f8d9d9a7945bb4896cdc4897865 Mon Sep 17 00:00:00 2001 From: sulhicader Date: Wed, 31 Jan 2024 12:45:55 +0530 Subject: [PATCH 23/24] Finalized workflow for university #2163 --- .github/workflows/publish-university.yml | 25 +++++++- .github/workflows/wave-bundle-university.yaml | 59 ------------------- .../workflows/wave-publish-university.yaml | 33 ----------- .github/workflows/wave-university.yaml | 30 ---------- 4 files changed, 23 insertions(+), 124 deletions(-) delete mode 100644 .github/workflows/wave-bundle-university.yaml delete mode 100644 .github/workflows/wave-publish-university.yaml delete mode 100644 .github/workflows/wave-university.yaml diff --git a/.github/workflows/publish-university.yml b/.github/workflows/publish-university.yml index 97fa710297..6cc8c40546 100644 --- a/.github/workflows/publish-university.yml +++ b/.github/workflows/publish-university.yml @@ -1,13 +1,34 @@ name: Publish Wave University on: - push: + workflow_dispatch: + inputs: + version: + description: 'Release Version' + required: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: "1.0.1" + VERSION: "${{ github.event.inputs.version }}" jobs: + publish: + name: Publish Wave University + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + token: ${{ secrets.GIT_TOKEN }} + + - name: Build university + run: make publish-university + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + packages_dir: university/dist + password: ${{ secrets.PYPI_UNIVERSITY_TOKEN }} + create-bundle: name: Create Wave Bundle runs-on: ubuntu-latest diff --git a/.github/workflows/wave-bundle-university.yaml b/.github/workflows/wave-bundle-university.yaml deleted file mode 100644 index 111e9a204e..0000000000 --- a/.github/workflows/wave-bundle-university.yaml +++ /dev/null @@ -1,59 +0,0 @@ -name: Wave Bundle - -on: - workflow_call: - inputs: - build-version: - type: string - description: The version of the application/image to be pushed - required: true - -jobs: - airgapped: - name: Create Wave Bundle - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Download H2O CLI - working-directory: ./university - run: mkdir -p .bin && curl -o .bin/h2o https://h2oai-cloud-release.s3.amazonaws.com/releases/ai/h2o/h2o-cloud/latest/cli/linux-amd64/h2o - - - name: Change permissions - working-directory: ./university - run: chmod +x .bin/h2o - - - name: Set version - working-directory: ./university - run: sed -i -r -e "s/\{\{VERSION\}\}/${{ inputs.build-version }}/g" app.toml - - - name: Get App Version - id: get-build-version - run: | - echo "VERSION=${{ inputs.build-version }}" >> "$GITHUB_OUTPUT" - - - name: Make air-gapped bundle - working-directory: ./university - run: | - .bin/h2o bundle \ - --docker-base-image 524466471676.dkr.ecr.us-east-1.amazonaws.com/q8s/launcher:v0.23.0-38 \ - --docker-use-buildkit \ - --generate-helm-charts \ - --helm-chart-version ${{ inputs.build-version }} \ - --helm-chart-name university \ - --helm-app-bundle-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/university-bundle \ - --helm-app-runtime-image-repo 524466471676.dkr.ecr.us-east-1.amazonaws.com/h2oai/university \ - --generate-dockerfile - - - uses: actions/upload-artifact@v3 - with: - name: wave-bundle - path: | - ./university/*.Dockerfile - ./university/*.wave - - - uses: actions/upload-artifact@v3 - with: - name: wave-bundle-helm - path: university/helm/ diff --git a/.github/workflows/wave-publish-university.yaml b/.github/workflows/wave-publish-university.yaml deleted file mode 100644 index 65c091a744..0000000000 --- a/.github/workflows/wave-publish-university.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: Publish Wave Bundle - -on: - workflow_call: - inputs: - build-version: - type: string - description: The version of the application/image to be pushed - required: true - -jobs: - retrieve-metadata: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Get App Version - id: get-build-version - run: | - echo "VERSION=${{ inputs.build-version }}" >> "$GITHUB_OUTPUT" - - build-and-publish: - needs: retrieve-metadata - permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout - - uses: ./.github/workflows/wave-bundle-docker-build-publish.yaml - with: - build-version: ${{ inputs.build-version }} - bundle-artifact: wave-bundle - wave-app-name: university diff --git a/.github/workflows/wave-university.yaml b/.github/workflows/wave-university.yaml deleted file mode 100644 index 89dab03308..0000000000 --- a/.github/workflows/wave-university.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Wave Bundle - -on: - workflow_call: - inputs: - build-version: - type: string - description: The version of the application/image to be pushed - required: true - -jobs: - bundle: - name: Create Wave Bundle - uses: ./.github/workflows/wave-bundle-university.yaml - with: - build-version: ${{ inputs.build-version }} - - publish: - needs: bundle - name: Build and Publish - uses: ./.github/workflows/wave-publish-university.yaml - with: - build-version: ${{ inputs.build-version }} - - helm-publish: - needs: bundle - name: Build and Publish Helm Chart - uses: ./.github/workflows/helm-release-university.yaml - with: - build-version: ${{ inputs.build-version }} From 3b2c8be0723db660b0003a001c68cb83727f2cae Mon Sep 17 00:00:00 2001 From: sulhicader Date: Wed, 31 Jan 2024 13:57:14 +0530 Subject: [PATCH 24/24] Changed the base image version --- .github/workflows/publish-university.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-university.yml b/.github/workflows/publish-university.yml index 6cc8c40546..dd11351aca 100644 --- a/.github/workflows/publish-university.yml +++ b/.github/workflows/publish-university.yml @@ -60,7 +60,7 @@ jobs: working-directory: ./university run: | .bin/h2o bundle \ - --docker-base-image 524466471676.dkr.ecr.us-east-1.amazonaws.com/q8s/launcher:v0.23.0-38 \ + --docker-base-image 524466471676.dkr.ecr.us-east-1.amazonaws.com/q8s/launcher:v0.31.0-310 \ --docker-use-buildkit \ --generate-helm-charts \ --helm-chart-version ${{ env.VERSION }} \