From 4b3d6db907007c8455d85ddbbf8c4481178637d7 Mon Sep 17 00:00:00 2001 From: Horusec Date: Thu, 15 Oct 2020 19:41:30 -0300 Subject: [PATCH 1/9] Adding migration dockerfile --- deployments/dockerfiles/migration/.semver.yaml | 4 ++++ deployments/dockerfiles/migration/Dockerfile | 5 +++++ 2 files changed, 9 insertions(+) create mode 100644 deployments/dockerfiles/migration/.semver.yaml create mode 100644 deployments/dockerfiles/migration/Dockerfile diff --git a/deployments/dockerfiles/migration/.semver.yaml b/deployments/dockerfiles/migration/.semver.yaml new file mode 100644 index 000000000..4a4fec0f8 --- /dev/null +++ b/deployments/dockerfiles/migration/.semver.yaml @@ -0,0 +1,4 @@ +alpha: 0 +beta: 0 +rc: 0 +release: v0.0.0 diff --git a/deployments/dockerfiles/migration/Dockerfile b/deployments/dockerfiles/migration/Dockerfile new file mode 100644 index 000000000..ad7f03ae1 --- /dev/null +++ b/deployments/dockerfiles/migration/Dockerfile @@ -0,0 +1,5 @@ +FROM migrate/migrate:v4.13.0 + +ADD development-kit/pkg/databases/relational/migration /horusec-migrations + +ENTRYPOINT [ "migrate", "-source /horusec-migrations" ] \ No newline at end of file From 5308f9130dd976892aec7fec195cf836fa127630 Mon Sep 17 00:00:00 2001 From: Horusec Date: Thu, 15 Oct 2020 20:02:35 -0300 Subject: [PATCH 2/9] Improving migration dockerfile --- deployments/dockerfiles/migration/Dockerfile | 5 ++++- deployments/dockerfiles/migration/migrate.sh | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 deployments/dockerfiles/migration/migrate.sh diff --git a/deployments/dockerfiles/migration/Dockerfile b/deployments/dockerfiles/migration/Dockerfile index ad7f03ae1..7d5e72b4d 100644 --- a/deployments/dockerfiles/migration/Dockerfile +++ b/deployments/dockerfiles/migration/Dockerfile @@ -1,5 +1,8 @@ FROM migrate/migrate:v4.13.0 ADD development-kit/pkg/databases/relational/migration /horusec-migrations +ADD deployments/dockerfiles/migration/migrate.sh /usr/local/bin -ENTRYPOINT [ "migrate", "-source /horusec-migrations" ] \ No newline at end of file +RUN chmod +x /usr/local/bin/migrate.sh + +ENTRYPOINT [ "migrate.sh" ] \ No newline at end of file diff --git a/deployments/dockerfiles/migration/migrate.sh b/deployments/dockerfiles/migration/migrate.sh new file mode 100644 index 000000000..6aae0c4f3 --- /dev/null +++ b/deployments/dockerfiles/migration/migrate.sh @@ -0,0 +1,2 @@ +#!/bin/sh +migrate -path "/horusec-migrations" "$@" \ No newline at end of file From 366f877cbc7bf08dc8260364a98e34e61ba9acd2 Mon Sep 17 00:00:00 2001 From: Horusec Date: Thu, 15 Oct 2020 20:34:36 -0300 Subject: [PATCH 3/9] Using env in migration --- deployments/dockerfiles/migration/migrate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/dockerfiles/migration/migrate.sh b/deployments/dockerfiles/migration/migrate.sh index 6aae0c4f3..2209bd28b 100644 --- a/deployments/dockerfiles/migration/migrate.sh +++ b/deployments/dockerfiles/migration/migrate.sh @@ -1,2 +1,2 @@ #!/bin/sh -migrate -path "/horusec-migrations" "$@" \ No newline at end of file +migrate -path "/horusec-migrations" -database "$HORUSEC_DATABASE_SQL_URI" "$@" \ No newline at end of file From d77b853b4f4e6a0225767ef87df5df79acdc7dfb Mon Sep 17 00:00:00 2001 From: Horusec Date: Thu, 15 Oct 2020 20:45:59 -0300 Subject: [PATCH 4/9] Adding migration template --- .../templates/databasemigration.yaml | 39 +++++++++++++++++++ .../templates/databasemigration.yaml | 39 +++++++++++++++++++ .../templates/databasemigration.yaml | 39 +++++++++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 horusec-account/deployments/helm/horusec-account/templates/databasemigration.yaml create mode 100644 horusec-analytic/deployments/helm/horusec-analytic/templates/databasemigration.yaml create mode 100644 horusec-api/deployments/helm/horusec-api/templates/databasemigration.yaml diff --git a/horusec-account/deployments/helm/horusec-account/templates/databasemigration.yaml b/horusec-account/deployments/helm/horusec-account/templates/databasemigration.yaml new file mode 100644 index 000000000..2313a7d33 --- /dev/null +++ b/horusec-account/deployments/helm/horusec-account/templates/databasemigration.yaml @@ -0,0 +1,39 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{ .Release.Name }}" + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": hook-succeeded +spec: + template: + metadata: + name: "{{ .Release.Name }}" + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + spec: + restartPolicy: Never + containers: + - name: horusec-database-migration + image: "horuszup/horusec-migration" + args: ["up"] + env: + {{- range .Values.env }} + - name: {{ .name }} + value: "{{ .value }}" + {{- end }} + {{- range .Values.envFromSecret }} + - name: {{ .name }} + valueFrom: + secretKeyRef: + name: {{ .key }} + key: {{ .key }} + {{- end }} diff --git a/horusec-analytic/deployments/helm/horusec-analytic/templates/databasemigration.yaml b/horusec-analytic/deployments/helm/horusec-analytic/templates/databasemigration.yaml new file mode 100644 index 000000000..2313a7d33 --- /dev/null +++ b/horusec-analytic/deployments/helm/horusec-analytic/templates/databasemigration.yaml @@ -0,0 +1,39 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{ .Release.Name }}" + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": hook-succeeded +spec: + template: + metadata: + name: "{{ .Release.Name }}" + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + spec: + restartPolicy: Never + containers: + - name: horusec-database-migration + image: "horuszup/horusec-migration" + args: ["up"] + env: + {{- range .Values.env }} + - name: {{ .name }} + value: "{{ .value }}" + {{- end }} + {{- range .Values.envFromSecret }} + - name: {{ .name }} + valueFrom: + secretKeyRef: + name: {{ .key }} + key: {{ .key }} + {{- end }} diff --git a/horusec-api/deployments/helm/horusec-api/templates/databasemigration.yaml b/horusec-api/deployments/helm/horusec-api/templates/databasemigration.yaml new file mode 100644 index 000000000..2313a7d33 --- /dev/null +++ b/horusec-api/deployments/helm/horusec-api/templates/databasemigration.yaml @@ -0,0 +1,39 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{ .Release.Name }}" + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": hook-succeeded +spec: + template: + metadata: + name: "{{ .Release.Name }}" + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + spec: + restartPolicy: Never + containers: + - name: horusec-database-migration + image: "horuszup/horusec-migration" + args: ["up"] + env: + {{- range .Values.env }} + - name: {{ .name }} + value: "{{ .value }}" + {{- end }} + {{- range .Values.envFromSecret }} + - name: {{ .name }} + valueFrom: + secretKeyRef: + name: {{ .key }} + key: {{ .key }} + {{- end }} From 64bbbbc8449d88c45b135ce37a82954c2177ec99 Mon Sep 17 00:00:00 2001 From: Horusec Date: Thu, 15 Oct 2020 20:47:51 -0300 Subject: [PATCH 5/9] Fixing migration template --- .../templates/databasemigration.yaml | 22 +++++++++---------- .../templates/databasemigration.yaml | 22 +++++++++---------- .../templates/databasemigration.yaml | 22 +++++++++---------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/horusec-account/deployments/helm/horusec-account/templates/databasemigration.yaml b/horusec-account/deployments/helm/horusec-account/templates/databasemigration.yaml index 2313a7d33..c578dddb7 100644 --- a/horusec-account/deployments/helm/horusec-account/templates/databasemigration.yaml +++ b/horusec-account/deployments/helm/horusec-account/templates/databasemigration.yaml @@ -26,14 +26,14 @@ spec: image: "horuszup/horusec-migration" args: ["up"] env: - {{- range .Values.env }} - - name: {{ .name }} - value: "{{ .value }}" - {{- end }} - {{- range .Values.envFromSecret }} - - name: {{ .name }} - valueFrom: - secretKeyRef: - name: {{ .key }} - key: {{ .key }} - {{- end }} + {{- range .Values.env }} + - name: {{ .name }} + value: "{{ .value }}" + {{- end }} + {{- range .Values.envFromSecret }} + - name: {{ .name }} + valueFrom: + secretKeyRef: + name: {{ .key }} + key: {{ .key }} + {{- end }} diff --git a/horusec-analytic/deployments/helm/horusec-analytic/templates/databasemigration.yaml b/horusec-analytic/deployments/helm/horusec-analytic/templates/databasemigration.yaml index 2313a7d33..c578dddb7 100644 --- a/horusec-analytic/deployments/helm/horusec-analytic/templates/databasemigration.yaml +++ b/horusec-analytic/deployments/helm/horusec-analytic/templates/databasemigration.yaml @@ -26,14 +26,14 @@ spec: image: "horuszup/horusec-migration" args: ["up"] env: - {{- range .Values.env }} - - name: {{ .name }} - value: "{{ .value }}" - {{- end }} - {{- range .Values.envFromSecret }} - - name: {{ .name }} - valueFrom: - secretKeyRef: - name: {{ .key }} - key: {{ .key }} - {{- end }} + {{- range .Values.env }} + - name: {{ .name }} + value: "{{ .value }}" + {{- end }} + {{- range .Values.envFromSecret }} + - name: {{ .name }} + valueFrom: + secretKeyRef: + name: {{ .key }} + key: {{ .key }} + {{- end }} diff --git a/horusec-api/deployments/helm/horusec-api/templates/databasemigration.yaml b/horusec-api/deployments/helm/horusec-api/templates/databasemigration.yaml index 2313a7d33..c578dddb7 100644 --- a/horusec-api/deployments/helm/horusec-api/templates/databasemigration.yaml +++ b/horusec-api/deployments/helm/horusec-api/templates/databasemigration.yaml @@ -26,14 +26,14 @@ spec: image: "horuszup/horusec-migration" args: ["up"] env: - {{- range .Values.env }} - - name: {{ .name }} - value: "{{ .value }}" - {{- end }} - {{- range .Values.envFromSecret }} - - name: {{ .name }} - valueFrom: - secretKeyRef: - name: {{ .key }} - key: {{ .key }} - {{- end }} + {{- range .Values.env }} + - name: {{ .name }} + value: "{{ .value }}" + {{- end }} + {{- range .Values.envFromSecret }} + - name: {{ .name }} + valueFrom: + secretKeyRef: + name: {{ .key }} + key: {{ .key }} + {{- end }} From 42a6075f3a3fe42aed8bf15eca9b317c2704dd45 Mon Sep 17 00:00:00 2001 From: Horusec Date: Fri, 16 Oct 2020 13:09:23 -0300 Subject: [PATCH 6/9] Adding migration image script --- deployments/scripts/update-image-service.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/deployments/scripts/update-image-service.sh b/deployments/scripts/update-image-service.sh index 3ba3fb459..fc6834d47 100755 --- a/deployments/scripts/update-image-service.sh +++ b/deployments/scripts/update-image-service.sh @@ -43,19 +43,22 @@ getDirectoryAndImageNameByToolName () { case "$SERVICE_NAME" in "horusec-account") IMAGE_NAME="horuszup/horusec-account" - DIRECTORY="./horusec-account";; + DIRECTORY="./horusec-account/deployments/Dockerfile";; "horusec-analytic") IMAGE_NAME="horuszup/horusec-analytic" - DIRECTORY="./horusec-analytic";; + DIRECTORY="./horusec-analytic/deployments/Dockerfile";; "horusec-api") IMAGE_NAME="horuszup/horusec-api" - DIRECTORY="./horusec-api";; + DIRECTORY="./horusec-api/deployments/Dockerfile";; "horusec-manager") IMAGE_NAME="horuszup/horusec-manager" - DIRECTORY="./horusec-manager";; + DIRECTORY="./horusec-manager/deployments/Dockerfile";; "horusec-messages") IMAGE_NAME="horuszup/horusec-messages" - DIRECTORY="./horusec-messages";; + DIRECTORY="./horusec-messages/deployments/Dockerfile";; + "horusec-migration") + IMAGE_NAME="horuszup/horusec-migration" + DIRECTORY="./deployments/dockerfiles/migration/Dockerfile";; *) echo "Param Service Name is invalid, please use the examples bellow allowed and try again!" echo "Params Service Name allowed: horusec-account, horusec-analytic, horusec-api, horusec-manager, horusec-messages" @@ -116,11 +119,11 @@ updateVersion () { if [ "$IS_TO_UPDATE_LATEST" == "true" ] then - docker build -t "$IMAGE_NAME:latest" -f $DIRECTORY/deployments/Dockerfile . + docker build -t "$IMAGE_NAME:latest" -f $DIRECTORY . docker push "$IMAGE_NAME:latest" fi - docker build -t "$IMAGE_NAME:$LATEST_VERSION" -f $DIRECTORY/deployments/Dockerfile . + docker build -t "$IMAGE_NAME:$LATEST_VERSION" -f $DIRECTORY . docker push "$IMAGE_NAME:$LATEST_VERSION" rollback_version_packagejson From a048b37e5fdc250eaec544f29f7899175e753d4c Mon Sep 17 00:00:00 2001 From: Horusec Date: Mon, 19 Oct 2020 12:10:46 -0300 Subject: [PATCH 7/9] Updating helm hook --- deployments/dockerfiles/migration/migrate.sh | 2 +- .../helm/horusec-account/templates/databasemigration.yaml | 4 ++-- .../helm/horusec-analytic/templates/databasemigration.yaml | 2 +- .../helm/horusec-api/templates/databasemigration.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deployments/dockerfiles/migration/migrate.sh b/deployments/dockerfiles/migration/migrate.sh index 2209bd28b..e7d35a9bc 100644 --- a/deployments/dockerfiles/migration/migrate.sh +++ b/deployments/dockerfiles/migration/migrate.sh @@ -1,2 +1,2 @@ #!/bin/sh -migrate -path "/horusec-migrations" -database "$HORUSEC_DATABASE_SQL_URI" "$@" \ No newline at end of file +migrate -path "/horusec-migrations" -database "$HORUSEC_DATABASE_SQL_URI" up "$@" diff --git a/horusec-account/deployments/helm/horusec-account/templates/databasemigration.yaml b/horusec-account/deployments/helm/horusec-account/templates/databasemigration.yaml index c578dddb7..62e45118e 100644 --- a/horusec-account/deployments/helm/horusec-account/templates/databasemigration.yaml +++ b/horusec-account/deployments/helm/horusec-account/templates/databasemigration.yaml @@ -10,7 +10,7 @@ metadata: annotations: "helm.sh/hook": pre-install,pre-upgrade "helm.sh/hook-weight": "-5" - "helm.sh/hook-delete-policy": hook-succeeded + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed spec: template: metadata: @@ -24,7 +24,7 @@ spec: containers: - name: horusec-database-migration image: "horuszup/horusec-migration" - args: ["up"] + command: ["migrate.sh"] env: {{- range .Values.env }} - name: {{ .name }} diff --git a/horusec-analytic/deployments/helm/horusec-analytic/templates/databasemigration.yaml b/horusec-analytic/deployments/helm/horusec-analytic/templates/databasemigration.yaml index c578dddb7..1d6151924 100644 --- a/horusec-analytic/deployments/helm/horusec-analytic/templates/databasemigration.yaml +++ b/horusec-analytic/deployments/helm/horusec-analytic/templates/databasemigration.yaml @@ -24,7 +24,7 @@ spec: containers: - name: horusec-database-migration image: "horuszup/horusec-migration" - args: ["up"] + command: ["migrate.sh"] env: {{- range .Values.env }} - name: {{ .name }} diff --git a/horusec-api/deployments/helm/horusec-api/templates/databasemigration.yaml b/horusec-api/deployments/helm/horusec-api/templates/databasemigration.yaml index c578dddb7..1d6151924 100644 --- a/horusec-api/deployments/helm/horusec-api/templates/databasemigration.yaml +++ b/horusec-api/deployments/helm/horusec-api/templates/databasemigration.yaml @@ -24,7 +24,7 @@ spec: containers: - name: horusec-database-migration image: "horuszup/horusec-migration" - args: ["up"] + command: ["migrate.sh"] env: {{- range .Values.env }} - name: {{ .name }} From c450ce8c0319b2d43aac58a570659d6bf3a54493 Mon Sep 17 00:00:00 2001 From: Horusec Date: Mon, 19 Oct 2020 16:45:28 -0300 Subject: [PATCH 8/9] Fixing mingration version --- deployments/dockerfiles/migration/.semver.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/dockerfiles/migration/.semver.yaml b/deployments/dockerfiles/migration/.semver.yaml index 4a4fec0f8..380baee26 100644 --- a/deployments/dockerfiles/migration/.semver.yaml +++ b/deployments/dockerfiles/migration/.semver.yaml @@ -1,4 +1,4 @@ alpha: 0 beta: 0 rc: 0 -release: v0.0.0 +release: v0.0.1 From 617e5f4fcb807a9d8f6911fa4a002f89e24b5d23 Mon Sep 17 00:00:00 2001 From: Horusec Date: Mon, 19 Oct 2020 20:24:15 -0300 Subject: [PATCH 9/9] Fixing service image script builder --- deployments/scripts/update-image-service.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/deployments/scripts/update-image-service.sh b/deployments/scripts/update-image-service.sh index fc6834d47..087ca3863 100755 --- a/deployments/scripts/update-image-service.sh +++ b/deployments/scripts/update-image-service.sh @@ -43,22 +43,22 @@ getDirectoryAndImageNameByToolName () { case "$SERVICE_NAME" in "horusec-account") IMAGE_NAME="horuszup/horusec-account" - DIRECTORY="./horusec-account/deployments/Dockerfile";; + DIRECTORY="./horusec-account/deployments";; "horusec-analytic") IMAGE_NAME="horuszup/horusec-analytic" - DIRECTORY="./horusec-analytic/deployments/Dockerfile";; + DIRECTORY="./horusec-analytic/deployments";; "horusec-api") IMAGE_NAME="horuszup/horusec-api" - DIRECTORY="./horusec-api/deployments/Dockerfile";; + DIRECTORY="./horusec-api/deployments";; "horusec-manager") IMAGE_NAME="horuszup/horusec-manager" - DIRECTORY="./horusec-manager/deployments/Dockerfile";; + DIRECTORY="./horusec-manager/deployments";; "horusec-messages") IMAGE_NAME="horuszup/horusec-messages" - DIRECTORY="./horusec-messages/deployments/Dockerfile";; + DIRECTORY="./horusec-messages/deployments";; "horusec-migration") IMAGE_NAME="horuszup/horusec-migration" - DIRECTORY="./deployments/dockerfiles/migration/Dockerfile";; + DIRECTORY="./deployments/dockerfiles/migration";; *) echo "Param Service Name is invalid, please use the examples bellow allowed and try again!" echo "Params Service Name allowed: horusec-account, horusec-analytic, horusec-api, horusec-manager, horusec-messages" @@ -119,11 +119,11 @@ updateVersion () { if [ "$IS_TO_UPDATE_LATEST" == "true" ] then - docker build -t "$IMAGE_NAME:latest" -f $DIRECTORY . + docker build -t "$IMAGE_NAME:latest" -f $DIRECTORY/Dockerfile . docker push "$IMAGE_NAME:latest" fi - docker build -t "$IMAGE_NAME:$LATEST_VERSION" -f $DIRECTORY . + docker build -t "$IMAGE_NAME:$LATEST_VERSION" -f $DIRECTORY/Dockerfile . docker push "$IMAGE_NAME:$LATEST_VERSION" rollback_version_packagejson