From bb2f07d34b4a2d4977e0aeaa749e77a542dee621 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sat, 4 Jan 2020 13:04:56 +0800 Subject: [PATCH 01/10] Use back revision variable in pom.xml So user or CI system can easily override revision from external sources such as Git tag name --- core/pom.xml | 2 +- ingestion/pom.xml | 2 +- sdk/java/pom.xml | 2 +- serving/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index c0d889e283..954c7c0018 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -23,7 +23,7 @@ dev.feast feast-parent - 0.3.6-SNAPSHOT + ${revision} Feast Core diff --git a/ingestion/pom.xml b/ingestion/pom.xml index fce645c51b..4908b54698 100644 --- a/ingestion/pom.xml +++ b/ingestion/pom.xml @@ -23,7 +23,7 @@ dev.feast feast-parent - 0.3.6-SNAPSHOT + ${revision} Feast Ingestion diff --git a/sdk/java/pom.xml b/sdk/java/pom.xml index b3543446e2..2970dae3ee 100644 --- a/sdk/java/pom.xml +++ b/sdk/java/pom.xml @@ -11,7 +11,7 @@ dev.feast feast-parent - 0.3.6-SNAPSHOT + ${revision} ../.. diff --git a/serving/pom.xml b/serving/pom.xml index 4a28eb739e..dc3391df62 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -23,7 +23,7 @@ dev.feast feast-parent - 0.3.6-SNAPSHOT + ${revision} feast-serving From faee6fa2bfd0df143d92e2e5fbd2739c7a60342f Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sat, 4 Jan 2020 13:06:24 +0800 Subject: [PATCH 02/10] Add flatten maven plugin This plugin is useful during deployment so the final pom is resolved without parent dependency, i.e. we do not necessarily need to upload parent library --- .gitignore | 4 ++++ pom.xml | 39 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 8aaa0b6920..4375b1b6b3 100644 --- a/.gitignore +++ b/.gitignore @@ -173,3 +173,7 @@ dmypy.json # Pyre type checker .pyre/ .vscode + +# .flattened-pom.xml is generated by flatten-maven-plugin. +# This pom should not be committed because it is only used during release / deployment. +.flattened-pom.xml diff --git a/pom.xml b/pom.xml index 882e24ab8e..424180df5d 100644 --- a/pom.xml +++ b/pom.xml @@ -24,7 +24,7 @@ dev.feast feast-parent - 0.3.6-SNAPSHOT + ${revision} pom @@ -35,6 +35,7 @@ + 0.3.6-SNAPSHOT https://github.com/gojek/feast UTF-8 @@ -47,7 +48,6 @@ 2.16.0 1.91.0 0.8.0 - 1.9.10 1.3 2.3.0 @@ -86,6 +86,10 @@ ossrh https://oss.sonatype.org/content/repositories/snapshots + + ossrh + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + @@ -277,7 +281,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.7 + 1.6.8 true ossrh @@ -285,6 +289,35 @@ false + + + org.codehaus.mojo + flatten-maven-plugin + 1.1.0 + + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + org.apache.maven.plugins maven-source-plugin From 0eaecd0c5001a42a9b9abe23f9c53f5bd2a87c45 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sat, 4 Jan 2020 13:18:34 +0800 Subject: [PATCH 03/10] Increase versions for maven source,javadoc,spotless plugins So it has newer features and more fixes --- pom.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 424180df5d..f210350416 100644 --- a/pom.xml +++ b/pom.xml @@ -81,6 +81,7 @@ ${github.url}/issues + ossrh @@ -321,7 +322,7 @@ org.apache.maven.plugins maven-source-plugin - 2.2.1 + 3.2.1 attach-sources @@ -334,7 +335,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.9.1 + 3.1.1 attach-javadocs @@ -347,7 +348,7 @@ com.diffplug.spotless spotless-maven-plugin - 1.26.0 + 1.26.1 From f2294c978b91e2eca8a1e1dcca429fe1f5ad9d78 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sat, 4 Jan 2020 14:16:47 +0800 Subject: [PATCH 04/10] Add gpg-plugin needed to sign releases --- pom.xml | 111 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 71 insertions(+), 40 deletions(-) diff --git a/pom.xml b/pom.xml index f210350416..1455d81fb9 100644 --- a/pom.xml +++ b/pom.xml @@ -279,46 +279,6 @@ - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.8 - true - - ossrh - https://oss.sonatype.org/ - false - - - - - org.codehaus.mojo - flatten-maven-plugin - 1.1.0 - - - - - flatten - process-resources - - flatten - - - - flatten.clean - clean - - clean - - - - org.apache.maven.plugins maven-source-plugin @@ -463,6 +423,77 @@ true + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.8 + true + + ossrh + https://oss.sonatype.org/ + + true + + + + + org.codehaus.mojo + flatten-maven-plugin + 1.1.0 + + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.5 + + + sign-artifacts + verify + + sign + + + + + --pinentry-mode + loopback + + + + ${gpg.passphrase} + + + + From 07ebfa8515ca4ac725f71d56117a9feb51253333 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sat, 4 Jan 2020 14:30:44 +0800 Subject: [PATCH 05/10] Use oss configure for flatten plugin, add developers info in pom.xml (required for releasing library --- pom.xml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1455d81fb9..1ed9e54c76 100644 --- a/pom.xml +++ b/pom.xml @@ -58,9 +58,18 @@ Gojek - https://www.gojek.io/ + https://www.gojek.com + + + Feast Authors + ${github.url} + Gojek + https://www.gojek.com + + + Apache License, Version 2.0 @@ -447,6 +456,7 @@ flatten-maven-plugin 1.1.0 + oss From cdc5065453828fad16e5e30347e59ea7982ceb92 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sat, 4 Jan 2020 15:00:59 +0800 Subject: [PATCH 06/10] Add publish-java-sdk script --- .prow/scripts/publish-java-sdk.sh | 61 +++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100755 .prow/scripts/publish-java-sdk.sh diff --git a/.prow/scripts/publish-java-sdk.sh b/.prow/scripts/publish-java-sdk.sh new file mode 100755 index 0000000000..497a3ee359 --- /dev/null +++ b/.prow/scripts/publish-java-sdk.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +GPG_KEY_IMPORT_DIR=/etc/gpg + +usage() +{ + echo "usage: publish-java-sdk.sh + + --revision Value for the revision e.g. '0.2.3' + --gpg-key-import-dir Directory containing existing GPG keys to import. + The directory should contain these 2 files: + - public-key + - private-key + The default value is '/etc/gpg' + + This script assumes the GPG private key is protected by a passphrase. + The passphrase can be specified in \$HOME/.m2/settings.xml: + + # ~/.m2/settings.xml + + + + ossrh + SONATYPE_USER + SONATYPE_PASSWORD + + + + + ossrh + + GPG_PASSPHRASE + + + + +" +} + +while [ "$1" != "" ]; do + case "$1" in + --revision ) REVISION="$2"; shift;; + --gpg-key-import-dir ) GPG_KEY_IMPORT_DIR="$2"; shift;; + -h | --help ) usage; exit;; + * ) usage; exit 1 + esac + shift +done + +if [ -z $REVISION ]; then usage; exit 1; fi + +echo $REVISION +echo $GPG_KEY_IMPORT_DIR + +gpg --import --batch --yes /etc/gpg/public-key +gpg --import --batch --yes /etc/gpg/private-key + +mvn --projects sdk/java -Drevision=$REVISION clean deploy From 0a956fe03601e4dc012b6f441330c62e8b728bb8 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sat, 4 Jan 2020 15:04:23 +0800 Subject: [PATCH 07/10] Add more logs to publish-java-sdk.sh --- .prow/scripts/publish-java-sdk.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.prow/scripts/publish-java-sdk.sh b/.prow/scripts/publish-java-sdk.sh index 497a3ee359..baa1f1209c 100755 --- a/.prow/scripts/publish-java-sdk.sh +++ b/.prow/scripts/publish-java-sdk.sh @@ -52,10 +52,19 @@ done if [ -z $REVISION ]; then usage; exit 1; fi -echo $REVISION -echo $GPG_KEY_IMPORT_DIR +echo "============================================================" +echo "Checking Maven and GPG versions" +echo "============================================================" +mvn --version +gpg --version +echo "============================================================" +echo "Importing GPG keys" +echo "============================================================" gpg --import --batch --yes /etc/gpg/public-key gpg --import --batch --yes /etc/gpg/private-key +echo "============================================================" +echo "Deploying Java SDK with revision: $REVISION" +echo "============================================================" mvn --projects sdk/java -Drevision=$REVISION clean deploy From 6240ce353b30f6e0a5cb1abd4bfbd5ca2c2fc4e3 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sat, 4 Jan 2020 16:16:35 +0800 Subject: [PATCH 08/10] Add ProwJob publish-java-sdk --- .prow/config.yaml | 29 +++++++++++++++++++++++++++++ .prow/scripts/publish-java-sdk.sh | 8 +++++--- pom.xml | 2 +- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/.prow/config.yaml b/.prow/config.yaml index 940e2562d7..c63d3dce79 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -169,6 +169,34 @@ postsubmits: # https://github.com/semver/semver/issues/232 - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ + - name: publish-java-sdk + decorate: true + spec: + containers: + - image: maven:3.6-jdk-8 + command: + - bash + - -c + - .prow/scripts/publish-java-sdk.sh --revision ${PULL_BASE_REF:1} + volumeMounts: + - name: gpg-keys + mountPath: /etc/gpg + readOnly: true + - name: maven-settings + mountPath: /root/.m2/settings.xml + subPath: settings.xml + readOnly: true + volumes: + - name: gpg-keys + secret: + secretName: gpg-keys + - name: maven-settings + secret: + secretName: maven-settings + branches: + # Filter on tags with semantic versioning, prefixed with "v" + - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ + - name: publish-docker-images decorate: true spec: @@ -278,4 +306,5 @@ postsubmits: secret: secretName: feast-service-account branches: + # Filter on tags with semantic versioning, prefixed with "v" - ^v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$ diff --git a/.prow/scripts/publish-java-sdk.sh b/.prow/scripts/publish-java-sdk.sh index baa1f1209c..0ee228a88a 100755 --- a/.prow/scripts/publish-java-sdk.sh +++ b/.prow/scripts/publish-java-sdk.sh @@ -17,9 +17,10 @@ usage() The default value is '/etc/gpg' This script assumes the GPG private key is protected by a passphrase. - The passphrase can be specified in \$HOME/.m2/settings.xml: + The passphrase can be specified in \$HOME/.m2/settings.xml. In the same xml + file, credentials to upload releases to Sonatype must also be provided. - # ~/.m2/settings.xml + # Example settings: ~/.m2/settings.xml @@ -56,6 +57,7 @@ echo "============================================================" echo "Checking Maven and GPG versions" echo "============================================================" mvn --version +echo "" gpg --version echo "============================================================" @@ -67,4 +69,4 @@ gpg --import --batch --yes /etc/gpg/private-key echo "============================================================" echo "Deploying Java SDK with revision: $REVISION" echo "============================================================" -mvn --projects sdk/java -Drevision=$REVISION clean deploy +mvn --projects sdk/java -Drevision=$REVISION --batch-mode clean deploy diff --git a/pom.xml b/pom.xml index 1ed9e54c76..9cd37d5338 100644 --- a/pom.xml +++ b/pom.xml @@ -479,7 +479,7 @@ org.apache.maven.plugins maven-gpg-plugin - 1.5 + 1.6 sign-artifacts From b38166aa409b0eceadc1b1520ad0aeab6e0fe0d7 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Sat, 4 Jan 2020 16:45:50 +0800 Subject: [PATCH 09/10] Use GPG_KEY_IMPORT_DIR variable --- .prow/scripts/publish-java-sdk.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.prow/scripts/publish-java-sdk.sh b/.prow/scripts/publish-java-sdk.sh index 0ee228a88a..17513d0eb0 100755 --- a/.prow/scripts/publish-java-sdk.sh +++ b/.prow/scripts/publish-java-sdk.sh @@ -63,8 +63,8 @@ gpg --version echo "============================================================" echo "Importing GPG keys" echo "============================================================" -gpg --import --batch --yes /etc/gpg/public-key -gpg --import --batch --yes /etc/gpg/private-key +gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/public-key +gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/private-key echo "============================================================" echo "Deploying Java SDK with revision: $REVISION" From 07311880798bdbfeed953631b6f1b7398fca7811 Mon Sep 17 00:00:00 2001 From: David Heryanto Date: Mon, 6 Jan 2020 18:14:26 +0800 Subject: [PATCH 10/10] Update revision in pom.xml to 0.4.2-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9cd37d5338..05fb701ac4 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ - 0.3.6-SNAPSHOT + 0.4.2-SNAPSHOT https://github.com/gojek/feast UTF-8