Skip to content

Commit

Permalink
HDDS-11057. Enable reproducible builds (#6856)
Browse files Browse the repository at this point in the history
  • Loading branch information
adoroszlai authored Oct 18, 2024
1 parent 86b7aae commit 721ae58
Show file tree
Hide file tree
Showing 25 changed files with 185 additions and 145 deletions.
61 changes: 61 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,7 @@ jobs:
retention-days: 1
- name: Store source tarball for compilation
uses: actions/upload-artifact@v4
if: needs.build-info.outputs.needs-compile == 'true'
with:
name: ozone-src
path: hadoop-ozone/dist/target/ozone-*-src.tar.gz
Expand Down Expand Up @@ -407,6 +408,66 @@ jobs:
name: ${{ github.job }}
path: target/${{ github.job }}
continue-on-error: true
repro:
needs:
- build-info
- build
runs-on: ubuntu-20.04
timeout-minutes: 30
steps:
- name: Checkout project
uses: actions/checkout@v4
- name: Cache for maven dependencies
uses: actions/cache/restore@v4
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/ozone
key: maven-repo-${{ hashFiles('**/pom.xml') }}
restore-keys: |
maven-repo-
- name: Download Ozone repo
id: download-ozone-repo
uses: actions/download-artifact@v4
with:
name: ozone-repo
path: |
~/.m2/repository/org/apache/ozone
- name: Download Ratis repo
if: ${{ inputs.ratis_args != '' }}
uses: actions/download-artifact@v4
with:
name: ratis-jars
path: |
~/.m2/repository/org/apache/ratis
- name: Setup java ${{ env.TEST_JAVA_VERSION }}
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: ${{ env.TEST_JAVA_VERSION }}
- name: Execute tests
run: |
hadoop-ozone/dev-support/checks/${{ github.job }}.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
continue-on-error: true
- name: Summary of failures
run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
if: ${{ !cancelled() }}
- name: Install diffoscope
run: |
sudo apt update -q
sudo apt install -y diffoscope
if: ${{ failure() }}
- name: Check artifact differences
run: |
hadoop-ozone/dev-support/checks/_diffoscope.sh
if: ${{ failure() }}
- name: Archive build results
uses: actions/upload-artifact@v4
if: always()
with:
name: ${{ github.job }}
path: target/${{ github.job }}
continue-on-error: true
acceptance:
needs:
- build-info
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,6 @@ public static Codec<DatanodeDetails> getCodec() {
private String version;
private long setupTime;
private String revision;
private String buildDate;
private volatile HddsProtos.NodeOperationalState persistedOpState;
private volatile long persistedOpStateExpiryEpochSec;
private int initialVersion;
Expand All @@ -112,7 +111,6 @@ private DatanodeDetails(Builder b) {
version = b.version;
setupTime = b.setupTime;
revision = b.revision;
buildDate = b.buildDate;
persistedOpState = b.persistedOpState;
persistedOpStateExpiryEpochSec = b.persistedOpStateExpiryEpochSec;
initialVersion = b.initialVersion;
Expand Down Expand Up @@ -141,7 +139,6 @@ public DatanodeDetails(DatanodeDetails datanodeDetails) {
this.version = datanodeDetails.version;
this.setupTime = datanodeDetails.setupTime;
this.revision = datanodeDetails.revision;
this.buildDate = datanodeDetails.buildDate;
this.persistedOpState = datanodeDetails.getPersistedOpState();
this.persistedOpStateExpiryEpochSec =
datanodeDetails.getPersistedOpStateExpiryEpochSec();
Expand Down Expand Up @@ -433,9 +430,6 @@ public static DatanodeDetails getFromProtoBuf(
if (extendedDetailsProto.hasRevision()) {
builder.setRevision(extendedDetailsProto.getRevision());
}
if (extendedDetailsProto.hasBuildDate()) {
builder.setBuildDate(extendedDetailsProto.getBuildDate());
}
return builder.build();
}

Expand Down Expand Up @@ -527,9 +521,6 @@ public ExtendedDatanodeDetailsProto getExtendedProtoBufMessage() {
if (!Strings.isNullOrEmpty(getRevision())) {
extendedBuilder.setRevision(getRevision());
}
if (!Strings.isNullOrEmpty(getBuildDate())) {
extendedBuilder.setBuildDate(getBuildDate());
}

return extendedBuilder.build();
}
Expand Down Expand Up @@ -622,7 +613,6 @@ public static final class Builder {
private String version;
private long setupTime;
private String revision;
private String buildDate;
private HddsProtos.NodeOperationalState persistedOpState;
private long persistedOpStateExpiryEpochSec = 0;
private int initialVersion;
Expand Down Expand Up @@ -654,7 +644,6 @@ public Builder setDatanodeDetails(DatanodeDetails details) {
this.version = details.getVersion();
this.setupTime = details.getSetupTime();
this.revision = details.getRevision();
this.buildDate = details.getBuildDate();
this.persistedOpState = details.getPersistedOpState();
this.persistedOpStateExpiryEpochSec =
details.getPersistedOpStateExpiryEpochSec();
Expand Down Expand Up @@ -801,18 +790,6 @@ public Builder setRevision(String rev) {
return this;
}

/**
* Sets the DataNode build date.
*
* @param date the build date of DataNode.
*
* @return DatanodeDetails.Builder
*/
public Builder setBuildDate(String date) {
this.buildDate = date;
return this;
}

/**
* Sets the DataNode setup time.
*
Expand Down Expand Up @@ -1054,24 +1031,6 @@ public void setRevision(String rev) {
this.revision = rev;
}

/**
* Returns the DataNode build date.
*
* @return DataNode build date
*/
public String getBuildDate() {
return buildDate;
}

/**
* Set DataNode build date.
*
* @param date DataNode build date
*/
public void setBuildDate(String date) {
this.buildDate = date;
}

@Override
public HddsProtos.NetworkNode toProtobuf(
int clientVersion) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,6 @@ public static void main(String[] args) {
System.out.println(
"Source code repository " + HDDS_VERSION_INFO.getUrl() + " -r " +
HDDS_VERSION_INFO.getRevision());
System.out.println("Compiled by " + HDDS_VERSION_INFO.getUser() + " on "
+ HDDS_VERSION_INFO.getDate());
System.out.println(
"Compiled with protoc " + HDDS_VERSION_INFO.getHadoopProtoc2Version() +
", " + HDDS_VERSION_INFO.getGrpcProtocVersion() +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,18 +65,6 @@ public String getRevision() {
return info.getProperty("revision", "Unknown");
}

public String getBranch() {
return info.getProperty("branch", "Unknown");
}

public String getDate() {
return info.getProperty("date", "Unknown");
}

public String getUser() {
return info.getProperty("user", "Unknown");
}

public String getUrl() {
return info.getProperty("url", "Unknown");
}
Expand Down Expand Up @@ -108,7 +96,6 @@ public String getCompilePlatform() {
public String getBuildVersion() {
return getVersion() +
" from " + getRevision() +
" by " + getUser() +
" source checksum " + getSrcChecksum();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,6 @@

version=${declared.hdds.version}
revision=${version-info.scm.commit}
branch=${version-info.scm.branch}
user=${user.name}
date=${version-info.build.time}
url=${version-info.scm.uri}
srcChecksum=${version-info.source.md5}
hadoopProtoc2Version=${proto2.hadooprpc.protobuf.version}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,6 @@ public void start() {
datanodeDetails.setSetupTime(Time.now());
datanodeDetails.setRevision(
HddsVersionInfo.HDDS_VERSION_INFO.getRevision());
datanodeDetails.setBuildDate(HddsVersionInfo.HDDS_VERSION_INFO.getDate());
TracingUtil.initTracing(
"HddsDatanodeService." + datanodeDetails.getUuidString()
.substring(0, 8), conf);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,6 @@ public interface ServiceRuntimeInfo {
*/
String getSoftwareVersion();

/**
* Get the compilation information which contains date, user and branch.
*
* @return the compilation information, as a JSON string.
*/
String getCompileInfo();

/**
* Gets the NN start time in milliseconds.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,6 @@ public String getSoftwareVersion() {
return versionInfo.getVersion();
}

@Override
public String getCompileInfo() {
return versionInfo.getDate() + " by " + versionInfo.getUser() + " from "
+ versionInfo.getBranch();
}

@Override
public long getStartedTimeInMillis() {
return startedTimeInMillis;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -742,9 +742,7 @@ public static String createStartupShutdownMessage(VersionInfo versionInfo,
" version = " + versionInfo.getVersion(),
" classpath = " + System.getProperty("java.class.path"),
" build = " + versionInfo.getUrl() + "/"
+ versionInfo.getRevision()
+ " ; compiled by '" + versionInfo.getUser()
+ "' on " + versionInfo.getDate(),
+ versionInfo.getRevision(),
" java = " + System.getProperty("java.version"),
" conf = " + conf);
}
Expand Down
2 changes: 1 addition & 1 deletion hadoop-hdds/interface-client/src/main/proto/hdds.proto
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ message ExtendedDatanodeDetailsProto {
optional string version = 2;
optional int64 setupTime = 3;
optional string revision = 4;
optional string buildDate = 5;
optional string buildDate = 5; // unused, reserved for compatibility
}

message MoveDataNodePairProto {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,6 @@ public static void main(String[] args) {
System.out.println(
"Source code repository " + OZONE_VERSION_INFO.getUrl() + " -r " +
OZONE_VERSION_INFO.getRevision());
System.out.println("Compiled by " + OZONE_VERSION_INFO.getUser() + " on "
+ OZONE_VERSION_INFO.getDate());
System.out.println(
"Compiled with protoc " + OZONE_VERSION_INFO.getHadoopProtoc2Version() +
", " + OZONE_VERSION_INFO.getGrpcProtocVersion() +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@
version=${declared.ozone.version}
release=${ozone.release}
revision=${version-info.scm.commit}
branch=${version-info.scm.branch}
user=${user.name}
date=${version-info.build.time}
url=${version-info.scm.uri}
srcChecksum=${version-info.source.md5}
hadoopProtoc2Version=${proto2.hadooprpc.protobuf.version}
Expand Down
31 changes: 31 additions & 0 deletions hadoop-ozone/dev-support/checks/_build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1

: ${OZONE_WITH_COVERAGE:="false"}

MAVEN_OPTIONS='-V -B -DskipTests -DskipDocs --no-transfer-progress'

if [[ "${OZONE_WITH_COVERAGE}" == "true" ]]; then
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Pcoverage"
else
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip"
fi

export MAVEN_OPTS="-Xmx4096m ${MAVEN_OPTS:-}"
mvn ${MAVEN_OPTIONS} clean "$@"
rc=$?
45 changes: 45 additions & 0 deletions hadoop-ozone/dev-support/checks/_diffoscope.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Helper script to compare jars reported by maven-artifact-plugin

set -e -u -o pipefail

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1

BASE_DIR="$(pwd -P)"
: ${OUTPUT_LOG:="${BASE_DIR}/target/repro/output.log"}

for jar in $(grep -o "investigate with diffoscope [^ ]*\.jar [^ ]*\.jar" "${OUTPUT_LOG}" | awk '{ print $NF }'); do
jarname=$(basename "$jar")
if [[ ! -e "$jar" ]]; then
echo "$jar does not exist"
continue
fi

ref=$(find target/reference -name "$jarname")
if [[ -z "$ref" ]]; then
ref=$(find ~/.m2/repository -name "$jarname")
fi

if [[ ! -e "$ref" ]]; then
echo "Reference not found for: $jarname"
continue
fi

diffoscope "$ref" "$jar"
done
18 changes: 2 additions & 16 deletions hadoop-ozone/dev-support/checks/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR/../../.." || exit 1

: ${OZONE_WITH_COVERAGE:="false"}

MAVEN_OPTIONS='-V -B -DskipTests -DskipDocs --no-transfer-progress'

if [[ "${OZONE_WITH_COVERAGE}" == "true" ]]; then
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Pcoverage"
else
MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip"
fi

export MAVEN_OPTS="-Xmx4096m ${MAVEN_OPTS:-}"
echo "${MAVEN_OPTIONS}"
mvn ${MAVEN_OPTIONS} clean install "$@"
exit $?
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "${DIR}"/_build.sh install "$@"
Loading

0 comments on commit 721ae58

Please sign in to comment.