feat(dev): Add release and verify scripts (apache#507)

Reason for this PR Add scripts for developer or release manager to easily release version or verify a version. What changes are included in this PR? Add release and verify scripts related document is updated to website, see Update the release and verify document, and add development document incubator-graphar-website#18 Are these changes tested? yes Are there any user-facing changes? no --------- Signed-off-by: acezen <qiaozi.zwb@alibaba-inc.com>
Elssky · Oct 8, 2024 · 2e68701 · 2e68701
1 parent b037185
commit 2e68701
Show file tree

Hide file tree

Showing 10 changed files with 446 additions and 4 deletions.
diff --git a/.devcontainer/graphar-dev.Dockerfile b/.devcontainer/graphar-dev.Dockerfile
@@ -40,6 +40,12 @@ RUN git clone --branch v1.8.3 https://github.com/google/benchmark.git /tmp/bench
     && make install \
     && rm -rf /tmp/benchmark
 
+RUN git clone --branch v3.6.0 https://github.com/catchorg/Catch2.git /tmp/catch2 --depth 1 \
+    && cd /tmp/catch2 \
+    && cmake -Bbuild -H. -DBUILD_TESTING=OFF \
+    && cmake --build build/ --target install \
+    && rm -rf /tmp/catch2
+
 ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64
 ENV JAVA_HOME=/usr/lib/jvm/default-java
 

diff --git a/LICENSE b/LICENSE
@@ -212,7 +212,7 @@ Apache-2.0 licenses
 The following components are provided under the Apache-2.0 License. See project link for details.
 The text of each license is the standard Apache 2.0 license.
 
-* spark 3.1.1 and 3.3.4 (https://github.com/apache/spark)
+* Apache Spark 3.1.1 and 3.3.4 (https://github.com/apache/spark)
     Files:
       maven-projects/spark/datasourcs-32/src/main/scala/org/apache/graphar/datasources/GarCommitProtocol.scala
       maven-projects/spark/datasourcs-32/src/main/scala/org/apache/graphar/datasources/GarDataSource.scala
@@ -234,9 +234,13 @@ The text of each license is the standard Apache 2.0 license.
       maven-projects/spark/datasourcs-33/src/main/scala/org/apache/graphar/datasources/orc/ORCOutputWriter.scala
       maven-projects/spark/datasourcs-33/src/main/scala/org/apache/graphar/datasources/orc/ORCWriteBuilder.scala
       maven-projects/spark/datasourcs-33/src/main/scala/org/apache/graphar/datasources/parquet/ParquetWriteBuilder.scala
-    are modified from spark.
+    are modified from Apache Spark.
+
+* Apache Arrow 12.0.0 (https://github.com/apache/arrow)
+    Files:
+      dev/release/setup-ubuntu.sh
+    are modified from Apache Arrow.
 
-* arrow 12.0.0 (https://github.com/apache/arrow)
 * fastFFI v0.1.2 (https://github.com/alibaba/fastFFI)
     Files:
       maven-projects/java/src/main/java/org/apache/graphar/stdcxx/StdString.java
@@ -251,6 +255,12 @@ The text of each license is the standard Apache 2.0 license.
       maven-projects/java/src/main/java/org/apache/graphar/stdcxx/StdUnorderedMap.java
     are modified from GraphScope.
 
+* Apache OpenDAL v0.45.1 (https://github.com/apache/opendal)
+    Files:
+      dev/release/release.py
+      dev/release/verify.py
+    are modified from OpenDAL.
+
 ================================================================
 MIT licenses
 ================================================================

diff --git a/NOTICE b/NOTICE
@@ -31,3 +31,11 @@ which includes the following in its NOTICE file:
 
   fastFFI 
   Copyright 1999-2021 Alibaba Group Holding Ltd.
+
+--------------------------------------------------------------------------------
+
+This product includes code from Apache OpenDAL, which includes the following in
+its NOTICE file:
+
+  Apache OpenDAL
+  Copyright 2022 and onwards The Apache Software Foundation.
diff --git a/dev/download_test_data.sh b/dev/download_test_data.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# A script to download test data for GraphAr
+
+if [ -n "${GAR_TEST_DATA}" ]; then
+    if [[ ! -d "$GAR_TEST_DATA" ]]; then
+        echo "GAR_TEST_DATA is set but the directory does not exist, cloning the test data to $GAR_TEST_DATA"
+        git clone https://github.com/apache/incubator-graphar-testing.git "$GAR_TEST_DATA" --depth 1 || true
+    fi
+else
+    echo "GAR_TEST_DATA is not set, cloning the test data to /tmp/graphar-testing"
+    git clone https://github.com/apache/incubator-graphar-testing.git /tmp/graphar-testing --depth 1 || true
+    echo "Test data has been cloned to /tmp/graphar-testing, please run"
+    echo "    export GAR_TEST_DATA=/tmp/graphar-testing"
+fi
diff --git a/dev/release/conda_env_cpp.txt b/dev/release/conda_env_cpp.txt
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake
+conda-forge::arrow-cpp=13.0.0
+make
+clangxx_linux-64
+conda-forge::catch2=3.6.0
diff --git a/dev/release/conda_env_scala.txt b/dev/release/conda_env_scala.txt
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+maven
+openjdk=11.0.13
diff --git a/dev/release/release.py b/dev/release/release.py
@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Derived from Apache OpenDAL v0.45.1
+# https://github.com/apache/opendal/blob/5079125/scripts/release.py
+
+import re
+import subprocess
+from pathlib import Path
+
+ROOT_DIR = Path(__file__).parent.parent.parent
+
+def get_package_version():
+    major_version = None
+    minor_version = None
+    patch_version = None
+    major_pattern = re.compile(r'set\s*\(\s*GRAPHAR_MAJOR_VERSION\s+(\d+)\s*\)', re.IGNORECASE)
+    minor_pattern = re.compile(r'set\s*\(\s*GRAPHAR_MINOR_VERSION\s+(\d+)\s*\)', re.IGNORECASE)
+    patch_pattern = re.compile(r'set\s*\(\s*GRAPHAR_PATCH_VERSION\s+(\d+)\s*\)', re.IGNORECASE)
+
+    file_path = ROOT_DIR / "cpp/CMakeLists.txt"
+    with open(file_path, 'r') as file:
+        for line in file:
+            major_match = major_pattern.search(line)
+            minor_match = minor_pattern.search(line)
+            patch_match = patch_pattern.search(line)
+
+            if major_match:
+                major_version = major_match.group(1)
+            if minor_match:
+                minor_version = minor_match.group(1)
+            if patch_match:
+                patch_version = patch_match.group(1)
+
+    if major_version and minor_version and patch_version:
+        return f"{major_version}.{minor_version}.{patch_version}"
+    else:
+        return None
+
+def archive_source_package():
+    print(f"Archive source package started")
+
+    version = get_package_version()
+    assert version, "Failed to get the package version"
+    name = f"apache-graphar-{version}-incubating-src"
+
+    archive_command = [
+        "git",
+        "archive",
+        "--prefix",
+        f"apache-graphar-{version}-incubating-src/",
+        "-o",
+        f"{ROOT_DIR}/dist/{name}.tar.gz",
+        "HEAD",
+    ]
+    subprocess.run(
+        archive_command,
+        cwd=ROOT_DIR,
+        check=True,
+    )
+
+    print(f"Archive source package to dist/{name}.tar.gz")
+
+
+def generate_signature():
+    for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"):
+        print(f"Generate signature for {i}")
+        subprocess.run(
+            ["gpg", "--yes", "--armor", "--output", f"{i}.asc", "--detach-sig", str(i)],
+            cwd=ROOT_DIR / "dist",
+            check=True,
+        )
+
+    for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"):
+        print(f"Check signature for {i}")
+        subprocess.run(
+            ["gpg", "--verify", f"{i}.asc", str(i)], cwd=ROOT_DIR / "dist", check=True
+        )
+
+
+def generate_checksum():
+    for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"):
+        print(f"Generate checksum for {i}")
+        subprocess.run(
+            ["sha512sum", str(i.relative_to(ROOT_DIR / "dist"))],
+            stdout=open(f"{i}.sha512", "w"),
+            cwd=ROOT_DIR / "dist",
+            check=True,
+        )
+
+    for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"):
+        print(f"Check checksum for {i}")
+        subprocess.run(
+            ["sha512sum", "--check", f"{str(i.relative_to(ROOT_DIR / 'dist'))}.sha512"],
+            cwd=ROOT_DIR / "dist",
+            check=True,
+        )
+
+
+if __name__ == "__main__":
+    (ROOT_DIR / "dist").mkdir(exist_ok=True)
+    archive_source_package()
+    generate_signature()
+    generate_checksum()
diff --git a/dev/release/setup-ubuntu.sh b/dev/release/setup-ubuntu.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Derived from Apache Arrow 12.0.0
+# https://github.com/apache/arrow/blob/9736dde/dev/release/setup-ubuntu.sh
+
+# A script to install dependencies required for release
+# verification on Ubuntu.
+
+set -exu
+
+codename=$(. /etc/os-release && echo ${UBUNTU_CODENAME})
+id=$(. /etc/os-release && echo ${ID})
+
+apt-get install -y -q --no-install-recommends \
+  build-essential \
+  cmake \
+  git \
+  gnupg \
+  libcurl4-openssl-dev \
+  maven \
+  openjdk-11-jdk \
+  wget \
+  pkg-config \
+  tzdata \
+  subversion
+
+wget -c https://apache.jfrog.io/artifactory/arrow/${id}/apache-arrow-apt-source-latest-${codename}.deb \
+    -P /tmp/
+apt-get install -y -q /tmp/apache-arrow-apt-source-latest-${codename}.deb
+apt-get update -y -q
+apt-get install -y -q --no-install-recommends \
+  libarrow-dev \
+  libarrow-dataset-dev \
+  libarrow-acero-dev \
+  libparquet-dev