feat(core): set Substrait version and producer

Signed-off-by: Niels Pardon <par@zurich.ibm.com>

Author: nielspardon

.github/workflows/pr.yml

@@ -69,7 +69,11 @@ jobs:
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v4
       - name: Build with Gradle
-        run: gradle build --rerun-tasks
+        run: |
+          # fetch submodule tags since actions/checkout@v4 does not
+          git submodule foreach 'git fetch --unshallow || true'
+
+          gradle build --rerun-tasks
   examples:
     name: Build Examples
     runs-on: ubuntu-latest
@@ -115,6 +119,9 @@ jobs:
       run: gu install native-image
     - name: Build with Gradle
       run: |
+        # fetch submodule tags since actions/checkout@v4 does not
+        git submodule foreach 'git fetch --unshallow || true'
+
         gradle nativeImage
     - name: Smoke Test
       run: |

.github/workflows/release.yml

@@ -35,7 +35,11 @@ jobs:
       - name: Install GraalVM native image
         run: gu install native-image
       - name: Build with Gradle
-        run: gradle nativeImage
+        run: |
+          # fetch submodule tags since actions/checkout@v4 does not
+          git submodule foreach 'git fetch --unshallow || true'
+
+          gradle nativeImage
       - name: Smoke Test
         run: |
           ./isthmus-cli/src/test/script/smoke.sh

.gitignore

@@ -12,3 +12,4 @@ out/**
 
 */bin
 .metals
+.bloop

ci/release/publish.sh

@@ -3,5 +3,8 @@
 
 set -euo pipefail
 
+# ensure the submodule tags exist
+git submodule foreach 'git fetch --unshallow || true'
+
 gradle wrapper
 ./gradlew clean :core:publishToSonatype :isthmus:publishToSonatype :spark:publishToSonatype closeAndReleaseSonatypeStagingRepository

core/build.gradle.kts

@@ -1,4 +1,9 @@
+import java.io.ByteArrayOutputStream
+import java.nio.charset.StandardCharsets
+import org.gradle.api.provider.ValueSource
+import org.gradle.api.provider.ValueSourceParameters
 import org.gradle.plugins.ide.idea.model.IdeaModel
+import org.slf4j.LoggerFactory
 
 plugins {
   `maven-publish`
@@ -105,6 +110,71 @@ configurations[JavaPlugin.API_CONFIGURATION_NAME].let { apiConfiguration ->
   apiConfiguration.setExtendsFrom(apiConfiguration.extendsFrom.filter { it.name != "antlr" })
 }
 
+abstract class SubstraitSpecVersionValueSource :
+  ValueSource<String, SubstraitSpecVersionValueSource.Parameters> {
+  companion object {
+    val logger = LoggerFactory.getLogger("SubstraitSpecVersionValueSource")
+  }
+
+  interface Parameters : ValueSourceParameters {
+    val substraitDirectory: Property<File>
+  }
+
+  @get:Inject abstract val execOperations: ExecOperations
+
+  override fun obtain(): String {
+    val stdOutput = ByteArrayOutputStream()
+    val errOutput = ByteArrayOutputStream()
+    execOperations.exec {
+      commandLine("git", "describe", "--tags")
+      standardOutput = stdOutput
+      errorOutput = errOutput
+      setIgnoreExitValue(true)
+      workingDir = parameters.substraitDirectory.get()
+    }
+
+    // capturing the error output and logging it to avoid issues with VS Code Spotless plugin
+    val error = String(errOutput.toByteArray())
+    if (error != "") {
+      logger.warn(error)
+    }
+
+    val cmdOut = String(stdOutput.toByteArray()).trim()
+
+    if (cmdOut.startsWith("v")) {
+      return cmdOut.substring(1)
+    }
+
+    return cmdOut
+  }
+}
+
+tasks.register("writeManifest") {
+  doLast {
+    val substraitSpecVersionProvider =
+      providers.of(SubstraitSpecVersionValueSource::class) {
+        parameters.substraitDirectory.set(project(":").file("substrait"))
+      }
+
+    val manifestFile =
+      layout.buildDirectory
+        .file("generated/sources/manifest/META-INF/MANIFEST.MF")
+        .get()
+        .getAsFile()
+    manifestFile.getParentFile().mkdirs()
+
+    manifestFile.printWriter(StandardCharsets.UTF_8).use {
+      it.println("Manifest-Version: 1.0")
+      it.println("Implementation-Title: substrait-java")
+      it.println("Implementation-Version: " + project.version)
+      it.println("Specification-Title: substrait")
+      it.println("Specification-Version: " + substraitSpecVersionProvider.get())
+    }
+  }
+}
+
+tasks.named("compileJava") { dependsOn("writeManifest") }
+
 tasks {
   shadowJar {
     archiveClassifier.set("") // to override ".jar" instead of producing "-all.jar"
@@ -114,6 +184,8 @@ tasks {
     // rename the shadowed deps so that they don't conflict with consumer's own deps
     relocate("org.antlr.v4.runtime", "io.substrait.org.antlr.v4.runtime")
   }
+
+  jar { manifest { from("build/generated/sources/manifest/META-INF/MANIFEST.MF") } }
 }
 
 java {
@@ -132,6 +204,7 @@ sourceSets {
   main {
     proto.srcDir("../substrait/proto")
     resources.srcDir("../substrait/extensions")
+    resources.srcDir("build/generated/sources/manifest/")
     java.srcDir(file("build/generated/sources/antlr/main/java/"))
   }
 }

core/src/main/java/io/substrait/plan/Plan.java

@@ -2,13 +2,21 @@
 
 import io.substrait.proto.AdvancedExtension;
 import io.substrait.relation.Rel;
+import java.io.IOException;
 import java.util.List;
 import java.util.Optional;
+import java.util.jar.Attributes.Name;
+import java.util.jar.Manifest;
 import org.immutables.value.Value;
 
 @Value.Immutable
 public abstract class Plan {
 
+  @Value.Default
+  public Version getVersion() {
+    return ImmutableVersion.builder().build();
+  }
+
   public abstract List<Root> getRoots();
 
   public abstract List<String> getExpectedTypeUrls();
@@ -19,6 +27,58 @@ public static ImmutablePlan.Builder builder() {
     return ImmutablePlan.builder();
   }
 
+  @Value.Immutable
+  public abstract static class Version {
+    private static final String[] VERSION_COMPONENTS = loadVersion();
+
+    @Value.Default
+    public int getMajor() {
+      return Integer.parseInt(VERSION_COMPONENTS[0]);
+    }
+
+    @Value.Default
+    public int getMinor() {
+      return Integer.parseInt(VERSION_COMPONENTS[1]);
+    }
+
+    @Value.Default
+    public int getPatch() {
+      return Integer.parseInt(VERSION_COMPONENTS[2]);
+    }
+
+    @SuppressWarnings("immutables:untype")
+    @Value.Default
+    public Optional<String> getGitHash() {
+      return Optional.ofNullable(null);
+    }
+
+    @SuppressWarnings("immutables:untype")
+    @Value.Default
+    public Optional<String> getProducer() {
+      return Optional.of("substrait-java");
+    }
+
+    private static String[] loadVersion() {
+      // load the specification version from the JAR manifest
+      String specificationVersion = Version.class.getPackage().getSpecificationVersion();
+
+      // load the manifest directly from the classpath if the specification version is null which is
+      // the case if the Version class is not in a JAR, e.g. during the Gradle build
+      if (specificationVersion == null) {
+        try {
+          Manifest manifest =
+              new Manifest(
+                  Version.class.getClassLoader().getResourceAsStream("META-INF/MANIFEST.MF"));
+          specificationVersion = manifest.getMainAttributes().getValue(Name.SPECIFICATION_VERSION);
+        } catch (IOException e) {
+          throw new IllegalStateException("Could not load version from manifest", e);
+        }
+      }
+
+      return specificationVersion.split("\\.");
+    }
+  }
+
   @Value.Immutable
   public abstract static class Root {
     public abstract Rel getInput();

core/src/main/java/io/substrait/plan/PlanProtoConverter.java

@@ -4,6 +4,7 @@
 import io.substrait.proto.Plan;
 import io.substrait.proto.PlanRel;
 import io.substrait.proto.Rel;
+import io.substrait.proto.Version;
 import io.substrait.relation.RelProtoConverter;
 import java.util.ArrayList;
 import java.util.List;
@@ -34,6 +35,18 @@ public Plan toProto(io.substrait.plan.Plan plan) {
     if (plan.getAdvancedExtension().isPresent()) {
       builder.setAdvancedExtensions(plan.getAdvancedExtension().get());
     }
+
+    Version.Builder versionBuilder =
+        Version.newBuilder()
+            .setMajorNumber(plan.getVersion().getMajor())
+            .setMinorNumber(plan.getVersion().getMinor())
+            .setPatchNumber(plan.getVersion().getPatch());
+
+    plan.getVersion().getGitHash().ifPresent(gh -> versionBuilder.setGitHash(gh));
+    plan.getVersion().getProducer().ifPresent(p -> versionBuilder.setProducer(p));
+
+    builder.setVersion(versionBuilder);
+
     return builder.build();
   }
 }

core/src/main/java/io/substrait/plan/ProtoPlanConverter.java

@@ -39,11 +39,29 @@ public Plan from(io.substrait.proto.Plan plan) {
       Rel rel = relConverter.from(root.getInput());
       roots.add(Plan.Root.builder().input(rel).names(root.getNamesList()).build());
     }
+
+    ImmutableVersion.Builder versionBuilder =
+        ImmutableVersion.builder()
+            .major(plan.getVersion().getMajorNumber())
+            .minor(plan.getVersion().getMinorNumber())
+            .patch(plan.getVersion().getPatchNumber());
+
+    // protobuf field 'git_hash' is an empty string by default
+    if (!"".equals(plan.getVersion().getGitHash())) {
+      versionBuilder.gitHash(Optional.of(plan.getVersion().getGitHash()));
+    }
+
+    // protobuf field 'producer' is an empty string by default
+    if (!"".equals(plan.getVersion().getProducer())) {
+      versionBuilder.producer(Optional.of(plan.getVersion().getProducer()));
+    }
+
     return Plan.builder()
         .roots(roots)
         .expectedTypeUrls(plan.getExpectedTypeUrlsList())
         .advancedExtension(
             Optional.ofNullable(plan.hasAdvancedExtensions() ? plan.getAdvancedExtensions() : null))
+        .version(versionBuilder.build())
         .build();
   }
 }

core/src/test/java/io/substrait/relation/SpecVersionTest.java

@@ -0,0 +1,21 @@
+package io.substrait.relation;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import io.substrait.plan.ImmutableVersion;
+import java.util.Optional;
+import org.junit.jupiter.api.Test;
+
+public class SpecVersionTest {
+  @Test
+  public void testSubstraitVersionDefaultValues() {
+    ImmutableVersion version = ImmutableVersion.builder().build();
+
+    assertNotNull(version.getMajor());
+    assertNotNull(version.getMinor());
+    assertNotNull(version.getPatch());
+
+    assertEquals(Optional.of("substrait-java"), version.getProducer());
+  }
+}

isthmus/src/main/java/io/substrait/isthmus/SqlToSubstrait.java

@@ -1,10 +1,9 @@
 package io.substrait.isthmus;
 
 import com.google.common.annotations.VisibleForTesting;
-import io.substrait.extension.ExtensionCollector;
+import io.substrait.plan.ImmutablePlan.Builder;
+import io.substrait.plan.PlanProtoConverter;
 import io.substrait.proto.Plan;
-import io.substrait.proto.PlanRel;
-import io.substrait.relation.RelProtoConverter;
 import java.util.List;
 import org.apache.calcite.plan.hep.HepPlanner;
 import org.apache.calcite.plan.hep.HepProgram;
@@ -56,22 +55,16 @@ List<RelRoot> sqlToRelNode(String sql, List<String> tables) throws SqlParseExcep
 
   private Plan executeInner(String sql, SqlValidator validator, Prepare.CatalogReader catalogReader)
       throws SqlParseException {
-    var plan = Plan.newBuilder();
-    ExtensionCollector functionCollector = new ExtensionCollector();
-    var relProtoConverter = new RelProtoConverter(functionCollector);
+    Builder builder = io.substrait.plan.Plan.builder();
+
     // TODO: consider case in which one sql passes conversion while others don't
-    sqlToRelNode(sql, validator, catalogReader)
-        .forEach(
-            root -> {
-              plan.addRelations(
-                  PlanRel.newBuilder()
-                      .setRoot(
-                          relProtoConverter.toProto(
-                              SubstraitRelVisitor.convert(
-                                  root, EXTENSION_COLLECTION, featureBoard))));
-            });
-    functionCollector.addExtensionsToPlan(plan);
-    return plan.build();
+    sqlToRelNode(sql, validator, catalogReader).stream()
+        .map(root -> SubstraitRelVisitor.convert(root, EXTENSION_COLLECTION, featureBoard))
+        .forEach(root -> builder.addRoots(root));
+
+    PlanProtoConverter planToProto = new PlanProtoConverter();
+
+    return planToProto.toProto(builder.build());
   }
 
   private List<RelRoot> sqlToRelNode(