diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0d90ef9cd..45c6911a6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -69,7 +69,11 @@ jobs: - name: Setup Gradle uses: gradle/actions/setup-gradle@v4 - name: Build with Gradle - run: gradle build --rerun-tasks + run: | + # fetch submodule tags since actions/checkout@v4 does not + git submodule foreach 'git fetch --unshallow || true' + + gradle build --rerun-tasks examples: name: Build Examples runs-on: ubuntu-latest @@ -115,6 +119,9 @@ jobs: run: gu install native-image - name: Build with Gradle run: | + # fetch submodule tags since actions/checkout@v4 does not + git submodule foreach 'git fetch --unshallow || true' + gradle nativeImage - name: Smoke Test run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 23824476b..d11cbd761 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,11 @@ jobs: - name: Install GraalVM native image run: gu install native-image - name: Build with Gradle - run: gradle nativeImage + run: | + # fetch submodule tags since actions/checkout@v4 does not + git submodule foreach 'git fetch --unshallow || true' + + gradle nativeImage - name: Smoke Test run: | ./isthmus-cli/src/test/script/smoke.sh diff --git a/.gitignore b/.gitignore index c9984ffef..63b93fd42 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ out/** */bin .metals +.bloop diff --git a/ci/release/publish.sh b/ci/release/publish.sh index 1768d0a00..920791386 100755 --- a/ci/release/publish.sh +++ b/ci/release/publish.sh @@ -3,5 +3,8 @@ set -euo pipefail +# ensure the submodule tags exist +git submodule foreach 'git fetch --unshallow || true' + gradle wrapper ./gradlew clean :core:publishToSonatype :isthmus:publishToSonatype :spark:publishToSonatype closeAndReleaseSonatypeStagingRepository diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 09ef4a1c0..de71d9ccf 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -1,4 +1,9 @@ +import java.io.ByteArrayOutputStream +import java.nio.charset.StandardCharsets +import org.gradle.api.provider.ValueSource +import org.gradle.api.provider.ValueSourceParameters import org.gradle.plugins.ide.idea.model.IdeaModel +import org.slf4j.LoggerFactory plugins { `maven-publish` @@ -105,6 +110,71 @@ configurations[JavaPlugin.API_CONFIGURATION_NAME].let { apiConfiguration -> apiConfiguration.setExtendsFrom(apiConfiguration.extendsFrom.filter { it.name != "antlr" }) } +abstract class SubstraitSpecVersionValueSource : + ValueSource { + companion object { + val logger = LoggerFactory.getLogger("SubstraitSpecVersionValueSource") + } + + interface Parameters : ValueSourceParameters { + val substraitDirectory: Property + } + + @get:Inject abstract val execOperations: ExecOperations + + override fun obtain(): String { + val stdOutput = ByteArrayOutputStream() + val errOutput = ByteArrayOutputStream() + execOperations.exec { + commandLine("git", "describe", "--tags") + standardOutput = stdOutput + errorOutput = errOutput + setIgnoreExitValue(true) + workingDir = parameters.substraitDirectory.get() + } + + // capturing the error output and logging it to avoid issues with VS Code Spotless plugin + val error = String(errOutput.toByteArray()) + if (error != "") { + logger.warn(error) + } + + val cmdOut = String(stdOutput.toByteArray()).trim() + + if (cmdOut.startsWith("v")) { + return cmdOut.substring(1) + } + + return cmdOut + } +} + +tasks.register("writeManifest") { + doLast { + val substraitSpecVersionProvider = + providers.of(SubstraitSpecVersionValueSource::class) { + parameters.substraitDirectory.set(project(":").file("substrait")) + } + + val manifestFile = + layout.buildDirectory + .file("generated/sources/manifest/META-INF/MANIFEST.MF") + .get() + .getAsFile() + manifestFile.getParentFile().mkdirs() + + manifestFile.printWriter(StandardCharsets.UTF_8).use { + it.println("Manifest-Version: 1.0") + it.println("Implementation-Title: substrait-java") + it.println("Implementation-Version: " + project.version) + it.println("Specification-Title: substrait") + it.println("Specification-Version: " + substraitSpecVersionProvider.get()) + } + } +} + +tasks.named("compileJava") { dependsOn("writeManifest") } + tasks { shadowJar { archiveClassifier.set("") // to override ".jar" instead of producing "-all.jar" @@ -114,6 +184,8 @@ tasks { // rename the shadowed deps so that they don't conflict with consumer's own deps relocate("org.antlr.v4.runtime", "io.substrait.org.antlr.v4.runtime") } + + jar { manifest { from("build/generated/sources/manifest/META-INF/MANIFEST.MF") } } } java { @@ -132,6 +204,7 @@ sourceSets { main { proto.srcDir("../substrait/proto") resources.srcDir("../substrait/extensions") + resources.srcDir("build/generated/sources/manifest/") java.srcDir(file("build/generated/sources/antlr/main/java/")) } } diff --git a/core/src/main/java/io/substrait/plan/Plan.java b/core/src/main/java/io/substrait/plan/Plan.java index 9d9bd3545..4fe63b00a 100644 --- a/core/src/main/java/io/substrait/plan/Plan.java +++ b/core/src/main/java/io/substrait/plan/Plan.java @@ -2,13 +2,21 @@ import io.substrait.proto.AdvancedExtension; import io.substrait.relation.Rel; +import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.jar.Attributes.Name; +import java.util.jar.Manifest; import org.immutables.value.Value; @Value.Immutable public abstract class Plan { + @Value.Default + public Version getVersion() { + return Version.DEFAULT_VERSION; + } + public abstract List getRoots(); public abstract List getExpectedTypeUrls(); @@ -19,6 +27,52 @@ public static ImmutablePlan.Builder builder() { return ImmutablePlan.builder(); } + @Value.Immutable + public abstract static class Version { + public static final Version DEFAULT_VERSION; + + static { + final String[] versionComponents = loadVersion(); + DEFAULT_VERSION = + ImmutableVersion.builder() + .major(Integer.parseInt(versionComponents[0])) + .minor(Integer.parseInt(versionComponents[1])) + .patch(Integer.parseInt(versionComponents[2])) + .producer(Optional.of("substrait-java")) + .build(); + } + + public abstract int getMajor(); + + public abstract int getMinor(); + + public abstract int getPatch(); + + public abstract Optional getGitHash(); + + public abstract Optional getProducer(); + + private static String[] loadVersion() { + // load the specification version from the JAR manifest + String specificationVersion = Version.class.getPackage().getSpecificationVersion(); + + // load the manifest directly from the classpath if the specification version is null which is + // the case if the Version class is not in a JAR, e.g. during the Gradle build + if (specificationVersion == null) { + try { + Manifest manifest = + new Manifest( + Version.class.getClassLoader().getResourceAsStream("META-INF/MANIFEST.MF")); + specificationVersion = manifest.getMainAttributes().getValue(Name.SPECIFICATION_VERSION); + } catch (IOException e) { + throw new IllegalStateException("Could not load version from manifest", e); + } + } + + return specificationVersion.split("\\."); + } + } + @Value.Immutable public abstract static class Root { public abstract Rel getInput(); diff --git a/core/src/main/java/io/substrait/plan/PlanProtoConverter.java b/core/src/main/java/io/substrait/plan/PlanProtoConverter.java index 0bdf7d68c..4746cda4d 100644 --- a/core/src/main/java/io/substrait/plan/PlanProtoConverter.java +++ b/core/src/main/java/io/substrait/plan/PlanProtoConverter.java @@ -4,6 +4,7 @@ import io.substrait.proto.Plan; import io.substrait.proto.PlanRel; import io.substrait.proto.Rel; +import io.substrait.proto.Version; import io.substrait.relation.RelProtoConverter; import java.util.ArrayList; import java.util.List; @@ -34,6 +35,18 @@ public Plan toProto(io.substrait.plan.Plan plan) { if (plan.getAdvancedExtension().isPresent()) { builder.setAdvancedExtensions(plan.getAdvancedExtension().get()); } + + Version.Builder versionBuilder = + Version.newBuilder() + .setMajorNumber(plan.getVersion().getMajor()) + .setMinorNumber(plan.getVersion().getMinor()) + .setPatchNumber(plan.getVersion().getPatch()); + + plan.getVersion().getGitHash().ifPresent(gh -> versionBuilder.setGitHash(gh)); + plan.getVersion().getProducer().ifPresent(p -> versionBuilder.setProducer(p)); + + builder.setVersion(versionBuilder); + return builder.build(); } } diff --git a/core/src/main/java/io/substrait/plan/ProtoPlanConverter.java b/core/src/main/java/io/substrait/plan/ProtoPlanConverter.java index 456e553a0..231243a0b 100644 --- a/core/src/main/java/io/substrait/plan/ProtoPlanConverter.java +++ b/core/src/main/java/io/substrait/plan/ProtoPlanConverter.java @@ -39,11 +39,29 @@ public Plan from(io.substrait.proto.Plan plan) { Rel rel = relConverter.from(root.getInput()); roots.add(Plan.Root.builder().input(rel).names(root.getNamesList()).build()); } + + ImmutableVersion.Builder versionBuilder = + ImmutableVersion.builder() + .major(plan.getVersion().getMajorNumber()) + .minor(plan.getVersion().getMinorNumber()) + .patch(plan.getVersion().getPatchNumber()); + + // protobuf field 'git_hash' is an empty string by default + if (!plan.getVersion().getGitHash().isEmpty()) { + versionBuilder.gitHash(Optional.of(plan.getVersion().getGitHash())); + } + + // protobuf field 'producer' is an empty string by default + if (!plan.getVersion().getProducer().isEmpty()) { + versionBuilder.producer(Optional.of(plan.getVersion().getProducer())); + } + return Plan.builder() .roots(roots) .expectedTypeUrls(plan.getExpectedTypeUrlsList()) .advancedExtension( Optional.ofNullable(plan.hasAdvancedExtensions() ? plan.getAdvancedExtensions() : null)) + .version(versionBuilder.build()) .build(); } } diff --git a/core/src/test/java/io/substrait/relation/SpecVersionTest.java b/core/src/test/java/io/substrait/relation/SpecVersionTest.java new file mode 100644 index 000000000..bb15ca8fb --- /dev/null +++ b/core/src/test/java/io/substrait/relation/SpecVersionTest.java @@ -0,0 +1,21 @@ +package io.substrait.relation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import io.substrait.plan.Plan.Version; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +public class SpecVersionTest { + @Test + public void testSubstraitVersionDefaultValues() { + Version version = Version.DEFAULT_VERSION; + + assertNotNull(version.getMajor()); + assertNotNull(version.getMinor()); + assertNotNull(version.getPatch()); + + assertEquals(Optional.of("substrait-java"), version.getProducer()); + } +} diff --git a/isthmus/src/main/java/io/substrait/isthmus/SqlToSubstrait.java b/isthmus/src/main/java/io/substrait/isthmus/SqlToSubstrait.java index 6fb64c6f7..7f2119e9e 100644 --- a/isthmus/src/main/java/io/substrait/isthmus/SqlToSubstrait.java +++ b/isthmus/src/main/java/io/substrait/isthmus/SqlToSubstrait.java @@ -1,10 +1,11 @@ package io.substrait.isthmus; import com.google.common.annotations.VisibleForTesting; -import io.substrait.extension.ExtensionCollector; +import io.substrait.plan.ImmutablePlan.Builder; +import io.substrait.plan.ImmutableVersion; +import io.substrait.plan.Plan.Version; +import io.substrait.plan.PlanProtoConverter; import io.substrait.proto.Plan; -import io.substrait.proto.PlanRel; -import io.substrait.relation.RelProtoConverter; import java.util.List; import org.apache.calcite.plan.hep.HepPlanner; import org.apache.calcite.plan.hep.HepProgram; @@ -56,22 +57,18 @@ List sqlToRelNode(String sql, List tables) throws SqlParseExcep private Plan executeInner(String sql, SqlValidator validator, Prepare.CatalogReader catalogReader) throws SqlParseException { - var plan = Plan.newBuilder(); - ExtensionCollector functionCollector = new ExtensionCollector(); - var relProtoConverter = new RelProtoConverter(functionCollector); + Builder builder = io.substrait.plan.Plan.builder(); + builder.version( + ImmutableVersion.builder().from(Version.DEFAULT_VERSION).producer("isthmus").build()); + // TODO: consider case in which one sql passes conversion while others don't - sqlToRelNode(sql, validator, catalogReader) - .forEach( - root -> { - plan.addRelations( - PlanRel.newBuilder() - .setRoot( - relProtoConverter.toProto( - SubstraitRelVisitor.convert( - root, EXTENSION_COLLECTION, featureBoard)))); - }); - functionCollector.addExtensionsToPlan(plan); - return plan.build(); + sqlToRelNode(sql, validator, catalogReader).stream() + .map(root -> SubstraitRelVisitor.convert(root, EXTENSION_COLLECTION, featureBoard)) + .forEach(root -> builder.addRoots(root)); + + PlanProtoConverter planToProto = new PlanProtoConverter(); + + return planToProto.toProto(builder.build()); } private List sqlToRelNode( diff --git a/spark/src/main/scala/io/substrait/spark/logical/ToSubstraitRel.scala b/spark/src/main/scala/io/substrait/spark/logical/ToSubstraitRel.scala index 96ed0e365..abf33c23d 100644 --- a/spark/src/main/scala/io/substrait/spark/logical/ToSubstraitRel.scala +++ b/spark/src/main/scala/io/substrait/spark/logical/ToSubstraitRel.scala @@ -38,7 +38,7 @@ import io.substrait.debug.TreePrinter import io.substrait.expression.{Expression => SExpression, ExpressionCreator} import io.substrait.extension.ExtensionCollector import io.substrait.hint.Hint -import io.substrait.plan.{ImmutableRoot, Plan} +import io.substrait.plan.{ImmutableRoot, ImmutableVersion, Plan} import io.substrait.relation.RelProtoConverter import io.substrait.relation.Set.SetOp import io.substrait.relation.files.{FileFormat, ImmutableFileOrFiles} @@ -543,6 +543,12 @@ class ToSubstraitRel extends AbstractLogicalPlanVisitor with Logging { def convert(p: LogicalPlan): Plan = { val rel = visit(p) Plan.builder + .version( + ImmutableVersion + .builder() + .from(Plan.Version.DEFAULT_VERSION) + .producer("substrait-spark") + .build()) .roots( Collections.singletonList( ImmutableRoot