Iceberg (#2031)

legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java

@@ -89,6 +89,7 @@ protected MutableList<RepositoryCodeStorage> buildRepositories(SourceLocationCon
  .with(this.buildCore("legend-engine-xts-service/legend-engine-language-pure-dsl-service-pure","service"))
  .with(this.buildCore("legend-engine-pure-ide/legend-engine-pure-ide-light-metadata-pure", "ide_metadata"))
  .with(this.buildCore("legend-engine-pure-code/legend-engine-pure-code-compiled-core", ""))
+ .with(this.buildCore("legend-engine-xts-iceberg/legend-engine-xt-iceberg-pure","external-tableformat-iceberg"))
  ;
  }
 

legend-engine-xts-iceberg/legend-engine-xt-iceberg-pure/pom.xml

@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2020 Goldman Sachs
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.finos.legend.engine</groupId>
+ <artifactId>legend-engine-xts-iceberg</artifactId>
+ <version>4.20.4-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <name>Legend Engine - XT - Table Format - Iceberg - PAR/JAVA</name>
+ <artifactId>legend-engine-xt-iceberg-pure</artifactId>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.finos.legend.pure</groupId>
+ <artifactId>legend-pure-maven-generation-par</artifactId>
+ <configuration>
+ <sourceDirectory>src/main/resources</sourceDirectory>
+ <purePlatformVersion>${legend.pure.version}</purePlatformVersion>
+ <repositories>
+ <repository>platform</repository>
+ <repository>core</repository>
+ <repository>core_external_tableformat_iceberg</repository>
+ </repositories>
+ <extraRepositories>
+ <extraRepository>${project.basedir}/src/main/resources/core_external_tableformat_iceberg.definition.json</extraRepository>
+ </extraRepositories>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>build-pure-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.finos.legend.pure</groupId>
+ <artifactId>legend-pure-m2-dsl-diagram-grammar</artifactId>
+ <version>${legend.pure.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.finos.legend.engine</groupId>
+ <artifactId>legend-engine-pure-code-compiled-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ <plugin>
+ <groupId>org.finos.legend.pure</groupId>
+ <artifactId>legend-pure-maven-generation-java</artifactId>
+ <executions>
+ <execution>
+ <phase>compile</phase>
+ <goals>
+ <goal>build-pure-compiled-jar</goal>
+ </goals>
+ <configuration>
+ <generateSources>true</generateSources>
+ <preventJavaCompilation>true</preventJavaCompilation>
+ <generationType>modular</generationType>
+ <useSingleDir>true</useSingleDir>
+ <repositories>
+ <repository>core_external_tableformat_iceberg</repository>
+ </repositories>
+ </configuration>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.finos.legend.pure</groupId>
+ <artifactId>legend-pure-m2-dsl-diagram-grammar</artifactId>
+ <version>${legend.pure.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.finos.legend.engine</groupId>
+ <artifactId>legend-engine-pure-code-compiled-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <!-- PURE -->
+ <dependency>
+ <groupId>org.finos.legend.pure</groupId>
+ <artifactId>legend-pure-m4</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.finos.legend.pure</groupId>
+ <artifactId>legend-pure-m3-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.finos.legend.pure</groupId>
+ <artifactId>legend-pure-runtime-java-engine-compiled</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.finos.legend.engine</groupId>
+ <artifactId>legend-engine-pure-platform-java</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.eclipse.collections</groupId>
+ <artifactId>eclipse-collections</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.collections</groupId>
+ <artifactId>eclipse-collections-api</artifactId>
+ </dependency>
+ <!-- PURE -->
+ </dependencies>
+</project>

legend-engine-xts-iceberg/legend-engine-xt-iceberg-pure/src/main/java/org/finos/legend/pure/code/core/CoreIcebergCodeRepositoryProvider.java

@@ -0,0 +1,28 @@
+// Copyright 2022 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.finos.legend.pure.code.core;
+
+import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepository;
+import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider;
+import org.finos.legend.pure.m3.serialization.filesystem.repository.GenericCodeRepository;
+
+public class CoreIcebergCodeRepositoryProvider implements CodeRepositoryProvider
+{
+ @Override
+ public CodeRepository repository()
+ {
+ return GenericCodeRepository.build("core_external_tableformat_iceberg.definition.json");
+ }
+}

legend-engine-xts-iceberg/legend-engine-xt-iceberg-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider

@@ -0,0 +1,17 @@
+#
+# Copyright 2023 Goldman Sachs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.finos.legend.pure.code.core.CoreIcebergCodeRepositoryProvider

legend-engine-xts-iceberg/legend-engine-xt-iceberg-pure/src/main/resources/core_external_tableformat_iceberg.definition.json

@@ -0,0 +1,8 @@
+{
+ "name": "core_external_tableformat_iceberg",
+ "pattern": "(meta::external::tableformat::iceberg|meta::protocols::pure)(::.*)?",
+ "dependencies": [
+ "platform",
+ "core"
+ ]
+}

legend-engine-xts-iceberg/legend-engine-xt-iceberg-pure/src/main/resources/core_external_tableformat_iceberg/metamodel.pure

@@ -0,0 +1,124 @@
+import meta::external::tableformat::iceberg::metamodel::*;
+
+Class meta::external::tableformat::iceberg::metamodel::Type
+{
+}
+
+// nested types
+
+Class meta::external::tableformat::iceberg::metamodel::NestedType extends meta::external::tableformat::iceberg::metamodel::Type
+{
+}
+
+Class meta::external::tableformat::iceberg::metamodel::StructType extends meta::external::tableformat::iceberg::metamodel::NestedType
+{
+ fields: meta::external::tableformat::iceberg::metamodel::NestedField[1..*];
+}
+
+Class meta::external::tableformat::iceberg::metamodel::ListType extends meta::external::tableformat::iceberg::metamodel::NestedType
+{
+ elementField: meta::external::tableformat::iceberg::metamodel::NestedField[1];
+}
+
+Class meta::external::tableformat::iceberg::metamodel::MapType extends meta::external::tableformat::iceberg::metamodel::NestedType
+{
+ keyField: meta::external::tableformat::iceberg::metamodel::NestedField[1];
+ valueField: meta::external::tableformat::iceberg::metamodel::NestedField[1];
+}
+
+Class meta::external::tableformat::iceberg::metamodel::NestedField
+{
+ id: Integer[1];
+ name: String[1];
+ type: meta::external::tableformat::iceberg::metamodel::Type[1];
+ required: Boolean[1];
+ doc: String[1];
+}
+
+// primitive types
+
+Class meta::external::tableformat::iceberg::metamodel::PrimitiveType extends meta::external::tableformat::iceberg::metamodel::Type
+{
+}
+
+Class meta::external::tableformat::iceberg::metamodel::BooleanType extends meta::external::tableformat::iceberg::metamodel::PrimitiveType
+{
+}
+
+Class
+{doc.doc = '32-bit signed integer'}
+meta::external::tableformat::iceberg::metamodel::IntegerType extends meta::external::tableformat::iceberg::metamodel::PrimitiveType
+{
+}
+
+Class
+{doc.doc = '64-bit signed integer'}
+meta::external::tableformat::iceberg::metamodel::LongType extends meta::external::tableformat::iceberg::metamodel::PrimitiveType
+{
+}
+
+Class
+{doc.doc = '32-bit IEEE 754 floating point'}
+meta::external::tableformat::iceberg::metamodel::FloatType extends meta::external::tableformat::iceberg::metamodel::PrimitiveType
+{
+}
+
+Class
+{doc.doc = '64-bit IEEE 754 floating point'}
+meta::external::tableformat::iceberg::metamodel::DoubleType extends meta::external::tableformat::iceberg::metamodel::PrimitiveType
+{
+}
+
+Class
+{doc.doc = 'Fixed point decimal'}
+meta::external::tableformat::iceberg::metamodel::DecimalType extends meta::external::tableformat::iceberg::metamodel::PrimitiveType
+[
+ precisionMaxLength: $this.precision <= 38
+]
+{
+ scale: Integer[1];
+ precision: Integer[1];
+}
+
+Class
+{doc.doc = 'Calendar date without timezone or time'}
+meta::external::tableformat::iceberg::metamodel::DateType extends meta::external::tableformat::iceberg::metamodel::PrimitiveType
+{
+}
+
+Class
+{doc.doc = 'Time of day without date or timezone'}
+meta::external::tableformat::iceberg::metamodel::TimeType extends meta::external::tableformat::iceberg::metamodel::PrimitiveType
+{
+}
+
+Class
+{doc.doc = 'Timestamp without timezone. These represent a date and time of day regardless of zone. Values are independent of zone adjustment and stored with microsecond precision'}
+meta::external::tableformat::iceberg::metamodel::TimestampType extends meta::external::tableformat::iceberg::metamodel::PrimitiveType
+{
+}
+
+Class
+{doc.doc = 'Timestamp with timzeone. These represent a point in time. Values are stored in UTC, do not retain source timezone, and have microsecond precision'}
+meta::external::tableformat::iceberg::metamodel::TimestampTzType extends meta::external::tableformat::iceberg::metamodel::PrimitiveType
+{
+}
+
+Class
+{doc.doc = 'Arbitrary length character sequences'}
+meta::external::tableformat::iceberg::metamodel::StringType extends meta::external::tableformat::iceberg::metamodel::PrimitiveType
+{
+}
+
+Class
+{doc.doc = 'Fixed-length byte array'}
+meta::external::tableformat::iceberg::metamodel::FixedType extends meta::external::tableformat::iceberg::metamodel::PrimitiveType
+{
+ length: Integer[1];
+}
+
+Class
+{doc.doc = 'Arbitrary length byte array'}
+meta::external::tableformat::iceberg::metamodel::BinaryType extends meta::external::tableformat::iceberg::metamodel::PrimitiveType
+{
+}

legend-engine-xts-iceberg/pom.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2022 Goldman Sachs
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.finos.legend.engine</groupId>
+ <artifactId>legend-engine</artifactId>
+ <version>4.20.4-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <name>Legend Engine - XTS - Iceberg</name>
+ <artifactId>legend-engine-xts-iceberg</artifactId>
+ <packaging>pom</packaging>
+
+ <properties>
+ <top.project>${project.basedir}/../</top.project>
+ </properties>
+
+ <modules>
+ <module>legend-engine-xt-iceberg-pure</module>
+ </modules>
+</project>

pom.xml

@@ -63,6 +63,9 @@
  <module>legend-engine-xts-json</module>
  <module>legend-engine-xts-avro</module>
 
+ <!-- Table Formats -->
+ <module>legend-engine-xts-iceberg</module>
+
  <!-- Languages -->
  <module>legend-engine-xts-rosetta</module>
  <module>legend-engine-xts-haskell</module>