[FEDE-6183] Upgrade arrow to 11.0.0

README.md

@@ -17,6 +17,74 @@
  under the License.
 -->
 
+# Siren fork of Arrow
+
+- The properties `drill.enable_unsafe_memory_access` and
+ `arrow.enable_unsafe_memory_access` are prefixed with `siren` and their
+ default value is set to `true`. The first property is deprecated.
+
+- In order to avoid conflict with a version of `netty` used in Elasticsearch, we
+ relocate the netty custom package and dependency in `memory` into a package
+ named `siren`. The relocation is achieved thanks to the maven shade plugin.
+
+- The Siren's fork of `netty` is used in `vector`. This means that `netty`
+ imports in that module need to be prefixed with `siren`.
+
+## Check that Siren version of Netty is used
+- In order to check that Siren version of Netty is being used, 
+ run the unit test `CheckAccessibleTest` in 
+ `https://github.com/sirensolutions/siren-platform/blob/master/core/src/test/java/io/siren/federate/core/common/CheckAccessibleTest.java`.
+- Note: the unit test `CheckAccessibleTest` is currently ignored, please set it again to ignore after running the test.
+ The unit test is ignored because the setting in `CheckAccessibleTest` is not taken into account when the whole unit test suite is run, therefore it fails. 
+ This could be because when the class is loaded, the default settings are used (which is a static block) and the new settings in the `CheckAccessibleTest` are
+ then not applied when the test suit is run.
+
+## Build
+
+To build the `memory`, `format` and `vector` modules:
+
+```sh
+$ cd java
+$ mvn clean package
+```
+
+Because of the default value change of `unsafe_memory_access` property, some
+tests in `vector` fail.
+
+```sh
+mvn -pl memory,memory/memory-core,memory/memory-netty,memory/memory-unsafe,format,vector install -Dsiren.arrow.enable_unsafe_memory_access=false -Dsiren.drill.enable_unsafe_memory_access=false
+```
+
+## Make a new release of Siren's Apache Arrow
+
+- Tests should pass.
+
+- Make a new version:
+
+```sh
+mvn versions:set -DnewVersion=siren-0.14.1-2
+```
+
+- tag the commit for the release
+
+```sh
+git tag --sign siren-0.14.1-2
+````
+
+- Deploy to Siren's artifactory
+
+```sh
+$ mvn deploy -DskipTests=true -P artifactory -Dartifactory_username=<USERNAME> -Dartifactory_password=<PASSWORD>
+```
+
+## Update to a new version of Siren's Apache Arrow
+Developer tips on updating to a new version of Netty can be found here: https://sirensolutions.atlassian.net/wiki/spaces/EN/pages/3108864001/Upgrading+Federate+Apache+Arrow+Version .
+
+- add `git@github.com:apache/arrow.git` as the `upstream` remote.
+- execute `git fetch --all --tags`
+- create a temporary branch from `siren-changes`
+- rebase against the new tag.
+
 # Apache Arrow
 
 [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/arrow.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:arrow)

java/adapter/orc/pom.xml

@@ -104,7 +104,7 @@
  <parent>
  <groupId>org.apache.arrow</groupId>
  <artifactId>arrow-java-root</artifactId>
- <version>11.0.0</version>
+ <version>siren-11.0.0-2-SNAPSHOT</version>
  <relativePath>../../pom.xml</relativePath>
  </parent>
 

java/format/pom.xml

@@ -15,7 +15,7 @@
 <parent>
  <artifactId>arrow-java-root</artifactId>
  <groupId>org.apache.arrow</groupId>
- <version>11.0.0</version>
+ <version>siren-11.0.0-2-SNAPSHOT</version>
 </parent>
 
 <artifactId>arrow-format</artifactId>

java/gandiva/pom.xml

@@ -14,7 +14,7 @@
  <parent>
  <groupId>org.apache.arrow</groupId>
  <artifactId>arrow-java-root</artifactId>
- <version>11.0.0</version>
+ <version>siren-11.0.0-2-SNAPSHOT</version>
  </parent>
 
  <groupId>org.apache.arrow.gandiva</groupId>

java/memory/memory-core/pom.xml

@@ -13,7 +13,7 @@
  <parent>
  <artifactId>arrow-memory</artifactId>
  <groupId>org.apache.arrow</groupId>
- <version>11.0.0</version>
+ <version>siren-11.0.0-2-SNAPSHOT</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
 

java/memory/memory-core/src/main/java/org/apache/arrow/memory/BoundsChecking.java

@@ -34,14 +34,15 @@ public class BoundsChecking {
  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BoundsChecking.class);
 
  static {
- String envProperty = System.getenv("ARROW_ENABLE_UNSAFE_MEMORY_ACCESS");
- String oldProperty = System.getProperty("drill.enable_unsafe_memory_access");
+ String envProperty = System.getenv().getOrDefault("SIREN_ARROW_ENABLE_UNSAFE_MEMORY_ACCESS", "true");
+ String oldProperty = System.getProperty("siren.drill.enable_unsafe_memory_access", "true");
  if (oldProperty != null) {
- logger.warn("\"drill.enable_unsafe_memory_access\" has been renamed to \"arrow.enable_unsafe_memory_access\"");
- logger.warn("\"arrow.enable_unsafe_memory_access\" can be set to: " +
- " true (to not check) or false (to check, default)");
+ logger.warn("\"siren.drill.enable_unsafe_memory_access\" has been renamed to " +
+ "\"siren.arrow.enable_unsafe_memory_access\"");
+ logger.warn("\"siren.arrow.enable_unsafe_memory_access\" can be set to: " +
+ " true (to not check, default) or false (to check)");
  }
- String newProperty = System.getProperty("arrow.enable_unsafe_memory_access");
+ String newProperty = System.getProperty("siren.arrow.enable_unsafe_memory_access", "true");
 
  // The priority of determining the unsafe flag:
  // 1. The system properties take precedence over the environmental variable.

java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java

@@ -81,7 +81,6 @@ public Object run() {
 
  // get the offset of the address field in a java.nio.Buffer object
  Field addressField = java.nio.Buffer.class.getDeclaredField("address");
- addressField.setAccessible(true);
  BYTE_BUFFER_ADDRESS_OFFSET = UNSAFE.objectFieldOffset(addressField);
 
  Constructor<?> directBufferConstructor;
@@ -99,10 +98,7 @@ public Object run() {
  constructor.setAccessible(true);
  logger.debug("Constructor for direct buffer found and made accessible");
  return constructor;
- } catch (NoSuchMethodException e) {
- logger.debug("Cannot get constructor for direct buffer allocation", e);
- return e;
- } catch (SecurityException e) {
+ } catch (Exception e) {
  logger.debug("Cannot get constructor for direct buffer allocation", e);
  return e;
  }

java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java

@@ -55,15 +55,15 @@ private boolean getFlagValue(ClassLoader classLoader) throws Exception {
  }
 
  /**
- * Ensure the flag for bounds checking is enabled by default.
- * This will protect users from JVM crashes.
+ * Siren: Ensure the flag for bounds checking is disabled by default.
+ * Enabling it will protect users from JVM crashes.
  */
  @Test
  public void testDefaultValue() throws Exception {
  ClassLoader classLoader = copyClassLoader();
  if (classLoader != null) {
  boolean boundsCheckingEnabled = getFlagValue(classLoader);
- Assert.assertTrue(boundsCheckingEnabled);
+ Assert.assertFalse(boundsCheckingEnabled);
  }
  }
 

java/memory/memory-netty/pom.xml

@@ -13,7 +13,7 @@
  <parent>
  <artifactId>arrow-memory</artifactId>
  <groupId>org.apache.arrow</groupId>
- <version>11.0.0</version>
+ <version>siren-11.0.0-2-SNAPSHOT</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
 
@@ -35,16 +35,67 @@
  <groupId>io.netty</groupId>
  <artifactId>netty-common</artifactId>
  </dependency>
+ <!--provided by elasticsearch-->
  <dependency>
  <groupId>org.slf4j</groupId>
  <artifactId>slf4j-api</artifactId>
+ <scope>provided</scope>
  </dependency>
  <dependency>
  <groupId>org.immutables</groupId>
  <artifactId>value</artifactId>
  </dependency>
  </dependencies>
 
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.1.0</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <dependencyReducedPomLocation>${project.build.directory}/dependency-reduced-pom.xml</dependencyReducedPomLocation>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ </transformers>
+ <relocations>
+ <relocation>
+ <pattern>io.netty</pattern>
+ <shadedPattern>siren.io.netty</shadedPattern>
+ </relocation>
+ </relocations>
+ <artifactSet>
+ <excludes>
+ <exclude>org.slf4j</exclude>
+ <exclude>com.google.code.findbugs</exclude>
+ <exclude>com.google.guava</exclude>
+ </excludes>
+ </artifactSet>
+ <!--to prevent the Invalid signature file error -->
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
  <profiles>
  <profile>
  <!-- This profile turns on integration testing. It activates the failsafe plugin and will run any tests

java/memory/memory-unsafe/pom.xml

@@ -13,7 +13,7 @@
  <parent>
  <artifactId>arrow-memory</artifactId>
  <groupId>org.apache.arrow</groupId>
- <version>11.0.0</version>
+ <version>siren-11.0.0-2-SNAPSHOT</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
 

java/memory/pom.xml

@@ -14,7 +14,7 @@
  <parent>
  <groupId>org.apache.arrow</groupId>
  <artifactId>arrow-java-root</artifactId>
- <version>11.0.0</version>
+ <version>siren-11.0.0-2-SNAPSHOT</version>
  </parent>
  <artifactId>arrow-memory</artifactId>
  <name>Arrow Memory</name>

java/pom.xml

@@ -20,7 +20,7 @@
 
  <groupId>org.apache.arrow</groupId>
  <artifactId>arrow-java-root</artifactId>
- <version>11.0.0</version>
+ <version>siren-11.0.0-2-SNAPSHOT</version>
  <packaging>pom</packaging>
 
  <name>Apache Arrow Java Root POM</name>
@@ -33,7 +33,7 @@
  <dep.junit.jupiter.version>5.9.0</dep.junit.jupiter.version>
  <dep.slf4j.version>1.7.25</dep.slf4j.version>
  <dep.guava-bom.version>31.1-jre</dep.guava-bom.version>
- <dep.netty-bom.version>4.1.82.Final</dep.netty-bom.version>
+ <dep.netty-bom.version>siren-4.1.82-1</dep.netty-bom.version>
  <dep.grpc-bom.version>1.49.1</dep.grpc-bom.version>
  <dep.protobuf-bom.version>3.21.6</dep.protobuf-bom.version>
  <dep.jackson-bom.version>2.13.4</dep.jackson-bom.version>
@@ -716,14 +716,14 @@
  <module>format</module>
  <module>memory</module>
  <module>vector</module>
- <module>tools</module>
- <module>adapter/jdbc</module>
- <module>plasma</module>
- <module>flight</module>
- <module>performance</module>
- <module>algorithm</module>
- <module>adapter/avro</module>
- <module>compression</module>
+ <!--<module>tools</module>-->
+ <!--<module>adapter/jdbc</module>-->
+ <!--<module>plasma</module>-->
+ <!--<module>flight</module>-->
+ <!--<module>performance</module>-->
+ <!--<module>algorithm</module>-->
+ <!--<module>adapter/avro</module>-->
+ <!--<module>compression</module>-->
  </modules>
 
  <profiles>
@@ -1213,6 +1213,24 @@
  </build>
  </profile>
 
+ <!-- To deploy to Siren artifactory -->
+ <profile>
+ <id>artifactory</id>
+
+ <distributionManagement>
+ <repository>
+ <id>artifactory-releases</id>
+ <name>artifactory-releases</name>
+ <url>${artifactory.url}/libs-release-local</url>
+ </repository>
+ <snapshotRepository>
+ <id>artifactory-snapshots</id>
+ <name>artifactory-snapshots</name>
+ <url>${artifactory.url}/libs-snapshot-local</url>
+ </snapshotRepository>
+ </distributionManagement>
+ </profile>
+
  </profiles>
 
 </project>

java/vector/pom.xml

@@ -14,7 +14,7 @@
  <parent>
  <groupId>org.apache.arrow</groupId>
  <artifactId>arrow-java-root</artifactId>
- <version>11.0.0</version>
+ <version>siren-11.0.0-2-SNAPSHOT</version>
  </parent>
  <artifactId>arrow-vector</artifactId>
  <name>Arrow Vectors</name>
@@ -57,7 +57,6 @@
  <groupId>org.apache.arrow</groupId>
  <artifactId>arrow-memory-netty</artifactId>
  <version>${project.version}</version>
- <scope>test</scope>
  </dependency>
  <dependency>
  <groupId>org.apache.arrow</groupId>
@@ -125,12 +124,6 @@
  <goals>
  <goal>test</goal>
  </goals>
- <configuration>
- <classpathDependencyExcludes>
- <classpathDependencyExclude>org.apache.arrow:arrow-memory-netty</classpathDependencyExclude>
- </classpathDependencyExcludes>
- <reportNameSuffix>netty</reportNameSuffix>
- </configuration>
  </execution>
  </executions>
  </plugin>
@@ -177,7 +170,7 @@
  <plugin>
  <groupId>org.apache.maven.plugins</groupId>
  <artifactId>maven-shade-plugin</artifactId>
- <version>3.1.1</version>
+ <version>3.3.0</version>
  <executions>
  <execution>
  <phase>package</phase>
@@ -187,14 +180,16 @@
  <configuration>
  <artifactSet>
  <includes>
+ <include>org.apache.arrow:arrow-memory-core</include>
+ <include>org.apache.arrow:arrow-memory-netty</include>
  <include>org.apache.arrow:arrow-format</include>
  <include>com.google.flatbuffers:*</include>
  </includes>
  </artifactSet>
  <shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>shade-format-flatbuffers</shadedClassifierName>
- <keepDependenciesWithProvidedScope>true</keepDependenciesWithProvidedScope>
- <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ <shadedClassifierName>shade</shadedClassifierName>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <useDependencyReducedPomInJar>true</useDependencyReducedPomInJar>
  <relocations>
  <relocation>
  <pattern>com.google.flatbuffers</pattern>