diff --git a/README.md b/README.md index e9e13537cc988..5e7de019ede72 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,73 @@ under the License. --> +# Siren fork of Arrow + +- The properties `drill.enable_unsafe_memory_access` and + `arrow.enable_unsafe_memory_access` are prefixed with `siren` and their + default value is set to `true`. The first property is deprecated. + +- In order to avoid conflict with a version of `netty` used in Elasticsearch, we + relocate the netty custom package and dependency in `memory` into a package + named `siren`. The relocation is achieved thanks to the maven shade plugin. + +- The Siren's fork of `netty` is used in `vector`. This means that `netty` + imports in that module need to be prefixed with `siren`. + +## Check that Siren version of Netty is used +- In order to check that Siren version of Netty is being used, + run the unit test `CheckAccessibleTest` in + `https://github.com/sirensolutions/siren-platform/blob/master/core/src/test/java/io/siren/federate/core/common/CheckAccessibleTest.java`. +- Note: the unit test `CheckAccessibleTest` is currently ignored, please set it again to ignore after running the test. + The unit test is ignored because the settings in `CheckAccessibleTest` is not taken into account when the whole unit test suite is run, therefore it fails. + This could be because when the class is loaded, the default settings is used (which is a static block) and the new settings in the `CheckAccessibleTest` is + then not applied when the test suit is run. + +## Build + +To build the `memory`, `format` and `vector` modules: + +```sh +$ cd java +$ mvn clean package +``` + +Because of the default value change of `unsafe_memory_access` property, some +tests in `vector` fail. + +```sh +mvn -pl memory,memory/memory-core,memory/memory-netty,memory/memory-unsafe,format,vector install -Dsiren.arrow.enable_unsafe_memory_access=false -Dsiren.drill.enable_unsafe_memory_access=false +``` + +## Make a new release + +- Tests should pass. + +- Make a new version: + +```sh +mvn versions:set -DnewVersion=siren-0.14.1-2 +``` + +- tag the commit for the release + +```sh +git tag --sign siren-0.14.1-2 +```` + +- Deploy to Siren's artifactory + +```sh +$ mvn deploy -DskipTests=true -P artifactory -Dartifactory_username= -Dartifactory_password= +``` + +## Update to a new version of Apache Arrow + +- add `git@github.com:apache/arrow.git` as the `upstream` remote. +- execute `git fetch --all --tags` +- create a temporary branch from `siren-changes` +- rebase against the new tag. + # Apache Arrow [![Build Status](https://ci.appveyor.com/api/projects/status/github/apache/arrow/branch/master?svg=true)](https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/branch/master) @@ -98,6 +165,37 @@ integrations in other projects, we'd be happy to have you involved: - [Learn the format][2] - Contribute code to one of the reference implementations +### How to Contribute + +We prefer to receive contributions in the form of GitHub pull requests. Please +send pull requests against the [github.com/apache/arrow][4] repository. + +If you are looking for some ideas on what to contribute, check out the [JIRA +issues][3] for the Apache Arrow project. Comment on the issue and/or contact +[dev@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/) +with your questions and ideas. + +If you’d like to report a bug but don’t have time to fix it, you can still post +it on JIRA, or email the mailing list +[dev@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/) + +To contribute a patch: + +1. Break your work into small, single-purpose patches if possible. It’s much +harder to merge in a large change with a lot of disjoint features. +2. Create a JIRA for your patch on the [Arrow Project +JIRA](https://issues.apache.org/jira/browse/ARROW). +3. Submit the patch as a GitHub pull request against the master branch. For a +tutorial, see the GitHub guides on forking a repo and sending a pull +request. Prefix your pull request name with the JIRA name (ex: +https://github.com/apache/arrow/pull/240). +4. Make sure that your code passes the unit tests. You can find instructions +how to run the unit tests for each Arrow component in its respective README +file. +5. Add new unit tests for your code. + +Thank you in advance for your contributions! + [1]: mailto:dev-subscribe@arrow.apache.org [2]: https://github.com/apache/arrow/tree/master/format [3]: https://issues.apache.org/jira/browse/ARROW diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml index f797c2a6030d9..d3dc89a3421d5 100644 --- a/java/adapter/orc/pom.xml +++ b/java/adapter/orc/pom.xml @@ -87,7 +87,7 @@ org.apache.arrow arrow-java-root - 4.0.0 + siren-4.0.0-1-SNAPSHOT ../../pom.xml diff --git a/java/format/pom.xml b/java/format/pom.xml index e363a2a4bb560..bba4a4a0eb356 100644 --- a/java/format/pom.xml +++ b/java/format/pom.xml @@ -15,7 +15,7 @@ arrow-java-root org.apache.arrow - 4.0.0 + siren-4.0.0-1-SNAPSHOT arrow-format diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index 66e1f10f61486..2bfd48f1cc5bb 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 4.0.0 + siren-4.0.0-1-SNAPSHOT org.apache.arrow.gandiva diff --git a/java/memory/memory-core/pom.xml b/java/memory/memory-core/pom.xml index be7faaf5f29a7..840695052343a 100644 --- a/java/memory/memory-core/pom.xml +++ b/java/memory/memory-core/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 4.0.0 + siren-4.0.0-1-SNAPSHOT 4.0.0 @@ -30,10 +30,12 @@ org.slf4j slf4j-api + provided org.immutables value + 2.8.2 diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BoundsChecking.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BoundsChecking.java index bbf7ff34dc964..a6d798bd093e3 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BoundsChecking.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BoundsChecking.java @@ -34,14 +34,15 @@ public class BoundsChecking { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BoundsChecking.class); static { - String envProperty = System.getenv("ARROW_ENABLE_UNSAFE_MEMORY_ACCESS"); - String oldProperty = System.getProperty("drill.enable_unsafe_memory_access"); + String envProperty = System.getenv().getOrDefault("SIREN_ARROW_ENABLE_UNSAFE_MEMORY_ACCESS", "true"); + String oldProperty = System.getProperty("siren.drill.enable_unsafe_memory_access", "true"); if (oldProperty != null) { - logger.warn("\"drill.enable_unsafe_memory_access\" has been renamed to \"arrow.enable_unsafe_memory_access\""); - logger.warn("\"arrow.enable_unsafe_memory_access\" can be set to: " + - " true (to not check) or false (to check, default)"); + logger.warn("\"siren.drill.enable_unsafe_memory_access\" has been renamed to " + + "\"siren.arrow.enable_unsafe_memory_access\""); + logger.warn("\"siren.arrow.enable_unsafe_memory_access\" can be set to: " + + " true (to not check, default) or false (to check)"); } - String newProperty = System.getProperty("arrow.enable_unsafe_memory_access"); + String newProperty = System.getProperty("siren.arrow.enable_unsafe_memory_access", "true"); // The priority of determining the unsafe flag: // 1. The system properties take precedence over the environmental variable. diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java index 16ef39702ca3e..ce7a4bd5e2105 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java @@ -81,7 +81,6 @@ public Object run() { // get the offset of the address field in a java.nio.Buffer object Field addressField = java.nio.Buffer.class.getDeclaredField("address"); - addressField.setAccessible(true); BYTE_BUFFER_ADDRESS_OFFSET = UNSAFE.objectFieldOffset(addressField); Constructor directBufferConstructor; @@ -99,10 +98,7 @@ public Object run() { constructor.setAccessible(true); logger.debug("Constructor for direct buffer found and made accessible"); return constructor; - } catch (NoSuchMethodException e) { - logger.debug("Cannot get constructor for direct buffer allocation", e); - return e; - } catch (SecurityException e) { + } catch (Exception e) { logger.debug("Cannot get constructor for direct buffer allocation", e); return e; } diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java index 5b86bed406b3a..2e423119a54df 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java @@ -55,7 +55,7 @@ private boolean getFlagValue(ClassLoader classLoader) throws Exception { } /** - * Ensure the flag for bounds checking is enabled by default. + * Siren: Ensure the flag for bounds checking is disabled by default. * This will protect users from JVM crashes. */ @Test @@ -63,7 +63,7 @@ public void testDefaultValue() throws Exception { ClassLoader classLoader = copyClassLoader(); if (classLoader != null) { boolean boundsCheckingEnabled = getFlagValue(classLoader); - Assert.assertTrue(boundsCheckingEnabled); + Assert.assertFalse(boundsCheckingEnabled); } } diff --git a/java/memory/memory-netty/pom.xml b/java/memory/memory-netty/pom.xml index 1c25e883d1ac8..6ce283625ff03 100644 --- a/java/memory/memory-netty/pom.xml +++ b/java/memory/memory-netty/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 4.0.0 + siren-4.0.0-1-SNAPSHOT 4.0.0 @@ -35,9 +35,11 @@ io.netty netty-common + org.slf4j slf4j-api + provided org.immutables @@ -62,6 +64,48 @@ UTC + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.0 + + + package + + shade + + + false + ${project.build.directory}/dependency-reduced-pom.xml + + + io.netty + siren.io.netty + + + + + org.slf4j + com.google.code.findbugs + com.google.guava + org.apache.arrow:arrow-memory-core:*:* + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + diff --git a/java/memory/memory-unsafe/pom.xml b/java/memory/memory-unsafe/pom.xml index 5c5346a4b23f5..b95a9906477df 100644 --- a/java/memory/memory-unsafe/pom.xml +++ b/java/memory/memory-unsafe/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 4.0.0 + siren-4.0.0-1-SNAPSHOT 4.0.0 diff --git a/java/memory/pom.xml b/java/memory/pom.xml index 17df5ee13e635..7a2d367793063 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 4.0.0 + siren-4.0.0-1-SNAPSHOT arrow-memory Arrow Memory diff --git a/java/pom.xml b/java/pom.xml index 1d93eb6577737..1b4ce1332e810 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -20,7 +20,7 @@ org.apache.arrow arrow-java-root - 4.0.0 + siren-4.0.0-1-SNAPSHOT pom Apache Arrow Java Root POM @@ -33,7 +33,7 @@ 5.4.0 1.7.25 20.0 - 4.1.48.Final + siren-4.1.48-1 2.11.4 2.7.1 1.12.0 @@ -657,15 +657,12 @@ format memory vector - tools - adapter/jdbc - plasma - flight/flight-core - flight/flight-grpc - performance - algorithm - adapter/avro - compression + + + + + + @@ -758,6 +755,25 @@ + + + + + artifactory + + + + artifactory-releases + artifactory-releases + ${artifactory.url}/libs-release-local + + + artifactory-snapshots + artifactory-snapshots + ${artifactory.url}/libs-snapshot-local + + + diff --git a/java/vector/pom.xml b/java/vector/pom.xml index fa3b24f71309f..854671d69b757 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 4.0.0 + siren-4.0.0-1-SNAPSHOT arrow-vector Arrow Vectors @@ -32,6 +32,12 @@ arrow-memory-core ${project.version} + + + org.apache.arrow + arrow-memory-netty + ${project.version} + com.fasterxml.jackson.core jackson-core @@ -49,12 +55,6 @@ commons-codec 1.10 - - org.apache.arrow - arrow-memory-netty - ${project.version} - test - org.apache.arrow arrow-memory-unsafe @@ -135,11 +135,6 @@ test - - - org.apache.arrow:arrow-memory-netty - - diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl index 2acd436155313..22f82bcfd9a9c 100644 --- a/java/vector/src/main/codegen/includes/vv_imports.ftl +++ b/java/vector/src/main/codegen/includes/vv_imports.ftl @@ -20,6 +20,8 @@ import static org.apache.arrow.util.Preconditions.checkState; import com.google.flatbuffers.FlatBufferBuilder; +import siren.io.netty.buffer.*; + import org.apache.arrow.memory.*; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.types.Types; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java index ea31a4f7846ef..1c592ecf687a1 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java @@ -37,7 +37,7 @@ import org.apache.arrow.vector.util.OversizedAllocationException; import org.apache.arrow.vector.util.TransferPair; -import io.netty.util.internal.PlatformDependent; +import siren.io.netty.util.internal.PlatformDependent; /** * BaseFixedWidthVector provides an abstract interface for diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java index 3bcfd983e6415..1935b45f7bd96 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -119,6 +119,11 @@ public void setInitialCapacity(int valueCount) { lastValueCapacity = valueCount; } + /** + * Get the current value capacity for the vector. + * + * @return number of elements that vector can hold. + */ @Override protected int getValueBufferValueCapacity() { return capAtMaxInt(valueBuffer.capacity() * 8); diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java index ec73382a0ef33..ae08e4f95d6f0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVectorHelper.java @@ -17,10 +17,10 @@ package org.apache.arrow.vector; -import static io.netty.util.internal.PlatformDependent.getByte; -import static io.netty.util.internal.PlatformDependent.getInt; -import static io.netty.util.internal.PlatformDependent.getLong; import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt; +import static siren.io.netty.util.internal.PlatformDependent.getByte; +import static siren.io.netty.util.internal.PlatformDependent.getInt; +import static siren.io.netty.util.internal.PlatformDependent.getLong; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BoundsChecking; @@ -28,7 +28,7 @@ import org.apache.arrow.vector.ipc.message.ArrowFieldNode; import org.apache.arrow.vector.util.DataSizeRoundingUtil; -import io.netty.util.internal.PlatformDependent; +import siren.io.netty.util.internal.PlatformDependent; /** * Helper class for performing generic operations on a bit vector buffer. diff --git a/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java index c5fef82d05276..550119200da63 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java @@ -35,7 +35,7 @@ import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.TransferPair; -import io.netty.util.internal.PlatformDependent; +import siren.io.netty.util.internal.PlatformDependent; /** * Decimal256Vector implements a fixed width vector (32 bytes) of diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java index f988f4f949023..f053f764de43f 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java @@ -35,7 +35,7 @@ import org.apache.arrow.vector.util.DecimalUtility; import org.apache.arrow.vector.util.TransferPair; -import io.netty.util.internal.PlatformDependent; +import siren.io.netty.util.internal.PlatformDependent; /** * DecimalVector implements a fixed width vector (16 bytes) of diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java index f778bcb209fa9..692714487fbfd 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java @@ -24,7 +24,7 @@ import org.apache.arrow.memory.ArrowBuf; -import io.netty.util.internal.PlatformDependent; +import siren.io.netty.util.internal.PlatformDependent; /** * Utility methods for configurable precision Decimal values (e.g. {@link BigDecimal}). diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java b/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java index 7703fed65a6b8..4b5ede6ce31cc 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/util/VectorAppender.java @@ -37,7 +37,7 @@ import org.apache.arrow.vector.complex.NonNullableStructVector; import org.apache.arrow.vector.complex.UnionVector; -import io.netty.util.internal.PlatformDependent; +import siren.io.netty.util.internal.PlatformDependent; /** * Utility to append two vectors together. diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java index 4b48876ff1627..11999db765ec7 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBitVectorHelper.java @@ -26,7 +26,7 @@ import org.apache.arrow.memory.RootAllocator; import org.junit.Test; -import io.netty.util.internal.PlatformDependent; +import siren.io.netty.util.internal.PlatformDependent; public class TestBitVectorHelper { @Test diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index 0be5e1e963c1f..3abecc5f270b0 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -1811,7 +1811,7 @@ public void testSetLastSetUsage() { assertEquals(40, vector.offsetBuffer.getInt(17 * BaseVariableWidthVector.OFFSET_WIDTH)); assertEquals(40, vector.offsetBuffer.getInt(18 * BaseVariableWidthVector.OFFSET_WIDTH)); assertEquals(40, vector.offsetBuffer.getInt(19 * BaseVariableWidthVector.OFFSET_WIDTH)); - + vector.set(19, STR6); assertArrayEquals(STR6, vector.get(19)); assertEquals(40, vector.offsetBuffer.getInt(19 * BaseVariableWidthVector.OFFSET_WIDTH));