diff --git a/README.md b/README.md index 664ada289bd4d..9ac81f3846966 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,62 @@ under the License. --> +# Siren fork of Arrow + +The properties `drill.enable_unsafe_memory_access` and + `arrow.enable_unsafe_memory_access` are prefixed with `siren` and their + default value is set to `true`. The first property is deprecated. + +## Check that Arrow uses Unsafe class to access off-heap memory for memory allocation +In order to check that Arrow uses Unsafe class for memory allocation, run the unit test `CheckArrowTest` in + `https://github.com/sirensolutions/siren-platform/blob/master/core/src/test/java/io/siren/federate/core/common/CheckArrowTest.java`. + +## Build + +To build the `memory`, `format` and `vector` modules: + +```sh +$ cd java +$ mvn clean package +``` + +Because of the default value change of `unsafe_memory_access` property, some +tests in `vector` fail. + +```sh +mvn -pl maven,maven/module-info-compiler-maven-plugin,memory,memory/memory-core,memory/memory-unsafe,format,vector install -Dsiren.arrow.enable_unsafe_memory_access=false -Dsiren.drill.enable_unsafe_memory_access=false +``` + +## Make a new release of Siren's Apache Arrow + +- Tests should pass. + +- Make a new version: + +```sh +mvn versions:set -DnewVersion=siren-0.14.1-2 +``` + +- tag the commit for the release + +```sh +git tag --sign siren-0.14.1-2 +```` + +- Deploy to Siren's artifactory + +```sh +$ mvn deploy -DskipTests=true -P artifactory -Dartifactory_username= -Dartifactory_password= +``` + +## Update to a new version of Siren's Apache Arrow +Developer tips on updating to a new version of Arrow can be found here: https://sirensolutions.atlassian.net/wiki/spaces/EN/pages/3108864001/Upgrading+Federate+Apache+Arrow+Version . + +- add `git@github.com:apache/arrow.git` as the `upstream` remote. +- execute `git fetch --all --tags` +- create a temporary branch from `siren-changes` +- rebase against the new tag. + # Apache Arrow [![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/arrow.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:arrow) diff --git a/cpp/submodules/parquet-testing b/cpp/submodules/parquet-testing deleted file mode 160000 index d69d979223e88..0000000000000 --- a/cpp/submodules/parquet-testing +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d69d979223e883faef9dc6fe3cf573087243c28a diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml index b3748586f2ea4..c34a9f6a85e6f 100644 --- a/java/adapter/orc/pom.xml +++ b/java/adapter/orc/pom.xml @@ -111,7 +111,7 @@ org.apache.arrow arrow-java-root - 15.0.0 + siren-15.0.0-2-SNAPSHOT ../../pom.xml diff --git a/java/format/pom.xml b/java/format/pom.xml index 94365224de94d..8c133e400a067 100644 --- a/java/format/pom.xml +++ b/java/format/pom.xml @@ -15,7 +15,7 @@ arrow-java-root org.apache.arrow - 15.0.0 + siren-15.0.0-2-SNAPSHOT arrow-format diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml index bb89d7ffb9a02..5297c09683460 100644 --- a/java/gandiva/pom.xml +++ b/java/gandiva/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0 + siren-15.0.0-2-SNAPSHOT org.apache.arrow.gandiva diff --git a/java/maven/module-info-compiler-maven-plugin/pom.xml b/java/maven/module-info-compiler-maven-plugin/pom.xml index ee384833a44fa..4be2d7915044e 100644 --- a/java/maven/module-info-compiler-maven-plugin/pom.xml +++ b/java/maven/module-info-compiler-maven-plugin/pom.xml @@ -16,7 +16,7 @@ org.apache.arrow.maven.plugins arrow-maven-plugins - 15.0.0 + siren-15.0.0-2-SNAPSHOT module-info-compiler-maven-plugin maven-plugin @@ -127,4 +127,4 @@ - \ No newline at end of file + diff --git a/java/maven/pom.xml b/java/maven/pom.xml index b1158e1318a33..8b019aaab0ddd 100644 --- a/java/maven/pom.xml +++ b/java/maven/pom.xml @@ -17,7 +17,7 @@ --> org.apache.arrow.maven.plugins arrow-maven-plugins - 15.0.0 + siren-15.0.0-2-SNAPSHOT Arrow Maven Plugins pom @@ -342,4 +342,25 @@ + + + + + artifactory + + + + artifactory-releases + artifactory-releases + ${artifactory.url}/libs-release-local + + + artifactory-snapshots + artifactory-snapshots + ${artifactory.url}/libs-snapshot-local + + + + + diff --git a/java/memory/memory-core/pom.xml b/java/memory/memory-core/pom.xml index dbd9b16aed259..b330d1c45eefa 100644 --- a/java/memory/memory-core/pom.xml +++ b/java/memory/memory-core/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 15.0.0 + siren-15.0.0-2-SNAPSHOT 4.0.0 diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BoundsChecking.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BoundsChecking.java index bbf7ff34dc964..a6d798bd093e3 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BoundsChecking.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/BoundsChecking.java @@ -34,14 +34,15 @@ public class BoundsChecking { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BoundsChecking.class); static { - String envProperty = System.getenv("ARROW_ENABLE_UNSAFE_MEMORY_ACCESS"); - String oldProperty = System.getProperty("drill.enable_unsafe_memory_access"); + String envProperty = System.getenv().getOrDefault("SIREN_ARROW_ENABLE_UNSAFE_MEMORY_ACCESS", "true"); + String oldProperty = System.getProperty("siren.drill.enable_unsafe_memory_access", "true"); if (oldProperty != null) { - logger.warn("\"drill.enable_unsafe_memory_access\" has been renamed to \"arrow.enable_unsafe_memory_access\""); - logger.warn("\"arrow.enable_unsafe_memory_access\" can be set to: " + - " true (to not check) or false (to check, default)"); + logger.warn("\"siren.drill.enable_unsafe_memory_access\" has been renamed to " + + "\"siren.arrow.enable_unsafe_memory_access\""); + logger.warn("\"siren.arrow.enable_unsafe_memory_access\" can be set to: " + + " true (to not check, default) or false (to check)"); } - String newProperty = System.getProperty("arrow.enable_unsafe_memory_access"); + String newProperty = System.getProperty("siren.arrow.enable_unsafe_memory_access", "true"); // The priority of determining the unsafe flag: // 1. The system properties take precedence over the environmental variable. diff --git a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java index cc615c5b38321..68b90325cc8b8 100644 --- a/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java +++ b/java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/MemoryUtil.java @@ -85,7 +85,6 @@ public Object run() { // get the offset of the address field in a java.nio.Buffer object Field addressField = java.nio.Buffer.class.getDeclaredField("address"); - addressField.setAccessible(true); BYTE_BUFFER_ADDRESS_OFFSET = UNSAFE.objectFieldOffset(addressField); Constructor directBufferConstructor; @@ -104,10 +103,7 @@ public Object run() { constructor.setAccessible(true); logger.debug("Constructor for direct buffer found and made accessible"); return constructor; - } catch (NoSuchMethodException e) { - logger.debug("Cannot get constructor for direct buffer allocation", e); - return e; - } catch (SecurityException e) { + } catch (Exception e) { logger.debug("Cannot get constructor for direct buffer allocation", e); return e; } diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java index 5b86bed406b3a..5809add14e7e3 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestBoundaryChecking.java @@ -55,15 +55,15 @@ private boolean getFlagValue(ClassLoader classLoader) throws Exception { } /** - * Ensure the flag for bounds checking is enabled by default. - * This will protect users from JVM crashes. + * Siren: Ensure the flag for bounds checking is disabled by default. + * Enabling it will protect users from JVM crashes. */ @Test public void testDefaultValue() throws Exception { ClassLoader classLoader = copyClassLoader(); if (classLoader != null) { boolean boundsCheckingEnabled = getFlagValue(classLoader); - Assert.assertTrue(boundsCheckingEnabled); + Assert.assertFalse(boundsCheckingEnabled); } } diff --git a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java index 7efc11a08acf2..7926aac19954e 100644 --- a/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java +++ b/java/memory/memory-core/src/test/java/org/apache/arrow/memory/TestOpens.java @@ -20,11 +20,13 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import org.junit.Ignore; import org.junit.Test; public class TestOpens { /** Instantiating the RootAllocator should poke MemoryUtil and fail. */ @Test + @Ignore public void testMemoryUtilFailsLoudly() { // This test is configured by Maven to run WITHOUT add-opens. So this should fail on JDK16+ // (where JEP396 means that add-opens is required to access JDK internals). diff --git a/java/memory/memory-netty/pom.xml b/java/memory/memory-netty/pom.xml index b7592e408f7b2..34666fde28260 100644 --- a/java/memory/memory-netty/pom.xml +++ b/java/memory/memory-netty/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 15.0.0 + siren-15.0.0-2-SNAPSHOT 4.0.0 @@ -34,9 +34,11 @@ io.netty netty-common + org.slf4j slf4j-api + provided ch.qos.logback diff --git a/java/memory/memory-unsafe/pom.xml b/java/memory/memory-unsafe/pom.xml index 4ccb0a38cc0ef..3ec57c471463a 100644 --- a/java/memory/memory-unsafe/pom.xml +++ b/java/memory/memory-unsafe/pom.xml @@ -13,7 +13,7 @@ arrow-memory org.apache.arrow - 15.0.0 + siren-15.0.0-2-SNAPSHOT 4.0.0 diff --git a/java/memory/pom.xml b/java/memory/pom.xml index 6e401580ac214..a596fd238513c 100644 --- a/java/memory/pom.xml +++ b/java/memory/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0 + siren-15.0.0-2-SNAPSHOT arrow-memory Arrow Memory diff --git a/java/pom.xml b/java/pom.xml index a8328576b1d28..83f96521813bb 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -20,7 +20,7 @@ org.apache.arrow arrow-java-root - 15.0.0 + siren-15.0.0-2-SNAPSHOT pom Apache Arrow Java Root POM @@ -349,7 +349,7 @@ true - true + false javax.annotation:javax.annotation-api:* @@ -602,7 +602,7 @@ org.apache.arrow arrow-bom - ${project.version} + 15.0.0 pom import @@ -792,17 +792,18 @@ maven - bom + format memory vector - tools - adapter/jdbc - flight - performance - algorithm - adapter/avro - compression + + + + + + + + @@ -1293,6 +1294,25 @@ + + + + artifactory + + + + artifactory-releases + artifactory-releases + ${artifactory.url}/libs-release-local + + + artifactory-snapshots + artifactory-snapshots + ${artifactory.url}/libs-snapshot-local + + + + diff --git a/java/vector/pom.xml b/java/vector/pom.xml index 63103063dffc5..5f993cfb642fd 100644 --- a/java/vector/pom.xml +++ b/java/vector/pom.xml @@ -14,7 +14,7 @@ org.apache.arrow arrow-java-root - 15.0.0 + siren-15.0.0-2-SNAPSHOT arrow-vector Arrow Vectors @@ -25,10 +25,12 @@ org.apache.arrow arrow-format + ${project.version} org.apache.arrow arrow-memory-core + ${project.version} com.fasterxml.jackson.core @@ -59,7 +61,7 @@ org.apache.arrow arrow-memory-unsafe - test + ${project.version} com.google.flatbuffers @@ -180,7 +182,7 @@ issues in the arrow-tools tests looking up FlatBuffer dependencies. --> - 3.2.4 + 3.3.0 package @@ -190,14 +192,16 @@ + org.apache.arrow:arrow-memory-core + org.apache.arrow:arrow-memory-unsafe org.apache.arrow:arrow-format com.google.flatbuffers:* true - shade-format-flatbuffers - true - true + shade + true + true com.google.flatbuffers diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java index 104819147b109..d8774087db4a0 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java @@ -111,6 +111,11 @@ public void setInitialCapacity(int valueCount) { lastValueCapacity = valueCount; } + /** + * Get the current value capacity for the vector. + * + * @return number of elements that vector can hold. + */ @Override protected int getValueBufferValueCapacity() { return capAtMaxInt(valueBuffer.capacity() * 8); diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java index fb96870804441..42053bee0ffb7 100644 --- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java +++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java @@ -1909,7 +1909,7 @@ public void testSetLastSetUsage() { assertEquals(40, vector.offsetBuffer.getInt(17 * BaseVariableWidthVector.OFFSET_WIDTH)); assertEquals(40, vector.offsetBuffer.getInt(18 * BaseVariableWidthVector.OFFSET_WIDTH)); assertEquals(40, vector.offsetBuffer.getInt(19 * BaseVariableWidthVector.OFFSET_WIDTH)); - + vector.set(19, STR6); assertArrayEquals(STR6, vector.get(19)); assertEquals(40, vector.offsetBuffer.getInt(19 * BaseVariableWidthVector.OFFSET_WIDTH)); diff --git a/testing b/testing deleted file mode 160000 index ad82a736c170e..0000000000000 --- a/testing +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ad82a736c170e97b7c8c035ebd8a801c17eec170