diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md index 652b5f05f52d6..1c4b6df46e25c 100644 --- a/CHANGELOG-3.0.md +++ b/CHANGELOG-3.0.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957)) - Added pull-based Ingestion (APIs, for ingestion source, a Kafka plugin, and IngestionEngine that pulls data from the ingestion source) ([#16958](https://github.com/opensearch-project/OpenSearch/pull/16958)) - Added ConfigurationUtils to core for the ease of configuration parsing [#17223](https://github.com/opensearch-project/OpenSearch/pull/17223) +- Arrow Flight RPC plugin with server bootstrap logic ([#16962](https://github.com/opensearch-project/OpenSearch/pull/16962)) ### Dependencies - Update Apache Lucene to 10.1.0 ([#16366](https://github.com/opensearch-project/OpenSearch/pull/16366)) diff --git a/codecov.yml b/codecov.yml index dac8f30956846..e22af90bcdbe1 100644 --- a/codecov.yml +++ b/codecov.yml @@ -4,6 +4,7 @@ codecov: ignore: - "test" - "benchmarks" + - "plugins/arrow-flight-rpc/**/org/apache/arrow/flight/**" coverage: precision: 2 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 32b8cb431afeb..be66db2c6b6b6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -82,7 +82,7 @@ opentelemetry = "1.46.0" opentelemetrysemconv = "1.29.0-alpha" # arrow dependencies -arrow = "17.0.0" +arrow = "18.1.0" flatbuffers = "2.0.0" [libraries] diff --git a/libs/arrow-spi/build.gradle b/libs/arrow-spi/build.gradle index d14b7e88cfb8c..90a4c162e428b 100644 --- a/libs/arrow-spi/build.gradle +++ b/libs/arrow-spi/build.gradle @@ -10,79 +10,11 @@ */ testingConventions.enabled = false + dependencies { api project(':libs:opensearch-core') - api "org.apache.arrow:arrow-vector:${versions.arrow}" - api "org.apache.arrow:arrow-format:${versions.arrow}" - api "org.apache.arrow:arrow-memory-core:${versions.arrow}" - runtimeOnly "org.apache.arrow:arrow-memory-netty-buffer-patch:${versions.arrow}" - runtimeOnly "org.apache.arrow:arrow-memory-netty:${versions.arrow}" - runtimeOnly "io.netty:netty-buffer:${versions.netty}" - runtimeOnly "io.netty:netty-common:${versions.netty}" - - runtimeOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" - runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}" - runtimeOnly "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" - api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" - - implementation "commons-codec:commons-codec:${versions.commonscodec}" } tasks.named('forbiddenApisMain').configure { replaceSignatureFiles 'jdk-signatures' } - -tasks.named('thirdPartyAudit').configure { - ignoreMissingClasses( - // Logging frameworks - 'org.apache.commons.logging.Log', - 'org.apache.commons.logging.LogFactory', - 'org.apache.log4j.Level', - 'org.apache.log4j.Logger', - 'org.slf4j.impl.StaticLoggerBinder', - 'org.slf4j.impl.StaticMDCBinder', - 'org.slf4j.impl.StaticMarkerBinder', - - // Reactor BlockHound - 'reactor.blockhound.BlockHound$Builder', - 'reactor.blockhound.integration.BlockHoundIntegration' - ) - - ignoreViolations( - "io.netty.util.internal.PlatformDependent0", - "io.netty.util.internal.PlatformDependent0\$1", - "io.netty.util.internal.PlatformDependent0\$2", - "io.netty.util.internal.PlatformDependent0\$3", - "io.netty.util.internal.PlatformDependent0\$4", - "io.netty.util.internal.PlatformDependent0\$6", - "io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef", - "io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef", - "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields", - "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields", - "io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields", - "io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode", - "io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField", - "io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField", - "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField", - "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField", - "io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField", - "io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess", - "io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess", - "io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess", - "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField", - "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField", - "io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField", - "org.apache.arrow.memory.ArrowBuf", - "org.apache.arrow.memory.util.ByteFunctionHelpers", - "org.apache.arrow.memory.util.MemoryUtil", - "org.apache.arrow.memory.util.MemoryUtil\$1", - "org.apache.arrow.memory.util.hash.MurmurHasher", - "org.apache.arrow.memory.util.hash.SimpleHasher", - "org.apache.arrow.vector.BaseFixedWidthVector", - "org.apache.arrow.vector.BitVectorHelper", - "org.apache.arrow.vector.Decimal256Vector", - "org.apache.arrow.vector.DecimalVector", - "org.apache.arrow.vector.util.DecimalUtility", - "org.apache.arrow.vector.util.VectorAppender" - ) -} diff --git a/libs/arrow-spi/licenses/arrow-format-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-format-17.0.0.jar.sha1 deleted file mode 100644 index 34fd4704eac91..0000000000000 --- a/libs/arrow-spi/licenses/arrow-format-17.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5d052f20fd1193840eb59818515e710156c364b2 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-core-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-core-17.0.0.jar.sha1 deleted file mode 100644 index ea312f4f5e51a..0000000000000 --- a/libs/arrow-spi/licenses/arrow-memory-core-17.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -51c5287ef5a624656bb38da7684078905b1a88c9 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-netty-17.0.0.jar.sha1 deleted file mode 100644 index f77b3d836b77b..0000000000000 --- a/libs/arrow-spi/licenses/arrow-memory-netty-17.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -de65a34dfeada4d47b161871fa39fa0a2ab4c39c \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-17.0.0.jar.sha1 deleted file mode 100644 index b21b4e8cc7d23..0000000000000 --- a/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-17.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cdfdaa1bd5135bd869515fc205392ba92dcc1509 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-vector-17.0.0.jar.sha1 b/libs/arrow-spi/licenses/arrow-vector-17.0.0.jar.sha1 deleted file mode 100644 index 8f9fddc882396..0000000000000 --- a/libs/arrow-spi/licenses/arrow-vector-17.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -16685545e4734382c1fcdaf12ac9b0a7d1fc06c0 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/jackson-databind-LICENSE.txt b/libs/arrow-spi/licenses/jackson-databind-LICENSE.txt deleted file mode 100644 index f5f45d26a49d6..0000000000000 --- a/libs/arrow-spi/licenses/jackson-databind-LICENSE.txt +++ /dev/null @@ -1,8 +0,0 @@ -This copy of Jackson JSON processor streaming parser/generator is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 diff --git a/libs/arrow-spi/licenses/jackson-databind-NOTICE.txt b/libs/arrow-spi/licenses/jackson-databind-NOTICE.txt deleted file mode 100644 index 4c976b7b4cc58..0000000000000 --- a/libs/arrow-spi/licenses/jackson-databind-NOTICE.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -## Licensing - -Jackson core and extension components may licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -## Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. diff --git a/libs/arrow-spi/licenses/netty-common-NOTICE.txt b/libs/arrow-spi/licenses/netty-common-NOTICE.txt deleted file mode 100644 index 971865b7c1c23..0000000000000 --- a/libs/arrow-spi/licenses/netty-common-NOTICE.txt +++ /dev/null @@ -1,264 +0,0 @@ - - The Netty Project - ================= - -Please visit the Netty web site for more information: - - * https://netty.io/ - -Copyright 2014 The Netty Project - -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. - -Also, please refer to each LICENSE..txt file, which is located in -the 'license' directory of the distribution file, for the license terms of the -components that this product depends on. - -------------------------------------------------------------------------------- -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - -This product contains a modified portion of 'Webbit', an event based -WebSocket and HTTP server, which can be obtained at: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - -This product contains a modified portion of 'SLF4J', a simple logging -facade for Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * https://www.slf4j.org/ - -This product contains a modified portion of 'Apache Harmony', an open source -Java SE, which can be obtained at: - - * NOTICE: - * license/NOTICE.harmony.txt - * LICENSE: - * license/LICENSE.harmony.txt (Apache License 2.0) - * HOMEPAGE: - * https://archive.apache.org/dist/harmony/ - -This product contains a modified portion of 'jbzip2', a Java bzip2 compression -and decompression library written by Matthew J. Francis. It can be obtained at: - - * LICENSE: - * license/LICENSE.jbzip2.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jbzip2/ - -This product contains a modified portion of 'libdivsufsort', a C API library to construct -the suffix array and the Burrows-Wheeler transformed string for any input string of -a constant-size alphabet written by Yuta Mori. It can be obtained at: - - * LICENSE: - * license/LICENSE.libdivsufsort.txt (MIT License) - * HOMEPAGE: - * https://github.com/y-256/libdivsufsort - -This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, - which can be obtained at: - - * LICENSE: - * license/LICENSE.jctools.txt (ASL2 License) - * HOMEPAGE: - * https://github.com/JCTools/JCTools - -This product optionally depends on 'JZlib', a re-implementation of zlib in -pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product optionally depends on 'Compress-LZF', a Java library for encoding and -decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: - - * LICENSE: - * license/LICENSE.compress-lzf.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/ning/compress - -This product optionally depends on 'lz4', a LZ4 Java compression -and decompression library written by Adrien Grand. It can be obtained at: - - * LICENSE: - * license/LICENSE.lz4.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jpountz/lz4-java - -This product optionally depends on 'lzma-java', a LZMA Java compression -and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.lzma-java.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jponge/lzma-java - -This product optionally depends on 'zstd-jni', a zstd-jni Java compression -and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.zstd-jni.txt (BSD) - * HOMEPAGE: - * https://github.com/luben/zstd-jni - -This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression -and decompression library written by William Kinney. It can be obtained at: - - * LICENSE: - * license/LICENSE.jfastlz.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jfastlz/ - -This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data -interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/protobuf - -This product optionally depends on 'Bouncy Castle Crypto APIs' to generate -a temporary self-signed X.509 certificate when the JVM does not provide the -equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * https://www.bouncycastle.org/ - -This product optionally depends on 'Snappy', a compression library produced -by Google Inc, which can be obtained at: - - * LICENSE: - * license/LICENSE.snappy.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/snappy - -This product optionally depends on 'JBoss Marshalling', an alternative Java -serialization API, which can be obtained at: - - * LICENSE: - * license/LICENSE.jboss-marshalling.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jboss-remoting/jboss-marshalling - -This product optionally depends on 'Caliper', Google's micro- -benchmarking framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.caliper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/google/caliper - -This product optionally depends on 'Apache Commons Logging', a logging -framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-logging.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/logging/ - -This product optionally depends on 'Apache Log4J', a logging framework, which -can be obtained at: - - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * https://logging.apache.org/log4j/ - -This product optionally depends on 'Aalto XML', an ultra-high performance -non-blocking XML processor, which can be obtained at: - - * LICENSE: - * license/LICENSE.aalto-xml.txt (Apache License 2.0) - * HOMEPAGE: - * https://wiki.fasterxml.com/AaltoHome - -This product contains a modified version of 'HPACK', a Java implementation of -the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: - - * LICENSE: - * license/LICENSE.hpack.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/twitter/hpack - -This product contains a modified version of 'HPACK', a Java implementation of -the HTTP/2 HPACK algorithm written by Cory Benfield. It can be obtained at: - - * LICENSE: - * license/LICENSE.hyper-hpack.txt (MIT License) - * HOMEPAGE: - * https://github.com/python-hyper/hpack/ - -This product contains a modified version of 'HPACK', a Java implementation of -the HTTP/2 HPACK algorithm written by Tatsuhiro Tsujikawa. It can be obtained at: - - * LICENSE: - * license/LICENSE.nghttp2-hpack.txt (MIT License) - * HOMEPAGE: - * https://github.com/nghttp2/nghttp2/ - -This product contains a modified portion of 'Apache Commons Lang', a Java library -provides utilities for the java.lang API, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-lang.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/proper/commons-lang/ - - -This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. - - * LICENSE: - * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/takari/maven-wrapper - -This product contains the dnsinfo.h header file, that provides a way to retrieve the system DNS configuration on MacOS. -This private header is also used by Apple's open source - mDNSResponder (https://opensource.apple.com/tarballs/mDNSResponder/). - - * LICENSE: - * license/LICENSE.dnsinfo.txt (Apple Public Source License 2.0) - * HOMEPAGE: - * https://www.opensource.apple.com/source/configd/configd-453.19/dnsinfo/dnsinfo.h - -This product optionally depends on 'Brotli4j', Brotli compression and -decompression for Java., which can be obtained at: - - * LICENSE: - * license/LICENSE.brotli4j.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/hyperxpro/Brotli4j diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamManager.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamManager.java index cdb83f032356a..3bee05f0110d1 100644 --- a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamManager.java +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamManager.java @@ -34,7 +34,7 @@ public interface StreamManager extends AutoCloseable { * @return A StreamTicket that can be used to access the stream * @throws IllegalArgumentException if producer is null or parentTaskId is invalid */ - StreamTicket registerStream(StreamProducer producer, TaskId parentTaskId); + StreamTicket registerStream(StreamProducer producer, TaskId parentTaskId); /** * Creates a stream reader for consuming Arrow data using a valid ticket. @@ -46,7 +46,7 @@ public interface StreamManager extends AutoCloseable { * @throws IllegalArgumentException if the ticket is invalid * @throws IllegalStateException if the stream has been cancelled or closed */ - StreamReader getStreamReader(StreamTicket ticket); + StreamReader getStreamReader(StreamTicket ticket); /** * Gets the StreamTicketFactory instance associated with this StreamManager. diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java index c5cd6f16adfdd..ea2ba1b114348 100644 --- a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamProducer.java @@ -8,9 +8,8 @@ package org.opensearch.arrow.spi; -import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.vector.VectorSchemaRoot; import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.common.unit.TimeValue; import org.opensearch.core.tasks.TaskId; import java.io.Closeable; @@ -77,7 +76,7 @@ * @see StreamReader */ @ExperimentalApi -public interface StreamProducer extends Closeable { +public interface StreamProducer extends Closeable { /** * Creates a VectorSchemaRoot that defines the schema for this stream. This schema will be used @@ -86,7 +85,7 @@ public interface StreamProducer extends Closeable { * @param allocator The allocator to use for creating vectors * @return A new VectorSchemaRoot instance */ - VectorSchemaRoot createRoot(BufferAllocator allocator); + VectorRoot createRoot(Allocator allocator); /** * Creates a job that will produce the stream data in batches. The job will populate @@ -95,7 +94,15 @@ public interface StreamProducer extends Closeable { * @param allocator The allocator to use for any additional memory allocations * @return A new BatchedJob instance */ - BatchedJob createJob(BufferAllocator allocator); + BatchedJob createJob(Allocator allocator); + + /** + * Returns the deadline for the job execution. + * After this deadline, the job should be considered expired. + * + * @return TimeValue representing the job's deadline + */ + TimeValue getJobDeadline(); /** * Provides an estimate of the total number of rows that will be produced. @@ -113,7 +120,7 @@ public interface StreamProducer extends Closeable { /** * BatchedJob interface for producing stream data in batches. */ - interface BatchedJob { + interface BatchedJob { /** * Executes the batch processing job. Implementations should populate the root with data @@ -122,7 +129,7 @@ interface BatchedJob { * @param root The VectorSchemaRoot to populate with data * @param flushSignal Signal to coordinate with consumers */ - void run(VectorSchemaRoot root, FlushSignal flushSignal); + void run(VectorRoot root, FlushSignal flushSignal); /** * Called to signal producer when the job is canceled. @@ -152,6 +159,6 @@ interface FlushSignal { * * @param timeout Maximum milliseconds to wait */ - void awaitConsumption(int timeout); + void awaitConsumption(TimeValue timeout); } } diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamReader.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamReader.java index b258652988b96..74ad3875238a9 100644 --- a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamReader.java +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/StreamReader.java @@ -8,7 +8,6 @@ package org.opensearch.arrow.spi; -import org.apache.arrow.vector.VectorSchemaRoot; import org.opensearch.common.annotation.ExperimentalApi; import java.io.Closeable; @@ -37,7 +36,7 @@ * @see StreamProducer */ @ExperimentalApi -public interface StreamReader extends Closeable { +public interface StreamReader extends Closeable { /** * Blocking request to load next batch into root. @@ -52,5 +51,5 @@ public interface StreamReader extends Closeable { * * @return the VectorSchemaRoot */ - VectorSchemaRoot getRoot(); + VectorRoot getRoot(); } diff --git a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/package-info.java b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/package-info.java index d075ecaa764bb..14227d69da8b0 100644 --- a/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/package-info.java +++ b/libs/arrow-spi/src/main/java/org/opensearch/arrow/spi/package-info.java @@ -7,6 +7,6 @@ */ /** - * Contains Apache Arrow related classes and Stream generic interfaces + * Contains Stream producer, consumer and manager generic interfaces */ package org.opensearch.arrow.spi; diff --git a/plugins/arrow-flight-rpc/build.gradle b/plugins/arrow-flight-rpc/build.gradle new file mode 100644 index 0000000000000..e3c36289d3449 --- /dev/null +++ b/plugins/arrow-flight-rpc/build.gradle @@ -0,0 +1,282 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +apply plugin: 'opensearch.internal-cluster-test' + +opensearchplugin { + description = 'Arrow flight based Stream implementation' + classname = 'org.opensearch.arrow.flight.bootstrap.FlightStreamPlugin' +} + +dependencies { + implementation project(':libs:opensearch-arrow-spi') + compileOnly 'org.checkerframework:checker-qual:3.44.0' + + implementation "org.apache.arrow:arrow-vector:${versions.arrow}" + implementation "org.apache.arrow:arrow-format:${versions.arrow}" + implementation "org.apache.arrow:flight-core:${versions.arrow}" + implementation "org.apache.arrow:arrow-memory-core:${versions.arrow}" + + runtimeOnly "org.apache.arrow:arrow-memory-netty:${versions.arrow}" + runtimeOnly "org.apache.arrow:arrow-memory-netty-buffer-patch:${versions.arrow}" + + implementation "io.netty:netty-buffer:${versions.netty}" + implementation "io.netty:netty-common:${versions.netty}" + + implementation "io.netty:netty-codec:${versions.netty}" + implementation "io.netty:netty-codec-http:${versions.netty}" + implementation "io.netty:netty-codec-http2:${versions.netty}" + implementation "io.netty:netty-handler:${versions.netty}" + implementation "io.netty:netty-resolver:${versions.netty}" + implementation "io.netty:netty-transport:${versions.netty}" + implementation "io.netty:netty-transport-native-unix-common:${versions.netty}" + implementation "io.netty:netty-transport-classes-epoll:${versions.netty}" + implementation "io.netty:netty-tcnative-classes:2.0.66.Final" + + implementation "org.slf4j:slf4j-api:${versions.slf4j}" + runtimeOnly "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}" + runtimeOnly "commons-codec:commons-codec:${versions.commonscodec}" + + implementation "io.grpc:grpc-api:${versions.grpc}" + runtimeOnly "io.grpc:grpc-core:${versions.grpc}" + implementation "io.grpc:grpc-stub:${versions.grpc}" + implementation "io.grpc:grpc-netty:${versions.grpc}" + + runtimeOnly group: 'com.google.code.findbugs', name: 'jsr305', version: '3.0.2' + compileOnly 'org.immutables:value:2.10.1' + annotationProcessor 'org.immutables:value:2.10.1' + + runtimeOnly 'io.perfmark:perfmark-api:0.27.0' + runtimeOnly 'org.apache.parquet:parquet-arrow:1.13.1' + runtimeOnly "io.grpc:grpc-protobuf-lite:${versions.grpc}" + runtimeOnly "io.grpc:grpc-protobuf:${versions.grpc}" + implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}" + implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + + runtimeOnly "com.google.guava:failureaccess:1.0.1" + compileOnly "com.google.errorprone:error_prone_annotations:2.31.0" + runtimeOnly('com.google.guava:guava:33.3.1-jre') { + attributes { + attribute(Attribute.of('org.gradle.jvm.environment', String), 'standard-jvm') + } + } +} + +tasks.internalClusterTest { + jvmArgs += ["--add-opens", "java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED"] +} + +tasks.named('test').configure { + jacoco { + excludes = ['org/apache/arrow/flight/**'] + } +} + +tasks.named("dependencyLicenses").configure { + mapping from: /netty-.*/, to: 'netty' + mapping from: /grpc-.*/, to: 'grpc' + mapping from: /jackson-.*/, to: 'jackson' +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' + + excludes = [ + 'org/apache/arrow/flight/OSFlightServer$Builder.class', + 'org/apache/arrow/flight/OSFlightClient$Builder.class', + 'org/opensearch/flight/bootstrap/server/ServerConfig$Netty4Configs.class', + 'org/opensearch/flight/bootstrap/server/ServerConfig.class', + 'org/opensearch/flight/bootstrap/tls/DefaultSslContextProvider.class', + 'org/apache/arrow/flight/OpenSearchFlightClient$Builder.class' + ] +} + +tasks.named('thirdPartyAudit').configure { + ignoreMissingClasses( + 'com.google.gson.stream.JsonReader', + 'com.google.gson.stream.JsonToken', + 'org.apache.parquet.schema.GroupType', + 'com.google.rpc.Status', + 'com.google.rpc.Status$Builder', + // Parquet Schema classes + 'org.apache.parquet.schema.LogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$DateLogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$DecimalLogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$IntLogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$IntervalLogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$ListLogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$LogicalTypeAnnotationVisitor', + 'org.apache.parquet.schema.LogicalTypeAnnotation$StringLogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$TimeLogicalTypeAnnotation', + 'org.apache.parquet.schema.LogicalTypeAnnotation$TimeUnit', + 'org.apache.parquet.schema.LogicalTypeAnnotation$TimestampLogicalTypeAnnotation', + 'org.apache.parquet.schema.MessageType', + 'org.apache.parquet.schema.OriginalType', + 'org.apache.parquet.schema.PrimitiveType', + 'org.apache.parquet.schema.PrimitiveType$PrimitiveTypeName', + 'org.apache.parquet.schema.PrimitiveType$PrimitiveTypeNameConverter', + 'org.apache.parquet.schema.Type', + 'org.apache.parquet.schema.Type$Repetition', + 'org.apache.parquet.schema.Types', + 'org.apache.parquet.schema.Types$BaseListBuilder', + 'org.apache.parquet.schema.Types$GroupBuilder', + 'org.apache.parquet.schema.Types$ListBuilder', + 'org.apache.parquet.schema.Types$PrimitiveBuilder', + + 'com.aayushatharva.brotli4j.Brotli4jLoader', + 'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status', + 'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper', + 'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel', + 'com.aayushatharva.brotli4j.encoder.Encoder$Mode', + 'com.aayushatharva.brotli4j.encoder.Encoder$Parameters', + // classes are missing + + // from io.netty.logging.CommonsLoggerFactory (netty) + 'org.apache.commons.logging.Log', + 'org.apache.commons.logging.LogFactory', + + 'org.slf4j.impl.StaticLoggerBinder', + 'org.slf4j.impl.StaticMDCBinder', + 'org.slf4j.impl.StaticMarkerBinder', + + // from Log4j (deliberate, Netty will fallback to Log4j 2) + 'org.apache.log4j.Level', + 'org.apache.log4j.Logger', + + // from io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (netty) + 'org.bouncycastle.cert.X509v3CertificateBuilder', + 'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter', + 'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder', + 'org.bouncycastle.openssl.PEMEncryptedKeyPair', + 'org.bouncycastle.openssl.PEMParser', + 'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter', + 'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder', + 'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder', + 'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo', + + // from io.netty.handler.ssl.JettyNpnSslEngine (netty) + 'org.eclipse.jetty.npn.NextProtoNego$ClientProvider', + 'org.eclipse.jetty.npn.NextProtoNego$ServerProvider', + 'org.eclipse.jetty.npn.NextProtoNego', + + // from io.netty.handler.codec.marshalling.ChannelBufferByteInput (netty) + 'org.jboss.marshalling.ByteInput', + + // from io.netty.handler.codec.marshalling.ChannelBufferByteOutput (netty) + 'org.jboss.marshalling.ByteOutput', + + // from io.netty.handler.codec.marshalling.CompatibleMarshallingEncoder (netty) + 'org.jboss.marshalling.Marshaller', + + // from io.netty.handler.codec.marshalling.ContextBoundUnmarshallerProvider (netty) + 'org.jboss.marshalling.MarshallerFactory', + 'org.jboss.marshalling.MarshallingConfiguration', + 'org.jboss.marshalling.Unmarshaller', + + 'com.google.protobuf.nano.CodedOutputByteBufferNano', + 'com.google.protobuf.nano.MessageNano', + 'com.ning.compress.BufferRecycler', + 'com.ning.compress.lzf.ChunkDecoder', + 'com.ning.compress.lzf.ChunkEncoder', + 'com.ning.compress.lzf.LZFChunk', + 'com.ning.compress.lzf.LZFEncoder', + 'com.ning.compress.lzf.util.ChunkDecoderFactory', + 'com.ning.compress.lzf.util.ChunkEncoderFactory', + 'lzma.sdk.lzma.Encoder', + 'net.jpountz.lz4.LZ4Compressor', + 'net.jpountz.lz4.LZ4Factory', + 'net.jpountz.lz4.LZ4FastDecompressor', + 'net.jpountz.xxhash.XXHash32', + 'net.jpountz.xxhash.XXHashFactory', + 'org.eclipse.jetty.alpn.ALPN$ClientProvider', + 'org.eclipse.jetty.alpn.ALPN$ServerProvider', + 'org.eclipse.jetty.alpn.ALPN', + + 'org.conscrypt.AllocatedBuffer', + 'org.conscrypt.BufferAllocator', + 'org.conscrypt.Conscrypt', + 'org.conscrypt.HandshakeListener', + + 'reactor.blockhound.BlockHound$Builder', + 'reactor.blockhound.integration.BlockHoundIntegration', + + 'com.google.protobuf.util.Timestamps' + ) + ignoreViolations( + // Guava internal classes + 'com.google.common.cache.Striped64', + 'com.google.common.cache.Striped64$1', + 'com.google.common.cache.Striped64$Cell', + 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray', + 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1', + 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2', + 'com.google.common.hash.Striped64', + 'com.google.common.hash.Striped64$1', + 'com.google.common.hash.Striped64$Cell', + 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', + 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', + 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper', + 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1', + + 'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator', + 'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$1', + 'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$2', + 'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$3', + 'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$4', + 'io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$5', + 'io.netty.util.internal.PlatformDependent0', + 'io.netty.util.internal.PlatformDependent0$1', + 'io.netty.util.internal.PlatformDependent0$2', + 'io.netty.util.internal.PlatformDependent0$3', + 'io.netty.util.internal.PlatformDependent0$4', + 'io.netty.util.internal.PlatformDependent0$6', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields', + 'io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode', + 'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField', + 'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', + 'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess', + 'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess', + 'io.netty.util.internal.PlatformDependent0', + 'io.netty.util.internal.PlatformDependent0$1', + 'io.netty.util.internal.PlatformDependent0$2', + 'io.netty.util.internal.PlatformDependent0$3', + 'io.netty.util.internal.PlatformDependent0$4', + 'io.netty.util.internal.PlatformDependent0$6', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields', + 'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields', + 'io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode', + 'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueConsumerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerIndexField', + 'io.netty.util.internal.shaded.org.jctools.queues.unpadded.MpscUnpaddedArrayQueueProducerLimitField', + 'io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess', + 'io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess', + 'io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess', + 'org.apache.arrow.memory.util.MemoryUtil', + 'org.apache.arrow.memory.util.MemoryUtil$1' + + ) +} diff --git a/plugins/arrow-flight-rpc/licenses/arrow-format-18.1.0.jar.sha1 b/plugins/arrow-flight-rpc/licenses/arrow-format-18.1.0.jar.sha1 new file mode 100644 index 0000000000000..6372bcd89eefd --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/arrow-format-18.1.0.jar.sha1 @@ -0,0 +1 @@ +9d356b6f20620f5619ff85b174f97ae507df4997 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-format-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/arrow-format-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-format-LICENSE.txt rename to plugins/arrow-flight-rpc/licenses/arrow-format-LICENSE.txt diff --git a/libs/arrow-spi/licenses/arrow-format-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/arrow-format-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-format-NOTICE.txt rename to plugins/arrow-flight-rpc/licenses/arrow-format-NOTICE.txt diff --git a/plugins/arrow-flight-rpc/licenses/arrow-memory-core-18.1.0.jar.sha1 b/plugins/arrow-flight-rpc/licenses/arrow-memory-core-18.1.0.jar.sha1 new file mode 100644 index 0000000000000..1a4da42973bfe --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/arrow-memory-core-18.1.0.jar.sha1 @@ -0,0 +1 @@ +35f4853d512f06759759b40b53bac850867886f8 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-core-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/arrow-memory-core-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-memory-core-LICENSE.txt rename to plugins/arrow-flight-rpc/licenses/arrow-memory-core-LICENSE.txt diff --git a/libs/arrow-spi/licenses/arrow-memory-core-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/arrow-memory-core-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-memory-core-NOTICE.txt rename to plugins/arrow-flight-rpc/licenses/arrow-memory-core-NOTICE.txt diff --git a/plugins/arrow-flight-rpc/licenses/arrow-memory-netty-18.1.0.jar.sha1 b/plugins/arrow-flight-rpc/licenses/arrow-memory-netty-18.1.0.jar.sha1 new file mode 100644 index 0000000000000..291d435138e30 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/arrow-memory-netty-18.1.0.jar.sha1 @@ -0,0 +1 @@ +9e9e08d0b548d2c02c632e5daaf176e588810d22 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/arrow-memory-netty-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-memory-netty-LICENSE.txt rename to plugins/arrow-flight-rpc/licenses/arrow-memory-netty-LICENSE.txt diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/arrow-memory-netty-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-memory-netty-NOTICE.txt rename to plugins/arrow-flight-rpc/licenses/arrow-memory-netty-NOTICE.txt diff --git a/plugins/arrow-flight-rpc/licenses/arrow-memory-netty-buffer-patch-18.1.0.jar.sha1 b/plugins/arrow-flight-rpc/licenses/arrow-memory-netty-buffer-patch-18.1.0.jar.sha1 new file mode 100644 index 0000000000000..40c7b2992d715 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/arrow-memory-netty-buffer-patch-18.1.0.jar.sha1 @@ -0,0 +1 @@ +86c8fbdb6ab220603ea3a215f48a7f793ac6a08d \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt rename to plugins/arrow-flight-rpc/licenses/arrow-memory-netty-buffer-patch-LICENSE.txt diff --git a/libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt rename to plugins/arrow-flight-rpc/licenses/arrow-memory-netty-buffer-patch-NOTICE.txt diff --git a/plugins/arrow-flight-rpc/licenses/arrow-vector-18.1.0.jar.sha1 b/plugins/arrow-flight-rpc/licenses/arrow-vector-18.1.0.jar.sha1 new file mode 100644 index 0000000000000..d526f82b6f06e --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/arrow-vector-18.1.0.jar.sha1 @@ -0,0 +1 @@ +b1fb77f4ef36fd52afe480ba12b7da77367eb88c \ No newline at end of file diff --git a/libs/arrow-spi/licenses/arrow-vector-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/arrow-vector-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-vector-LICENSE.txt rename to plugins/arrow-flight-rpc/licenses/arrow-vector-LICENSE.txt diff --git a/libs/arrow-spi/licenses/arrow-vector-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/arrow-vector-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/arrow-vector-NOTICE.txt rename to plugins/arrow-flight-rpc/licenses/arrow-vector-NOTICE.txt diff --git a/libs/arrow-spi/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/arrow-flight-rpc/licenses/commons-codec-1.16.1.jar.sha1 similarity index 100% rename from libs/arrow-spi/licenses/commons-codec-1.16.1.jar.sha1 rename to plugins/arrow-flight-rpc/licenses/commons-codec-1.16.1.jar.sha1 diff --git a/libs/arrow-spi/licenses/commons-codec-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/commons-codec-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/commons-codec-LICENSE.txt rename to plugins/arrow-flight-rpc/licenses/commons-codec-LICENSE.txt diff --git a/libs/arrow-spi/licenses/commons-codec-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/commons-codec-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/commons-codec-NOTICE.txt rename to plugins/arrow-flight-rpc/licenses/commons-codec-NOTICE.txt diff --git a/plugins/arrow-flight-rpc/licenses/failureaccess-1.0.1.jar.sha1 b/plugins/arrow-flight-rpc/licenses/failureaccess-1.0.1.jar.sha1 new file mode 100644 index 0000000000000..4798b37e20691 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/failureaccess-1.0.1.jar.sha1 @@ -0,0 +1 @@ +1dcf1de382a0bf95a3d8b0849546c88bac1292c9 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/failureaccess-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/failureaccess-LICENSE.txt new file mode 100644 index 0000000000000..7a4a3ea2424c0 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/failureaccess-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/libs/arrow-spi/licenses/flatbuffers-java-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/failureaccess-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/flatbuffers-java-NOTICE.txt rename to plugins/arrow-flight-rpc/licenses/failureaccess-NOTICE.txt diff --git a/libs/arrow-spi/licenses/flatbuffers-java-2.0.0.jar.sha1 b/plugins/arrow-flight-rpc/licenses/flatbuffers-java-2.0.0.jar.sha1 similarity index 100% rename from libs/arrow-spi/licenses/flatbuffers-java-2.0.0.jar.sha1 rename to plugins/arrow-flight-rpc/licenses/flatbuffers-java-2.0.0.jar.sha1 diff --git a/libs/arrow-spi/licenses/flatbuffers-java-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/flatbuffers-java-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/flatbuffers-java-LICENSE.txt rename to plugins/arrow-flight-rpc/licenses/flatbuffers-java-LICENSE.txt diff --git a/libs/arrow-spi/licenses/slf4j-api-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/flatbuffers-java-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/slf4j-api-NOTICE.txt rename to plugins/arrow-flight-rpc/licenses/flatbuffers-java-NOTICE.txt diff --git a/plugins/arrow-flight-rpc/licenses/flight-core-18.1.0.jar.sha1 b/plugins/arrow-flight-rpc/licenses/flight-core-18.1.0.jar.sha1 new file mode 100644 index 0000000000000..fc2e34539cf04 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/flight-core-18.1.0.jar.sha1 @@ -0,0 +1 @@ +82494895fcb0656967680442f63ce1214e532d52 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/flight-core-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/flight-core-LICENSE.txt new file mode 100644 index 0000000000000..7bb1330a1002b --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/flight-core-LICENSE.txt @@ -0,0 +1,2261 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +src/arrow/util (some portions): Apache 2.0, and 3-clause BSD + +Some portions of this module are derived from code in the Chromium project, +copyright (c) Google inc and (c) The Chromium Authors and licensed under the +Apache 2.0 License or the under the 3-clause BSD license: + + Copyright (c) 2013 The Chromium Authors. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from Daniel Lemire's FrameOfReference project. + +https://github.com/lemire/FrameOfReference/blob/6ccaf9e97160f9a3b299e23a8ef739e711ef0c71/src/bpacking.cpp +https://github.com/lemire/FrameOfReference/blob/146948b6058a976bc7767262ad3a2ce201486b93/scripts/turbopacking64.py + +Copyright: 2013 Daniel Lemire +Home page: http://lemire.me/en/ +Project page: https://github.com/lemire/FrameOfReference +License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the TensorFlow project + +Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the NumPy project. + +https://github.com/numpy/numpy/blob/e1f191c46f2eebd6cb892a4bfe14d9dd43a06c4e/numpy/core/src/multiarray/multiarraymodule.c#L2910 + +https://github.com/numpy/numpy/blob/68fd82271b9ea5a9e50d4e761061dfcca851382a/numpy/core/src/multiarray/datetime.c + +Copyright (c) 2005-2017, NumPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the Boost project + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from the FlatBuffers project + +Copyright 2014 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the tslib project + +Copyright 2015 Microsoft Corporation. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from the jemalloc project + +https://github.com/jemalloc/jemalloc + +Copyright (C) 2002-2017 Jason Evans . +All rights reserved. +Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. +Copyright (C) 2009-2017 Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice(s), + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice(s), + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- + +This project includes code from the Go project, BSD 3-clause license + PATENTS +weak patent termination clause +(https://github.com/golang/go/blob/master/PATENTS). + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project includes code from the hs2client + +https://github.com/cloudera/hs2client + +Copyright 2016 Cloudera Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +The script ci/scripts/util_wait_for_it.sh has the following license + +Copyright (c) 2016 Giles Hall + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The script r/configure has the following license (MIT) + +Copyright (c) 2017, Jeroen Ooms and Jim Hester + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +cpp/src/arrow/util/logging.cc, cpp/src/arrow/util/logging.h and +cpp/src/arrow/util/logging-test.cc are adapted from +Ray Project (https://github.com/ray-project/ray) (Apache 2.0). + +Copyright (c) 2016 Ray Project (https://github.com/ray-project/ray) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- +The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h, +cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h, +cpp/src/arrow/vendored/datetime/ios.mm, +cpp/src/arrow/vendored/datetime/tz.cpp are adapted from +Howard Hinnant's date library (https://github.com/HowardHinnant/date) +It is licensed under MIT license. + +The MIT License (MIT) +Copyright (c) 2015, 2016, 2017 Howard Hinnant +Copyright (c) 2016 Adrian Colomitchi +Copyright (c) 2017 Florian Dang +Copyright (c) 2017 Paul Thompson +Copyright (c) 2018 Tomasz Kamiński + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/src/arrow/util/utf8.h includes code adapted from the page + https://bjoern.hoehrmann.de/utf-8/decoder/dfa/ +with the following license (MIT) + +Copyright (c) 2008-2009 Bjoern Hoehrmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/xxhash/ have the following license +(BSD 2-Clause License) + +xxHash Library +Copyright (c) 2012-2014, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash homepage: http://www.xxhash.com +- xxHash source repository : https://github.com/Cyan4973/xxHash + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/double-conversion/ have the following license +(BSD 3-Clause License) + +Copyright 2006-2011, the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/uriparser/ have the following license +(BSD 3-Clause License) + +uriparser - RFC 3986 URI parsing library + +Copyright (C) 2007, Weijia Song +Copyright (C) 2007, Sebastian Pipping +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the following + disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials + provided with the distribution. + + * Neither the name of the nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files under dev/tasks/conda-recipes have the following license + +BSD 3-clause license +Copyright (c) 2015-2018, conda-forge +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/utfcpp/ have the following license + +Copyright 2006-2018 Nemanja Trifunovic + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This project includes code from Apache Kudu. + + * cpp/cmake_modules/CompilerInfo.cmake is based on Kudu's cmake_modules/CompilerInfo.cmake + +Copyright: 2016 The Apache Software Foundation. +Home page: https://kudu.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Apache Impala (incubating), formerly +Impala. The Impala code and rights were donated to the ASF as part of the +Incubator process after the initial code imports into Apache Parquet. + +Copyright: 2012 Cloudera, Inc. +Copyright: 2016 The Apache Software Foundation. +Home page: http://impala.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Apache Aurora. + +* dev/release/{release,changelog,release-candidate} are based on the scripts from + Apache Aurora + +Copyright: 2016 The Apache Software Foundation. +Home page: https://aurora.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from the Google styleguide. + +* cpp/build-support/cpplint.py is based on the scripts from the Google styleguide. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/styleguide +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This project includes code from Snappy. + +* cpp/cmake_modules/{SnappyCMakeLists.txt,SnappyConfig.h} are based on code + from Google's Snappy project. + +Copyright: 2009 Google Inc. All rights reserved. +Homepage: https://github.com/google/snappy +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +This project includes code from the manylinux project. + +* python/manylinux1/scripts/{build_python.sh,python-tag-abi-tag.py, + requirements.txt} are based on code from the manylinux project. + +Copyright: 2016 manylinux +Homepage: https://github.com/pypa/manylinux +License: The MIT License (MIT) + +-------------------------------------------------------------------------------- + +This project includes code from the cymove project: + +* python/pyarrow/includes/common.pxd includes code from the cymove project + +The MIT License (MIT) +Copyright (c) 2019 Omer Ozarslan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The projects includes code from the Ursabot project under the dev/archery +directory. + +License: BSD 2-Clause + +Copyright 2019 RStudio, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +This project include code from mingw-w64. + +* cpp/src/arrow/util/cpu-info.cc has a polyfill for mingw-w64 < 5 + +Copyright (c) 2009 - 2013 by the mingw-w64 project +Homepage: https://mingw-w64.org +License: Zope Public License (ZPL) Version 2.1. + +--------------------------------------------------------------------------------- + +This project include code from Google's Asylo project. + +* cpp/src/arrow/result.h is based on status_or.h + +Copyright (c) Copyright 2017 Asylo authors +Homepage: https://asylo.dev/ +License: Apache 2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Google's protobuf project + +* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN +* cpp/src/arrow/util/bit_stream_utils.h contains code from wire_format_lite.h + +Copyright 2008 Google Inc. All rights reserved. +Homepage: https://developers.google.com/protocol-buffers/ +License: + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +-------------------------------------------------------------------------------- + +3rdparty dependency LLVM is statically linked in certain binary distributions. +Additionally some sections of source code have been derived from sources in LLVM +and have been clearly labeled as such. LLVM has the following license: + +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +-------------------------------------------------------------------------------- + +3rdparty dependency gRPC is statically linked in certain binary +distributions, like the python wheels. gRPC has the following license: + +Copyright 2014 gRPC authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency Apache Thrift is statically linked in certain binary +distributions, like the python wheels. Apache Thrift has the following license: + +Apache Thrift +Copyright (C) 2006 - 2019, The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency Apache ORC is statically linked in certain binary +distributions, like the python wheels. Apache ORC has the following license: + +Apache ORC +Copyright 2013-2019 The Apache Software Foundation + +This product includes software developed by The Apache Software +Foundation (http://www.apache.org/). + +This product includes software developed by Hewlett-Packard: +(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------------- + +3rdparty dependency zstd is statically linked in certain binary +distributions, like the python wheels. ZSTD has the following license: + +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency lz4 is statically linked in certain binary +distributions, like the python wheels. lz4 has the following license: + +LZ4 Library +Copyright (c) 2011-2016, Yann Collet +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency Brotli is statically linked in certain binary +distributions, like the python wheels. Brotli has the following license: + +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency rapidjson is statically linked in certain binary +distributions, like the python wheels. rapidjson and its dependencies have the +following licenses: + +Tencent is pleased to support the open source community by making RapidJSON +available. + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +All rights reserved. + +If you have downloaded a copy of the RapidJSON binary from Tencent, please note +that the RapidJSON binary is licensed under the MIT License. +If you have downloaded a copy of the RapidJSON source code from Tencent, please +note that RapidJSON source code is licensed under the MIT License, except for +the third-party components listed below which are subject to different license +terms. Your integration of RapidJSON into your own projects may require +compliance with the MIT License, as well as the other licenses applicable to +the third-party components included within RapidJSON. To avoid the problematic +JSON license in your own projects, it's sufficient to exclude the +bin/jsonchecker/ directory, as it's the only code under the JSON license. +A copy of the MIT License is included in this file. + +Other dependencies and licenses: + + Open Source Software Licensed Under the BSD License: + -------------------------------------------------------------------- + + The msinttypes r29 + Copyright (c) 2006-2013 Alexander Chemeris + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + + Terms of the MIT License: + -------------------------------------------------------------------- + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency snappy is statically linked in certain binary +distributions, like the python wheels. snappy has the following license: + +Copyright 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Google Inc. nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +=== + +Some of the benchmark data in testdata/ is licensed differently: + + - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and + is licensed under the Creative Commons Attribution 3.0 license + (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ + for more information. + + - kppkn.gtb is taken from the Gaviota chess tablebase set, and + is licensed under the MIT License. See + https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 + for more information. + + - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper + “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA + Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, + which is licensed under the CC-BY license. See + http://www.ploscompbiol.org/static/license for more ifnormation. + + - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project + Gutenberg. The first three have expired copyrights and are in the public + domain; the latter does not have expired copyright, but is still in the + public domain according to the license information + (http://www.gutenberg.org/ebooks/53). + +-------------------------------------------------------------------------------- + +3rdparty dependency gflags is statically linked in certain binary +distributions, like the python wheels. gflags has the following license: + +Copyright (c) 2006, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency glog is statically linked in certain binary +distributions, like the python wheels. glog has the following license: + +Copyright (c) 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +A function gettimeofday in utilities.cc is based on + +http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd + +The license of this code is: + +Copyright (c) 2003-2008, Jouni Malinen and contributors +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name(s) of the above-listed copyright holder(s) nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency re2 is statically linked in certain binary +distributions, like the python wheels. re2 has the following license: + +Copyright (c) 2009 The RE2 Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its contributors + may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +3rdparty dependency c-ares is statically linked in certain binary +distributions, like the python wheels. c-ares has the following license: + +# c-ares license + +Copyright (c) 2007 - 2018, Daniel Stenberg with many contributors, see AUTHORS +file. + +Copyright 1998 by the Massachusetts Institute of Technology. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, provided that +the above copyright notice appear in all copies and that both that copyright +notice and this permission notice appear in supporting documentation, and that +the name of M.I.T. not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior permission. +M.I.T. makes no representations about the suitability of this software for any +purpose. It is provided "as is" without express or implied warranty. + +-------------------------------------------------------------------------------- + +3rdparty dependency zlib is redistributed as a dynamically linked shared +library in certain binary distributions, like the python wheels. In the future +this will likely change to static linkage. zlib has the following license: + +zlib.h -- interface of the 'zlib' general purpose compression library + version 1.2.11, January 15th, 2017 + + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + +-------------------------------------------------------------------------------- + +3rdparty dependency openssl is redistributed as a dynamically linked shared +library in certain binary distributions, like the python wheels. openssl +preceding version 3 has the following license: + + LICENSE ISSUES + ============== + + The OpenSSL toolkit stays under a double license, i.e. both the conditions of + the OpenSSL License and the original SSLeay license apply to the toolkit. + See below for the actual license texts. + + OpenSSL License + --------------- + +/* ==================================================================== + * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + + Original SSLeay License + ----------------------- + +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ + +-------------------------------------------------------------------------------- + +This project includes code from the rtools-backports project. + +* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code + from the rtools-backports project. + +Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms. +All rights reserved. +Homepage: https://github.com/r-windows/rtools-backports +License: 3-clause BSD + +-------------------------------------------------------------------------------- + +Some code from pandas has been adapted for the pyarrow codebase. pandas is +available under the 3-clause BSD license, which follows: + +pandas license +============== + +Copyright (c) 2011-2012, Lambda Foundry, Inc. and PyData Development Team +All rights reserved. + +Copyright (c) 2008-2011 AQR Capital Management, LLC +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the copyright holder nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +Some bits from DyND, in particular aspects of the build system, have been +adapted from libdynd and dynd-python under the terms of the BSD 2-clause +license + +The BSD 2-Clause License + + Copyright (C) 2011-12, Dynamic NDArray Developers + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Dynamic NDArray Developers list: + + * Mark Wiebe + * Continuum Analytics + +-------------------------------------------------------------------------------- + +Some source code from Ibis (https://github.com/cloudera/ibis) has been adapted +for PyArrow. Ibis is released under the Apache License, Version 2.0. + +-------------------------------------------------------------------------------- + +dev/tasks/homebrew-formulae/apache-arrow.rb has the following license: + +BSD 2-Clause License + +Copyright (c) 2009-present, Homebrew contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- + +cpp/src/arrow/vendored/base64.cpp has the following license + +ZLIB License + +Copyright (C) 2004-2017 René Nyffenegger + +This source code is provided 'as-is', without any express or implied +warranty. In no event will the author be held liable for any damages arising +from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including +commercial applications, and to alter it and redistribute it freely, subject to +the following restrictions: + +1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. + +3. This notice may not be removed or altered from any source distribution. + +René Nyffenegger rene.nyffenegger@adp-gmbh.ch + +-------------------------------------------------------------------------------- + +This project includes code from Folly. + + * cpp/src/arrow/vendored/ProducerConsumerQueue.h + +is based on Folly's + + * folly/Portability.h + * folly/lang/Align.h + * folly/ProducerConsumerQueue.h + +Copyright: Copyright (c) Facebook, Inc. and its affiliates. +Home page: https://github.com/facebook/folly +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +The file cpp/src/arrow/vendored/musl/strptime.c has the following license + +Copyright © 2005-2020 Rich Felker, et al. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +The file cpp/cmake_modules/BuildUtils.cmake contains code from + +https://gist.github.com/cristianadam/ef920342939a89fae3e8a85ca9459b49 + +which is made available under the MIT license + +Copyright (c) 2019 Cristian Adam + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/portable-snippets/ contain code from + +https://github.com/nemequ/portable-snippets + +and have the following copyright notice: + +Each source file contains a preamble explaining the license situation +for that file, which takes priority over this file. With the +exception of some code pulled in from other repositories (such as +µnit, an MIT-licensed project which is used for testing), the code is +public domain, released using the CC0 1.0 Universal dedication (*). + +(*) https://creativecommons.org/publicdomain/zero/1.0/legalcode + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/fast_float/ contain code from + +https://github.com/lemire/fast_float + +which is made available under the Apache License 2.0. + +-------------------------------------------------------------------------------- + +The file python/pyarrow/vendored/docscrape.py contains code from + +https://github.com/numpy/numpydoc/ + +which is made available under the BSD 2-clause license. + +-------------------------------------------------------------------------------- + +The file python/pyarrow/vendored/version.py contains code from + +https://github.com/pypa/packaging/ + +which is made available under both the Apache license v2.0 and the +BSD 2-clause license. + +-------------------------------------------------------------------------------- + +The files in cpp/src/arrow/vendored/pcg contain code from + +https://github.com/imneme/pcg-cpp + +and have the following copyright notice: + +Copyright 2014-2019 Melissa O'Neill , + and the PCG Project contributors. + +SPDX-License-Identifier: (Apache-2.0 OR MIT) + +Licensed under the Apache License, Version 2.0 (provided in +LICENSE-APACHE.txt and at http://www.apache.org/licenses/LICENSE-2.0) +or under the MIT license (provided in LICENSE-MIT.txt and at +http://opensource.org/licenses/MIT), at your option. This file may not +be copied, modified, or distributed except according to those terms. + +Distributed on an "AS IS" BASIS, WITHOUT WARRANTY OF ANY KIND, either +express or implied. See your chosen license for details. + +-------------------------------------------------------------------------------- +r/R/dplyr-count-tally.R (some portions) + +Some portions of this file are derived from code from + +https://github.com/tidyverse/dplyr/ + +which is made available under the MIT license + +Copyright (c) 2013-2019 RStudio and others. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the “Software”), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +The file src/arrow/util/io_util.cc contains code from the CPython project +which is made available under the Python Software Foundation License Version 2. + +-------------------------------------------------------------------------------- + +3rdparty dependency opentelemetry-cpp is statically linked in certain binary +distributions. opentelemetry-cpp is made available under the Apache License 2.0. + +Copyright The OpenTelemetry Authors +SPDX-License-Identifier: Apache-2.0 + +-------------------------------------------------------------------------------- + +ci/conan/ is based on code from Conan Package and Dependency Manager. + +Copyright (c) 2019 Conan.io + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- + +3rdparty dependency UCX is redistributed as a dynamically linked shared +library in certain binary distributions. UCX has the following license: + +Copyright (c) 2014-2015 UT-Battelle, LLC. All rights reserved. +Copyright (C) 2014-2020 Mellanox Technologies Ltd. All rights reserved. +Copyright (C) 2014-2015 The University of Houston System. All rights reserved. +Copyright (C) 2015 The University of Tennessee and The University + of Tennessee Research Foundation. All rights reserved. +Copyright (C) 2016-2020 ARM Ltd. All rights reserved. +Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved. +Copyright (C) 2016-2020 Advanced Micro Devices, Inc. All rights reserved. +Copyright (C) 2019 UChicago Argonne, LLC. All rights reserved. +Copyright (c) 2018-2020 NVIDIA CORPORATION. All rights reserved. +Copyright (C) 2020 Huawei Technologies Co., Ltd. All rights reserved. +Copyright (C) 2016-2020 Stony Brook University. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- + +The file dev/tasks/r/github.packages.yml contains code from + +https://github.com/ursa-labs/arrow-r-nightly + +which is made available under the Apache License 2.0. + +-------------------------------------------------------------------------------- +.github/actions/sync-nightlies/action.yml (some portions) + +Some portions of this file are derived from code from + +https://github.com/JoshPiper/rsync-docker + +which is made available under the MIT license + +Copyright (c) 2020 Joshua Piper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +.github/actions/sync-nightlies/action.yml (some portions) + +Some portions of this file are derived from code from + +https://github.com/burnett01/rsync-deployments + +which is made available under the MIT license + +Copyright (c) 2019-2022 Contention +Copyright (c) 2019-2022 Burnett01 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectHashMap.java +java/vector/src/main/java/org/apache/arrow/vector/util/IntObjectMap.java + +These file are derived from code from Netty, which is made available under the +Apache License 2.0. diff --git a/plugins/arrow-flight-rpc/licenses/flight-core-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/flight-core-NOTICE.txt new file mode 100644 index 0000000000000..2089c6fb20358 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/flight-core-NOTICE.txt @@ -0,0 +1,84 @@ +Apache Arrow +Copyright 2016-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This product includes software from the SFrame project (BSD, 3-clause). +* Copyright (C) 2015 Dato, Inc. +* Copyright (c) 2009 Carnegie Mellon University. + +This product includes software from the Feather project (Apache 2.0) +https://github.com/wesm/feather + +This product includes software from the DyND project (BSD 2-clause) +https://github.com/libdynd + +This product includes software from the LLVM project + * distributed under the University of Illinois Open Source + +This product includes software from the google-lint project + * Copyright (c) 2009 Google Inc. All rights reserved. + +This product includes software from the mman-win32 project + * Copyright https://code.google.com/p/mman-win32/ + * Licensed under the MIT License; + +This product includes software from the LevelDB project + * Copyright (c) 2011 The LevelDB Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * Moved from Kudu http://github.com/cloudera/kudu + +This product includes software from the CMake project + * Copyright 2001-2009 Kitware, Inc. + * Copyright 2012-2014 Continuum Analytics, Inc. + * All rights reserved. + +This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause) + * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved. + +This product includes software from the Ibis project (Apache 2.0) + * Copyright (c) 2015 Cloudera, Inc. + * https://github.com/cloudera/ibis + +This product includes software from Dremio (Apache 2.0) + * Copyright (C) 2017-2018 Dremio Corporation + * https://github.com/dremio/dremio-oss + +This product includes software from Google Guava (Apache 2.0) + * Copyright (C) 2007 The Guava Authors + * https://github.com/google/guava + +This product include software from CMake (BSD 3-Clause) + * CMake - Cross Platform Makefile Generator + * Copyright 2000-2019 Kitware, Inc. and Contributors + +The web site includes files generated by Jekyll. + +-------------------------------------------------------------------------------- + +This product includes code from Apache Kudu, which includes the following in +its NOTICE file: + + Apache Kudu + Copyright 2016 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Portions of this software were developed at + Cloudera, Inc (http://www.cloudera.com/). + +-------------------------------------------------------------------------------- + +This product includes code from Apache ORC, which includes the following in +its NOTICE file: + + Apache ORC + Copyright 2013-2019 The Apache Software Foundation + + This product includes software developed by The Apache Software + Foundation (http://www.apache.org/). + + This product includes software developed by Hewlett-Packard: + (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P diff --git a/libs/arrow-spi/licenses/netty-common-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/grpc-LICENSE.txt similarity index 99% rename from libs/arrow-spi/licenses/netty-common-LICENSE.txt rename to plugins/arrow-flight-rpc/licenses/grpc-LICENSE.txt index 62589edd12a37..d645695673349 100644 --- a/libs/arrow-spi/licenses/netty-common-LICENSE.txt +++ b/plugins/arrow-flight-rpc/licenses/grpc-LICENSE.txt @@ -1,7 +1,7 @@ Apache License Version 2.0, January 2004 - https://www.apache.org/licenses/ + http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION @@ -193,7 +193,7 @@ you may not use this file except in compliance with the License. You may obtain a copy of the License at - https://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/plugins/arrow-flight-rpc/licenses/grpc-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/grpc-NOTICE.txt new file mode 100644 index 0000000000000..f70c5620cf75a --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-NOTICE.txt @@ -0,0 +1,62 @@ +Copyright 2014 The gRPC Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +----------------------------------------------------------------------- + +This product contains a modified portion of 'OkHttp', an open source +HTTP & SPDY client for Android and Java applications, which can be obtained +at: + + * LICENSE: + * okhttp/third_party/okhttp/LICENSE (Apache License 2.0) + * HOMEPAGE: + * https://github.com/square/okhttp + * LOCATION_IN_GRPC: + * okhttp/third_party/okhttp + +This product contains a modified portion of 'Envoy', an open source +cloud-native high-performance edge/middle/service proxy, which can be +obtained at: + + * LICENSE: + * xds/third_party/envoy/LICENSE (Apache License 2.0) + * NOTICE: + * xds/third_party/envoy/NOTICE + * HOMEPAGE: + * https://www.envoyproxy.io + * LOCATION_IN_GRPC: + * xds/third_party/envoy + +This product contains a modified portion of 'protoc-gen-validate (PGV)', +an open source protoc plugin to generate polyglot message validators, +which can be obtained at: + + * LICENSE: + * xds/third_party/protoc-gen-validate/LICENSE (Apache License 2.0) + * NOTICE: + * xds/third_party/protoc-gen-validate/NOTICE + * HOMEPAGE: + * https://github.com/envoyproxy/protoc-gen-validate + * LOCATION_IN_GRPC: + * xds/third_party/protoc-gen-validate + +This product contains a modified portion of 'udpa', +an open source universal data plane API, which can be obtained at: + + * LICENSE: + * xds/third_party/udpa/LICENSE (Apache License 2.0) + * HOMEPAGE: + * https://github.com/cncf/udpa + * LOCATION_IN_GRPC: + * xds/third_party/udpa diff --git a/plugins/arrow-flight-rpc/licenses/grpc-api-1.68.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/grpc-api-1.68.2.jar.sha1 new file mode 100644 index 0000000000000..1844172dec982 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-api-1.68.2.jar.sha1 @@ -0,0 +1 @@ +a257a5dd25dda1c97a99b56d5b9c1e56c12ae554 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/grpc-core-1.68.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/grpc-core-1.68.2.jar.sha1 new file mode 100644 index 0000000000000..e20345d29e914 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-core-1.68.2.jar.sha1 @@ -0,0 +1 @@ +b0fd51a1c029785d1c9ae2cfc80a296b60dfcfdb \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/grpc-netty-1.68.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/grpc-netty-1.68.2.jar.sha1 new file mode 100644 index 0000000000000..36be00ed13330 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-netty-1.68.2.jar.sha1 @@ -0,0 +1 @@ +3c3279d2e3520195fd26e0c3d9aca2ed1157d8c3 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/grpc-protobuf-1.68.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/grpc-protobuf-1.68.2.jar.sha1 new file mode 100644 index 0000000000000..e861b41837f33 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-protobuf-1.68.2.jar.sha1 @@ -0,0 +1 @@ +35b28e0d57874021cd31e76dd4a795f76a82471e \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/grpc-protobuf-lite-1.68.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/grpc-protobuf-lite-1.68.2.jar.sha1 new file mode 100644 index 0000000000000..b2401f9752829 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-protobuf-lite-1.68.2.jar.sha1 @@ -0,0 +1 @@ +a53064b896adcfefe74362a33e111492351dfc03 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/grpc-stub-1.68.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/grpc-stub-1.68.2.jar.sha1 new file mode 100644 index 0000000000000..118464f8f48ff --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/grpc-stub-1.68.2.jar.sha1 @@ -0,0 +1 @@ +d58ee1cf723b4b5536d44b67e328c163580a8d98 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/guava-33.3.1-jre.jar.sha1 b/plugins/arrow-flight-rpc/licenses/guava-33.3.1-jre.jar.sha1 new file mode 100644 index 0000000000000..ce59350c0d430 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/guava-33.3.1-jre.jar.sha1 @@ -0,0 +1 @@ +852f8b363da0111e819460021ca693cacca3e8db \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/guava-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/guava-LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/guava-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/arrow-flight-rpc/licenses/guava-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/guava-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/libs/arrow-spi/licenses/jackson-annotations-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/jackson-LICENSE similarity index 100% rename from libs/arrow-spi/licenses/jackson-annotations-LICENSE.txt rename to plugins/arrow-flight-rpc/licenses/jackson-LICENSE diff --git a/libs/arrow-spi/licenses/jackson-annotations-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/jackson-NOTICE similarity index 100% rename from libs/arrow-spi/licenses/jackson-annotations-NOTICE.txt rename to plugins/arrow-flight-rpc/licenses/jackson-NOTICE diff --git a/libs/arrow-spi/licenses/jackson-annotations-2.18.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/jackson-annotations-2.18.2.jar.sha1 similarity index 100% rename from libs/arrow-spi/licenses/jackson-annotations-2.18.2.jar.sha1 rename to plugins/arrow-flight-rpc/licenses/jackson-annotations-2.18.2.jar.sha1 diff --git a/libs/arrow-spi/licenses/jackson-databind-2.18.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/jackson-databind-2.18.2.jar.sha1 similarity index 100% rename from libs/arrow-spi/licenses/jackson-databind-2.18.2.jar.sha1 rename to plugins/arrow-flight-rpc/licenses/jackson-databind-2.18.2.jar.sha1 diff --git a/plugins/arrow-flight-rpc/licenses/jsr305-3.0.2.jar.sha1 b/plugins/arrow-flight-rpc/licenses/jsr305-3.0.2.jar.sha1 new file mode 100644 index 0000000000000..c5c92d87b9d6c --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/jsr305-3.0.2.jar.sha1 @@ -0,0 +1 @@ +25ea2e8b0c338a877313bd4672d3fe056ea78f0d \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/jsr305-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/jsr305-LICENSE.txt new file mode 100644 index 0000000000000..0cb8710c4b3e5 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/jsr305-LICENSE.txt @@ -0,0 +1,29 @@ +Copyright (c) 2007-2009, JSR305 expert group +All rights reserved. + +http://www.opensource.org/licenses/bsd-license.php + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the JSR305 expert group nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + diff --git a/plugins/arrow-flight-rpc/licenses/jsr305-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/jsr305-NOTICE.txt new file mode 100644 index 0000000000000..8d1c8b69c3fce --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/jsr305-NOTICE.txt @@ -0,0 +1 @@ + diff --git a/libs/arrow-spi/licenses/netty-buffer-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/netty-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/netty-buffer-LICENSE.txt rename to plugins/arrow-flight-rpc/licenses/netty-LICENSE.txt diff --git a/libs/arrow-spi/licenses/netty-buffer-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/netty-NOTICE.txt similarity index 100% rename from libs/arrow-spi/licenses/netty-buffer-NOTICE.txt rename to plugins/arrow-flight-rpc/licenses/netty-NOTICE.txt diff --git a/libs/arrow-spi/licenses/netty-buffer-4.1.118.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-buffer-4.1.118.Final.jar.sha1 similarity index 100% rename from libs/arrow-spi/licenses/netty-buffer-4.1.118.Final.jar.sha1 rename to plugins/arrow-flight-rpc/licenses/netty-buffer-4.1.118.Final.jar.sha1 diff --git a/plugins/arrow-flight-rpc/licenses/netty-codec-4.1.118.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-codec-4.1.118.Final.jar.sha1 new file mode 100644 index 0000000000000..7964f25f0372a --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-codec-4.1.118.Final.jar.sha1 @@ -0,0 +1 @@ +307f665c08ce57333121de4f460479fc0c3c94d4 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-codec-http-4.1.118.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-codec-http-4.1.118.Final.jar.sha1 new file mode 100644 index 0000000000000..7cb43dd276c8a --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-codec-http-4.1.118.Final.jar.sha1 @@ -0,0 +1 @@ +eda08a71294afe78c779b85fd696bc13491507a8 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-codec-http2-4.1.118.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-codec-http2-4.1.118.Final.jar.sha1 new file mode 100644 index 0000000000000..fab58dee2dfbf --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-codec-http2-4.1.118.Final.jar.sha1 @@ -0,0 +1 @@ +e3c35c0685ec9e84c4f84b79feea7c9d185a08d3 \ No newline at end of file diff --git a/libs/arrow-spi/licenses/netty-common-4.1.118.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-common-4.1.118.Final.jar.sha1 similarity index 100% rename from libs/arrow-spi/licenses/netty-common-4.1.118.Final.jar.sha1 rename to plugins/arrow-flight-rpc/licenses/netty-common-4.1.118.Final.jar.sha1 diff --git a/plugins/arrow-flight-rpc/licenses/netty-handler-4.1.118.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-handler-4.1.118.Final.jar.sha1 new file mode 100644 index 0000000000000..d6eea2494813e --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-handler-4.1.118.Final.jar.sha1 @@ -0,0 +1 @@ +30ebb05b6b0fb071dbfcf713017c4a767a97bb9b \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-resolver-4.1.118.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-resolver-4.1.118.Final.jar.sha1 new file mode 100644 index 0000000000000..19fbdbbb19b04 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-resolver-4.1.118.Final.jar.sha1 @@ -0,0 +1 @@ +28c378c19c1779eca1104b400452627f3ebc4aea \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-tcnative-classes-2.0.66.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-tcnative-classes-2.0.66.Final.jar.sha1 new file mode 100644 index 0000000000000..7bc4213520498 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-tcnative-classes-2.0.66.Final.jar.sha1 @@ -0,0 +1 @@ +9588bd2f891157538a78d86c945aa34bf9308dda \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-transport-4.1.118.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-transport-4.1.118.Final.jar.sha1 new file mode 100644 index 0000000000000..f3b714539e61b --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-transport-4.1.118.Final.jar.sha1 @@ -0,0 +1 @@ +5a27232e5d08218722d94ca14f0b1b4576e7711c \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-transport-classes-epoll-4.1.118.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-transport-classes-epoll-4.1.118.Final.jar.sha1 new file mode 100644 index 0000000000000..d53656cd3b7dc --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-transport-classes-epoll-4.1.118.Final.jar.sha1 @@ -0,0 +1 @@ +376ce95507066f0e755d97c1c8bcd6c33f657617 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/netty-transport-native-unix-common-4.1.118.Final.jar.sha1 b/plugins/arrow-flight-rpc/licenses/netty-transport-native-unix-common-4.1.118.Final.jar.sha1 new file mode 100644 index 0000000000000..f1562364e2848 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/netty-transport-native-unix-common-4.1.118.Final.jar.sha1 @@ -0,0 +1 @@ +9da25a94e6a0edac90da0bc7894e5a54efcb866b \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/parquet-arrow-1.13.1.jar.sha1 b/plugins/arrow-flight-rpc/licenses/parquet-arrow-1.13.1.jar.sha1 new file mode 100644 index 0000000000000..a1b89891ca8e1 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/parquet-arrow-1.13.1.jar.sha1 @@ -0,0 +1 @@ +9e59add52791af8b05c1aefe2a2f8865602c9368 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/parquet-arrow-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/parquet-arrow-LICENSE.txt new file mode 100644 index 0000000000000..b0065815a5e92 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/parquet-arrow-LICENSE.txt @@ -0,0 +1,218 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +-------------------------------------------------------------------------------- + +This product includes code from Apache Avro. + +Copyright: 2014 The Apache Software Foundation. +Home page: https://avro.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This project includes code from Daniel Lemire's JavaFastPFOR project. The +"Lemire" bit packing source code produced by parquet-generator is derived from +the JavaFastPFOR project. + +Copyright: 2013 Daniel Lemire +Home page: http://lemire.me/en/ +Project page: https://github.com/lemire/JavaFastPFOR +License: Apache License Version 2.0 http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Apache Spark. + +* dev/merge_parquet_pr.py is based on Spark's dev/merge_spark_pr.py + +Copyright: 2014 The Apache Software Foundation. +Home page: https://spark.apache.org/ +License: http://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Twitter's ElephantBird project. + +* parquet-hadoop's UnmaterializableRecordCounter.java includes code from + ElephantBird's LzoRecordReader.java + +Copyright: 2012-2014 Twitter +Home page: https://github.com/twitter/elephant-bird +License: http://www.apache.org/licenses/LICENSE-2.0 + diff --git a/plugins/arrow-flight-rpc/licenses/parquet-arrow-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/parquet-arrow-NOTICE.txt new file mode 100644 index 0000000000000..46300d6cd98fd --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/parquet-arrow-NOTICE.txt @@ -0,0 +1,94 @@ + +Apache Parquet Java +Copyright 2014-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +-------------------------------------------------------------------------------- + +This product includes parquet-tools, initially developed at ARRIS, Inc. with +the following copyright notice: + + Copyright 2013 ARRIS, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +This product includes parquet-protobuf, initially developed by Lukas Nalezenc +with the following copyright notice: + + Copyright 2013 Lukas Nalezenec. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +This product includes code from Apache Avro, which includes the following in +its NOTICE file: + + Apache Avro + Copyright 2010-2015 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + +-------------------------------------------------------------------------------- + +This project includes code from Kite, developed at Cloudera, Inc. with +the following copyright notice: + +| Copyright 2013 Cloudera Inc. +| +| Licensed under the Apache License, Version 2.0 (the "License"); +| you may not use this file except in compliance with the License. +| You may obtain a copy of the License at +| +| http://www.apache.org/licenses/LICENSE-2.0 +| +| Unless required by applicable law or agreed to in writing, software +| distributed under the License is distributed on an "AS IS" BASIS, +| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +| See the License for the specific language governing permissions and +| limitations under the License. + +-------------------------------------------------------------------------------- + +This project includes code from Netflix, Inc. with the following copyright +notice: + +| Copyright 2016 Netflix, Inc. +| +| Licensed under the Apache License, Version 2.0 (the "License"); +| you may not use this file except in compliance with the License. +| You may obtain a copy of the License at +| +| http://www.apache.org/licenses/LICENSE-2.0 +| +| Unless required by applicable law or agreed to in writing, software +| distributed under the License is distributed on an "AS IS" BASIS, +| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +| See the License for the specific language governing permissions and +| limitations under the License. + diff --git a/plugins/arrow-flight-rpc/licenses/perfmark-api-0.27.0.jar.sha1 b/plugins/arrow-flight-rpc/licenses/perfmark-api-0.27.0.jar.sha1 new file mode 100644 index 0000000000000..c85ee41fd9bbd --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/perfmark-api-0.27.0.jar.sha1 @@ -0,0 +1 @@ +f86f575a41b091786a4b027cd9c0c1d2e3fc1c01 \ No newline at end of file diff --git a/plugins/arrow-flight-rpc/licenses/perfmark-api-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/perfmark-api-LICENSE.txt new file mode 100644 index 0000000000000..261eeb9e9f8b2 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/perfmark-api-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/arrow-flight-rpc/licenses/perfmark-api-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/perfmark-api-NOTICE.txt new file mode 100644 index 0000000000000..04fbb4e692e51 --- /dev/null +++ b/plugins/arrow-flight-rpc/licenses/perfmark-api-NOTICE.txt @@ -0,0 +1,41 @@ + +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +----------------------------------------------------------------------- + +This product contains a modified portion of 'Catapult', an open source +Trace Event viewer for Chome, Linux, and Android applications, which can +be obtained at: + + * LICENSE: + * traceviewer/src/main/resources/io/perfmark/traceviewer/third_party/catapult/LICENSE (New BSD License) + * HOMEPAGE: + * https://github.com/catapult-project/catapult + +This product contains a modified portion of 'Polymer', a library for Web +Components, which can be obtained at: + * LICENSE: + * traceviewer/src/main/resources/io/perfmark/traceviewer/third_party/polymer/LICENSE (New BSD License) + * HOMEPAGE: + * https://github.com/Polymer/polymer + + +This product contains a modified portion of 'ASM', an open source +Java Bytecode library, which can be obtained at: + + * LICENSE: + * agent/src/main/resources/io/perfmark/agent/third_party/asm/LICENSE (BSD style License) + * HOMEPAGE: + * https://asm.ow2.io/ \ No newline at end of file diff --git a/libs/arrow-spi/licenses/slf4j-api-1.7.36.jar.sha1 b/plugins/arrow-flight-rpc/licenses/slf4j-api-1.7.36.jar.sha1 similarity index 100% rename from libs/arrow-spi/licenses/slf4j-api-1.7.36.jar.sha1 rename to plugins/arrow-flight-rpc/licenses/slf4j-api-1.7.36.jar.sha1 diff --git a/libs/arrow-spi/licenses/slf4j-api-LICENSE.txt b/plugins/arrow-flight-rpc/licenses/slf4j-api-LICENSE.txt similarity index 100% rename from libs/arrow-spi/licenses/slf4j-api-LICENSE.txt rename to plugins/arrow-flight-rpc/licenses/slf4j-api-LICENSE.txt diff --git a/plugins/arrow-flight-rpc/licenses/slf4j-api-NOTICE.txt b/plugins/arrow-flight-rpc/licenses/slf4j-api-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/arrow-flight-rpc/src/internalClusterTest/java/org/opensearch/arrow/flight/ArrowFlightServerIT.java b/plugins/arrow-flight-rpc/src/internalClusterTest/java/org/opensearch/arrow/flight/ArrowFlightServerIT.java new file mode 100644 index 0000000000000..f3aae77bfabf0 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/internalClusterTest/java/org/opensearch/arrow/flight/ArrowFlightServerIT.java @@ -0,0 +1,307 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight; + +import org.apache.arrow.flight.CallOptions; +import org.apache.arrow.flight.FlightDescriptor; +import org.apache.arrow.flight.FlightInfo; +import org.apache.arrow.flight.FlightRuntimeException; +import org.apache.arrow.flight.OSFlightClient; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.IntVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.opensearch.arrow.flight.bootstrap.FlightClientManager; +import org.opensearch.arrow.flight.bootstrap.FlightService; +import org.opensearch.arrow.flight.bootstrap.FlightStreamPlugin; +import org.opensearch.arrow.spi.StreamManager; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.arrow.spi.StreamReader; +import org.opensearch.arrow.spi.StreamTicket; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.junit.BeforeClass; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 3) +public class ArrowFlightServerIT extends OpenSearchIntegTestCase { + + @BeforeClass + public static void setupFeatureFlags() { + FeatureFlagSetter.set(FeatureFlags.ARROW_STREAMS_SETTING.getKey()); + } + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(FlightStreamPlugin.class); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + ensureGreen(); + Thread.sleep(1000); + } + + public void testArrowFlightEndpoint() { + for (DiscoveryNode node : getClusterState().nodes()) { + FlightService flightService = internalCluster().getInstance(FlightService.class, node.getName()); + FlightClientManager flightClientManager = flightService.getFlightClientManager(); + OSFlightClient flightClient = flightClientManager.getFlightClient(node.getId()).get(); + assertNotNull(flightClient); + flightClient.handshake(CallOptions.timeout(5000L, TimeUnit.MILLISECONDS)); + } + } + + public void testFlightStreamReader() throws Exception { + for (DiscoveryNode node : getClusterState().nodes()) { + StreamManager streamManagerRandomNode = getStreamManagerRandomNode(); + StreamTicket ticket = streamManagerRandomNode.registerStream(getStreamProducer(), null); + StreamManager streamManagerCurrentNode = getStreamManager(node.getName()); + // reader should be accessible from any node in the cluster due to the use ProxyStreamProducer + try (StreamReader reader = streamManagerCurrentNode.getStreamReader(ticket)) { + int totalBatches = 0; + assertNotNull(reader.getRoot().getVector("docID")); + while (reader.next()) { + IntVector docIDVector = (IntVector) reader.getRoot().getVector("docID"); + assertEquals(10, docIDVector.getValueCount()); + for (int i = 0; i < 10; i++) { + assertEquals(docIDVector.toString(), i + (totalBatches * 10L), docIDVector.get(i)); + } + totalBatches++; + } + assertEquals(10, totalBatches); + } + } + } + + public void testEarlyCancel() throws Exception { + DiscoveryNode previousNode = null; + for (DiscoveryNode node : getClusterState().nodes()) { + if (previousNode == null) { + previousNode = node; + continue; + } + StreamManager streamManagerServer = getStreamManager(node.getName()); + TestStreamProducer streamProducer = getStreamProducer(); + StreamTicket ticket = streamManagerServer.registerStream(streamProducer, null); + StreamManager streamManagerClient = getStreamManager(previousNode.getName()); + + CountDownLatch readerComplete = new CountDownLatch(1); + AtomicReference readerException = new AtomicReference<>(); + AtomicReference> readerRef = new AtomicReference<>(); + + // Start reader thread + Thread readerThread = new Thread(() -> { + try { + StreamReader reader = streamManagerClient.getStreamReader(ticket); + readerRef.set(reader); + assertNotNull(reader.getRoot()); + IntVector docIDVector = (IntVector) reader.getRoot().getVector("docID"); + assertNotNull(docIDVector); + + // Read first batch + reader.next(); + assertEquals(10, docIDVector.getValueCount()); + for (int i = 0; i < 10; i++) { + assertEquals(docIDVector.toString(), i, docIDVector.get(i)); + } + reader.close(); + } catch (Exception e) { + readerException.set(e); + } finally { + readerComplete.countDown(); + } + }, "flight-reader-thread"); + + readerThread.start(); + assertTrue("Reader thread did not complete in time", readerComplete.await(1, TimeUnit.SECONDS)); + + if (readerException.get() != null) { + throw readerException.get(); + } + + StreamReader reader = readerRef.get(); + + try { + reader.next(); + fail("Expected FlightRuntimeException"); + } catch (FlightRuntimeException e) { + assertEquals("CANCELLED", e.status().code().name()); + assertEquals("Stream closed before end", e.getMessage()); + reader.close(); + } + + // Wait for close to complete + // Due to https://github.com/grpc/grpc-java/issues/5882, there is a logic in FlightStream.java + // where it exhausts the stream on the server side before it is actually cancelled. + assertTrue( + "Timeout waiting for stream cancellation on server [" + node.getName() + "]", + streamProducer.waitForClose(2, TimeUnit.SECONDS) + ); + previousNode = node; + } + } + + public void testFlightStreamServerError() throws Exception { + DiscoveryNode previousNode = null; + for (DiscoveryNode node : getClusterState().nodes()) { + if (previousNode == null) { + previousNode = node; + continue; + } + StreamManager streamManagerServer = getStreamManager(node.getName()); + TestStreamProducer streamProducer = getStreamProducer(); + streamProducer.setProduceError(true); + StreamTicket ticket = streamManagerServer.registerStream(streamProducer, null); + StreamManager streamManagerClient = getStreamManager(previousNode.getName()); + try (StreamReader reader = streamManagerClient.getStreamReader(ticket)) { + int totalBatches = 0; + assertNotNull(reader.getRoot().getVector("docID")); + try { + while (reader.next()) { + IntVector docIDVector = (IntVector) reader.getRoot().getVector("docID"); + assertEquals(10, docIDVector.getValueCount()); + totalBatches++; + } + fail("Expected FlightRuntimeException"); + } catch (FlightRuntimeException e) { + assertEquals("INTERNAL", e.status().code().name()); + assertEquals("There was an error servicing your request.", e.getMessage()); + } + assertEquals(1, totalBatches); + } + previousNode = node; + } + } + + public void testFlightGetInfo() throws Exception { + StreamTicket ticket = null; + for (DiscoveryNode node : getClusterState().nodes()) { + FlightService flightService = internalCluster().getInstance(FlightService.class, node.getName()); + StreamManager streamManager = flightService.getStreamManager(); + if (ticket == null) { + ticket = streamManager.registerStream(getStreamProducer(), null); + } + FlightClientManager flightClientManager = flightService.getFlightClientManager(); + OSFlightClient flightClient = flightClientManager.getFlightClient(node.getId()).get(); + assertNotNull(flightClient); + FlightDescriptor flightDescriptor = FlightDescriptor.command(ticket.toBytes()); + FlightInfo flightInfo = flightClient.getInfo(flightDescriptor, CallOptions.timeout(5000L, TimeUnit.MILLISECONDS)); + assertNotNull(flightInfo); + assertEquals(100, flightInfo.getRecords()); + } + } + + private StreamManager getStreamManager(String nodeName) { + FlightService flightService = internalCluster().getInstance(FlightService.class, nodeName); + return flightService.getStreamManager(); + } + + private StreamManager getStreamManagerRandomNode() { + FlightService flightService = internalCluster().getInstance(FlightService.class); + return flightService.getStreamManager(); + } + + private TestStreamProducer getStreamProducer() { + return new TestStreamProducer(); + } + + private static class TestStreamProducer implements StreamProducer { + volatile boolean isClosed = false; + private final CountDownLatch closeLatch = new CountDownLatch(1); + TimeValue deadline = TimeValue.timeValueSeconds(5); + private volatile boolean produceError = false; + + public void setProduceError(boolean produceError) { + this.produceError = produceError; + } + + TestStreamProducer() {} + + VectorSchemaRoot root; + + @Override + public VectorSchemaRoot createRoot(BufferAllocator allocator) { + IntVector docIDVector = new IntVector("docID", allocator); + FieldVector[] vectors = new FieldVector[] { docIDVector }; + root = new VectorSchemaRoot(Arrays.asList(vectors)); + return root; + } + + @Override + public BatchedJob createJob(BufferAllocator allocator) { + return new BatchedJob<>() { + @Override + public void run(VectorSchemaRoot root, FlushSignal flushSignal) { + IntVector docIDVector = (IntVector) root.getVector("docID"); + root.setRowCount(10); + for (int i = 0; i < 100; i++) { + docIDVector.setSafe(i % 10, i); + if ((i + 1) % 10 == 0) { + flushSignal.awaitConsumption(TimeValue.timeValueMillis(1000)); + docIDVector.clear(); + root.setRowCount(10); + if (produceError) { + throw new RuntimeException("Server error while producing batch"); + } + } + } + } + + @Override + public void onCancel() { + root.close(); + isClosed = true; + } + + @Override + public boolean isCancelled() { + return isClosed; + } + }; + } + + @Override + public TimeValue getJobDeadline() { + return deadline; + } + + @Override + public int estimatedRowCount() { + return 100; + } + + @Override + public String getAction() { + return ""; + } + + @Override + public void close() { + root.close(); + closeLatch.countDown(); + isClosed = true; + } + + public boolean waitForClose(long timeout, TimeUnit unit) throws InterruptedException { + return closeLatch.await(timeout, unit); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/OSFlightClient.java b/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/OSFlightClient.java new file mode 100644 index 0000000000000..88675f23b7059 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/OSFlightClient.java @@ -0,0 +1,948 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.apache.arrow.flight; + +import org.apache.arrow.flight.FlightProducer.StreamListener; +import org.apache.arrow.flight.auth.BasicClientAuthHandler; +import org.apache.arrow.flight.auth.ClientAuthHandler; +import org.apache.arrow.flight.auth.ClientAuthInterceptor; +import org.apache.arrow.flight.auth.ClientAuthWrapper; +import org.apache.arrow.flight.auth2.BasicAuthCredentialWriter; +import org.apache.arrow.flight.auth2.ClientBearerHeaderHandler; +import org.apache.arrow.flight.auth2.ClientHandshakeWrapper; +import org.apache.arrow.flight.auth2.ClientIncomingAuthHeaderMiddleware; +import org.apache.arrow.flight.grpc.ClientInterceptorAdapter; +import org.apache.arrow.flight.grpc.CredentialCallOption; +import org.apache.arrow.flight.grpc.StatusUtils; +import org.apache.arrow.flight.impl.Flight; +import org.apache.arrow.flight.impl.Flight.Empty; +import org.apache.arrow.flight.impl.FlightServiceGrpc; +import org.apache.arrow.flight.impl.FlightServiceGrpc.FlightServiceBlockingStub; +import org.apache.arrow.flight.impl.FlightServiceGrpc.FlightServiceStub; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider; + +import javax.net.ssl.SSLException; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.BooleanSupplier; + +import io.grpc.Channel; +import io.grpc.ClientCall; +import io.grpc.ClientInterceptor; +import io.grpc.ClientInterceptors; +import io.grpc.ManagedChannel; +import io.grpc.MethodDescriptor; +import io.grpc.StatusRuntimeException; +import io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.NettyChannelBuilder; +import io.grpc.stub.ClientCallStreamObserver; +import io.grpc.stub.ClientCalls; +import io.grpc.stub.ClientResponseObserver; +import io.grpc.stub.StreamObserver; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.ServerChannel; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; + +/** + * Clone of {@link FlightClient} to support setting SslContext directly. It can be discarded once + * FlightClient.Builder supports setting SslContext directly. + */ +@SuppressWarnings("forbiddenApisMain") +public class OSFlightClient implements AutoCloseable { + private static final int PENDING_REQUESTS = 5; + /** + * The maximum number of trace events to keep on the gRPC Channel. This value disables channel + * tracing. + */ + private static final int MAX_CHANNEL_TRACE_EVENTS = 0; + + private final BufferAllocator allocator; + private final ManagedChannel channel; + + private final FlightServiceBlockingStub blockingStub; + private final FlightServiceStub asyncStub; + private final ClientAuthInterceptor authInterceptor = new ClientAuthInterceptor(); + private final MethodDescriptor doGetDescriptor; + private final MethodDescriptor doPutDescriptor; + private final MethodDescriptor doExchangeDescriptor; + private final List middleware; + + /** Create a Flight client from an allocator and a gRPC channel. */ + OSFlightClient(BufferAllocator incomingAllocator, ManagedChannel channel, List middleware) { + this.allocator = incomingAllocator.newChildAllocator("flight-client", 0, Long.MAX_VALUE); + this.channel = channel; + this.middleware = middleware; + + final ClientInterceptor[] interceptors; + interceptors = new ClientInterceptor[] { authInterceptor, new ClientInterceptorAdapter(middleware) }; + + // Create a channel with interceptors pre-applied for DoGet and DoPut + Channel interceptedChannel = ClientInterceptors.intercept(channel, interceptors); + + blockingStub = FlightServiceGrpc.newBlockingStub(interceptedChannel); + asyncStub = FlightServiceGrpc.newStub(interceptedChannel); + doGetDescriptor = FlightBindingService.getDoGetDescriptor(allocator); + doPutDescriptor = FlightBindingService.getDoPutDescriptor(allocator); + doExchangeDescriptor = FlightBindingService.getDoExchangeDescriptor(allocator); + } + + /** + * Get a list of available flights. + * + * @param criteria Criteria for selecting flights + * @param options RPC-layer hints for the call. + * @return FlightInfo Iterable + */ + public Iterable listFlights(Criteria criteria, CallOption... options) { + final Iterator flights; + try { + flights = CallOptions.wrapStub(blockingStub, options).listFlights(criteria.asCriteria()); + } catch (StatusRuntimeException sre) { + throw StatusUtils.fromGrpcRuntimeException(sre); + } + return () -> StatusUtils.wrapIterator(flights, t -> { + try { + return new FlightInfo(t); + } catch (URISyntaxException e) { + // We don't expect this will happen for conforming Flight implementations. For + // instance, a Java server + // itself wouldn't be able to construct an invalid Location. + throw new RuntimeException(e); + } + }); + } + + /** + * Lists actions available on the Flight service. + * + * @param options RPC-layer hints for the call. + */ + public Iterable listActions(CallOption... options) { + final Iterator actions; + try { + actions = CallOptions.wrapStub(blockingStub, options).listActions(Empty.getDefaultInstance()); + } catch (StatusRuntimeException sre) { + throw StatusUtils.fromGrpcRuntimeException(sre); + } + return () -> StatusUtils.wrapIterator(actions, ActionType::new); + } + + /** + * Performs an action on the Flight service. + * + * @param action The action to perform. + * @param options RPC-layer hints for this call. + * @return An iterator of results. + */ + public Iterator doAction(Action action, CallOption... options) { + return StatusUtils.wrapIterator(CallOptions.wrapStub(blockingStub, options).doAction(action.toProtocol()), Result::new); + } + + /** Authenticates with a username and password. */ + public void authenticateBasic(String username, String password) { + BasicClientAuthHandler basicClient = new BasicClientAuthHandler(username, password); + authenticate(basicClient); + } + + /** + * Authenticates against the Flight service. + * + * @param options RPC-layer hints for this call. + * @param handler The auth mechanism to use. + */ + public void authenticate(ClientAuthHandler handler, CallOption... options) { + Preconditions.checkArgument(!authInterceptor.hasAuthHandler(), "Auth already completed."); + ClientAuthWrapper.doClientAuth(handler, CallOptions.wrapStub(asyncStub, options)); + authInterceptor.setAuthHandler(handler); + } + + /** + * Authenticates with a username and password. + * + * @param username the username. + * @param password the password. + * @return a CredentialCallOption containing a bearer token if the server emitted one, or empty if + * no bearer token was returned. This can be used in subsequent API calls. + */ + public Optional authenticateBasicToken(String username, String password) { + final ClientIncomingAuthHeaderMiddleware.Factory clientAuthMiddleware = new ClientIncomingAuthHeaderMiddleware.Factory( + new ClientBearerHeaderHandler() + ); + middleware.add(clientAuthMiddleware); + handshake(new CredentialCallOption(new BasicAuthCredentialWriter(username, password))); + + return Optional.ofNullable(clientAuthMiddleware.getCredentialCallOption()); + } + + /** + * Executes the handshake against the Flight service. + * + * @param options RPC-layer hints for this call. + */ + public void handshake(CallOption... options) { + ClientHandshakeWrapper.doClientHandshake(CallOptions.wrapStub(asyncStub, options)); + } + + /** + * Create or append a descriptor with another stream. + * + * @param descriptor FlightDescriptor the descriptor for the data + * @param root VectorSchemaRoot the root containing data + * @param metadataListener A handler for metadata messages from the server. This will be passed + * buffers that will be freed after {@link StreamListener#onNext(Object)} is called! + * @param options RPC-layer hints for this call. + * @return ClientStreamListener an interface to control uploading data + */ + public ClientStreamListener startPut( + FlightDescriptor descriptor, + VectorSchemaRoot root, + PutListener metadataListener, + CallOption... options + ) { + return startPut(descriptor, root, new MapDictionaryProvider(), metadataListener, options); + } + + /** + * Create or append a descriptor with another stream. + * + * @param descriptor FlightDescriptor the descriptor for the data + * @param root VectorSchemaRoot the root containing data + * @param metadataListener A handler for metadata messages from the server. + * @param options RPC-layer hints for this call. + * @return ClientStreamListener an interface to control uploading data. {@link + * ClientStreamListener#start(VectorSchemaRoot, DictionaryProvider)} will already have been + * called. + */ + public ClientStreamListener startPut( + FlightDescriptor descriptor, + VectorSchemaRoot root, + DictionaryProvider provider, + PutListener metadataListener, + CallOption... options + ) { + Preconditions.checkNotNull(root, "root must not be null"); + Preconditions.checkNotNull(provider, "provider must not be null"); + final ClientStreamListener writer = startPut(descriptor, metadataListener, options); + writer.start(root, provider); + return writer; + } + + /** + * Create or append a descriptor with another stream. + * + * @param descriptor FlightDescriptor the descriptor for the data + * @param metadataListener A handler for metadata messages from the server. + * @param options RPC-layer hints for this call. + * @return ClientStreamListener an interface to control uploading data. {@link + * ClientStreamListener#start(VectorSchemaRoot, DictionaryProvider)} will NOT already have + * been called. + */ + public ClientStreamListener startPut(FlightDescriptor descriptor, PutListener metadataListener, CallOption... options) { + Preconditions.checkNotNull(descriptor, "descriptor must not be null"); + Preconditions.checkNotNull(metadataListener, "metadataListener must not be null"); + + try { + final ClientCall call = asyncStubNewCall(doPutDescriptor, options); + final SetStreamObserver resultObserver = new SetStreamObserver(allocator, metadataListener); + ClientCallStreamObserver observer = (ClientCallStreamObserver) ClientCalls.asyncBidiStreamingCall( + call, + resultObserver + ); + return new PutObserver(descriptor, observer, metadataListener::isCancelled, metadataListener::getResult); + } catch (StatusRuntimeException sre) { + throw StatusUtils.fromGrpcRuntimeException(sre); + } + } + + /** + * Get info on a stream. + * + * @param descriptor The descriptor for the stream. + * @param options RPC-layer hints for this call. + */ + public FlightInfo getInfo(FlightDescriptor descriptor, CallOption... options) { + try { + return new FlightInfo(CallOptions.wrapStub(blockingStub, options).getFlightInfo(descriptor.toProtocol())); + } catch (URISyntaxException e) { + // We don't expect this will happen for conforming Flight implementations. For instance, a + // Java server + // itself wouldn't be able to construct an invalid Location. + throw new RuntimeException(e); + } catch (StatusRuntimeException sre) { + throw StatusUtils.fromGrpcRuntimeException(sre); + } + } + + /** + * Start or get info on execution of a long-running query. + * + * @param descriptor The descriptor for the stream. + * @param options RPC-layer hints for this call. + * @return Metadata about execution. + */ + public PollInfo pollInfo(FlightDescriptor descriptor, CallOption... options) { + try { + return new PollInfo(CallOptions.wrapStub(blockingStub, options).pollFlightInfo(descriptor.toProtocol())); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } catch (StatusRuntimeException sre) { + throw StatusUtils.fromGrpcRuntimeException(sre); + } + } + + /** + * Get schema for a stream. + * + * @param descriptor The descriptor for the stream. + * @param options RPC-layer hints for this call. + */ + public SchemaResult getSchema(FlightDescriptor descriptor, CallOption... options) { + try { + return SchemaResult.fromProtocol(CallOptions.wrapStub(blockingStub, options).getSchema(descriptor.toProtocol())); + } catch (StatusRuntimeException sre) { + throw StatusUtils.fromGrpcRuntimeException(sre); + } + } + + /** + * Retrieve a stream from the server. + * + * @param ticket The ticket granting access to the data stream. + * @param options RPC-layer hints for this call. + */ + public FlightStream getStream(Ticket ticket, CallOption... options) { + final ClientCall call = asyncStubNewCall(doGetDescriptor, options); + FlightStream stream = new FlightStream( + allocator, + PENDING_REQUESTS, + (String message, Throwable cause) -> call.cancel(message, cause), + (count) -> call.request(count) + ); + + final StreamObserver delegate = stream.asObserver(); + ClientResponseObserver clientResponseObserver = new ClientResponseObserver< + Flight.Ticket, + ArrowMessage>() { + + @Override + public void beforeStart(ClientCallStreamObserver requestStream) { + requestStream.disableAutoInboundFlowControl(); + } + + @Override + public void onNext(ArrowMessage value) { + delegate.onNext(value); + } + + @Override + public void onError(Throwable t) { + delegate.onError(StatusUtils.toGrpcException(t)); + } + + @Override + public void onCompleted() { + delegate.onCompleted(); + } + }; + + ClientCalls.asyncServerStreamingCall(call, ticket.toProtocol(), clientResponseObserver); + return stream; + } + + /** + * Initiate a bidirectional data exchange with the server. + * + * @param descriptor A descriptor for the data stream. + * @param options RPC call options. + * @return A pair of a readable stream and a writable stream. + */ + public ExchangeReaderWriter doExchange(FlightDescriptor descriptor, CallOption... options) { + Preconditions.checkNotNull(descriptor, "descriptor must not be null"); + + try { + final ClientCall call = asyncStubNewCall(doExchangeDescriptor, options); + final FlightStream stream = new FlightStream(allocator, PENDING_REQUESTS, call::cancel, call::request); + final ClientCallStreamObserver observer = (ClientCallStreamObserver) ClientCalls + .asyncBidiStreamingCall(call, stream.asObserver()); + final ClientStreamListener writer = new PutObserver(descriptor, observer, stream.cancelled::isDone, () -> { + try { + stream.completed.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw CallStatus.INTERNAL.withDescription("Client error: interrupted while completing call") + .withCause(e) + .toRuntimeException(); + } catch (ExecutionException e) { + throw CallStatus.INTERNAL.withDescription("Client error: internal while completing call") + .withCause(e) + .toRuntimeException(); + } + }); + // Send the descriptor to start. + try (final ArrowMessage message = new ArrowMessage(descriptor.toProtocol())) { + observer.onNext(message); + } catch (Exception e) { + throw CallStatus.INTERNAL.withCause(e).withDescription("Could not write descriptor " + descriptor).toRuntimeException(); + } + return new ExchangeReaderWriter(stream, writer); + } catch (StatusRuntimeException sre) { + throw StatusUtils.fromGrpcRuntimeException(sre); + } + } + + /** A pair of a reader and a writer for a DoExchange call. */ + public static class ExchangeReaderWriter implements AutoCloseable { + private final FlightStream reader; + private final ClientStreamListener writer; + + ExchangeReaderWriter(FlightStream reader, ClientStreamListener writer) { + this.reader = reader; + this.writer = writer; + } + + /** Get the reader for the call. */ + public FlightStream getReader() { + return reader; + } + + /** Get the writer for the call. */ + public ClientStreamListener getWriter() { + return writer; + } + + /** + * Make sure stream is drained. You must call this to be notified of any errors that may have + * happened after the exchange is complete. This should be called after + * `getWriter().completed()` and instead of `getWriter().getResult()`. + */ + public void getResult() { + // After exchange is complete, make sure stream is drained to propagate errors through reader + while (reader.next()) { + } + } + + /** Shut down the streams in this call. */ + @Override + public void close() throws Exception { + reader.close(); + } + } + + /** A stream observer for Flight.PutResult. */ + private static class SetStreamObserver implements StreamObserver { + private final BufferAllocator allocator; + private final StreamListener listener; + + SetStreamObserver(BufferAllocator allocator, StreamListener listener) { + super(); + this.allocator = allocator; + this.listener = listener == null ? NoOpStreamListener.getInstance() : listener; + } + + @Override + public void onNext(Flight.PutResult value) { + try (final PutResult message = PutResult.fromProtocol(allocator, value)) { + listener.onNext(message); + } + } + + @Override + public void onError(Throwable t) { + listener.onError(StatusUtils.fromThrowable(t)); + } + + @Override + public void onCompleted() { + listener.onCompleted(); + } + } + + /** The implementation of a {@link ClientStreamListener} for writing data to a Flight server. */ + static class PutObserver extends OutboundStreamListenerImpl implements ClientStreamListener { + private final BooleanSupplier isCancelled; + private final Runnable getResult; + + /** + * Create a new client stream listener. + * + * @param descriptor The descriptor for the stream. + * @param observer The write-side gRPC StreamObserver. + * @param isCancelled A flag to check if the call has been cancelled. + * @param getResult A flag that blocks until the overall call completes. + */ + PutObserver( + FlightDescriptor descriptor, + ClientCallStreamObserver observer, + BooleanSupplier isCancelled, + Runnable getResult + ) { + super(descriptor, observer); + Preconditions.checkNotNull(descriptor, "descriptor must be provided"); + Preconditions.checkNotNull(isCancelled, "isCancelled must be provided"); + Preconditions.checkNotNull(getResult, "getResult must be provided"); + this.isCancelled = isCancelled; + this.getResult = getResult; + this.unloader = null; + } + + @Override + protected void waitUntilStreamReady() { + // Check isCancelled as well to avoid inadvertently blocking forever + // (so long as PutListener properly implements it) + while (!responseObserver.isReady() && !isCancelled.getAsBoolean()) { + /* busy wait */ + } + } + + @Override + public void getResult() { + getResult.run(); + } + } + + /** + * Cancel execution of a distributed query. + * + * @param request The query to cancel. + * @param options Call options. + * @return The server response. + */ + public CancelFlightInfoResult cancelFlightInfo(CancelFlightInfoRequest request, CallOption... options) { + Action action = new Action(FlightConstants.CANCEL_FLIGHT_INFO.getType(), request.serialize().array()); + Iterator results = doAction(action, options); + if (!results.hasNext()) { + throw CallStatus.INTERNAL.withDescription("Server did not return a response").toRuntimeException(); + } + + CancelFlightInfoResult result; + try { + result = CancelFlightInfoResult.deserialize(ByteBuffer.wrap(results.next().getBody())); + } catch (IOException e) { + throw CallStatus.INTERNAL.withDescription("Failed to parse server response: " + e).withCause(e).toRuntimeException(); + } + results.forEachRemaining((ignored) -> {}); + return result; + } + + /** + * Request the server to extend the lifetime of a query result set. + * + * @param request The result set partition. + * @param options Call options. + * @return The new endpoint with an updated expiration time. + */ + public FlightEndpoint renewFlightEndpoint(RenewFlightEndpointRequest request, CallOption... options) { + Action action = new Action(FlightConstants.RENEW_FLIGHT_ENDPOINT.getType(), request.serialize().array()); + Iterator results = doAction(action, options); + if (!results.hasNext()) { + throw CallStatus.INTERNAL.withDescription("Server did not return a response").toRuntimeException(); + } + + FlightEndpoint result; + try { + result = FlightEndpoint.deserialize(ByteBuffer.wrap(results.next().getBody())); + } catch (IOException | URISyntaxException e) { + throw CallStatus.INTERNAL.withDescription("Failed to parse server response: " + e).withCause(e).toRuntimeException(); + } + results.forEachRemaining((ignored) -> {}); + return result; + } + + /** + * Set server session option(s) by name/value. + * + *

Sessions are generally persisted via HTTP cookies. + * + * @param request The session options to set on the server. + * @param options Call options. + * @return The result containing per-value error statuses, if any. + */ + public SetSessionOptionsResult setSessionOptions(SetSessionOptionsRequest request, CallOption... options) { + Action action = new Action(FlightConstants.SET_SESSION_OPTIONS.getType(), request.serialize().array()); + Iterator results = doAction(action, options); + if (!results.hasNext()) { + throw CallStatus.INTERNAL.withDescription("Server did not return a response").toRuntimeException(); + } + + SetSessionOptionsResult result; + try { + result = SetSessionOptionsResult.deserialize(ByteBuffer.wrap(results.next().getBody())); + } catch (IOException e) { + throw CallStatus.INTERNAL.withDescription("Failed to parse server response: " + e).withCause(e).toRuntimeException(); + } + results.forEachRemaining((ignored) -> {}); + return result; + } + + /** + * Get the current server session options. + * + *

The session is generally accessed via an HTTP cookie. + * + * @param request The (empty) GetSessionOptionsRequest. + * @param options Call options. + * @return The result containing the set of session options configured on the server. + */ + public GetSessionOptionsResult getSessionOptions(GetSessionOptionsRequest request, CallOption... options) { + Action action = new Action(FlightConstants.GET_SESSION_OPTIONS.getType(), request.serialize().array()); + Iterator results = doAction(action, options); + if (!results.hasNext()) { + throw CallStatus.INTERNAL.withDescription("Server did not return a response").toRuntimeException(); + } + + GetSessionOptionsResult result; + try { + result = GetSessionOptionsResult.deserialize(ByteBuffer.wrap(results.next().getBody())); + } catch (IOException e) { + throw CallStatus.INTERNAL.withDescription("Failed to parse server response: " + e).withCause(e).toRuntimeException(); + } + results.forEachRemaining((ignored) -> {}); + return result; + } + + /** + * Close/invalidate the current server session. + * + *

The session is generally accessed via an HTTP cookie. + * + * @param request The (empty) CloseSessionRequest. + * @param options Call options. + * @return The result containing the status of the close operation. + */ + public CloseSessionResult closeSession(CloseSessionRequest request, CallOption... options) { + Action action = new Action(FlightConstants.CLOSE_SESSION.getType(), request.serialize().array()); + Iterator results = doAction(action, options); + if (!results.hasNext()) { + throw CallStatus.INTERNAL.withDescription("Server did not return a response").toRuntimeException(); + } + + CloseSessionResult result; + try { + result = CloseSessionResult.deserialize(ByteBuffer.wrap(results.next().getBody())); + } catch (IOException e) { + throw CallStatus.INTERNAL.withDescription("Failed to parse server response: " + e).withCause(e).toRuntimeException(); + } + results.forEachRemaining((ignored) -> {}); + return result; + } + + /** Interface for writers to an Arrow data stream. */ + public interface ClientStreamListener extends OutboundStreamListener { + + /** + * Wait for the stream to finish on the server side. You must call this to be notified of any + * errors that may have happened during the upload. + */ + void getResult(); + } + + /** + * A handler for server-sent application metadata messages during a Flight DoPut operation. + * + *

Generally, instead of implementing this yourself, you should use {@link AsyncPutListener} or + * {@link SyncPutListener}. + */ + public interface PutListener extends StreamListener { + + /** + * Wait for the stream to finish on the server side. You must call this to be notified of any + * errors that may have happened during the upload. + */ + void getResult(); + + /** + * Called when a message from the server is received. + * + * @param val The application metadata. This buffer will be reclaimed once onNext returns; you + * must retain a reference to use it outside this method. + */ + @Override + void onNext(PutResult val); + + /** + * Check if the call has been cancelled. + * + *

By default, this always returns false. Implementations should provide an appropriate + * implementation, as otherwise, a DoPut operation may inadvertently block forever. + */ + default boolean isCancelled() { + return false; + } + } + + /** Shut down this client. */ + @Override + public void close() throws InterruptedException { + channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); + allocator.close(); + } + + /** Create a builder for a Flight client. */ + public static Builder builder() { + return new Builder(); + } + + /** + * Create a builder for a Flight client. + * + * @param allocator The allocator to use for the client. + * @param location The location to connect to. + */ + public static Builder builder(BufferAllocator allocator, Location location) { + return new Builder(allocator, location); + } + + public static Builder builder( + BufferAllocator allocator, + Location location, + Class channelType, + ExecutorService executorService, + EventLoopGroup workerELG, + SslContext sslContext + ) { + Builder builder = new Builder(allocator, location); + builder.channelType(channelType); + builder.executor(executorService); + builder.eventLoopGroup(workerELG); + if (sslContext != null) { + builder.useTLS(sslContext); + } + return builder; + } + + /** A builder for Flight clients. */ + public static final class Builder { + private BufferAllocator allocator; + private Location location; + private boolean forceTls = false; + private int maxInboundMessageSize = FlightServer.MAX_GRPC_MESSAGE_SIZE; + private InputStream trustedCertificates = null; + private InputStream clientCertificate = null; + private InputStream clientKey = null; + private String overrideHostname = null; + private List middleware = new ArrayList<>(); + private boolean verifyServer = true; + private EventLoopGroup workerELG; + private ExecutorService executorService; + private Class channelType; + private SslContext sslContext; + + private Builder() {} + + private Builder(BufferAllocator allocator, Location location) { + this.allocator = Preconditions.checkNotNull(allocator); + this.location = Preconditions.checkNotNull(location); + } + + /** Force the client to connect over TLS. */ + public Builder useTls() { + this.forceTls = true; + return this; + } + + /** Override the hostname checked for TLS. Use with caution in production. */ + public Builder overrideHostname(final String hostname) { + this.overrideHostname = hostname; + return this; + } + + /** Set the maximum inbound message size. */ + public Builder maxInboundMessageSize(int maxSize) { + Preconditions.checkArgument(maxSize > 0); + this.maxInboundMessageSize = maxSize; + return this; + } + + /** Set the trusted TLS certificates. */ + public Builder trustedCertificates(final InputStream stream) { + this.trustedCertificates = Preconditions.checkNotNull(stream); + return this; + } + + /** Set the trusted TLS certificates. */ + public Builder clientCertificate(final InputStream clientCertificate, final InputStream clientKey) { + Preconditions.checkNotNull(clientKey); + this.clientCertificate = Preconditions.checkNotNull(clientCertificate); + this.clientKey = Preconditions.checkNotNull(clientKey); + return this; + } + + public Builder allocator(BufferAllocator allocator) { + this.allocator = Preconditions.checkNotNull(allocator); + return this; + } + + public Builder location(Location location) { + this.location = Preconditions.checkNotNull(location); + return this; + } + + public Builder intercept(FlightClientMiddleware.Factory factory) { + middleware.add(factory); + return this; + } + + public Builder verifyServer(boolean verifyServer) { + this.verifyServer = verifyServer; + return this; + } + + public Builder eventLoopGroup(EventLoopGroup elg) { + this.workerELG = elg; + return this; + } + + public Builder executor(ExecutorService executorService) { + this.executorService = executorService; + return this; + } + + public Builder channelType(Class channelType) { + this.channelType = channelType; + return this; + } + + public Builder useTLS(SslContext sslContext) { + this.sslContext = Objects.requireNonNull(sslContext); + return this; + } + + /** Create the client from this builder. */ + public OSFlightClient build() { + final NettyChannelBuilder builder; + + switch (location.getUri().getScheme()) { + case LocationSchemes.GRPC: + case LocationSchemes.GRPC_INSECURE: + case LocationSchemes.GRPC_TLS: { + builder = NettyChannelBuilder.forAddress(location.toSocketAddress()); + if (workerELG != null) { + builder.eventLoopGroup(workerELG); + } + if (executorService != null) { + builder.executor(executorService); + } + if (channelType != null) { + builder.channelType(channelType); + } + if (sslContext != null) { + builder.sslContext(sslContext); + } + break; + } + case LocationSchemes.GRPC_DOMAIN_SOCKET: { + // The implementation is platform-specific, so we have to find the classes at runtime + builder = NettyChannelBuilder.forAddress(location.toSocketAddress()); + try { + try { + // Linux + builder.channelType( + Class.forName("io.netty.channel.epoll.EpollDomainSocketChannel").asSubclass(ServerChannel.class) + ); + final EventLoopGroup elg = Class.forName("io.netty.channel.epoll.EpollEventLoopGroup") + .asSubclass(EventLoopGroup.class) + .getDeclaredConstructor() + .newInstance(); + builder.eventLoopGroup(elg); + } catch (ClassNotFoundException e) { + // BSD + // this might not work as io.netty.channel.kqueue classes aren't present in grpc-netty-shaded + builder.channelType( + Class.forName("io.netty.channel.kqueue.KQueueDomainSocketChannel").asSubclass(ServerChannel.class) + ); + final EventLoopGroup elg = Class.forName("io.netty.channel.kqueue.KQueueEventLoopGroup") + .asSubclass(EventLoopGroup.class) + .getDeclaredConstructor() + .newInstance(); + builder.eventLoopGroup(elg); + } + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException + | InvocationTargetException e) { + throw new UnsupportedOperationException( + "Could not find suitable Netty native transport implementation for domain socket address." + ); + } + break; + } + default: + throw new IllegalArgumentException("Scheme is not supported: " + location.getUri().getScheme()); + } + + if (this.forceTls || LocationSchemes.GRPC_TLS.equals(location.getUri().getScheme())) { + builder.useTransportSecurity(); + + final boolean hasTrustedCerts = this.trustedCertificates != null; + final boolean hasKeyCertPair = this.clientCertificate != null && this.clientKey != null; + if (!this.verifyServer && (hasTrustedCerts || hasKeyCertPair)) { + throw new IllegalArgumentException( + "FlightClient has been configured to disable server verification, " + "but certificate options have been specified." + ); + } + + final SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient(); + + if (!this.verifyServer) { + sslContextBuilder.trustManager(InsecureTrustManagerFactory.INSTANCE); + } else if (this.trustedCertificates != null || this.clientCertificate != null || this.clientKey != null) { + if (this.trustedCertificates != null) { + sslContextBuilder.trustManager(this.trustedCertificates); + } + if (this.clientCertificate != null && this.clientKey != null) { + sslContextBuilder.keyManager(this.clientCertificate, this.clientKey); + } + } + try { + builder.sslContext(sslContextBuilder.build()); + } catch (SSLException e) { + throw new RuntimeException(e); + } + + if (this.overrideHostname != null) { + builder.overrideAuthority(this.overrideHostname); + } + } else { + builder.usePlaintext(); + } + + builder.maxTraceEvents(MAX_CHANNEL_TRACE_EVENTS) + .maxInboundMessageSize(maxInboundMessageSize) + .maxInboundMetadataSize(maxInboundMessageSize); + return new OSFlightClient(allocator, builder.build(), middleware); + } + } + + /** + * Helper method to create a call from the asyncStub, method descriptor, and list of calling + * options. + */ + private ClientCall asyncStubNewCall( + MethodDescriptor descriptor, + CallOption... options + ) { + FlightServiceStub wrappedStub = CallOptions.wrapStub(asyncStub, options); + return wrappedStub.getChannel().newCall(descriptor, wrappedStub.getCallOptions()); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/OSFlightServer.java b/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/OSFlightServer.java new file mode 100644 index 0000000000000..b423e9d8ae533 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/OSFlightServer.java @@ -0,0 +1,556 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.apache.arrow.flight; + +import org.apache.arrow.flight.auth.ServerAuthHandler; +import org.apache.arrow.flight.auth.ServerAuthInterceptor; +import org.apache.arrow.flight.auth2.Auth2Constants; +import org.apache.arrow.flight.auth2.CallHeaderAuthenticator; +import org.apache.arrow.flight.auth2.ServerCallHeaderAuthMiddleware; +import org.apache.arrow.flight.grpc.ServerBackpressureThresholdInterceptor; +import org.apache.arrow.flight.grpc.ServerInterceptorAdapter; +import org.apache.arrow.flight.grpc.ServerInterceptorAdapter.KeyFactory; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.util.VisibleForTesting; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.net.ssl.SSLException; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import io.grpc.Server; +import io.grpc.ServerInterceptors; +import io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.NettyServerBuilder; +import io.netty.channel.Channel; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.ServerChannel; +import io.netty.handler.ssl.ClientAuth; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; + +/** + * Clone of {@link FlightServer} to support setting SslContext directly. It can be discarded once + * FlightServer.Builder supports setting SslContext directly. + */ +public class OSFlightServer implements AutoCloseable { + + private static final Logger logger = LogManager.getLogger(OSFlightServer.class); + + private final Location location; + private final Server server; + // The executor used by the gRPC server. We don't use it here, but we do need to clean it up with + // the server. + // May be null, if a user-supplied executor was provided (as we do not want to clean that up) + @VisibleForTesting + final ExecutorService grpcExecutor; + + /** The maximum size of an individual gRPC message. This effectively disables the limit. */ + static final int MAX_GRPC_MESSAGE_SIZE = Integer.MAX_VALUE; + + /** The default number of bytes that can be queued on an output stream before blocking. */ + public static final int DEFAULT_BACKPRESSURE_THRESHOLD = 10 * 1024 * 1024; // 10MB + + /** Create a new instance from a gRPC server. For internal use only. */ + private OSFlightServer(Location location, Server server, ExecutorService grpcExecutor) { + this.location = location; + this.server = server; + this.grpcExecutor = grpcExecutor; + } + + /** Start the server. */ + public OSFlightServer start() throws IOException { + server.start(); + return this; + } + + /** Get the port the server is running on (if applicable). */ + public int getPort() { + return server.getPort(); + } + + /** Get the location for this server. */ + public Location getLocation() { + if (location.getUri().getPort() == 0) { + // If the server was bound to port 0, replace the port in the location with the real port. + final URI uri = location.getUri(); + try { + return new Location( + new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), getPort(), uri.getPath(), uri.getQuery(), uri.getFragment()) + ); + } catch (URISyntaxException e) { + // We don't expect this to happen + throw new RuntimeException(e); + } + } + return location; + } + + /** Block until the server shuts down. */ + public void awaitTermination() throws InterruptedException { + server.awaitTermination(); + } + + /** Request that the server shut down. */ + public void shutdown() { + server.shutdown(); + if (grpcExecutor != null) { + grpcExecutor.shutdown(); + } + } + + /** + * Wait for the server to shut down with a timeout. + * + * @return true if the server shut down successfully. + */ + public boolean awaitTermination(final long timeout, final TimeUnit unit) throws InterruptedException { + return server.awaitTermination(timeout, unit); + } + + /** Shutdown the server, waits for up to 6 seconds for successful shutdown before returning. */ + @Override + public void close() throws InterruptedException { + shutdown(); + final boolean terminated = awaitTermination(3000, TimeUnit.MILLISECONDS); + if (terminated) { + logger.debug("Server was terminated within 3s"); + return; + } + + // get more aggressive in termination. + server.shutdownNow(); + + int count = 0; + while (!server.isTerminated() && count < 30) { + count++; + logger.debug("Waiting for termination"); + Thread.sleep(100); + } + + if (!server.isTerminated()) { + logger.warn("Couldn't shutdown server, resources likely will be leaked."); + } + } + + /** Create a builder for a Flight server. */ + public static Builder builder() { + return new Builder(); + } + + /** Create a builder for a Flight server. */ + public static Builder builder(BufferAllocator allocator, Location location, FlightProducer producer) { + return new Builder(allocator, location, producer); + } + + public static Builder builder( + BufferAllocator allocator, + Location location, + FlightProducer producer, + SslContext sslContext, + Class channelType, + EventLoopGroup bossELG, + EventLoopGroup workerELG, + ExecutorService grpcExecutor + ) { + Builder builder = new Builder(allocator, location, producer); + if (sslContext != null) { + builder.useTls(sslContext); + } + builder.transportHint("netty.channelType", channelType); + builder.transportHint("netty.bossEventLoopGroup", bossELG); + builder.transportHint("netty.workerEventLoopGroup", workerELG); + builder.executor(grpcExecutor); + return builder; + } + + /** A builder for Flight servers. */ + public static final class Builder { + private BufferAllocator allocator; + private Location location; + private FlightProducer producer; + private final Map builderOptions; + private ServerAuthHandler authHandler = ServerAuthHandler.NO_OP; + private CallHeaderAuthenticator headerAuthenticator = CallHeaderAuthenticator.NO_OP; + private ExecutorService executor = null; + private int maxInboundMessageSize = MAX_GRPC_MESSAGE_SIZE; + private int maxHeaderListSize = MAX_GRPC_MESSAGE_SIZE; + private int backpressureThreshold = DEFAULT_BACKPRESSURE_THRESHOLD; + private InputStream certChain; + private InputStream key; + private InputStream mTlsCACert; + private SslContext sslContext; + private final List> interceptors; + // Keep track of inserted interceptors + private final Set interceptorKeys; + + Builder() { + builderOptions = new HashMap<>(); + interceptors = new ArrayList<>(); + interceptorKeys = new HashSet<>(); + } + + Builder(BufferAllocator allocator, Location location, FlightProducer producer) { + this(); + this.allocator = Preconditions.checkNotNull(allocator); + this.location = Preconditions.checkNotNull(location); + this.producer = Preconditions.checkNotNull(producer); + } + + /** Create the server for this builder. */ + @SuppressWarnings("unchecked") + public OSFlightServer build() { + // Add the auth middleware if applicable. + if (headerAuthenticator != CallHeaderAuthenticator.NO_OP) { + this.middleware( + FlightServerMiddleware.Key.of(Auth2Constants.AUTHORIZATION_HEADER), + new ServerCallHeaderAuthMiddleware.Factory(headerAuthenticator) + ); + } + + this.middleware(FlightConstants.HEADER_KEY, new ServerHeaderMiddleware.Factory()); + + final NettyServerBuilder builder; + switch (location.getUri().getScheme()) { + case LocationSchemes.GRPC_DOMAIN_SOCKET: { + // The implementation is platform-specific, so we have to find the classes at runtime + builder = NettyServerBuilder.forAddress(location.toSocketAddress()); + try { + try { + // Linux + builder.channelType( + Class.forName("io.netty.channel.epoll.EpollServerDomainSocketChannel").asSubclass(ServerChannel.class) + ); + final EventLoopGroup elg = Class.forName("io.netty.channel.epoll.EpollEventLoopGroup") + .asSubclass(EventLoopGroup.class) + .getConstructor() + .newInstance(); + builder.bossEventLoopGroup(elg).workerEventLoopGroup(elg); + } catch (ClassNotFoundException e) { + // BSD + // below logic may not work as the kqueue classes aren't present in grpc-netty-shaded + builder.channelType( + Class.forName("io.netty.channel.kqueue.KQueueServerDomainSocketChannel").asSubclass(ServerChannel.class) + ); + final EventLoopGroup elg = Class.forName("io.netty.channel.kqueue.KQueueEventLoopGroup") + .asSubclass(EventLoopGroup.class) + .getConstructor() + .newInstance(); + builder.bossEventLoopGroup(elg).workerEventLoopGroup(elg); + } + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException + | InvocationTargetException e) { + throw new UnsupportedOperationException( + "Could not find suitable Netty native transport implementation for domain socket address." + ); + } + break; + } + case LocationSchemes.GRPC: + case LocationSchemes.GRPC_INSECURE: { + builder = NettyServerBuilder.forAddress(location.toSocketAddress()); + break; + } + case LocationSchemes.GRPC_TLS: { + if (certChain == null) { + throw new IllegalArgumentException("Must provide a certificate and key to serve gRPC over TLS"); + } + builder = NettyServerBuilder.forAddress(location.toSocketAddress()); + break; + } + default: + throw new IllegalArgumentException("Scheme is not supported: " + location.getUri().getScheme()); + } + + if (sslContext != null && certChain != null) { + SslContextBuilder sslContextBuilder = GrpcSslContexts.forServer(certChain, key); + + if (mTlsCACert != null) { + sslContextBuilder.clientAuth(ClientAuth.REQUIRE).trustManager(mTlsCACert); + } + try { + sslContext = sslContextBuilder.build(); + } catch (SSLException e) { + throw new RuntimeException(e); + } finally { + closeMTlsCACert(); + closeCertChain(); + closeKey(); + } + + builder.sslContext(sslContext); + } + + // Share one executor between the gRPC service, DoPut, and Handshake + final ExecutorService exec; + // We only want to have FlightServer close the gRPC executor if we created it here. We should + // not close + // user-supplied executors. + final ExecutorService grpcExecutor; + if (executor != null) { + exec = executor; + grpcExecutor = null; + } else { + throw new IllegalStateException("GRPC executor must be passed to start Flight server."); + } + + final FlightBindingService flightService = new FlightBindingService(allocator, producer, authHandler, exec); + builder.executor(exec) + .maxInboundMessageSize(maxInboundMessageSize) + .maxInboundMetadataSize(maxHeaderListSize) + .addService( + ServerInterceptors.intercept( + flightService, + new ServerBackpressureThresholdInterceptor(backpressureThreshold), + new ServerAuthInterceptor(authHandler) + ) + ); + + // Allow hooking into the gRPC builder. This is not guaranteed to be available on all Arrow + // versions or + // Flight implementations. + builderOptions.computeIfPresent("grpc.builderConsumer", (key, builderConsumer) -> { + final Consumer consumer = (Consumer) builderConsumer; + consumer.accept(builder); + return null; + }); + + // Allow explicitly setting some Netty-specific options + builderOptions.computeIfPresent("netty.channelType", (key, channelType) -> { + builder.channelType((Class) channelType); + return null; + }); + builderOptions.computeIfPresent("netty.bossEventLoopGroup", (key, elg) -> { + builder.bossEventLoopGroup((EventLoopGroup) elg); + return null; + }); + builderOptions.computeIfPresent("netty.workerEventLoopGroup", (key, elg) -> { + builder.workerEventLoopGroup((EventLoopGroup) elg); + return null; + }); + + builder.intercept(new ServerInterceptorAdapter(interceptors)); + return new OSFlightServer(location, builder.build(), grpcExecutor); + } + + public Builder setMaxHeaderListSize(int maxHeaderListSize) { + this.maxHeaderListSize = maxHeaderListSize; + return this; + } + + /** + * Set the maximum size of a message. Defaults to "unlimited", depending on the underlying + * transport. + */ + public Builder maxInboundMessageSize(int maxMessageSize) { + this.maxInboundMessageSize = maxMessageSize; + return this; + } + + /** + * Set the number of bytes that may be queued on a server output stream before writes are + * blocked. + */ + public Builder backpressureThreshold(int backpressureThreshold) { + Preconditions.checkArgument(backpressureThreshold > 0); + this.backpressureThreshold = backpressureThreshold; + return this; + } + + /** + * A small utility function to ensure that InputStream attributes. are closed if they are not + * null + * + * @param stream The InputStream to close (if it is not null). + */ + private void closeInputStreamIfNotNull(InputStream stream) { + if (stream != null) { + try { + stream.close(); + } catch (IOException expected) { + // stream closes gracefully, doesn't expect an exception. + } + } + } + + /** + * A small utility function to ensure that the certChain attribute is closed if it is not null. + * It then sets the attribute to null. + */ + private void closeCertChain() { + closeInputStreamIfNotNull(certChain); + certChain = null; + } + + /** + * A small utility function to ensure that the key attribute is closed if it is not null. It + * then sets the attribute to null. + */ + private void closeKey() { + closeInputStreamIfNotNull(key); + key = null; + } + + /** + * A small utility function to ensure that the mTlsCACert attribute is closed if it is not null. + * It then sets the attribute to null. + */ + private void closeMTlsCACert() { + closeInputStreamIfNotNull(mTlsCACert); + mTlsCACert = null; + } + + /** + * Enable TLS on the server. + * + * @param certChain The certificate chain to use. + * @param key The private key to use. + */ + public Builder useTls(final File certChain, final File key) throws IOException { + closeCertChain(); + this.certChain = new FileInputStream(certChain); + + closeKey(); + this.key = new FileInputStream(key); + + return this; + } + + /** + * Enable Client Verification via mTLS on the server. + * + * @param mTlsCACert The CA certificate to use for verifying clients. + */ + public Builder useMTlsClientVerification(final File mTlsCACert) throws IOException { + closeMTlsCACert(); + this.mTlsCACert = new FileInputStream(mTlsCACert); + return this; + } + + /** + * Enable TLS on the server. + * + * @param certChain The certificate chain to use. + * @param key The private key to use. + */ + public Builder useTls(final InputStream certChain, final InputStream key) throws IOException { + closeCertChain(); + this.certChain = certChain; + + closeKey(); + this.key = key; + + return this; + } + + /** + * Enable TLS on the server. + * @param sslContext SslContext to use. + */ + public Builder useTls(SslContext sslContext) { + this.sslContext = Objects.requireNonNull(sslContext); + return this; + } + + /** + * Enable mTLS on the server. + * + * @param mTlsCACert The CA certificate to use for verifying clients. + */ + public Builder useMTlsClientVerification(final InputStream mTlsCACert) throws IOException { + closeMTlsCACert(); + this.mTlsCACert = mTlsCACert; + return this; + } + + /** + * Set the executor used by the server. + * + *

Flight will NOT take ownership of the executor. The application must clean it up if one is + * provided. (If not provided, Flight will use a default executor which it will clean up.) + */ + public Builder executor(ExecutorService executor) { + this.executor = executor; + return this; + } + + /** Set the authentication handler. */ + public Builder authHandler(ServerAuthHandler authHandler) { + this.authHandler = authHandler; + return this; + } + + /** Set the header-based authentication mechanism. */ + public Builder headerAuthenticator(CallHeaderAuthenticator headerAuthenticator) { + this.headerAuthenticator = headerAuthenticator; + return this; + } + + /** Provide a transport-specific option. Not guaranteed to have any effect. */ + public Builder transportHint(final String key, Object option) { + builderOptions.put(key, option); + return this; + } + + /** + * Add a Flight middleware component to inspect and modify requests to this service. + * + * @param key An identifier for this middleware component. Service implementations can retrieve + * the middleware instance for the current call using {@link + * org.apache.arrow.flight.FlightProducer.CallContext}. + * @param factory A factory for the middleware. + * @param The middleware type. + * @throws IllegalArgumentException if the key already exists + */ + public Builder middleware( + final FlightServerMiddleware.Key key, + final FlightServerMiddleware.Factory factory + ) { + if (interceptorKeys.contains(key.key)) { + throw new IllegalArgumentException("Key already exists: " + key.key); + } + interceptors.add(new KeyFactory<>(key, factory)); + interceptorKeys.add(key.key); + return this; + } + + public Builder allocator(BufferAllocator allocator) { + this.allocator = Preconditions.checkNotNull(allocator); + return this; + } + + public Builder location(Location location) { + this.location = Preconditions.checkNotNull(location); + return this; + } + + public Builder producer(FlightProducer producer) { + this.producer = Preconditions.checkNotNull(producer); + return this; + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/package-info.java b/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/package-info.java new file mode 100644 index 0000000000000..789a88a2d1159 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/apache/arrow/flight/package-info.java @@ -0,0 +1,13 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Clone of FlightServer and FlightClient due to package private access of + * certain configurations. + */ +package org.apache.arrow.flight; diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/FlightServerInfoAction.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/FlightServerInfoAction.java new file mode 100644 index 0000000000000..c988090081266 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/FlightServerInfoAction.java @@ -0,0 +1,65 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.transport.client.node.NodeClient; + +import java.util.List; + +import static org.opensearch.rest.RestRequest.Method.GET; + +/** + * It handles GET requests for retrieving Flight server information. + */ +public class FlightServerInfoAction extends BaseRestHandler { + + /** + * Constructor for FlightServerInfoAction. + */ + public FlightServerInfoAction() {} + + /** + * Returns the name of the action. + * @return The name of the action. + */ + @Override + public String getName() { + return "flight_server_info_action"; + } + + /** + * Returns the list of routes for the action. + * @return The list of routes for the action. + */ + @Override + public List routes() { + return List.of(new Route(GET, "/_flight/info"), new Route(GET, "/_flight/info/{nodeId}")); + } + + /** + * Prepares the request for the action. + * @param request The REST request. + * @param client The node client. + * @return The rest channel consumer. + */ + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { + String nodeId = request.param("nodeId"); + if (nodeId != null) { + // Query specific node + NodesFlightInfoRequest nodesRequest = new NodesFlightInfoRequest(nodeId); + return channel -> client.execute(NodesFlightInfoAction.INSTANCE, nodesRequest, new RestToXContentListener<>(channel)); + } else { + NodesFlightInfoRequest nodesRequest = new NodesFlightInfoRequest(); + return channel -> client.execute(NodesFlightInfoAction.INSTANCE, nodesRequest, new RestToXContentListener<>(channel)); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodeFlightInfo.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodeFlightInfo.java new file mode 100644 index 0000000000000..23163bfac8c2e --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodeFlightInfo.java @@ -0,0 +1,99 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.action.support.nodes.BaseNodeResponse; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + +/** + * Represents the response for a node's flight information. + */ +public class NodeFlightInfo extends BaseNodeResponse implements ToXContentObject { + private final BoundTransportAddress boundAddress; + + /** + * Constructor for NodeFlightInfo. + * @param in The stream input to read from. + * @throws IOException If an I/O error occurs. + */ + public NodeFlightInfo(StreamInput in) throws IOException { + super(in); + boundAddress = new BoundTransportAddress(in); + } + + /** + * Constructor for NodeFlightInfo. + * @param node The discovery node. + * @param boundAddress The bound transport address. + */ + public NodeFlightInfo(DiscoveryNode node, BoundTransportAddress boundAddress) { + super(node); + this.boundAddress = boundAddress; + } + + /** + * Writes the node flight information to the stream. + * @param out The stream output to write to. + * @throws IOException If an I/O error occurs. + */ + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + boundAddress.writeTo(out); + } + + /** + * Returns the bound transport address. + * @return The bound transport address. + */ + public BoundTransportAddress getBoundAddress() { + return boundAddress; + } + + /** + * Converts the node flight information to XContent. + * @param builder The XContent builder. + * @param params The parameters for the XContent conversion. + * @return The XContent builder. + * @throws IOException If an I/O error occurs. + */ + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(); + builder.startObject("flight_server"); + + builder.startArray("bound_addresses"); + for (TransportAddress address : boundAddress.boundAddresses()) { + builder.startObject(); + builder.field("host", address.address().getHostString()); + builder.field("port", address.address().getPort()); + builder.endObject(); + } + builder.endArray(); + + TransportAddress publishAddress = boundAddress.publishAddress(); + builder.startObject("publish_address"); + builder.field("host", publishAddress.address().getHostString()); + builder.field("port", publishAddress.address().getPort()); + builder.endObject(); + + builder.endObject(); + builder.endObject(); + return builder; + } + +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoAction.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoAction.java new file mode 100644 index 0000000000000..3c3a9965459cb --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoAction.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.action.ActionType; + +/** + * Action to retrieve flight info from nodes + */ +public class NodesFlightInfoAction extends ActionType { + /** + * Singleton instance of NodesFlightInfoAction. + */ + public static final NodesFlightInfoAction INSTANCE = new NodesFlightInfoAction(); + /** + * Name of this action. + */ + public static final String NAME = "cluster:admin/flight/info"; + + NodesFlightInfoAction() { + super(NAME, NodesFlightInfoResponse::new); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoRequest.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoRequest.java new file mode 100644 index 0000000000000..43bf38a096b57 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoRequest.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.action.support.nodes.BaseNodesRequest; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.transport.TransportRequest; + +import java.io.IOException; + +/** + * Flight Info Request + */ +public class NodesFlightInfoRequest extends BaseNodesRequest { + + /** + * Constructor for NodesFlightInfoRequest + * @param in StreamInput + * @throws IOException If an I/O error occurs + */ + public NodesFlightInfoRequest(StreamInput in) throws IOException { + super(in); + } + + /** + * Constructor for NodesFlightInfoRequest + * @param nodesIds String array of node IDs + */ + public NodesFlightInfoRequest(String... nodesIds) { + super(nodesIds); + } + + /** + * Writes the request to the given StreamOutput + */ + public static class NodeFlightInfoRequest extends TransportRequest { + NodesFlightInfoRequest request; + + /** + * Constructor for NodeFlightInfoRequest + * @param in StreamInput to read from + * @throws IOException If an I/O error occurs + */ + public NodeFlightInfoRequest(StreamInput in) throws IOException { + super(in); + } + + /** + * Constructor for NodeFlightInfoRequest + * @param request NodesFlightInfoRequest + */ + public NodeFlightInfoRequest(NodesFlightInfoRequest request) { + this.request = request; + } + } + + /** + * Writes the request to the given StreamOutput + * @param out StreamOutput to write to + * @throws IOException If an I/O error occurs + */ + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoResponse.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoResponse.java new file mode 100644 index 0000000000000..805aa188ce37a --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoResponse.java @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.action.FailedNodeException; +import org.opensearch.action.support.nodes.BaseNodesResponse; +import org.opensearch.cluster.ClusterName; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; + +/** + * Represents the response for nodes flight information. + */ +public class NodesFlightInfoResponse extends BaseNodesResponse implements ToXContentObject { + /** + * Constructs a new NodesFlightInfoResponse instance. + * + * @param in The stream input to read from. + * @throws IOException If an I/O error occurs. + */ + public NodesFlightInfoResponse(StreamInput in) throws IOException { + super(in); + } + + /** + * Constructs a new NodesFlightInfoResponse instance. + * + * @param clusterName The cluster name. + * @param nodes The list of node flight information. + * @param failures The list of failed node exceptions. + */ + public NodesFlightInfoResponse(ClusterName clusterName, List nodes, List failures) { + super(clusterName, nodes, failures); + } + + /** + * Reads the nodes from the given stream input. + * + * @param in The stream input to read from. + * @return The list of node flight information. + * @throws IOException If an I/O error occurs. + */ + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readList(NodeFlightInfo::new); + } + + /** + * Writes the nodes to the given stream output. + * + * @param out The stream output to write to. + * @param nodes The list of node flight information. + * @throws IOException If an I/O error occurs. + */ + @Override + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeList(nodes); + } + + /** + * Converts the nodes flight information response to XContent. + * @param builder The XContent builder. + * @param params The parameters for the XContent conversion. + * @return The XContent builder. + * @throws IOException If an I/O error occurs. + */ + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + + builder.startObject(); + builder.startObject("_nodes"); + builder.field("total", getNodes().size()); + builder.field("successful", getNodes().size()); + builder.field("failed", failures().size()); + builder.endObject(); + + builder.field("cluster_name", getClusterName().value()); + + builder.startObject("nodes"); + for (NodeFlightInfo nodeInfo : getNodes()) { + builder.field(nodeInfo.getNode().getId()); + nodeInfo.toXContent(builder, params); + } + builder.endObject(); + + if (!failures().isEmpty()) { + builder.startArray("failures"); + for (FailedNodeException failure : failures()) { + builder.startObject(); + builder.field("node_id", failure.nodeId()); + builder.field("reason", failure.getMessage()); + builder.endObject(); + } + builder.endArray(); + } + + builder.endObject(); + return builder; + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/TransportNodesFlightInfoAction.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/TransportNodesFlightInfoAction.java new file mode 100644 index 0000000000000..51f4cc05b8001 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/TransportNodesFlightInfoAction.java @@ -0,0 +1,113 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.action.FailedNodeException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.nodes.TransportNodesAction; +import org.opensearch.arrow.flight.bootstrap.FlightService; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +import java.io.IOException; +import java.util.List; + +/** + * Transport action for getting flight information from nodes + */ +public class TransportNodesFlightInfoAction extends TransportNodesAction< + NodesFlightInfoRequest, + NodesFlightInfoResponse, + NodesFlightInfoRequest.NodeFlightInfoRequest, + NodeFlightInfo> { + + private final FlightService flightService; + + /** + * Constructor for TransportNodesFlightInfoAction + * @param settings The settings for the action + * @param threadPool The thread pool for the action + * @param clusterService The cluster service for the action + * @param transportService The transport service for the action + * @param actionFilters The action filters for the action + * @param flightService The flight service for the action + */ + @Inject + public TransportNodesFlightInfoAction( + Settings settings, + ThreadPool threadPool, + ClusterService clusterService, + TransportService transportService, + ActionFilters actionFilters, + FlightService flightService + ) { + super( + NodesFlightInfoAction.NAME, + threadPool, + clusterService, + transportService, + actionFilters, + NodesFlightInfoRequest::new, + NodesFlightInfoRequest.NodeFlightInfoRequest::new, + ThreadPool.Names.MANAGEMENT, + NodeFlightInfo.class + ); + this.flightService = flightService; + } + + /** + * Creates a new response object for the action. + * @param request The associated request. + * @param nodeFlightInfos All successful node-level responses. + * @param failures All node-level failures. + * @return The response object. + */ + @Override + protected NodesFlightInfoResponse newResponse( + NodesFlightInfoRequest request, + List nodeFlightInfos, + List failures + ) { + return new NodesFlightInfoResponse(clusterService.getClusterName(), nodeFlightInfos, failures); + } + + /** + * Creates a new request object for a node. + * @param request The associated request. + * @return The request object. + */ + @Override + protected NodesFlightInfoRequest.NodeFlightInfoRequest newNodeRequest(NodesFlightInfoRequest request) { + return new NodesFlightInfoRequest.NodeFlightInfoRequest(request); + } + + /** + * Creates a new response object for a node. + * @param in The stream input to read from. + * @return The response object. + */ + @Override + protected NodeFlightInfo newNodeResponse(StreamInput in) throws IOException { + return new NodeFlightInfo(in); + } + + /** + * Creates a new response object for a node. + * @param request The associated request. + * @return The response object. + */ + @Override + protected NodeFlightInfo nodeOperation(NodesFlightInfoRequest.NodeFlightInfoRequest request) { + return new NodeFlightInfo(clusterService.localNode(), flightService.getBoundAddress()); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/package-info.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/package-info.java new file mode 100644 index 0000000000000..19dde32f32e8f --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/api/flightinfo/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Action to retrieve flight info from nodes + */ +package org.opensearch.arrow.flight.api.flightinfo; diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightClientManager.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightClientManager.java new file mode 100644 index 0000000000000..f99e0dc15fcf1 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightClientManager.java @@ -0,0 +1,400 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.arrow.flight.bootstrap; + +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.OSFlightClient; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.VisibleForTesting; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.Version; +import org.opensearch.arrow.flight.api.flightinfo.NodeFlightInfo; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoAction; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoRequest; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoResponse; +import org.opensearch.arrow.flight.bootstrap.tls.SslContextProvider; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterStateListener; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Nullable; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.client.Client; + +import java.util.Collection; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +import io.netty.channel.EventLoopGroup; + +import static org.opensearch.common.util.FeatureFlags.ARROW_STREAMS_SETTING; + +/** + * Manages Flight client connections to OpenSearch nodes in a cluster. + * This class maintains a pool of Flight clients for internode communication, + * handles client lifecycle, and responds to cluster state changes. + * + *

The manager implements ClusterStateListener to automatically update + * client connections when nodes join or leave the cluster.

+ */ +public class FlightClientManager implements ClusterStateListener, AutoCloseable { + private static final Version MIN_SUPPORTED_VERSION = Version.fromString("2.19.0"); + private static final Logger logger = LogManager.getLogger(FlightClientManager.class); + static final int LOCATION_TIMEOUT_MS = 1000; + private final ExecutorService grpcExecutor; + private final ClientConfiguration clientConfig; + private final Map flightClients = new ConcurrentHashMap<>(); + private final Client client; + private static final long CLIENT_BUILD_TIMEOUT_MS = TimeUnit.MINUTES.toMillis(1); + + /** + * Creates a new FlightClientManager instance. + * + * @param allocator Supplier for buffer allocation + * @param clusterService Service for cluster state management + * @param sslContextProvider Provider for SSL/TLS context configuration + * @param elg Event loop group for network operations + * @param threadPool Thread pool for executing tasks asynchronously + * @param client OpenSearch client + */ + public FlightClientManager( + BufferAllocator allocator, + ClusterService clusterService, + @Nullable SslContextProvider sslContextProvider, + EventLoopGroup elg, + ThreadPool threadPool, + Client client + ) { + grpcExecutor = threadPool.executor(ServerConfig.FLIGHT_CLIENT_THREAD_POOL_NAME); + this.clientConfig = new ClientConfiguration( + Objects.requireNonNull(allocator, "BufferAllocator cannot be null"), + Objects.requireNonNull(clusterService, "ClusterService cannot be null"), + sslContextProvider, + Objects.requireNonNull(elg, "EventLoopGroup cannot be null"), + Objects.requireNonNull(grpcExecutor, "ExecutorService cannot be null") + ); + this.client = Objects.requireNonNull(client, "Client cannot be null"); + clusterService.addListener(this); + } + + /** + * Returns the location of a Flight client for a given node ID. + * + * @param nodeId The ID of the node for which to retrieve the location + * @return The Location of the Flight client for the specified node + */ + public Location getFlightClientLocation(String nodeId) { + ClientHolder clientHolder = flightClients.get(nodeId); + if (clientHolder != null && clientHolder.location != null) { + return clientHolder.location; + } + buildClientAsync(nodeId); + return null; + } + + /** + * Returns a Flight client for a given node ID. + * + * @param nodeId The ID of the node for which to retrieve the Flight client + * @return An OpenSearchFlightClient instance for the specified node + */ + public Optional getFlightClient(String nodeId) { + if (nodeId == null || nodeId.isEmpty()) { + throw new IllegalArgumentException("Node ID cannot be null or empty"); + } + + ClientHolder holder = flightClients.get(nodeId); + + if (holder == null) { + buildClientAsync(nodeId); + return Optional.empty(); + } + + if (holder.state == BuildState.COMPLETE) { + return Optional.ofNullable(holder.flightClient); + } + + if (holder.isStale()) { + logger.warn("Detected stale building state for node [{}], triggering rebuild", nodeId); + if (flightClients.remove(nodeId, holder)) { + try { + holder.close(); + } catch (Exception e) { + logger.warn("Error closing stale client holder for node [{}]. {}", nodeId, e.getMessage()); + } + buildClientAsync(nodeId); + } + } + + return Optional.empty(); + } + + /** + * Represents the state and metadata of a Flight client + */ + private record ClientHolder(OSFlightClient flightClient, Location location, long buildStartTime, BuildState state) + implements + AutoCloseable { + + private static ClientHolder building() { + return new ClientHolder(null, null, System.currentTimeMillis(), BuildState.BUILDING); + } + + private static ClientHolder complete(OSFlightClient client, Location location) { + return new ClientHolder(client, location, System.currentTimeMillis(), BuildState.COMPLETE); + } + + boolean isStale() { + return state == BuildState.BUILDING && (System.currentTimeMillis() - buildStartTime) > CLIENT_BUILD_TIMEOUT_MS; + } + + /** + * Closes the client holder and logs the operation + * @param nodeId The ID of the node this holder belongs to + * @param reason The reason for closing + */ + void close(String nodeId, String reason) { + try { + if (flightClient != null) { + flightClient.close(); + } + if (state == BuildState.BUILDING) { + logger.info("Cleaned up building state for node [{}]: {}", nodeId, reason); + } else { + logger.info("Closed client for node [{}]: {}", nodeId, reason); + } + } catch (Exception e) { + logger.error("Failed to close client for node [{}] ({}): {}", nodeId, reason, e.getMessage()); + } + } + + @Override + public void close() throws Exception { + if (flightClient != null) { + flightClient.close(); + } + } + } + + private enum BuildState { + BUILDING, + COMPLETE + } + + /** + * Initiates async build of a flight client for the given node + */ + void buildClientAsync(String nodeId) { + // Try to put a building placeholder + ClientHolder placeholder = ClientHolder.building(); + if (flightClients.putIfAbsent(nodeId, placeholder) != null) { + return; // Another thread is already handling this node + } + + CompletableFuture locationFuture = new CompletableFuture<>(); + locationFuture.thenAccept(location -> { + try { + DiscoveryNode node = getNodeFromClusterState(nodeId); + if (!isValidNode(node)) { + logger.warn("Node [{}] is not valid for client creation", nodeId); + flightClients.remove(nodeId, placeholder); + return; + } + + OSFlightClient flightClient = buildClient(location); + ClientHolder newHolder = ClientHolder.complete(flightClient, location); + + if (!flightClients.replace(nodeId, placeholder, newHolder)) { + // Something changed while we were building + logger.warn("Failed to store new client for node [{}], state changed during build", nodeId); + flightClient.close(); + } + } catch (Exception e) { + logger.error("Failed to build Flight client for node [{}]. {}", nodeId, e); + flightClients.remove(nodeId, placeholder); + throw new RuntimeException(e); + } + }).exceptionally(throwable -> { + flightClients.remove(nodeId, placeholder); + logger.error("Failed to get Flight server location for node [{}] {}", nodeId, throwable); + throw new CompletionException(throwable); + }); + + requestNodeLocation(nodeId, locationFuture); + } + + Collection getClients() { + return flightClients.values(); + } + + private void requestNodeLocation(String nodeId, CompletableFuture future) { + NodesFlightInfoRequest request = new NodesFlightInfoRequest(nodeId); + client.execute(NodesFlightInfoAction.INSTANCE, request, new ActionListener<>() { + @Override + public void onResponse(NodesFlightInfoResponse response) { + NodeFlightInfo nodeInfo = response.getNodesMap().get(nodeId); + if (nodeInfo != null) { + TransportAddress publishAddress = nodeInfo.getBoundAddress().publishAddress(); + String address = publishAddress.getAddress(); + int flightPort = publishAddress.address().getPort(); + Location location = clientConfig.sslContextProvider != null + ? Location.forGrpcTls(address, flightPort) + : Location.forGrpcInsecure(address, flightPort); + + future.complete(location); + } else { + future.completeExceptionally(new IllegalStateException("No Flight info received for node: [" + nodeId + "]")); + } + } + + @Override + public void onFailure(Exception e) { + future.completeExceptionally(e); + logger.error("Failed to get Flight server info for node: [{}] {}", nodeId, e); + } + }); + } + + private OSFlightClient buildClient(Location location) { + return OSFlightClient.builder( + clientConfig.allocator, + location, + ServerConfig.clientChannelType(), + clientConfig.grpcExecutor, + clientConfig.workerELG, + clientConfig.sslContextProvider != null ? clientConfig.sslContextProvider.getClientSslContext() : null + ).build(); + } + + private DiscoveryNode getNodeFromClusterState(String nodeId) { + return Objects.requireNonNull(clientConfig.clusterService).state().nodes().get(nodeId); + } + + /** + * Closes the FlightClientManager and all associated Flight clients. + */ + @Override + public void close() throws Exception { + for (ClientHolder clientHolder : flightClients.values()) { + clientHolder.close(); + } + flightClients.clear(); + grpcExecutor.shutdown(); + } + + /** + * Returns the ID of the local node in the cluster. + * + * @return String representing the local node ID + */ + public String getLocalNodeId() { + return Objects.requireNonNull(clientConfig.clusterService).state().nodes().getLocalNodeId(); + } + + /** + * Handles cluster state changes by updating node locations and managing client connections. + * + * @param event The ClusterChangedEvent containing information about the cluster state change + */ + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (!event.nodesChanged()) { + return; + } + + final DiscoveryNodes nodes = event.state().nodes(); + + cleanupStaleBuilding(); + removeStaleClients(nodes); + updateExistingClients(nodes); + } + + private void removeStaleClients(DiscoveryNodes nodes) { + flightClients.entrySet().removeIf(entry -> { + String nodeId = entry.getKey(); + ClientHolder holder = entry.getValue(); + + if (!nodes.nodeExists(nodeId)) { + holder.close(nodeId, "node no longer exists"); + return true; + } + + if (holder.state == BuildState.BUILDING && holder.isStale()) { + holder.close(nodeId, "client build state is stale"); + return true; + } + + return false; + }); + } + + /** + * Updates clients for existing nodes based on their validity + */ + private void updateExistingClients(DiscoveryNodes nodes) { + for (DiscoveryNode node : nodes) { + String nodeId = node.getId(); + + if (isValidNode(node)) { + ClientHolder existingHolder = flightClients.get(nodeId); + + if (existingHolder == null) { + buildClientAsync(nodeId); + } else if (existingHolder.state == BuildState.BUILDING && existingHolder.isStale()) { + if (flightClients.remove(nodeId, existingHolder)) { + existingHolder.close(nodeId, "rebuilding stale client"); + buildClientAsync(nodeId); + } + } + } else { + ClientHolder holder = flightClients.remove(nodeId); + if (holder != null) { + holder.close(nodeId, "node is no longer valid"); + } + } + } + } + + /** + * Cleans up any clients that are in a stale BUILDING state + */ + private void cleanupStaleBuilding() { + flightClients.entrySet().removeIf(entry -> { + ClientHolder holder = entry.getValue(); + if (holder.state == BuildState.BUILDING && holder.isStale()) { + holder.close(entry.getKey(), "cleaning up stale building state"); + return true; + } + return false; + }); + } + + private static boolean isValidNode(DiscoveryNode node) { + return node != null && !node.getVersion().before(MIN_SUPPORTED_VERSION) && FeatureFlags.isEnabled(ARROW_STREAMS_SETTING); + } + + @VisibleForTesting + Map getFlightClients() { + return flightClients; + } + + private record ClientConfiguration(BufferAllocator allocator, ClusterService clusterService, SslContextProvider sslContextProvider, + EventLoopGroup workerELG, ExecutorService grpcExecutor) { + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightService.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightService.java new file mode 100644 index 0000000000000..1d75135eacc1c --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightService.java @@ -0,0 +1,175 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.bootstrap; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.util.AutoCloseables; +import org.apache.arrow.util.VisibleForTesting; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.arrow.flight.bootstrap.tls.DefaultSslContextProvider; +import org.opensearch.arrow.flight.bootstrap.tls.SslContextProvider; +import org.opensearch.arrow.flight.impl.BaseFlightProducer; +import org.opensearch.arrow.flight.impl.FlightStreamManager; +import org.opensearch.arrow.spi.StreamManager; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.plugins.NetworkPlugin; +import org.opensearch.plugins.SecureTransportSettingsProvider; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.client.Client; + +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Objects; + +/** + * FlightService manages the Arrow Flight server and client for OpenSearch. + * It handles the initialization, startup, and shutdown of the Flight server and client, + * as well as managing the stream operations through a FlightStreamManager. + */ +public class FlightService extends NetworkPlugin.AuxTransport { + private static final Logger logger = LogManager.getLogger(FlightService.class); + private final ServerComponents serverComponents; + private StreamManager streamManager; + private Client client; + private FlightClientManager clientManager; + private SecureTransportSettingsProvider secureTransportSettingsProvider; + private BufferAllocator allocator; + private ThreadPool threadPool; + + /** + * Constructor for FlightService. + * @param settings The settings for the FlightService. + */ + public FlightService(Settings settings) { + Objects.requireNonNull(settings, "Settings cannot be null"); + try { + ServerConfig.init(settings); + } catch (Exception e) { + throw new RuntimeException("Failed to initialize Arrow Flight server", e); + } + this.serverComponents = new ServerComponents(settings); + } + + void setClusterService(ClusterService clusterService) { + serverComponents.setClusterService(Objects.requireNonNull(clusterService, "ClusterService cannot be null")); + } + + void setNetworkService(NetworkService networkService) { + serverComponents.setNetworkService(Objects.requireNonNull(networkService, "NetworkService cannot be null")); + } + + void setThreadPool(ThreadPool threadPool) { + this.threadPool = Objects.requireNonNull(threadPool, "ThreadPool cannot be null"); + serverComponents.setThreadPool(threadPool); + } + + void setClient(Client client) { + this.client = client; + } + + void setSecureTransportSettingsProvider(SecureTransportSettingsProvider secureTransportSettingsProvider) { + this.secureTransportSettingsProvider = secureTransportSettingsProvider; + } + + /** + * Starts the FlightService by initializing the stream manager. + */ + @SuppressWarnings("removal") + @Override + protected void doStart() { + try { + allocator = AccessController.doPrivileged((PrivilegedAction) () -> new RootAllocator(Integer.MAX_VALUE)); + serverComponents.setAllocator(allocator); + SslContextProvider sslContextProvider = ServerConfig.isSslEnabled() + ? new DefaultSslContextProvider(secureTransportSettingsProvider) + : null; + serverComponents.setSslContextProvider(sslContextProvider); + serverComponents.initComponents(); + clientManager = new FlightClientManager( + allocator, // sharing the same allocator between server and client + serverComponents.clusterService, + sslContextProvider, + serverComponents.workerEventLoopGroup, // sharing the same worker ELG between server and client + threadPool, + client + ); + initializeStreamManager(clientManager); + serverComponents.setFlightProducer(new BaseFlightProducer(clientManager, (FlightStreamManager) streamManager, allocator)); + serverComponents.start(); + + } catch (Exception e) { + logger.error("Failed to start Flight server", e); + doClose(); + throw new RuntimeException("Failed to start Flight server", e); + } + } + + /** + * Retrieves the FlightClientManager used by the FlightService. + * @return The FlightClientManager instance. + */ + public FlightClientManager getFlightClientManager() { + return clientManager; + } + + /** + * Retrieves the StreamManager used by the FlightService. + * @return The StreamManager instance. + */ + public StreamManager getStreamManager() { + return streamManager; + } + + /** + * Retrieves the bound address of the FlightService. + * @return The BoundTransportAddress instance. + */ + public BoundTransportAddress getBoundAddress() { + return serverComponents.getBoundAddress(); + } + + @VisibleForTesting + SslContextProvider getSslContextProvider() { + return serverComponents.getSslContextProvider(); + } + + /** + * Stops the FlightService by closing the server components and network resources. + */ + @Override + protected void doStop() { + try { + AutoCloseables.close(serverComponents); + AutoCloseables.close(streamManager); + AutoCloseables.close(clientManager); + AutoCloseables.close(allocator); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * doStop() ensures all resources are cleaned up and resources are recreated on + * doStart() + */ + @Override + protected void doClose() { + doStop(); + } + + private void initializeStreamManager(FlightClientManager clientManager) { + streamManager = new FlightStreamManager(() -> allocator); + ((FlightStreamManager) streamManager).setClientManager(clientManager); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightStreamPlugin.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightStreamPlugin.java new file mode 100644 index 0000000000000..63136a32f6d37 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/FlightStreamPlugin.java @@ -0,0 +1,234 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.bootstrap; + +import org.opensearch.arrow.flight.api.flightinfo.FlightServerInfoAction; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoAction; +import org.opensearch.arrow.flight.api.flightinfo.TransportNodesFlightInfoAction; +import org.opensearch.arrow.spi.StreamManager; +import org.opensearch.cluster.metadata.IndexNameExpressionResolver; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.IndexScopedSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.common.util.PageCacheRecycler; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.indices.breaker.CircuitBreakerService; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; +import org.opensearch.plugins.ActionPlugin; +import org.opensearch.plugins.ClusterPlugin; +import org.opensearch.plugins.NetworkPlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SecureTransportSettingsProvider; +import org.opensearch.plugins.StreamManagerPlugin; +import org.opensearch.repositories.RepositoriesService; +import org.opensearch.rest.RestController; +import org.opensearch.rest.RestHandler; +import org.opensearch.script.ScriptService; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.threadpool.ExecutorBuilder; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.Transport; +import org.opensearch.transport.client.Client; +import org.opensearch.watcher.ResourceWatcherService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +/** + * FlightStreamPlugin class extends BaseFlightStreamPlugin and provides implementation for FlightStream plugin. + */ +public class FlightStreamPlugin extends Plugin implements StreamManagerPlugin, NetworkPlugin, ActionPlugin, ClusterPlugin { + + private final FlightService flightService; + + /** + * Constructor for FlightStreamPluginImpl. + * @param settings The settings for the FlightStreamPlugin. + */ + public FlightStreamPlugin(Settings settings) { + this.flightService = new FlightService(settings); + } + + /** + * Creates components for the FlightStream plugin. + * @param client The client instance. + * @param clusterService The cluster service instance. + * @param threadPool The thread pool instance. + * @param resourceWatcherService The resource watcher service instance. + * @param scriptService The script service instance. + * @param xContentRegistry The named XContent registry. + * @param environment The environment instance. + * @param nodeEnvironment The node environment instance. + * @param namedWriteableRegistry The named writeable registry. + * @param indexNameExpressionResolver The index name expression resolver instance. + * @param repositoriesServiceSupplier The supplier for the repositories service. + * @return FlightService + */ + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + flightService.setClusterService(clusterService); + flightService.setThreadPool(threadPool); + flightService.setClient(client); + return List.of(flightService); + } + + /** + * Gets the secure transports for the FlightStream plugin. + * @param settings The settings for the plugin. + * @param threadPool The thread pool instance. + * @param pageCacheRecycler The page cache recycler instance. + * @param circuitBreakerService The circuit breaker service instance. + * @param namedWriteableRegistry The named writeable registry. + * @param networkService The network service instance. + * @param secureTransportSettingsProvider The secure transport settings provider. + * @param tracer The tracer instance. + * @return A map of secure transports. + */ + @Override + public Map> getSecureTransports( + Settings settings, + ThreadPool threadPool, + PageCacheRecycler pageCacheRecycler, + CircuitBreakerService circuitBreakerService, + NamedWriteableRegistry namedWriteableRegistry, + NetworkService networkService, + SecureTransportSettingsProvider secureTransportSettingsProvider, + Tracer tracer + ) { + flightService.setSecureTransportSettingsProvider(secureTransportSettingsProvider); + return Collections.emptyMap(); + } + + /** + * Gets the auxiliary transports for the FlightStream plugin. + * @param settings The settings for the plugin. + * @param threadPool The thread pool instance. + * @param circuitBreakerService The circuit breaker service instance. + * @param networkService The network service instance. + * @param clusterSettings The cluster settings instance. + * @param tracer The tracer instance. + * @return A map of auxiliary transports. + */ + @Override + public Map> getAuxTransports( + Settings settings, + ThreadPool threadPool, + CircuitBreakerService circuitBreakerService, + NetworkService networkService, + ClusterSettings clusterSettings, + Tracer tracer + ) { + flightService.setNetworkService(networkService); + return Collections.singletonMap(FlightService.AUX_TRANSPORT_TYPES_KEY, () -> flightService); + } + + /** + * Gets the REST handlers for the FlightStream plugin. + * @param settings The settings for the plugin. + * @param restController The REST controller instance. + * @param clusterSettings The cluster settings instance. + * @param indexScopedSettings The index scoped settings instance. + * @param settingsFilter The settings filter instance. + * @param indexNameExpressionResolver The index name expression resolver instance. + * @param nodesInCluster The supplier for the discovery nodes. + * @return A list of REST handlers. + */ + @Override + public List getRestHandlers( + Settings settings, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster + ) { + return List.of(new FlightServerInfoAction()); + } + + /** + * Gets the list of action handlers for the FlightStream plugin. + * @return A list of action handlers. + */ + @Override + public List> getActions() { + return List.of(new ActionHandler<>(NodesFlightInfoAction.INSTANCE, TransportNodesFlightInfoAction.class)); + } + + /** + * Called when node is started. DiscoveryNode argument is passed to allow referring localNode value inside plugin + * + * @param localNode local Node info + */ + @Override + public void onNodeStarted(DiscoveryNode localNode) { + flightService.getFlightClientManager().buildClientAsync(localNode.getId()); + } + + /** + * Gets the StreamManager instance for managing flight streams. + */ + @Override + public Supplier getStreamManager() { + return flightService::getStreamManager; + } + + /** + * Gets the list of ExecutorBuilder instances for building thread pools used for FlightServer. + * @param settings The settings for the plugin + */ + @Override + public List> getExecutorBuilders(Settings settings) { + return List.of(ServerConfig.getServerExecutorBuilder(), ServerConfig.getClientExecutorBuilder()); + } + + /** + * Gets the list of settings for the Flight plugin. + */ + @Override + public List> getSettings() { + return new ArrayList<>( + Arrays.asList( + ServerComponents.SETTING_FLIGHT_PORTS, + ServerComponents.SETTING_FLIGHT_HOST, + ServerComponents.SETTING_FLIGHT_BIND_HOST, + ServerComponents.SETTING_FLIGHT_PUBLISH_HOST + ) + ) { + { + addAll(ServerConfig.getSettings()); + } + }; + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/ServerComponents.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/ServerComponents.java new file mode 100644 index 0000000000000..6beb61d811b8d --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/ServerComponents.java @@ -0,0 +1,285 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.bootstrap; + +import org.apache.arrow.flight.FlightProducer; +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.OSFlightServer; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.util.AutoCloseables; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.arrow.flight.bootstrap.tls.SslContextProvider; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Nullable; +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.transport.PortsRange; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.BindTransportException; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; + +import io.netty.channel.EventLoopGroup; +import io.netty.util.NettyRuntime; +import io.netty.util.concurrent.Future; + +import static java.util.Collections.emptyList; +import static org.opensearch.common.settings.Setting.intSetting; +import static org.opensearch.common.settings.Setting.listSetting; +import static org.opensearch.plugins.NetworkPlugin.AuxTransport.AUX_TRANSPORT_PORT; +import static org.opensearch.transport.Transport.resolveTransportPublishPort; + +@SuppressWarnings("removal") +final class ServerComponents implements AutoCloseable { + + public static final Setting> SETTING_FLIGHT_HOST = listSetting( + "flight.host", + emptyList(), + Function.identity(), + Setting.Property.NodeScope + ); + + public static final Setting> SETTING_FLIGHT_BIND_HOST = listSetting( + "flight.bind_host", + SETTING_FLIGHT_HOST, + Function.identity(), + Setting.Property.NodeScope + ); + + public static final Setting> SETTING_FLIGHT_PUBLISH_HOST = listSetting( + "flight.publish_host", + SETTING_FLIGHT_HOST, + Function.identity(), + Setting.Property.NodeScope + ); + + public static final Setting SETTING_FLIGHT_PUBLISH_PORT = intSetting( + "flight.publish_port", + -1, + -1, + Setting.Property.NodeScope + ); + + private static final Logger logger = LogManager.getLogger(ServerComponents.class); + + private static final String GRPC_WORKER_ELG = "os-grpc-worker-ELG"; + private static final String GRPC_BOSS_ELG = "os-grpc-boss-ELG"; + private static final int SHUTDOWN_TIMEOUT_SECONDS = 5; + + public static final String FLIGHT_TRANSPORT_SETTING_KEY = "transport-flight"; + public static final Setting SETTING_FLIGHT_PORTS = AUX_TRANSPORT_PORT.getConcreteSettingForNamespace( + FLIGHT_TRANSPORT_SETTING_KEY + ); + + private final Settings settings; + private final PortsRange port; + private final String[] bindHosts; + private final String[] publishHosts; + private volatile BoundTransportAddress boundAddress; + + private OSFlightServer server; + private BufferAllocator allocator; + ClusterService clusterService; + private NetworkService networkService; + private ThreadPool threadPool; + private SslContextProvider sslContextProvider; + private FlightProducer flightProducer; + + private EventLoopGroup bossEventLoopGroup; + EventLoopGroup workerEventLoopGroup; + private ExecutorService serverExecutor; + + ServerComponents(Settings settings) { + this.settings = settings; + this.port = SETTING_FLIGHT_PORTS.get(settings); + + List bindHosts = SETTING_FLIGHT_BIND_HOST.get(settings); + this.bindHosts = bindHosts.toArray(new String[0]); + + List publishHosts = SETTING_FLIGHT_PUBLISH_HOST.get(settings); + this.publishHosts = publishHosts.toArray(new String[0]); + } + + void setAllocator(BufferAllocator allocator) { + this.allocator = allocator; + } + + void setClusterService(ClusterService clusterService) { + this.clusterService = Objects.requireNonNull(clusterService); + } + + void setNetworkService(NetworkService networkService) { + this.networkService = Objects.requireNonNull(networkService); + } + + void setThreadPool(ThreadPool threadPool) { + this.threadPool = Objects.requireNonNull(threadPool); + } + + void setSslContextProvider(@Nullable SslContextProvider sslContextProvider) { + this.sslContextProvider = sslContextProvider; + } + + void setFlightProducer(FlightProducer flightProducer) { + this.flightProducer = Objects.requireNonNull(flightProducer); + } + + private OSFlightServer buildAndStartServer(Location location, FlightProducer producer) throws IOException { + OSFlightServer server = OSFlightServer.builder( + allocator, + location, + producer, + sslContextProvider != null ? sslContextProvider.getServerSslContext() : null, + ServerConfig.serverChannelType(), + bossEventLoopGroup, + workerEventLoopGroup, + serverExecutor + ).build(); + AccessController.doPrivileged((PrivilegedAction) () -> { + try { + server.start(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return null; + }); + return server; + } + + SslContextProvider getSslContextProvider() { + return sslContextProvider; + } + + BoundTransportAddress getBoundAddress() { + return boundAddress; + } + + void start() { + InetAddress[] hostAddresses; + try { + hostAddresses = networkService.resolveBindHostAddresses(bindHosts); + } catch (IOException e) { + throw new BindTransportException("Failed to resolve host [" + Arrays.toString(bindHosts) + "]", e); + } + + List boundAddresses = new ArrayList<>(hostAddresses.length); + for (InetAddress address : hostAddresses) { + AccessController.doPrivileged((PrivilegedAction) () -> { + boundAddresses.add(bindAddress(address, port)); + return null; + }); + } + + final InetAddress publishInetAddress; + try { + publishInetAddress = networkService.resolvePublishHostAddresses(publishHosts); + } catch (Exception e) { + throw new BindTransportException("Failed to resolve publish address", e); + } + + final int publishPort = resolveTransportPublishPort(SETTING_FLIGHT_PUBLISH_PORT.get(settings), boundAddresses, publishInetAddress); + + if (publishPort < 0) { + throw new BindTransportException( + "Failed to auto-resolve flight publish port, multiple bound addresses " + + boundAddresses + + " with distinct ports and none of them matched the publish address (" + + publishInetAddress + + "). Please specify a unique port by setting " + + SETTING_FLIGHT_PUBLISH_PORT.getKey() + ); + } + + TransportAddress publishAddress = new TransportAddress(new InetSocketAddress(publishInetAddress, publishPort)); + this.boundAddress = new BoundTransportAddress(boundAddresses.toArray(new TransportAddress[0]), publishAddress); + } + + void initComponents() throws Exception { + bossEventLoopGroup = ServerConfig.createELG(GRPC_BOSS_ELG, 1); + workerEventLoopGroup = ServerConfig.createELG(GRPC_WORKER_ELG, NettyRuntime.availableProcessors() * 2); + serverExecutor = threadPool.executor(ServerConfig.FLIGHT_SERVER_THREAD_POOL_NAME); + } + + @Override + public void close() { + try { + AutoCloseables.close(server); + gracefullyShutdownELG(bossEventLoopGroup, GRPC_BOSS_ELG); + gracefullyShutdownELG(workerEventLoopGroup, GRPC_WORKER_ELG); + if (serverExecutor != null) { + serverExecutor.shutdown(); + } + } catch (Exception e) { + logger.error("Error while closing server components", e); + } + } + + private TransportAddress bindAddress(final InetAddress hostAddress, final PortsRange portsRange) { + final AtomicReference lastException = new AtomicReference<>(); + final AtomicReference boundSocket = new AtomicReference<>(); + final TransportAddress[] address = new TransportAddress[1]; + boolean success = portsRange.iterate(portNumber -> { + boundSocket.set(new InetSocketAddress(hostAddress, portNumber)); + address[0] = new TransportAddress(boundSocket.get()); + try { + return startFlightServer(address[0]); + } catch (Exception e) { + lastException.set(e); + return false; + } + }); + + if (!success) { + throw new BindTransportException("Failed to bind to [" + hostAddress + "]", lastException.get()); + } + return address[0]; + } + + private boolean startFlightServer(TransportAddress transportAddress) { + InetSocketAddress address = transportAddress.address(); + Location serverLocation = sslContextProvider != null + ? Location.forGrpcTls(address.getHostString(), address.getPort()) + : Location.forGrpcInsecure(address.getHostString(), address.getPort()); + try { + this.server = buildAndStartServer(serverLocation, flightProducer); + logger.info("Arrow Flight server started. Listening at {}", serverLocation); + return true; + } catch (Exception e) { + String errorMsg = "Failed to start Arrow Flight server at " + serverLocation; + logger.error(errorMsg, e); + return false; + } + } + + private void gracefullyShutdownELG(EventLoopGroup group, String groupName) { + if (group != null) { + Future shutdownFuture = group.shutdownGracefully(0, SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS); + shutdownFuture.awaitUninterruptibly(); + if (!shutdownFuture.isSuccess()) { + logger.warn("Error closing {} netty event loop group {}", groupName, shutdownFuture.cause()); + } + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/ServerConfig.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/ServerConfig.java new file mode 100644 index 0000000000000..88513fa6e3bb8 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/ServerConfig.java @@ -0,0 +1,227 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.bootstrap; + +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.threadpool.ScalingExecutorBuilder; + +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import io.netty.channel.Channel; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.epoll.Epoll; +import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.epoll.EpollServerSocketChannel; +import io.netty.channel.epoll.EpollSocketChannel; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.util.concurrent.DefaultThreadFactory; + +/** + * Configuration class for OpenSearch Flight server settings. + * This class manages server-side configurations including port settings, Arrow memory settings, + * thread pool configurations, and SSL/TLS settings. + */ +public class ServerConfig { + /** + * Creates a new instance of the server configuration with default settings. + */ + public ServerConfig() {} + + static final Setting ARROW_ALLOCATION_MANAGER_TYPE = Setting.simpleString( + "arrow.allocation.manager.type", + "Netty", + Setting.Property.NodeScope + ); + + static final Setting ARROW_ENABLE_NULL_CHECK_FOR_GET = Setting.boolSetting( + "arrow.enable_null_check_for_get", + false, + Setting.Property.NodeScope + ); + + static final Setting ARROW_ENABLE_DEBUG_ALLOCATOR = Setting.boolSetting( + "arrow.memory.debug.allocator", + false, + Setting.Property.NodeScope + ); + + static final Setting ARROW_ENABLE_UNSAFE_MEMORY_ACCESS = Setting.boolSetting( + "arrow.enable_unsafe_memory_access", + true, + Setting.Property.NodeScope + ); + + static final Setting FLIGHT_THREAD_POOL_MIN_SIZE = Setting.intSetting( + "thread_pool.flight-server.min", + 0, + 0, + Setting.Property.NodeScope + ); + + static final Setting FLIGHT_THREAD_POOL_MAX_SIZE = Setting.intSetting( + "thread_pool.flight-server.max", + 100000, // TODO depends on max concurrent streams per node, decide after benchmark. To be controlled by admission control layer. + 1, + Setting.Property.NodeScope + ); + + static final Setting FLIGHT_THREAD_POOL_KEEP_ALIVE = Setting.timeSetting( + "thread_pool.flight-server.keep_alive", + TimeValue.timeValueSeconds(30), + Setting.Property.NodeScope + ); + + static final Setting ARROW_SSL_ENABLE = Setting.boolSetting( + "arrow.ssl.enable", + false, // TODO: get default from security enabled + Setting.Property.NodeScope + ); + + /** + * The thread pool name for the Flight server. + */ + public static final String FLIGHT_SERVER_THREAD_POOL_NAME = "flight-server"; + + /** + * The thread pool name for the Flight client. + */ + public static final String FLIGHT_CLIENT_THREAD_POOL_NAME = "flight-client"; + + private static final String host = "localhost"; + private static boolean enableSsl; + private static int threadPoolMin; + private static int threadPoolMax; + private static TimeValue keepAlive; + + /** + * Initializes the server configuration with the provided settings. + * Sets system properties for Arrow memory management and configures thread pool settings. + * + * @param settings The OpenSearch settings to initialize the server with + */ + @SuppressForbidden(reason = "required for arrow allocator") + @SuppressWarnings("removal") + public static void init(Settings settings) { + AccessController.doPrivileged((PrivilegedAction) () -> { + System.setProperty("arrow.allocation.manager.type", ARROW_ALLOCATION_MANAGER_TYPE.get(settings)); + System.setProperty("arrow.enable_null_check_for_get", Boolean.toString(ARROW_ENABLE_NULL_CHECK_FOR_GET.get(settings))); + System.setProperty("arrow.enable_unsafe_memory_access", Boolean.toString(ARROW_ENABLE_UNSAFE_MEMORY_ACCESS.get(settings))); + System.setProperty("arrow.memory.debug.allocator", Boolean.toString(ARROW_ENABLE_DEBUG_ALLOCATOR.get(settings))); + Netty4Configs.init(settings); + return null; + }); + enableSsl = ARROW_SSL_ENABLE.get(settings); + threadPoolMin = FLIGHT_THREAD_POOL_MIN_SIZE.get(settings); + threadPoolMax = FLIGHT_THREAD_POOL_MAX_SIZE.get(settings); + keepAlive = FLIGHT_THREAD_POOL_KEEP_ALIVE.get(settings); + } + + /** + * Checks if SSL/TLS is enabled for the Flight server. + * + * @return true if SSL is enabled, false otherwise + */ + public static boolean isSslEnabled() { + return enableSsl; + } + + /** + * Gets the thread pool executor builder configured for the Flight server. + * + * @return The configured ScalingExecutorBuilder instance + */ + public static ScalingExecutorBuilder getServerExecutorBuilder() { + return new ScalingExecutorBuilder(FLIGHT_SERVER_THREAD_POOL_NAME, threadPoolMin, threadPoolMax, keepAlive); + } + + /** + * Gets the thread pool executor builder configured for the Flight server. + * + * @return The configured ScalingExecutorBuilder instance + */ + public static ScalingExecutorBuilder getClientExecutorBuilder() { + return new ScalingExecutorBuilder(FLIGHT_CLIENT_THREAD_POOL_NAME, threadPoolMin, threadPoolMax, keepAlive); + } + + /** + * Returns a list of all settings managed by this configuration class. + * + * @return List of Setting instances + */ + public static List> getSettings() { + return new ArrayList<>( + Arrays.asList( + ARROW_ALLOCATION_MANAGER_TYPE, + ARROW_ENABLE_NULL_CHECK_FOR_GET, + ARROW_ENABLE_DEBUG_ALLOCATOR, + ARROW_ENABLE_UNSAFE_MEMORY_ACCESS, + ARROW_SSL_ENABLE + ) + ) { + { + addAll(Netty4Configs.getSettings()); + } + }; + } + + static EventLoopGroup createELG(String name, int eventLoopThreads) { + + return Epoll.isAvailable() + ? new EpollEventLoopGroup(eventLoopThreads, new DefaultThreadFactory(name, true)) + : new NioEventLoopGroup(eventLoopThreads, new DefaultThreadFactory(name, true)); + } + + static Class serverChannelType() { + return Epoll.isAvailable() ? EpollServerSocketChannel.class : NioServerSocketChannel.class; + } + + static Class clientChannelType() { + return Epoll.isAvailable() ? EpollSocketChannel.class : NioSocketChannel.class; + } + + private static class Netty4Configs { + public static final Setting NETTY_ALLOCATOR_NUM_DIRECT_ARENAS = Setting.intSetting( + "io.netty.allocator.numDirectArenas", + 1, // TODO - 2 * the number of available processors; to be confirmed and set after running benchmarks + 1, + Setting.Property.NodeScope + ); + + public static final Setting NETTY_TRY_REFLECTION_SET_ACCESSIBLE = Setting.boolSetting( + "io.netty.tryReflectionSetAccessible", + true, + Setting.Property.NodeScope + ); + + public static final Setting NETTY_NO_UNSAFE = Setting.boolSetting("io.netty.noUnsafe", false, Setting.Property.NodeScope); + + public static final Setting NETTY_TRY_UNSAFE = Setting.boolSetting("io.netty.tryUnsafe", true, Setting.Property.NodeScope); + + @SuppressForbidden(reason = "required for netty allocator configuration") + public static void init(Settings settings) { + System.setProperty("io.netty.allocator.numDirectArenas", Integer.toString(NETTY_ALLOCATOR_NUM_DIRECT_ARENAS.get(settings))); + System.setProperty("io.netty.noUnsafe", Boolean.toString(NETTY_NO_UNSAFE.get(settings))); + System.setProperty("io.netty.tryUnsafe", Boolean.toString(NETTY_TRY_UNSAFE.get(settings))); + System.setProperty("io.netty.tryReflectionSetAccessible", Boolean.toString(NETTY_TRY_REFLECTION_SET_ACCESSIBLE.get(settings))); + } + + public static List> getSettings() { + return Arrays.asList(NETTY_TRY_REFLECTION_SET_ACCESSIBLE, NETTY_ALLOCATOR_NUM_DIRECT_ARENAS, NETTY_NO_UNSAFE, NETTY_TRY_UNSAFE); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/package-info.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/package-info.java new file mode 100644 index 0000000000000..3ee247809b0c0 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Bootstrap classes for initializing and configuring OpenSearch Flight service. + */ +package org.opensearch.arrow.flight.bootstrap; diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/DefaultSslContextProvider.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/DefaultSslContextProvider.java new file mode 100644 index 0000000000000..187124911fc5f --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/DefaultSslContextProvider.java @@ -0,0 +1,104 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.bootstrap.tls; + +import org.opensearch.plugins.SecureTransportSettingsProvider; + +import javax.net.ssl.SSLException; + +import java.util.Locale; + +import io.netty.handler.ssl.ApplicationProtocolConfig; +import io.netty.handler.ssl.ApplicationProtocolNames; +import io.netty.handler.ssl.ClientAuth; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; +import io.netty.handler.ssl.SslProvider; +import io.netty.handler.ssl.SupportedCipherSuiteFilter; + +/** + * DefaultSslContextProvider is an implementation of the SslContextProvider interface that provides SSL contexts based on the provided SecureTransportSettingsProvider. + */ +public class DefaultSslContextProvider implements SslContextProvider { + + private final SecureTransportSettingsProvider secureTransportSettingsProvider; + + /** + * Constructor for DefaultSslContextProvider. + * @param secureTransportSettingsProvider The SecureTransportSettingsProvider instance. + */ + public DefaultSslContextProvider(SecureTransportSettingsProvider secureTransportSettingsProvider) { + this.secureTransportSettingsProvider = secureTransportSettingsProvider; + } + + // TODO - handle certificates reload + /** + * Creates and returns the server SSL context based on the provided SecureTransportSettingsProvider. + * @return The server SSL context. + */ + @Override + public SslContext getServerSslContext() { + try { + SecureTransportSettingsProvider.SecureTransportParameters parameters = secureTransportSettingsProvider.parameters(null).get(); + return SslContextBuilder.forServer(parameters.keyManagerFactory().get()) + .sslProvider(SslProvider.valueOf(parameters.sslProvider().get().toUpperCase(Locale.ROOT))) + .clientAuth(ClientAuth.valueOf(parameters.clientAuth().get().toUpperCase(Locale.ROOT))) + .protocols(parameters.protocols()) + .ciphers(parameters.cipherSuites(), SupportedCipherSuiteFilter.INSTANCE) + .sessionCacheSize(0) + .sessionTimeout(0) + .applicationProtocolConfig( + new ApplicationProtocolConfig( + ApplicationProtocolConfig.Protocol.ALPN, + // NO_ADVERTISE is currently the only mode supported by both OpenSsl and JDK providers. + ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, + // ACCEPT is currently the only mode supported by both OpenSsl and JDK providers. + ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, + ApplicationProtocolNames.HTTP_2, + ApplicationProtocolNames.HTTP_1_1 + ) + ) + .trustManager(parameters.trustManagerFactory().get()) + .build(); + } catch (SSLException e) { + throw new RuntimeException(e); + } + } + + /** + * Returns the client SSL context based on the provided SecureTransportSettingsProvider. + * @return The client SSL context. + */ + @Override + public SslContext getClientSslContext() { + try { + SecureTransportSettingsProvider.SecureTransportParameters parameters = secureTransportSettingsProvider.parameters(null).get(); + return SslContextBuilder.forClient() + .sslProvider(SslProvider.valueOf(parameters.sslProvider().get().toUpperCase(Locale.ROOT))) + .protocols(parameters.protocols()) + .ciphers(parameters.cipherSuites(), SupportedCipherSuiteFilter.INSTANCE) + .applicationProtocolConfig( + new ApplicationProtocolConfig( + ApplicationProtocolConfig.Protocol.ALPN, + ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, + ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, + ApplicationProtocolNames.HTTP_2, + ApplicationProtocolNames.HTTP_1_1 + ) + ) + .sessionCacheSize(0) + .sessionTimeout(0) + .keyManager(parameters.keyManagerFactory().get()) + .trustManager(parameters.trustManagerFactory().get()) + .build(); + } catch (SSLException e) { + throw new RuntimeException(e); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/SslContextProvider.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/SslContextProvider.java new file mode 100644 index 0000000000000..2cd38bc3c1dd5 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/SslContextProvider.java @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.bootstrap.tls; + +import io.netty.handler.ssl.SslContext; + +/** + * Provider interface for SSL/TLS context configuration in OpenSearch Flight. + * This interface defines methods for managing SSL contexts for both server and client-side + * Flight communications. + */ +public interface SslContextProvider { + + /** + * Gets the SSL context configuration for the Flight server. + * This context is used to secure incoming connections to the Flight server. + * + * @return SslContext configured for server-side TLS + */ + SslContext getServerSslContext(); + + /** + * Gets the SSL context configuration for Flight clients. + * This context is used when making outbound connections to other Flight servers. + * + * @return SslContext configured for client-side TLS + */ + SslContext getClientSslContext(); +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/package-info.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/package-info.java new file mode 100644 index 0000000000000..2ad8ae734c2da --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/bootstrap/tls/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * TLS/SSL configuration and security components for OpenSearch Flight service. + */ +package org.opensearch.arrow.flight.bootstrap.tls; diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseBackpressureStrategy.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseBackpressureStrategy.java new file mode 100644 index 0000000000000..d06d2eda5f23f --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseBackpressureStrategy.java @@ -0,0 +1,44 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.BackpressureStrategy; + +/** + * Base class for backpressure strategy. + */ +public class BaseBackpressureStrategy extends BackpressureStrategy.CallbackBackpressureStrategy { + private final Runnable readyCallback; + private final Runnable cancelCallback; + + /** + * Constructor for BaseBackpressureStrategy. + * + * @param readyCallback Callback to execute when the listener is ready. + * @param cancelCallback Callback to execute when the listener is cancelled. + */ + BaseBackpressureStrategy(Runnable readyCallback, Runnable cancelCallback) { + this.readyCallback = readyCallback; + this.cancelCallback = cancelCallback; + } + + /** Callback to execute when the listener is ready. */ + protected void readyCallback() { + if (readyCallback != null) { + readyCallback.run(); + } + } + + /** Callback to execute when the listener is cancelled. */ + protected void cancelCallback() { + if (cancelCallback != null) { + cancelCallback.run(); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseFlightProducer.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseFlightProducer.java new file mode 100644 index 0000000000000..9752c5b09d3cc --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/BaseFlightProducer.java @@ -0,0 +1,175 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.BackpressureStrategy; +import org.apache.arrow.flight.CallStatus; +import org.apache.arrow.flight.FlightDescriptor; +import org.apache.arrow.flight.FlightEndpoint; +import org.apache.arrow.flight.FlightInfo; +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.NoOpFlightProducer; +import org.apache.arrow.flight.OSFlightClient; +import org.apache.arrow.flight.Ticket; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.arrow.flight.bootstrap.FlightClientManager; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.arrow.spi.StreamTicket; + +import java.util.Collections; +import java.util.Optional; + +/** + * BaseFlightProducer extends NoOpFlightProducer to provide stream management functionality + * for Arrow Flight in OpenSearch. This class handles the retrieval and streaming of data + * based on provided tickets, managing backpressure, and coordinating between the stream + * provider and the server stream listener. + */ +public class BaseFlightProducer extends NoOpFlightProducer { + private final FlightClientManager flightClientManager; + private final FlightStreamManager streamManager; + private final BufferAllocator allocator; + private static final Logger logger = LogManager.getLogger(BaseFlightProducer.class); + + /** + * Constructs a new BaseFlightProducer. + * + * @param flightClientManager The FlightClientManager to handle client connections. + * @param streamManager The StreamManager to handle stream operations, including + * retrieving and removing streams based on tickets. + * @param allocator The BufferAllocator for memory management in Arrow operations. + */ + public BaseFlightProducer(FlightClientManager flightClientManager, FlightStreamManager streamManager, BufferAllocator allocator) { + this.flightClientManager = flightClientManager; + this.streamManager = streamManager; + this.allocator = allocator; + } + + /** + * Handles the retrieval and streaming of data based on the provided ticket. + * This method orchestrates the entire process of setting up the stream, + * managing backpressure, and handling data flow to the client. + * + * @param context The call context (unused in this implementation) + * @param ticket The ticket containing stream information + * @param listener The server stream listener to handle the data flow + */ + @Override + public void getStream(CallContext context, Ticket ticket, ServerStreamListener listener) { + StreamTicket streamTicket = streamManager.getStreamTicketFactory().fromBytes(ticket.getBytes()); + Optional streamProducerHolder = Optional.empty(); + try { + if (streamTicket.getNodeId().equals(flightClientManager.getLocalNodeId())) { + streamProducerHolder = streamManager.removeStreamProducer(streamTicket); + } else { + Optional remoteClient = flightClientManager.getFlightClient(streamTicket.getNodeId()); + if (remoteClient.isEmpty()) { + listener.error( + CallStatus.UNAVAILABLE.withDescription("Either server is not up yet or node does not support Streams.").cause() + ); + return; + } + StreamProducer proxyProvider = new ProxyStreamProducer( + new FlightStreamReader(remoteClient.get().getStream(ticket)) + ); + streamProducerHolder = Optional.of(FlightStreamManager.StreamProducerHolder.create(proxyProvider, allocator)); + } + if (streamProducerHolder.isEmpty()) { + listener.error(CallStatus.NOT_FOUND.withDescription("Stream not found").toRuntimeException()); + return; + } + try (StreamProducer producer = streamProducerHolder.get().producer()) { + StreamProducer.BatchedJob batchedJob = producer.createJob(allocator); + if (context.isCancelled()) { + batchedJob.onCancel(); + listener.error(CallStatus.CANCELLED.cause()); + return; + } + listener.setOnCancelHandler(batchedJob::onCancel); + BackpressureStrategy backpressureStrategy = new BaseBackpressureStrategy(null, batchedJob::onCancel); + backpressureStrategy.register(listener); + StreamProducer.FlushSignal flushSignal = (timeout) -> { + BackpressureStrategy.WaitResult result = backpressureStrategy.waitForListener(timeout.millis()); + if (result.equals(BackpressureStrategy.WaitResult.READY)) { + listener.putNext(); + } else if (result.equals(BackpressureStrategy.WaitResult.TIMEOUT)) { + listener.error(CallStatus.TIMED_OUT.cause()); + throw new RuntimeException("Stream deadline exceeded for consumption"); + } else if (result.equals(BackpressureStrategy.WaitResult.CANCELLED)) { + batchedJob.onCancel(); + listener.error(CallStatus.CANCELLED.cause()); + throw new RuntimeException("Stream cancelled by client"); + } else if (result.equals(BackpressureStrategy.WaitResult.OTHER)) { + batchedJob.onCancel(); + listener.error(CallStatus.INTERNAL.toRuntimeException()); + throw new RuntimeException("Error while waiting for client: " + result); + } else { + batchedJob.onCancel(); + listener.error(CallStatus.INTERNAL.toRuntimeException()); + throw new RuntimeException("Error while waiting for client: " + result); + } + }; + try (VectorSchemaRoot root = streamProducerHolder.get().getRoot()) { + listener.start(root); + batchedJob.run(root, flushSignal); + } + listener.completed(); + } + } catch (Exception e) { + listener.error(CallStatus.INTERNAL.withDescription(e.getMessage()).withCause(e).cause()); + logger.error(e); + throw new RuntimeException(e); + } + } + + /** + * Retrieves FlightInfo for the given FlightDescriptor, handling both local and remote cases. + * + * @param context The call context + * @param descriptor The FlightDescriptor containing stream information + * @return FlightInfo for the requested stream + */ + @Override + public FlightInfo getFlightInfo(CallContext context, FlightDescriptor descriptor) { + // TODO: this api should only be used internally + StreamTicket streamTicket = streamManager.getStreamTicketFactory().fromBytes(descriptor.getCommand()); + if (streamTicket.getNodeId().equals(flightClientManager.getLocalNodeId())) { + Optional streamProducerHolder = streamManager.getStreamProducer(streamTicket); + if (streamProducerHolder.isEmpty()) { + throw CallStatus.NOT_FOUND.withDescription("FlightInfo not found").toRuntimeException(); + } + Location location = flightClientManager.getFlightClientLocation(streamTicket.getNodeId()); + if (location == null) { + throw CallStatus.UNAVAILABLE.withDescription("Internal error while determining location information from ticket.") + .toRuntimeException(); + } + FlightEndpoint endpoint = new FlightEndpoint(new Ticket(descriptor.getCommand()), location); + FlightInfo.Builder infoBuilder; + try { + infoBuilder = FlightInfo.builder( + streamProducerHolder.get().getRoot().getSchema(), + descriptor, + Collections.singletonList(endpoint) + ).setRecords(streamProducerHolder.get().producer().estimatedRowCount()); + } catch (Exception e) { + throw CallStatus.INTERNAL.withDescription("Internal error while creating VectorSchemaRoot.").toRuntimeException(); + } + return infoBuilder.build(); + } else { + Optional remoteClient = flightClientManager.getFlightClient(streamTicket.getNodeId()); + if (remoteClient.isEmpty()) { + throw CallStatus.UNAVAILABLE.withDescription("Client doesn't support Stream").toRuntimeException(); + } + return remoteClient.get().getInfo(descriptor); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamManager.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamManager.java new file mode 100644 index 0000000000000..da35ed44aacac --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamManager.java @@ -0,0 +1,201 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.flight.OSFlightClient; +import org.apache.arrow.flight.Ticket; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.arrow.flight.bootstrap.FlightClientManager; +import org.opensearch.arrow.spi.StreamManager; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.arrow.spi.StreamReader; +import org.opensearch.arrow.spi.StreamTicket; +import org.opensearch.arrow.spi.StreamTicketFactory; +import org.opensearch.common.SetOnce; +import org.opensearch.common.cache.Cache; +import org.opensearch.common.cache.CacheBuilder; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.tasks.TaskId; + +import java.io.IOException; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Supplier; + +/** + * FlightStreamManager is a concrete implementation of StreamManager that provides + * an abstraction layer for managing Arrow Flight streams in OpenSearch. + * It encapsulates the details of Flight client operations, allowing consumers to + * work with streams without direct exposure to Flight internals. + */ +public class FlightStreamManager implements StreamManager { + private static final Logger logger = LogManager.getLogger(FlightStreamManager.class); + + private FlightStreamTicketFactory ticketFactory; + private FlightClientManager clientManager; + private final Supplier allocatorSupplier; + private final Cache streamProducers; + // TODO read from setting + private static final TimeValue DEFAULT_CACHE_EXPIRE = TimeValue.timeValueMinutes(10); + private static final int MAX_WEIGHT = 1000; + + /** + * Holds a StreamProducer along with its metadata and resources + */ + record StreamProducerHolder(StreamProducer producer, BufferAllocator allocator, long creationTime, + SetOnce root) { + public StreamProducerHolder { + Objects.requireNonNull(producer, "StreamProducer cannot be null"); + Objects.requireNonNull(allocator, "BufferAllocator cannot be null"); + } + + static StreamProducerHolder create(StreamProducer producer, BufferAllocator allocator) { + return new StreamProducerHolder(producer, allocator, System.currentTimeMillis(), new SetOnce<>()); + } + + boolean isExpired() { + return System.currentTimeMillis() - creationTime > producer.getJobDeadline().getMillis(); + } + + /** + * Gets the VectorSchemaRoot associated with the StreamProducer. + * If the root is not set, it creates a new one using the provided BufferAllocator. + */ + public VectorSchemaRoot getRoot() { + root.trySet(producer.createRoot(allocator)); + return root.get(); + } + } + + /** + * Constructs a new FlightStreamManager. + * @param allocatorSupplier The supplier for BufferAllocator instances used for memory management. + * This parameter is required to be non-null. + + */ + public FlightStreamManager(Supplier allocatorSupplier) { + this.allocatorSupplier = allocatorSupplier; + this.streamProducers = CacheBuilder.builder() + .setExpireAfterWrite(DEFAULT_CACHE_EXPIRE) + .setMaximumWeight(MAX_WEIGHT) + .build(); + } + + /** + * Sets the FlightClientManager for this FlightStreamManager. + * @param clientManager The FlightClientManager instance to use for Flight client operations. + * This parameter is required to be non-null. + */ + public void setClientManager(FlightClientManager clientManager) { + this.clientManager = clientManager; + this.ticketFactory = new FlightStreamTicketFactory(clientManager::getLocalNodeId); + } + + /** + * Registers a new stream producer with the StreamManager. + * @param provider The StreamProducer instance to register. + * @param parentTaskId The parent task ID associated with the stream. + * @return A StreamTicket representing the registered stream. + */ + @Override + @SuppressWarnings("unchecked") + public StreamTicket registerStream(StreamProducer provider, TaskId parentTaskId) { + Objects.requireNonNull(provider, "StreamProducer cannot be null"); + StreamTicket ticket = ticketFactory.newTicket(); + streamProducers.put( + ticket.getTicketId(), + StreamProducerHolder.create((StreamProducer) provider, allocatorSupplier.get()) + ); + return ticket; + } + + /** + * Retrieves a StreamReader for the given StreamTicket. + * @param ticket The StreamTicket representing the stream to retrieve. + * @return A StreamReader instance for the specified stream. + */ + @Override + @SuppressWarnings("unchecked") + public StreamReader getStreamReader(StreamTicket ticket) { + Optional flightClient = clientManager.getFlightClient(ticket.getNodeId()); + if (flightClient.isEmpty()) { + throw new RuntimeException("Flight client not found for node [" + ticket.getNodeId() + "]."); + } + FlightStream stream = flightClient.get().getStream(new Ticket(ticket.toBytes())); + return (StreamReader) new FlightStreamReader(stream); + } + + /** + * Retrieves the StreamTicketFactory used by this StreamManager. + * @return The StreamTicketFactory instance associated with this StreamManager. + */ + @Override + public StreamTicketFactory getStreamTicketFactory() { + return ticketFactory; + } + + /** + * Gets the StreamProducer associated with a ticket if it hasn't expired based on its deadline. + * + * @param ticket The StreamTicket identifying the stream + * @return Optional of StreamProducerHolder containing the producer if found and not expired + */ + public Optional getStreamProducer(StreamTicket ticket) { + Objects.requireNonNull(ticket, "StreamTicket cannot be null"); + StreamProducerHolder holder = streamProducers.get(ticket.getTicketId()); + if (holder != null) { + if (holder.isExpired()) { + removeStreamProducer(ticket); + return Optional.empty(); + } + return Optional.of(holder); + } + return Optional.empty(); + } + + /** + * Gets and removes the StreamProducer associated with a ticket. + * Ensure that close is called on the StreamProducer after use. + * @param ticket The StreamTicket identifying the stream + * @return Optional of StreamProducerHolder containing the producer if found + */ + public Optional removeStreamProducer(StreamTicket ticket) { + Objects.requireNonNull(ticket, "StreamTicket cannot be null"); + + String ticketId = ticket.getTicketId(); + StreamProducerHolder holder = streamProducers.get(ticketId); + + if (holder != null) { + streamProducers.invalidate(ticketId); + return Optional.of(holder); + } + return Optional.empty(); + } + + /** + * Closes the StreamManager and cancels all associated streams. + * This method should be called when the StreamManager is no longer needed to clean up resources. + * It is recommended to implement this method to cancel all threads and clear the streamManager queue. + */ + @Override + public void close() throws Exception { + streamProducers.values().forEach(holder -> { + try { + holder.producer().close(); + } catch (IOException e) { + logger.error("Error closing stream producer, this may cause memory leaks.", e); + } + }); + streamProducers.invalidateAll(); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamReader.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamReader.java new file mode 100644 index 0000000000000..d9e366dca30e2 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamReader.java @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.FlightRuntimeException; +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.opensearch.ExceptionsHelper; +import org.opensearch.arrow.spi.StreamReader; + +/** + * FlightStreamReader is a wrapper class that adapts the FlightStream interface + * to the StreamReader interface. + */ +public class FlightStreamReader implements StreamReader { + + private final FlightStream flightStream; + + /** + * Constructs a FlightStreamReader with the given FlightStream. + * + * @param flightStream The FlightStream to be adapted. + */ + public FlightStreamReader(FlightStream flightStream) { + this.flightStream = flightStream; + } + + /** + * Moves the flightStream to the next batch of data. + * @return true if there is a next batch of data, false otherwise. + * @throws FlightRuntimeException if an error occurs while advancing to the next batch like early termination of stream + */ + @Override + public boolean next() throws FlightRuntimeException { + return flightStream.next(); + } + + /** + * Returns the VectorSchemaRoot containing the current batch of data. + * @return The VectorSchemaRoot containing the current batch of data. + * @throws FlightRuntimeException if an error occurs while retrieving the root like early termination of stream + */ + @Override + public VectorSchemaRoot getRoot() throws FlightRuntimeException { + return flightStream.getRoot(); + } + + /** + * Closes the flightStream. + */ + @Override + public void close() { + ExceptionsHelper.catchAsRuntimeException(flightStream::close); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamTicket.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamTicket.java new file mode 100644 index 0000000000000..baa9e79fec6a1 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamTicket.java @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.opensearch.arrow.spi.StreamTicket; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Objects; + +class FlightStreamTicket implements StreamTicket { + private static final int MAX_TOTAL_SIZE = 4096; + private static final int MAX_ID_LENGTH = 256; + + private final String ticketID; + private final String nodeID; + + public FlightStreamTicket(String ticketID, String nodeID) { + this.ticketID = ticketID; + this.nodeID = nodeID; + } + + @Override + public String getTicketId() { + return ticketID; + } + + @Override + public String getNodeId() { + return nodeID; + } + + @Override + public byte[] toBytes() { + byte[] ticketIDBytes = ticketID.getBytes(StandardCharsets.UTF_8); + byte[] nodeIDBytes = nodeID.getBytes(StandardCharsets.UTF_8); + + if (ticketIDBytes.length > Short.MAX_VALUE || nodeIDBytes.length > Short.MAX_VALUE) { + throw new IllegalArgumentException("Field lengths exceed the maximum allowed size."); + } + ByteBuffer buffer = ByteBuffer.allocate(2 + ticketIDBytes.length + 2 + nodeIDBytes.length); + buffer.putShort((short) ticketIDBytes.length); + buffer.putShort((short) nodeIDBytes.length); + buffer.put(ticketIDBytes); + buffer.put(nodeIDBytes); + return Base64.getEncoder().encode(buffer.array()); + } + + static StreamTicket fromBytes(byte[] bytes) { + if (bytes == null || bytes.length < 4) { + throw new IllegalArgumentException("Invalid byte array input."); + } + + if (bytes.length > MAX_TOTAL_SIZE) { + throw new IllegalArgumentException("Input exceeds maximum allowed size"); + } + + ByteBuffer buffer = ByteBuffer.wrap(Base64.getDecoder().decode(bytes)); + + short ticketIDLength = buffer.getShort(); + if (ticketIDLength < 0 || ticketIDLength > MAX_ID_LENGTH) { + throw new IllegalArgumentException("Invalid ticketID length: " + ticketIDLength); + } + + short nodeIDLength = buffer.getShort(); + if (nodeIDLength < 0 || nodeIDLength > MAX_ID_LENGTH) { + throw new IllegalArgumentException("Invalid nodeID length: " + nodeIDLength); + } + + byte[] ticketIDBytes = new byte[ticketIDLength]; + if (buffer.remaining() < ticketIDLength) { + throw new IllegalArgumentException("Malformed byte array. Not enough data for TicketId."); + } + buffer.get(ticketIDBytes); + + byte[] nodeIDBytes = new byte[nodeIDLength]; + if (buffer.remaining() < nodeIDLength) { + throw new IllegalArgumentException("Malformed byte array. Not enough data for NodeId."); + } + buffer.get(nodeIDBytes); + + String ticketID = new String(ticketIDBytes, StandardCharsets.UTF_8); + String nodeID = new String(nodeIDBytes, StandardCharsets.UTF_8); + return new FlightStreamTicket(ticketID, nodeID); + } + + @Override + public int hashCode() { + return Objects.hash(ticketID, nodeID); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + FlightStreamTicket that = (FlightStreamTicket) obj; + return Objects.equals(ticketID, that.ticketID) && Objects.equals(nodeID, that.nodeID); + } + + @Override + public String toString() { + return "FlightStreamTicket{ticketID='" + ticketID + "', nodeID='" + nodeID + "'}"; + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamTicketFactory.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamTicketFactory.java new file mode 100644 index 0000000000000..473eb92cf2db3 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/FlightStreamTicketFactory.java @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.opensearch.arrow.spi.StreamTicket; +import org.opensearch.arrow.spi.StreamTicketFactory; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.util.UUID; +import java.util.function.Supplier; + +/** + * Default implementation of StreamTicketFactory + */ +@ExperimentalApi +public class FlightStreamTicketFactory implements StreamTicketFactory { + + private final Supplier nodeId; + + /** + * Constructs a new DefaultStreamTicketFactory instance. + * + * @param nodeId A Supplier that provides the node ID for the StreamTicket + */ + public FlightStreamTicketFactory(Supplier nodeId) { + this.nodeId = nodeId; + } + + /** + * Creates a new StreamTicket with a unique ticket ID. + * + * @return A new StreamTicket instance + */ + @Override + public StreamTicket newTicket() { + return new FlightStreamTicket(generateUniqueTicket(), nodeId.get()); + } + + /** + * Deserializes a StreamTicket from its byte representation. + * + * @param bytes The byte array containing the serialized ticket data + * @return A StreamTicket instance reconstructed from the byte array + * @throws IllegalArgumentException if bytes is null or invalid + */ + @Override + public StreamTicket fromBytes(byte[] bytes) { + return FlightStreamTicket.fromBytes(bytes); + } + + private String generateUniqueTicket() { + return UUID.randomUUID().toString(); + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/ProxyStreamProducer.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/ProxyStreamProducer.java new file mode 100644 index 0000000000000..a97f4697571e0 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/ProxyStreamProducer.java @@ -0,0 +1,123 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.opensearch.ExceptionsHelper; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.arrow.spi.StreamReader; +import org.opensearch.arrow.spi.StreamTicket; +import org.opensearch.common.unit.TimeValue; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * ProxyStreamProvider acts as forward proxy for FlightStream. + * It creates a BatchedJob to handle the streaming of data from the remote FlightStream. + * This is useful when stream is not present locally and needs to be fetched from a node + * retrieved using {@link StreamTicket#getNodeId()} where it is present. + */ +public class ProxyStreamProducer implements StreamProducer { + + private final StreamReader remoteStream; + + /** + * Constructs a new ProxyStreamProducer instance. + * + * @param remoteStream The remote FlightStream to be proxied. + */ + public ProxyStreamProducer(StreamReader remoteStream) { + this.remoteStream = remoteStream; + } + + /** + * Creates a VectorSchemaRoot for the remote FlightStream. + * @param allocator The allocator to use for creating vectors + * @return A VectorSchemaRoot representing the schema of the remote FlightStream + */ + @Override + public VectorSchemaRoot createRoot(BufferAllocator allocator) { + return remoteStream.getRoot(); + } + + /** + * Creates a BatchedJob + * @param allocator The allocator to use for any additional memory allocations + */ + @Override + public BatchedJob createJob(BufferAllocator allocator) { + return new ProxyBatchedJob(remoteStream); + } + + /** + * Returns the deadline for the remote FlightStream. + * Since the stream is not present locally, the deadline is set to -1. It piggybacks on remote stream expiration + * @return The deadline for the remote FlightStream + */ + @Override + public TimeValue getJobDeadline() { + return TimeValue.MINUS_ONE; + } + + /** + * Provides an estimate of the total number of rows that will be produced. + */ + @Override + public int estimatedRowCount() { + // TODO get it from remote flight stream + return -1; + } + + /** + * Task action name + */ + @Override + public String getAction() { + // TODO get it from remote flight stream + return ""; + } + + /** + * Closes the remote FlightStream. + */ + @Override + public void close() { + ExceptionsHelper.catchAsRuntimeException(remoteStream::close); + } + + static class ProxyBatchedJob implements BatchedJob { + + private final StreamReader remoteStream; + private final AtomicBoolean isCancelled = new AtomicBoolean(false); + + ProxyBatchedJob(StreamReader remoteStream) { + this.remoteStream = remoteStream; + } + + @Override + public void run(VectorSchemaRoot root, FlushSignal flushSignal) { + while (!isCancelled.get() && remoteStream.next()) { + flushSignal.awaitConsumption(TimeValue.timeValueMillis(1000)); + } + } + + @Override + public void onCancel() { + isCancelled.set(true); + } + + @Override + public boolean isCancelled() { + // Proxy stream don't have any business logic to set this flag, + // they piggyback on remote stream getting cancelled. + return isCancelled.get(); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/package-info.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/package-info.java new file mode 100644 index 0000000000000..90ca54b44a55d --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/impl/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Core components and implementations for OpenSearch Flight service, including base producers and consumers. + */ +package org.opensearch.arrow.flight.impl; diff --git a/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/package-info.java b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/package-info.java new file mode 100644 index 0000000000000..2341a24d0be85 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/java/org/opensearch/arrow/flight/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Root package for OpenSearch Flight functionality, providing core flight service integration with OpenSearch. + */ +package org.opensearch.arrow.flight; diff --git a/plugins/arrow-flight-rpc/src/main/plugin-metadata/plugin-security.policy b/plugins/arrow-flight-rpc/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000000..803350a578009 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +grant codeBase "${codebase.netty-common}" { + permission java.net.SocketPermission "*", "accept,connect,listen,resolve"; + permission java.lang.RuntimePermission "*", "setContextClassLoader"; +}; + +grant codeBase "${codebase.grpc-core}" { + permission java.net.SocketPermission "*", "accept,connect,listen,resolve"; + permission java.lang.RuntimePermission "*", "setContextClassLoader"; +}; + +grant { + // arrow flight service permissions + permission java.util.PropertyPermission "arrow.allocation.manager.type", "write"; + permission java.util.PropertyPermission "arrow.enable_null_check_for_get", "write"; + permission java.util.PropertyPermission "arrow.enable_unsafe_memory_access", "write"; + permission java.util.PropertyPermission "arrow.memory.debug.allocator", "write"; + + permission java.util.PropertyPermission "io.netty.tryReflectionSetAccessible", "write"; + permission java.util.PropertyPermission "io.netty.allocator.numDirectArenas", "write"; + permission java.util.PropertyPermission "io.netty.noUnsafe", "write"; + permission java.util.PropertyPermission "io.netty.tryUnsafe", "write"; + + // Needed for netty based arrow flight server for netty configs related to buffer allocator + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + permission java.util.PropertyPermission "arrow.allocation.manager.type", "write"; + + permission java.lang.RuntimePermission "modifyThreadGroup"; + permission java.lang.RuntimePermission "modifyThread"; + permission java.net.SocketPermission "*", "accept,connect,listen,resolve"; + + // Reflection access needed by Arrow + permission java.lang.RuntimePermission "accessDeclaredMembers"; + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; + + // Memory access + permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; +}; diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/FlightStreamPluginTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/FlightStreamPluginTests.java new file mode 100644 index 0000000000000..a3f0d1ca99b25 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/FlightStreamPluginTests.java @@ -0,0 +1,104 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight; + +import org.opensearch.arrow.flight.api.flightinfo.FlightServerInfoAction; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoAction; +import org.opensearch.arrow.flight.bootstrap.FlightService; +import org.opensearch.arrow.flight.bootstrap.FlightStreamPlugin; +import org.opensearch.arrow.spi.StreamManager; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.plugins.SecureTransportSettingsProvider; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ExecutorBuilder; +import org.opensearch.threadpool.ThreadPool; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.function.Supplier; + +import static org.opensearch.common.util.FeatureFlags.ARROW_STREAMS_SETTING; +import static org.opensearch.plugins.NetworkPlugin.AuxTransport.AUX_TRANSPORT_TYPES_KEY; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FlightStreamPluginTests extends OpenSearchTestCase { + private Settings settings; + private ClusterService clusterService; + + @Override + public void setUp() throws Exception { + super.setUp(); + settings = Settings.builder().put(ARROW_STREAMS_SETTING.getKey(), true).build(); + clusterService = mock(ClusterService.class); + ClusterState clusterState = mock(ClusterState.class); + DiscoveryNodes nodes = mock(DiscoveryNodes.class); + when(clusterService.state()).thenReturn(clusterState); + when(clusterState.nodes()).thenReturn(nodes); + when(nodes.getLocalNodeId()).thenReturn("test-node"); + } + + public void testPluginEnabled() throws IOException { + FeatureFlags.initializeFeatureFlags(settings); + FeatureFlagSetter.set(ARROW_STREAMS_SETTING.getKey()); + FlightStreamPlugin plugin = new FlightStreamPlugin(settings); + Collection components = plugin.createComponents( + null, + clusterService, + mock(ThreadPool.class), + null, + null, + null, + null, + null, + null, + null, + null + ); + + assertNotNull(components); + assertFalse(components.isEmpty()); + assertEquals(1, components.size()); + assertTrue(components.iterator().next() instanceof FlightService); + + List> executorBuilders = plugin.getExecutorBuilders(settings); + assertNotNull(executorBuilders); + assertFalse(executorBuilders.isEmpty()); + assertEquals(2, executorBuilders.size()); + + Supplier streamManager = plugin.getStreamManager(); + assertNotNull(streamManager); + + List> settings = plugin.getSettings(); + assertNotNull(settings); + assertFalse(settings.isEmpty()); + + assertNotNull(plugin.getSecureTransports(null, null, null, null, null, null, mock(SecureTransportSettingsProvider.class), null)); + + assertTrue( + plugin.getAuxTransports(null, null, null, new NetworkService(List.of()), null, null) + .get(AUX_TRANSPORT_TYPES_KEY) + .get() instanceof FlightService + ); + assertEquals(1, plugin.getRestHandlers(null, null, null, null, null, null, null).size()); + assertTrue(plugin.getRestHandlers(null, null, null, null, null, null, null).get(0) instanceof FlightServerInfoAction); + assertEquals(1, plugin.getActions().size()); + assertEquals(NodesFlightInfoAction.INSTANCE.name(), plugin.getActions().get(0).getAction().name()); + + plugin.close(); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/FlightServerInfoActionTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/FlightServerInfoActionTests.java new file mode 100644 index 0000000000000..d3115fc745475 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/FlightServerInfoActionTests.java @@ -0,0 +1,101 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.SetOnce; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.rest.RestRequest; +import org.opensearch.test.rest.FakeRestRequest; +import org.opensearch.test.rest.RestActionTestCase; +import org.junit.Before; + +import java.util.Collections; + +import static org.mockito.Mockito.mock; + +public class FlightServerInfoActionTests extends RestActionTestCase { + private FlightServerInfoAction handler; + + @Before + public void setUpAction() { + handler = new FlightServerInfoAction(); + controller().registerHandler(handler); + } + + public void testGetName() { + assertEquals("flight_server_info_action", handler.getName()); + } + + public void testRoutes() { + var routes = handler.routes(); + assertEquals(2, routes.size()); + assertTrue( + routes.stream().anyMatch(route -> route.getPath().equals("/_flight/info") && route.getMethod() == RestRequest.Method.GET) + ); + assertTrue( + routes.stream() + .anyMatch(route -> route.getPath().equals("/_flight/info/{nodeId}") && route.getMethod() == RestRequest.Method.GET) + ); + } + + public void testFlightInfoRequest() { + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET) + .withPath("/_flight/info") + .build(); + SetOnce executeCalled = new SetOnce<>(); + verifyingClient.setExecuteVerifier((action, actionRequest) -> { + assertEquals(NodesFlightInfoAction.INSTANCE.name(), action.name()); + assertNotNull(actionRequest); + executeCalled.set(true); + return new NodesFlightInfoResponse( + new ClusterName("test-cluster"), + Collections.singletonList(new NodeFlightInfo(mock(DiscoveryNode.class), mock(BoundTransportAddress.class))), + Collections.emptyList() + ); + }); + dispatchRequest(request); + assertEquals(Boolean.TRUE, executeCalled.get()); + } + + public void testFlightInfoRequestWithNodeId() throws Exception { + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET) + .withPath("/_flight/info/local_node") + .build(); + SetOnce executeCalled = new SetOnce<>(); + verifyingClient.setExecuteVerifier((action, actionRequest) -> { + assertEquals(NodesFlightInfoAction.INSTANCE.name(), action.name()); + assertNotNull(actionRequest); + executeCalled.set(true); + return null; + }); + dispatchRequest(request); + assertEquals(Boolean.TRUE, executeCalled.get()); + } + + public void testFlightInfoRequestWithInvalidPath() throws Exception { + RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET) + .withPath("/_flight/invalid_path") + .build(); + SetOnce executeCalled = new SetOnce<>(); + verifyingClient.setExecuteVerifier((action, actionRequest) -> { + assertEquals(NodesFlightInfoAction.INSTANCE.name(), action.name()); + assertNotNull(actionRequest); + executeCalled.set(true); + return new NodesFlightInfoResponse( + new ClusterName("test-cluster"), + Collections.singletonList(new NodeFlightInfo(mock(DiscoveryNode.class), mock(BoundTransportAddress.class))), + Collections.emptyList() + ); + }); + dispatchRequest(request); + assertNull(executeCalled.get()); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodeFlightInfoTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodeFlightInfoTests.java new file mode 100644 index 0000000000000..59e695313c16e --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodeFlightInfoTests.java @@ -0,0 +1,160 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.Version; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.net.InetAddress; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +@SuppressWarnings("unchecked") +public class NodeFlightInfoTests extends OpenSearchTestCase { + + public void testNodeFlightInfoSerialization() throws Exception { + DiscoveryNode node = new DiscoveryNode( + "test_node", + "test_node", + "hostname", + "localhost", + "127.0.0.1", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + new HashMap<>(), + new HashSet<>(), + Version.CURRENT + ); + + TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 47470); + BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + + NodeFlightInfo originalInfo = new NodeFlightInfo(node, boundAddress); + + BytesStreamOutput output = new BytesStreamOutput(); + originalInfo.writeTo(output); + + StreamInput input = output.bytes().streamInput(); + NodeFlightInfo deserializedInfo = new NodeFlightInfo(input); + + assertEquals(originalInfo.getNode(), deserializedInfo.getNode()); + assertEquals(originalInfo.getBoundAddress().boundAddresses().length, deserializedInfo.getBoundAddress().boundAddresses().length); + assertEquals(originalInfo.getBoundAddress().boundAddresses()[0], deserializedInfo.getBoundAddress().boundAddresses()[0]); + assertEquals(originalInfo.getBoundAddress().publishAddress(), deserializedInfo.getBoundAddress().publishAddress()); + } + + public void testNodeFlightInfoEquality() throws Exception { + DiscoveryNode node = new DiscoveryNode( + "test_node", + "test_node", + "hostname", + "localhost", + "127.0.0.1", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + new HashMap<>(), + new HashSet<>(), + Version.CURRENT + ); + + TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 47470); + BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + + NodeFlightInfo info1 = new NodeFlightInfo(node, boundAddress); + NodeFlightInfo info2 = new NodeFlightInfo(node, boundAddress); + + assertEquals(info1.getBoundAddress(), info2.getBoundAddress()); + } + + public void testGetters() throws Exception { + DiscoveryNode node = new DiscoveryNode( + "test_node", + "test_node", + "hostname", + "localhost", + "127.0.0.1", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + new HashMap<>(), + new HashSet<>(), + Version.CURRENT + ); + + TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 47470); + BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + + NodeFlightInfo info = new NodeFlightInfo(node, boundAddress); + + assertEquals(node, info.getNode()); + assertEquals(boundAddress, info.getBoundAddress()); + } + + public void testToXContent() throws Exception { + TransportAddress boundAddress1 = new TransportAddress(InetAddress.getLoopbackAddress(), 47470); + TransportAddress boundAddress2 = new TransportAddress(InetAddress.getLoopbackAddress(), 47471); + TransportAddress publishAddress = new TransportAddress(InetAddress.getLoopbackAddress(), 47472); + + BoundTransportAddress boundAddress = new BoundTransportAddress( + new TransportAddress[] { boundAddress1, boundAddress2 }, + publishAddress + ); + + NodeFlightInfo info = new NodeFlightInfo( + new DiscoveryNode( + "test_node", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + Collections.emptyMap(), + Collections.emptySet(), + Version.CURRENT + ), + boundAddress + ); + + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + builder.field("node_info"); + info.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map responseMap = parser.map(); + + Map nodeInfo = (Map) responseMap.get("node_info"); + assertNotNull("node_info object should exist", nodeInfo); + + Map flightServer = (Map) nodeInfo.get("flight_server"); + assertNotNull("flight_server object should exist", flightServer); + + List> boundAddresses = (List>) flightServer.get("bound_addresses"); + assertNotNull("bound_addresses array should exist", boundAddresses); + assertEquals("Should have 2 bound addresses", 2, boundAddresses.size()); + + assertEquals("localhost", boundAddresses.get(0).get("host")); + assertEquals(47470, boundAddresses.get(0).get("port")); + + assertEquals("localhost", boundAddresses.get(1).get("host")); + assertEquals(47471, boundAddresses.get(1).get("port")); + + Map publishAddressMap = (Map) flightServer.get("publish_address"); + assertNotNull("publish_address object should exist", publishAddressMap); + assertEquals("localhost", publishAddressMap.get("host")); + assertEquals(47472, publishAddressMap.get("port")); + } + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoRequestTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoRequestTests.java new file mode 100644 index 0000000000000..ef8f88b78c3ee --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoRequestTests.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.test.OpenSearchTestCase; + +public class NodesFlightInfoRequestTests extends OpenSearchTestCase { + + public void testNodesFlightInfoRequestSerialization() throws Exception { + NodesFlightInfoRequest originalRequest = new NodesFlightInfoRequest("node1", "node2"); + + BytesStreamOutput output = new BytesStreamOutput(); + originalRequest.writeTo(output); + + StreamInput input = output.bytes().streamInput(); + NodesFlightInfoRequest deserializedRequest = new NodesFlightInfoRequest(input); + + assertArrayEquals(originalRequest.nodesIds(), deserializedRequest.nodesIds()); + } + + public void testNodesFlightInfoRequestConcreteNodes() { + String[] nodeIds = new String[] { "node1", "node2" }; + NodesFlightInfoRequest request = new NodesFlightInfoRequest(nodeIds); + assertArrayEquals(nodeIds, request.nodesIds()); + } + + public void testNodesFlightInfoRequestAllNodes() { + NodesFlightInfoRequest request = new NodesFlightInfoRequest(); + assertEquals(0, request.nodesIds().length); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoResponseTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoResponseTests.java new file mode 100644 index 0000000000000..707a222fe381f --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/NodesFlightInfoResponseTests.java @@ -0,0 +1,241 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo; + +import org.opensearch.Version; +import org.opensearch.action.FailedNodeException; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.net.ConnectException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +@SuppressWarnings("unchecked") +public class NodesFlightInfoResponseTests extends OpenSearchTestCase { + + public void testNodesFlightInfoResponseSerialization() throws Exception { + ClusterName clusterName = new ClusterName("test-cluster"); + List nodes = new ArrayList<>(); + + DiscoveryNode node1 = createTestNode("node1"); + DiscoveryNode node2 = createTestNode("node2"); + + nodes.add(createNodeFlightInfo(node1, 47470)); + nodes.add(createNodeFlightInfo(node2, 47471)); + + NodesFlightInfoResponse originalResponse = new NodesFlightInfoResponse(clusterName, nodes, List.of()); + + BytesStreamOutput output = new BytesStreamOutput(); + originalResponse.writeTo(output); + + StreamInput input = output.bytes().streamInput(); + NodesFlightInfoResponse deserializedResponse = new NodesFlightInfoResponse(input); + assertEquals(originalResponse.getNodes().size(), deserializedResponse.getNodes().size()); + + for (int i = 0; i < originalResponse.getNodes().size(); i++) { + NodeFlightInfo originalNode = originalResponse.getNodes().get(i); + NodeFlightInfo deserializedNode = deserializedResponse.getNodes().get(i); + + assertEquals(originalNode.getNode().getId(), deserializedNode.getNode().getId()); + assertEquals(originalNode.getNode().getName(), deserializedNode.getNode().getName()); + assertEquals(originalNode.getBoundAddress().publishAddress(), deserializedNode.getBoundAddress().publishAddress()); + } + assertEquals(originalResponse.getClusterName(), deserializedResponse.getClusterName()); + } + + public void testNodesFlightInfoResponseEmpty() { + ClusterName clusterName = new ClusterName("test-cluster"); + List nodes = new ArrayList<>(); + + NodesFlightInfoResponse response = new NodesFlightInfoResponse(clusterName, nodes, List.of()); + + assertTrue(response.getNodes().isEmpty()); + assertEquals(clusterName, response.getClusterName()); + } + + public void testToXContentWithFailures() throws Exception { + NodesFlightInfoResponse response = getNodesFlightInfoResponse(); + + XContentBuilder builder = JsonXContent.contentBuilder(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map responseMap = parser.map(); + + Map nodesStats = (Map) responseMap.get("_nodes"); + assertNotNull("_nodes object should exist", nodesStats); + assertEquals(2, nodesStats.get("total")); + assertEquals(2, nodesStats.get("successful")); + assertEquals(2, nodesStats.get("failed")); + + assertEquals("test-cluster", responseMap.get("cluster_name")); + + Map nodes = (Map) responseMap.get("nodes"); + assertNotNull("nodes object should exist", nodes); + assertEquals(2, nodes.size()); + + Map firstNode = (Map) nodes.get("successful_node_1"); + assertNotNull(firstNode); + Map firstNodeFlightServer = (Map) firstNode.get("flight_server"); + assertNotNull(firstNodeFlightServer); + Map firstNodePublishAddress = (Map) firstNodeFlightServer.get("publish_address"); + assertEquals("localhost", firstNodePublishAddress.get("host")); + assertEquals(47470, firstNodePublishAddress.get("port")); + + Map secondNode = (Map) nodes.get("successful_node_2"); + assertNotNull(secondNode); + Map secondNodeFlightServer = (Map) secondNode.get("flight_server"); + assertNotNull(secondNodeFlightServer); + Map secondNodePublishAddress = (Map) secondNodeFlightServer.get("publish_address"); + assertEquals("localhost", secondNodePublishAddress.get("host")); + assertEquals(47471, secondNodePublishAddress.get("port")); + + List> failuresList = (List>) responseMap.get("failures"); + assertNotNull("failures array should exist", failuresList); + assertEquals(2, failuresList.size()); + + Map firstFailure = failuresList.get(0); + assertEquals("failed_node_1", firstFailure.get("node_id")); + assertEquals("Connection refused", firstFailure.get("reason")); + + Map secondFailure = failuresList.get(1); + assertEquals("failed_node_2", secondFailure.get("node_id")); + assertEquals("Node not found", secondFailure.get("reason")); + } + } + + private static NodesFlightInfoResponse getNodesFlightInfoResponse() { + DiscoveryNode node1 = new DiscoveryNode( + "successful_node_1", + "successful_node_1", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + Collections.emptyMap(), + Collections.emptySet(), + Version.CURRENT + ); + + List successfulNodes = getNodeFlightInfos(node1); + + return getNodesFlightInfoResponse(successfulNodes); + } + + private static NodesFlightInfoResponse getNodesFlightInfoResponse(List successfulNodes) { + List failures = Arrays.asList( + new FailedNodeException("failed_node_1", "Connection refused", new ConnectException("Connection refused")), + new FailedNodeException("failed_node_2", "Node not found", new Exception("Node not found")) + ); + + return new NodesFlightInfoResponse(new ClusterName("test-cluster"), successfulNodes, failures); + } + + private static List getNodeFlightInfos(DiscoveryNode node1) { + DiscoveryNode node2 = new DiscoveryNode( + "successful_node_2", + "successful_node_2", + new TransportAddress(InetAddress.getLoopbackAddress(), 9301), + Collections.emptyMap(), + Collections.emptySet(), + Version.CURRENT + ); + + TransportAddress address1 = new TransportAddress(InetAddress.getLoopbackAddress(), 47470); + return getNodeFlightInfos(node1, address1, node2); + } + + private static List getNodeFlightInfos(DiscoveryNode node1, TransportAddress address1, DiscoveryNode node2) { + BoundTransportAddress boundAddress1 = new BoundTransportAddress(new TransportAddress[] { address1 }, address1); + + TransportAddress address2 = new TransportAddress(InetAddress.getLoopbackAddress(), 47471); + BoundTransportAddress boundAddress2 = new BoundTransportAddress(new TransportAddress[] { address2 }, address2); + + return Arrays.asList(new NodeFlightInfo(node1, boundAddress1), new NodeFlightInfo(node2, boundAddress2)); + } + + public void testToXContentWithNoFailures() throws Exception { + NodesFlightInfoResponse response = getFlightInfoResponse(); + + XContentBuilder builder = JsonXContent.contentBuilder(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + Map responseMap = parser.map(); + + Map nodesStats = (Map) responseMap.get("_nodes"); + assertNotNull(nodesStats); + assertEquals(1, nodesStats.get("total")); + assertEquals(1, nodesStats.get("successful")); + assertEquals(0, nodesStats.get("failed")); + + assertEquals("test-cluster", responseMap.get("cluster_name")); + + Map nodes = (Map) responseMap.get("nodes"); + assertNotNull(nodes); + assertEquals(1, nodes.size()); + + assertNull("failures array should not exist", responseMap.get("failures")); + } + } + + private static NodesFlightInfoResponse getFlightInfoResponse() { + DiscoveryNode node = new DiscoveryNode( + "successful_node", + "successful_node", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + Collections.emptyMap(), + Collections.emptySet(), + Version.CURRENT + ); + + TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 47470); + BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + + return new NodesFlightInfoResponse( + new ClusterName("test-cluster"), + Collections.singletonList(new NodeFlightInfo(node, boundAddress)), + Collections.emptyList() + ); + } + + private DiscoveryNode createTestNode(String nodeId) { + return new DiscoveryNode( + nodeId, + nodeId, + "host" + nodeId, + "localhost", + "127.0.0.1", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + new HashMap<>(), + new HashSet<>(), + Version.CURRENT + ); + } + + private NodeFlightInfo createNodeFlightInfo(DiscoveryNode node, int port) { + TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), port); + BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + return new NodeFlightInfo(node, boundAddress); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/TransportNodesFlightInfoActionTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/TransportNodesFlightInfoActionTests.java new file mode 100644 index 0000000000000..ca4c4bf0c28c8 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/api/flightinfo/TransportNodesFlightInfoActionTests.java @@ -0,0 +1,182 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.api.flightinfo;/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import org.opensearch.Version; +import org.opensearch.action.FailedNodeException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.arrow.flight.bootstrap.FlightService; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; +import org.junit.Before; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class TransportNodesFlightInfoActionTests extends OpenSearchTestCase { + + private DiscoveryNode localNode; + private TransportNodesFlightInfoAction action; + private BoundTransportAddress boundAddress; + + @Before + public void setUp() throws Exception { + super.setUp(); + + localNode = new DiscoveryNode( + "local_node", + "local_node", + "host", + "localhost", + "127.0.0.1", + new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + new HashMap<>(), + new HashSet<>(), + Version.CURRENT + ); + + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterName()).thenReturn(new ClusterName("test-cluster")); + when(clusterService.localNode()).thenReturn(localNode); + + TransportAddress address = new TransportAddress(InetAddress.getLoopbackAddress(), 47470); + boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + + FlightService flightService = mock(FlightService.class); + when(flightService.getBoundAddress()).thenReturn(boundAddress); + + action = new TransportNodesFlightInfoAction( + Settings.EMPTY, + mock(ThreadPool.class), + clusterService, + mock(TransportService.class), + new ActionFilters(Collections.emptySet()), + flightService + ); + } + + public void testNewResponse() { + NodesFlightInfoRequest request = new NodesFlightInfoRequest(); + List nodeFlightInfos = Collections.singletonList(new NodeFlightInfo(localNode, boundAddress)); + List failures = Collections.emptyList(); + + NodesFlightInfoResponse response = action.newResponse(request, nodeFlightInfos, failures); + + assertNotNull(response); + assertEquals("test-cluster", response.getClusterName().value()); + assertEquals(1, response.getNodes().size()); + assertEquals(0, response.failures().size()); + + NodeFlightInfo nodeInfo = response.getNodes().get(0); + assertEquals(localNode, nodeInfo.getNode()); + assertEquals(boundAddress, nodeInfo.getBoundAddress()); + } + + public void testNewResponseWithFailures() { + NodesFlightInfoRequest request = new NodesFlightInfoRequest(); + List nodeFlightInfos = Collections.emptyList(); + List failures = Collections.singletonList(new FailedNodeException("failed_node", "test failure", null)); + + NodesFlightInfoResponse response = action.newResponse(request, nodeFlightInfos, failures); + + assertNotNull(response); + assertEquals("test-cluster", response.getClusterName().value()); + assertEquals(0, response.getNodes().size()); + assertEquals(1, response.failures().size()); + assertEquals("failed_node", response.failures().get(0).nodeId()); + assertEquals("test failure", response.failures().get(0).getMessage()); + } + + public void testNewNodeRequest() { + NodesFlightInfoRequest request = new NodesFlightInfoRequest("node1", "node2"); + NodesFlightInfoRequest.NodeFlightInfoRequest nodeRequest = action.newNodeRequest(request); + + assertNotNull(nodeRequest); + assertArrayEquals(new String[] { "node1", "node2" }, nodeRequest.request.nodesIds()); + } + + public void testNewNodeResponse() throws IOException { + NodeFlightInfo nodeInfo = new NodeFlightInfo(localNode, boundAddress); + BytesStreamOutput out = new BytesStreamOutput(); + nodeInfo.writeTo(out); + StreamInput in = out.bytes().streamInput(); + + NodeFlightInfo deserializedInfo = action.newNodeResponse(in); + + assertNotNull(deserializedInfo); + assertEquals(nodeInfo.getNode(), deserializedInfo.getNode()); + assertEquals(nodeInfo.getBoundAddress().publishAddress(), deserializedInfo.getBoundAddress().publishAddress()); + } + + public void testNodeOperation() { + NodesFlightInfoRequest.NodeFlightInfoRequest nodeRequest = new NodesFlightInfoRequest.NodeFlightInfoRequest( + new NodesFlightInfoRequest() + ); + + NodeFlightInfo response = action.nodeOperation(nodeRequest); + + assertNotNull(response); + assertEquals(localNode, response.getNode()); + assertEquals(boundAddress.publishAddress(), response.getBoundAddress().publishAddress()); + } + + public void testNodeOperationWithSpecificNodes() throws IOException { + NodesFlightInfoRequest request = new NodesFlightInfoRequest("local_node"); + NodesFlightInfoRequest.NodeFlightInfoRequest nodeRequest = new NodesFlightInfoRequest.NodeFlightInfoRequest(request); + + NodeFlightInfo response = action.nodeOperation(nodeRequest); + + assertNotNull(response); + assertEquals(localNode, response.getNode()); + assertEquals(boundAddress, response.getBoundAddress()); + } + + public void testNodeOperationWithInvalidNode() throws IOException { + NodesFlightInfoRequest request = new NodesFlightInfoRequest("invalid_node"); + NodesFlightInfoRequest.NodeFlightInfoRequest nodeRequest = new NodesFlightInfoRequest.NodeFlightInfoRequest(request); + + NodeFlightInfo response = action.nodeOperation(nodeRequest); + + assertNotNull(response); + assertEquals(localNode, response.getNode()); + assertEquals(boundAddress, response.getBoundAddress()); + } + + public void testSerialization() throws IOException { + NodesFlightInfoRequest request = new NodesFlightInfoRequest("node1", "node2"); + BytesStreamOutput out = new BytesStreamOutput(); + request.writeTo(out); + StreamInput in = out.bytes().streamInput(); + NodesFlightInfoRequest deserializedRequest = new NodesFlightInfoRequest(in); + + assertArrayEquals(request.nodesIds(), deserializedRequest.nodesIds()); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/FlightClientManagerTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/FlightClientManagerTests.java new file mode 100644 index 0000000000000..49f41a22b2ced --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/FlightClientManagerTests.java @@ -0,0 +1,400 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.arrow.flight.bootstrap; + +import org.apache.arrow.flight.Location; +import org.apache.arrow.flight.OSFlightClient; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.opensearch.Version; +import org.opensearch.arrow.flight.api.flightinfo.NodeFlightInfo; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoAction; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoRequest; +import org.opensearch.arrow.flight.api.flightinfo.NodesFlightInfoResponse; +import org.opensearch.arrow.flight.bootstrap.tls.SslContextProvider; +import org.opensearch.cluster.ClusterChangedEvent; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.transport.BoundTransportAddress; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.client.Client; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import io.netty.channel.EventLoopGroup; +import io.netty.util.NettyRuntime; + +import static org.opensearch.arrow.flight.bootstrap.FlightClientManager.LOCATION_TIMEOUT_MS; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.when; + +@SuppressWarnings("unchecked") +public class FlightClientManagerTests extends OpenSearchTestCase { + + private static BufferAllocator allocator; + private static EventLoopGroup elg; + private static ExecutorService executorService; + private static final AtomicInteger port = new AtomicInteger(0); + + private ClusterService clusterService; + private Client client; + private ClusterState state; + private FlightClientManager clientManager; + private ScheduledExecutorService locationUpdaterExecutor; + + @BeforeClass + public static void setupClass() throws Exception { + ServerConfig.init(Settings.EMPTY); + allocator = new RootAllocator(); + elg = ServerConfig.createELG("test-grpc-worker-elg", NettyRuntime.availableProcessors() * 2); + executorService = ServerConfig.createELG("test-grpc-worker", NettyRuntime.availableProcessors() * 2); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + locationUpdaterExecutor = Executors.newScheduledThreadPool(1); + + FeatureFlagSetter.set(FeatureFlags.ARROW_STREAMS_SETTING.getKey()); + clusterService = mock(ClusterService.class); + client = mock(Client.class); + state = getDefaultState(); + when(clusterService.state()).thenReturn(state); + + mockFlightInfoResponse(state.nodes(), 0); + + SslContextProvider sslContextProvider = null; + + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.executor(ServerConfig.FLIGHT_CLIENT_THREAD_POOL_NAME)).thenReturn(executorService); + clientManager = new FlightClientManager(allocator, clusterService, sslContextProvider, elg, threadPool, client); + ClusterChangedEvent event = new ClusterChangedEvent("test", state, ClusterState.EMPTY_STATE); + clientManager.clusterChanged(event); + assertBusy(() -> { + assertEquals("Flight client isn't built in time limit", 2, clientManager.getClients().size()); + assertTrue("local_node should exist", clientManager.getFlightClient("local_node").isPresent()); + assertNotNull("local_node should exist", clientManager.getFlightClient("local_node").get()); + assertTrue("remote_node should exist", clientManager.getFlightClient("remote_node").isPresent()); + assertNotNull("remote_node should exist", clientManager.getFlightClient("remote_node").get()); + }, 2, TimeUnit.SECONDS); + } + + private void mockFlightInfoResponse(DiscoveryNodes nodes, int sleepDuration) { + doAnswer(invocation -> { + locationUpdaterExecutor.schedule(() -> { + try { + NodesFlightInfoRequest request = invocation.getArgument(1); + ActionListener listener = invocation.getArgument(2); + + List nodeInfos = new ArrayList<>(); + for (DiscoveryNode node : nodes) { + if (request.nodesIds().length == 0 || Arrays.asList(request.nodesIds()).contains(node.getId())) { + int flightPort = getBaseStreamPort() + port.addAndGet(2); + TransportAddress address = new TransportAddress( + InetAddress.getByName(node.getAddress().getAddress()), + flightPort + ); + BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + NodeFlightInfo nodeInfo = new NodeFlightInfo(node, boundAddress); + nodeInfos.add(nodeInfo); + } + } + NodesFlightInfoResponse response = new NodesFlightInfoResponse(ClusterName.DEFAULT, nodeInfos, Collections.emptyList()); + listener.onResponse(response); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + }, sleepDuration, TimeUnit.MILLISECONDS); + return null; + }).when(client).execute(eq(NodesFlightInfoAction.INSTANCE), any(NodesFlightInfoRequest.class), any(ActionListener.class)); + + } + + @Override + public void tearDown() throws Exception { + locationUpdaterExecutor.shutdown(); + super.tearDown(); + clientManager.close(); + } + + private ClusterState getDefaultState() throws Exception { + int testPort = getBasePort() + port.addAndGet(2); + + DiscoveryNode localNode = createNode("local_node", "127.0.0.1", testPort); + DiscoveryNode remoteNode = createNode("remote_node", "127.0.0.2", testPort + 1); + + // Setup initial cluster state + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + nodesBuilder.add(remoteNode); + nodesBuilder.add(localNode); + nodesBuilder.localNodeId(localNode.getId()); + DiscoveryNodes nodes = nodesBuilder.build(); + + return ClusterState.builder(new ClusterName("test")).nodes(nodes).build(); + } + + private DiscoveryNode createNode(String nodeId, String host, int port) throws Exception { + TransportAddress address = new TransportAddress(InetAddress.getByName(host), port); + Map attributes = new HashMap<>(); + attributes.put("arrow.streams.enabled", "true"); + Set roles = Collections.singleton(DiscoveryNodeRole.DATA_ROLE); + return new DiscoveryNode(nodeId, address, attributes, roles, Version.CURRENT); + } + + @AfterClass + public static void tearClass() { + allocator.close(); + } + + public void testGetFlightClientForExistingNode() { + validateNodes(); + } + + public void testGetFlightClientLocation() { + for (DiscoveryNode node : state.nodes()) { + Location location = clientManager.getFlightClientLocation(node.getId()); + assertNotNull("Flight client location should be returned", location); + assertEquals("Location host should match", node.getHostAddress(), location.getUri().getHost()); + } + } + + public void testGetFlightClientForNonExistentNode() throws Exception { + assertTrue(clientManager.getFlightClient("non_existent_node").isEmpty()); + } + + public void testClusterChangedWithNodesChanged() throws Exception { + DiscoveryNode newNode = createNode("new_node", "127.0.0.3", getBasePort() + port.addAndGet(1)); + DiscoveryNodes.Builder newNodesBuilder = DiscoveryNodes.builder(); + + for (DiscoveryNode node : state.nodes()) { + newNodesBuilder.add(node); + } + newNodesBuilder.localNodeId("local_node"); + // Update cluster state with new node + newNodesBuilder.add(newNode); + DiscoveryNodes newNodes = newNodesBuilder.build(); + + ClusterState newState = ClusterState.builder(new ClusterName("test")).nodes(newNodes).build(); + mockFlightInfoResponse(newNodes, 0); + when(clusterService.state()).thenReturn(newState); + clientManager.clusterChanged(new ClusterChangedEvent("test", newState, state)); + + for (DiscoveryNode node : newState.nodes()) { + assertBusy( + () -> { assertNotNull("Flight client isn't built in time limit", clientManager.getFlightClient(node.getId())); }, + 5, + TimeUnit.SECONDS + ); + } + } + + public void testClusterChangedWithNoNodesChanged() throws Exception { + ClusterChangedEvent event = new ClusterChangedEvent("test", state, state); + clientManager.clusterChanged(event); + + // Verify original client still exists + for (DiscoveryNode node : state.nodes()) { + assertNotNull(clientManager.getFlightClient(node.getId())); + } + } + + public void testGetLocalNodeId() throws Exception { + assertEquals("Local node ID should match", "local_node", clientManager.getLocalNodeId()); + } + + public void testCloseWithActiveClients() throws Exception { + for (DiscoveryNode node : state.nodes()) { + OSFlightClient client = clientManager.getFlightClient(node.getId()).get(); + assertNotNull(client); + } + + clientManager.close(); + assertEquals(0, clientManager.getFlightClients().size()); + } + + public void testIncompatibleNodeVersion() throws Exception { + Map attributes = new HashMap<>(); + attributes.put("arrow.streams.enabled", "true"); + DiscoveryNode oldVersionNode = new DiscoveryNode( + "old_version_node", + new TransportAddress(InetAddress.getByName("127.0.0.3"), getBasePort() + port.addAndGet(1)), + attributes, + Collections.singleton(DiscoveryNodeRole.DATA_ROLE), + Version.fromString("2.18.0") // Version before Arrow Flight introduction + ); + + // Update cluster state with old version node + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + nodesBuilder.add(oldVersionNode); + nodesBuilder.localNodeId("local_node"); + DiscoveryNodes nodes = nodesBuilder.build(); + ClusterState oldVersionState = ClusterState.builder(new ClusterName("test")).nodes(nodes).build(); + + when(clusterService.state()).thenReturn(oldVersionState); + mockFlightInfoResponse(nodes, 0); + + assertFalse(clientManager.getFlightClient(oldVersionNode.getId()).isPresent()); + } + + public void testGetFlightClientLocationTimeout() throws Exception { + reset(client); + + String nodeId = "test_node"; + DiscoveryNode testNode = createNode(nodeId, "127.0.0.1", getBasePort() + port.addAndGet(2)); + + // Update cluster state with the test node + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + nodesBuilder.add(testNode); + nodesBuilder.localNodeId(nodeId); + ClusterState newState = ClusterState.builder(new ClusterName("test")).nodes(nodesBuilder.build()).build(); + when(clusterService.state()).thenReturn(newState); + // Mock a delayed response that will cause timeout + mockFlightInfoResponse(newState.nodes(), LOCATION_TIMEOUT_MS + 100); + + ClusterChangedEvent event = new ClusterChangedEvent("test", newState, ClusterState.EMPTY_STATE); + clientManager.clusterChanged(event); + assertFalse(clientManager.getFlightClient(nodeId).isPresent()); + } + + public void testGetFlightClientLocationExecutionError() throws Exception { + reset(client); + + String nodeId = "test_node"; + DiscoveryNode testNode = createNode(nodeId, "127.0.0.1", getBasePort() + port.addAndGet(2)); + + // Update cluster state with the test node + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + nodesBuilder.add(testNode); + nodesBuilder.localNodeId(nodeId); + ClusterState newState = ClusterState.builder(new ClusterName("test")).nodes(nodesBuilder.build()).build(); + + when(clusterService.state()).thenReturn(newState); + + // Mock failure + doAnswer(invocation -> { + ActionListener listener = invocation.getArgument(2); + listener.onFailure(new RuntimeException("Test execution error")); + return null; + }).when(client).execute(eq(NodesFlightInfoAction.INSTANCE), any(NodesFlightInfoRequest.class), any(ActionListener.class)); + + ClusterChangedEvent event = new ClusterChangedEvent("test", newState, ClusterState.EMPTY_STATE); + clientManager.clusterChanged(event); + + assertFalse(clientManager.getFlightClient(nodeId).isPresent()); + } + + public void testFailedClusterUpdateButSuccessfulDirectRequest() throws Exception { + reset(client); + + String nodeId = "test_node"; + DiscoveryNode testNode = createNode(nodeId, "127.0.0.1", getBasePort() + port.addAndGet(2)); + + // Update cluster state with the test node + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + nodesBuilder.add(testNode); + nodesBuilder.localNodeId(nodeId); + ClusterState newState = ClusterState.builder(new ClusterName("test")).nodes(nodesBuilder.build()).build(); + + when(clusterService.state()).thenReturn(newState); + + // First mock call fails during cluster update + AtomicBoolean firstCall = new AtomicBoolean(true); + doAnswer(invocation -> { + locationUpdaterExecutor.schedule(() -> { + ActionListener listener = invocation.getArgument(2); + if (firstCall.getAndSet(false)) { + // Fail on first call (during cluster update) + listener.onFailure(new RuntimeException("Failed during cluster update")); + } else { + // Succeed on second call (direct request) + try { + NodesFlightInfoRequest request = invocation.getArgument(1); + List nodeInfos = new ArrayList<>(); + for (DiscoveryNode node : newState.nodes()) { + if (request.nodesIds().length == 0 || Arrays.asList(request.nodesIds()).contains(node.getId())) { + int flightPort = getBaseStreamPort() + port.addAndGet(2); + TransportAddress address = new TransportAddress( + InetAddress.getByName(node.getAddress().getAddress()), + flightPort + ); + BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { address }, address); + NodeFlightInfo nodeInfo = new NodeFlightInfo(node, boundAddress); + nodeInfos.add(nodeInfo); + } + } + NodesFlightInfoResponse response = new NodesFlightInfoResponse( + ClusterName.DEFAULT, + nodeInfos, + Collections.emptyList() + ); + listener.onResponse(response); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + } + }, 0, TimeUnit.MICROSECONDS); + return null; + }).when(client).execute(eq(NodesFlightInfoAction.INSTANCE), any(NodesFlightInfoRequest.class), any(ActionListener.class)); + + ClusterChangedEvent event = new ClusterChangedEvent("test", newState, ClusterState.EMPTY_STATE); + clientManager.clusterChanged(event); + assertBusy(() -> { assertFalse("first call should be invoked", firstCall.get()); }, 5, TimeUnit.SECONDS); + // Verify that the client can still be created successfully on direct request + clientManager.buildClientAsync(nodeId); + assertBusy( + () -> { + assertNotNull("Flight client should be created successfully on direct request", clientManager.getFlightClient(nodeId)); + }, + 5, + TimeUnit.SECONDS + ); + } + + private void validateNodes() { + for (DiscoveryNode node : state.nodes()) { + Optional client = clientManager.getFlightClient(node.getId()); + assertTrue("Flight client should be created for node [" + node.getId() + "].", client.isPresent()); + assertNotNull("Flight client should be created for node [" + node.getId() + "].", client.get()); + } + } + + protected static int getBaseStreamPort() { + return getBasePort(9401); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/FlightServiceTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/FlightServiceTests.java new file mode 100644 index 0000000000000..35badb7b452eb --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/FlightServiceTests.java @@ -0,0 +1,160 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.arrow.flight.bootstrap; + +import org.opensearch.Version; +import org.opensearch.arrow.flight.bootstrap.tls.SslContextProvider; +import org.opensearch.cluster.ClusterName; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.network.NetworkService; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.client.Client; + +import java.net.InetAddress; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FlightServiceTests extends OpenSearchTestCase { + + private Settings settings; + private ClusterService clusterService; + private NetworkService networkService; + private ThreadPool threadPool; + private final AtomicInteger port = new AtomicInteger(0); + private DiscoveryNode localNode; + + @Override + public void setUp() throws Exception { + super.setUp(); + FeatureFlagSetter.set(FeatureFlags.ARROW_STREAMS_SETTING.getKey()); + int availablePort = getBasePort(9500) + port.addAndGet(1); + settings = Settings.EMPTY; + localNode = createNode(availablePort); + + // Setup initial cluster state + DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(); + nodesBuilder.localNodeId(localNode.getId()); + nodesBuilder.add(localNode); + DiscoveryNodes nodes = nodesBuilder.build(); + ClusterState clusterState = ClusterState.builder(new ClusterName("test")).nodes(nodes).build(); + clusterService = mock(ClusterService.class); + when(clusterService.state()).thenReturn(clusterState); + + threadPool = mock(ThreadPool.class); + when(threadPool.executor(ServerConfig.FLIGHT_SERVER_THREAD_POOL_NAME)).thenReturn(mock(ExecutorService.class)); + when(threadPool.executor(ServerConfig.FLIGHT_CLIENT_THREAD_POOL_NAME)).thenReturn(mock(ExecutorService.class)); + networkService = new NetworkService(Collections.emptyList()); + } + + public void testInitializeWithSslDisabled() throws Exception { + + Settings noSslSettings = Settings.builder().put("arrow.ssl.enable", false).build(); + + try (FlightService noSslService = new FlightService(noSslSettings)) { + noSslService.setClusterService(clusterService); + noSslService.setThreadPool(threadPool); + noSslService.setClient(mock(Client.class)); + noSslService.setNetworkService(networkService); + noSslService.start(); + SslContextProvider sslContextProvider = noSslService.getSslContextProvider(); + assertNull("SSL context provider should be null", sslContextProvider); + assertNotNull(noSslService.getFlightClientManager()); + assertNotNull(noSslService.getBoundAddress()); + } + } + + public void testStartAndStop() throws Exception { + try (FlightService testService = new FlightService(Settings.EMPTY)) { + testService.setClusterService(clusterService); + testService.setThreadPool(threadPool); + testService.setClient(mock(Client.class)); + testService.setNetworkService(networkService); + testService.start(); + testService.stop(); + testService.start(); + assertNotNull(testService.getStreamManager()); + } + } + + public void testInitializeWithoutSecureTransportSettingsProvider() { + Settings sslSettings = Settings.builder().put(settings).put("arrow.ssl.enable", true).build(); + + try (FlightService sslService = new FlightService(sslSettings)) { + // Should throw exception when initializing without provider + expectThrows(RuntimeException.class, () -> { + sslService.setClusterService(clusterService); + sslService.setThreadPool(threadPool); + sslService.setClient(mock(Client.class)); + sslService.setNetworkService(networkService); + sslService.start(); + }); + } + } + + public void testServerStartupFailure() { + Settings invalidSettings = Settings.builder() + .put(ServerComponents.SETTING_FLIGHT_PUBLISH_PORT.getKey(), "-100") // Invalid port + .build(); + try (FlightService invalidService = new FlightService(invalidSettings)) { + invalidService.setClusterService(clusterService); + invalidService.setThreadPool(threadPool); + invalidService.setClient(mock(Client.class)); + invalidService.setNetworkService(networkService); + expectThrows(RuntimeException.class, () -> { invalidService.doStart(); }); + } + } + + public void testLifecycleStateTransitions() throws Exception { + // Find new port for this test + try (FlightService testService = new FlightService(Settings.EMPTY)) { + testService.setClusterService(clusterService); + testService.setThreadPool(threadPool); + testService.setClient(mock(Client.class)); + testService.setNetworkService(networkService); + // Test all state transitions + testService.start(); + assertEquals("STARTED", testService.lifecycleState().toString()); + + testService.stop(); + assertEquals("STOPPED", testService.lifecycleState().toString()); + + testService.close(); + assertEquals("CLOSED", testService.lifecycleState().toString()); + } + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + private DiscoveryNode createNode(int port) throws Exception { + TransportAddress address = new TransportAddress(InetAddress.getByName("127.0.0.1"), port); + Map attributes = new HashMap<>(); + attributes.put("arrow.streams.enabled", "true"); + + Set roles = Collections.singleton(DiscoveryNodeRole.DATA_ROLE); + return new DiscoveryNode("local_node", address, attributes, roles, Version.CURRENT); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/ServerConfigTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/ServerConfigTests.java new file mode 100644 index 0000000000000..9419e26318046 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/bootstrap/ServerConfigTests.java @@ -0,0 +1,80 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.arrow.flight.bootstrap; + +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.ScalingExecutorBuilder; + +import static org.opensearch.arrow.flight.bootstrap.ServerComponents.SETTING_FLIGHT_PUBLISH_PORT; + +public class ServerConfigTests extends OpenSearchTestCase { + + private Settings settings; + + @Override + public void setUp() throws Exception { + super.setUp(); + settings = Settings.builder() + .put("arrow.allocation.manager.type", "Netty") + .put("arrow.enable_null_check_for_get", false) + .put("arrow.enable_unsafe_memory_access", true) + .put("arrow.memory.debug.allocator", false) + .put("arrow.ssl.enable", true) + .put("thread_pool.flight-server.min", 1) + .put("thread_pool.flight-server.max", 4) + .put("thread_pool.flight-server.keep_alive", TimeValue.timeValueMinutes(5)) + .build(); + } + + public void testInit() { + ServerConfig.init(settings); + + // Verify system properties are set correctly + assertEquals("Netty", System.getProperty("arrow.allocation.manager.type")); + assertEquals("false", System.getProperty("arrow.enable_null_check_for_get")); + assertEquals("true", System.getProperty("arrow.enable_unsafe_memory_access")); + assertEquals("false", System.getProperty("arrow.memory.debug.allocator")); + + // Verify SSL settings + assertTrue(ServerConfig.isSslEnabled()); + + ScalingExecutorBuilder executorBuilder = ServerConfig.getServerExecutorBuilder(); + assertNotNull(executorBuilder); + assertEquals(3, executorBuilder.getRegisteredSettings().size()); + assertEquals(1, executorBuilder.getRegisteredSettings().get(0).get(settings)); // min + assertEquals(4, executorBuilder.getRegisteredSettings().get(1).get(settings)); // max + assertEquals(TimeValue.timeValueMinutes(5), executorBuilder.getRegisteredSettings().get(2).get(settings)); // keep alive + } + + public void testGetSettings() { + var settings = ServerConfig.getSettings(); + assertNotNull(settings); + assertFalse(settings.isEmpty()); + + assertTrue(settings.contains(ServerConfig.ARROW_ALLOCATION_MANAGER_TYPE)); + assertTrue(settings.contains(ServerConfig.ARROW_ENABLE_NULL_CHECK_FOR_GET)); + assertTrue(settings.contains(ServerConfig.ARROW_ENABLE_UNSAFE_MEMORY_ACCESS)); + assertTrue(settings.contains(ServerConfig.ARROW_ENABLE_DEBUG_ALLOCATOR)); + assertTrue(settings.contains(ServerConfig.ARROW_SSL_ENABLE)); + } + + public void testDefaultSettings() { + Settings defaultSettings = Settings.EMPTY; + ServerConfig.init(defaultSettings); + + // Verify default values + assertEquals(-1, SETTING_FLIGHT_PUBLISH_PORT.get(defaultSettings).intValue()); + assertEquals("Netty", ServerConfig.ARROW_ALLOCATION_MANAGER_TYPE.get(defaultSettings)); + assertFalse(ServerConfig.ARROW_ENABLE_NULL_CHECK_FOR_GET.get(defaultSettings)); + assertTrue(ServerConfig.ARROW_ENABLE_UNSAFE_MEMORY_ACCESS.get(defaultSettings)); + assertFalse(ServerConfig.ARROW_ENABLE_DEBUG_ALLOCATOR.get(defaultSettings)); + assertFalse(ServerConfig.ARROW_SSL_ENABLE.get(defaultSettings)); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/BaseFlightProducerTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/BaseFlightProducerTests.java new file mode 100644 index 0000000000000..479b89127ced8 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/BaseFlightProducerTests.java @@ -0,0 +1,463 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.FlightProducer; +import org.apache.arrow.flight.Ticket; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.dictionary.DictionaryProvider; +import org.apache.arrow.vector.ipc.message.IpcOption; +import org.opensearch.arrow.flight.bootstrap.FlightClientManager; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class BaseFlightProducerTests extends OpenSearchTestCase { + + private BaseFlightProducer baseFlightProducer; + private FlightStreamManager streamManager; + private StreamProducer streamProducer; + private StreamProducer.BatchedJob batchedJob; + private static final String LOCAL_NODE_ID = "localNodeId"; + private static final FlightClientManager flightClientManager = mock(FlightClientManager.class); + private final Ticket ticket = new Ticket((new FlightStreamTicket("test-ticket", LOCAL_NODE_ID)).toBytes()); + private BufferAllocator allocator; + + @Override + @SuppressWarnings("unchecked") + public void setUp() throws Exception { + super.setUp(); + FeatureFlagSetter.set(FeatureFlags.ARROW_STREAMS_SETTING.getKey()); + streamManager = mock(FlightStreamManager.class); + when(streamManager.getStreamTicketFactory()).thenReturn(new FlightStreamTicketFactory(() -> LOCAL_NODE_ID)); + when(flightClientManager.getLocalNodeId()).thenReturn(LOCAL_NODE_ID); + allocator = mock(BufferAllocator.class); + streamProducer = mock(StreamProducer.class); + batchedJob = mock(StreamProducer.BatchedJob.class); + baseFlightProducer = new BaseFlightProducer(flightClientManager, streamManager, allocator); + } + + private static class TestServerStreamListener implements FlightProducer.ServerStreamListener { + private final CountDownLatch completionLatch = new CountDownLatch(1); + private final AtomicInteger putNextCount = new AtomicInteger(0); + private final AtomicBoolean isCancelled = new AtomicBoolean(false); + private Throwable error; + private final AtomicBoolean dataConsumed = new AtomicBoolean(false); + private final AtomicBoolean ready = new AtomicBoolean(false); + private Runnable onReadyHandler; + private Runnable onCancelHandler; + + @Override + public void putNext() { + assertFalse(dataConsumed.get()); + putNextCount.incrementAndGet(); + dataConsumed.set(true); + } + + @Override + public boolean isReady() { + return ready.get(); + } + + public void setReady(boolean val) { + ready.set(val); + if (this.onReadyHandler != null) { + this.onReadyHandler.run(); + } + } + + @Override + public void start(VectorSchemaRoot root) { + // No-op for this test + } + + @Override + public void start(VectorSchemaRoot root, DictionaryProvider dictionaries, IpcOption option) {} + + @Override + public void putNext(ArrowBuf metadata) { + putNext(); + } + + @Override + public void putMetadata(ArrowBuf metadata) { + + } + + @Override + public void completed() { + completionLatch.countDown(); + } + + @Override + public void error(Throwable t) { + error = t; + completionLatch.countDown(); + } + + @Override + public boolean isCancelled() { + return isCancelled.get(); + } + + @Override + public void setOnReadyHandler(Runnable handler) { + this.onReadyHandler = handler; + } + + @Override + public void setOnCancelHandler(Runnable handler) { + this.onCancelHandler = handler; + } + + public void resetConsumptionLatch() { + dataConsumed.set(false); + } + + public boolean getDataConsumed() { + return dataConsumed.get(); + } + + public int getPutNextCount() { + return putNextCount.get(); + } + + public Throwable getError() { + return error; + } + + public void cancel() { + isCancelled.set(true); + if (this.onCancelHandler != null) { + this.onCancelHandler.run(); + } + } + } + + public void testGetStream_SuccessfulFlow() throws Exception { + final VectorSchemaRoot root = mock(VectorSchemaRoot.class); + when(streamManager.removeStreamProducer(any(FlightStreamTicket.class))).thenReturn( + Optional.of(FlightStreamManager.StreamProducerHolder.create(streamProducer, allocator)) + ); + when(streamProducer.createJob(any(BufferAllocator.class))).thenReturn(batchedJob); + when(streamProducer.createRoot(any(BufferAllocator.class))).thenReturn(root); + + AtomicInteger flushCount = new AtomicInteger(0); + TestServerStreamListener listener = new TestServerStreamListener(); + doAnswer(invocation -> { + StreamProducer.FlushSignal flushSignal = invocation.getArgument(1); + for (int i = 0; i < 3; i++) { + Thread clientThread = new Thread(() -> { + listener.setReady(false); + listener.setReady(true); + }); + listener.setReady(false); + clientThread.start(); + flushSignal.awaitConsumption(TimeValue.timeValueMillis(100)); + assertTrue(listener.getDataConsumed()); + flushCount.incrementAndGet(); + listener.resetConsumptionLatch(); + } + return null; + }).when(batchedJob).run(any(VectorSchemaRoot.class), any(StreamProducer.FlushSignal.class)); + baseFlightProducer.getStream(mock(FlightProducer.CallContext.class), ticket, listener); + + assertNull(listener.getError()); + assertEquals(3, listener.getPutNextCount()); + assertEquals(3, flushCount.get()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + verify(root).close(); + } + + public void testGetStream_WithSlowClient() throws Exception { + final VectorSchemaRoot root = mock(VectorSchemaRoot.class); + + when(streamManager.removeStreamProducer(any(FlightStreamTicket.class))).thenReturn( + Optional.of(FlightStreamManager.StreamProducerHolder.create(streamProducer, allocator)) + ); + when(streamProducer.createJob(any(BufferAllocator.class))).thenReturn(batchedJob); + when(streamProducer.createRoot(any(BufferAllocator.class))).thenReturn(root); + + AtomicInteger flushCount = new AtomicInteger(0); + TestServerStreamListener listener = new TestServerStreamListener(); + + doAnswer(invocation -> { + StreamProducer.FlushSignal flushSignal = invocation.getArgument(1); + for (int i = 0; i < 5; i++) { + Thread clientThread = new Thread(() -> { + try { + listener.setReady(false); + Thread.sleep(100); + listener.setReady(true); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + listener.setReady(false); + clientThread.start(); + flushSignal.awaitConsumption(TimeValue.timeValueMillis(300)); // waiting for consumption for more than client thread sleep + assertTrue(listener.getDataConsumed()); + flushCount.incrementAndGet(); + listener.resetConsumptionLatch(); + } + return null; + }).when(batchedJob).run(any(), any()); + + baseFlightProducer.getStream(mock(FlightProducer.CallContext.class), ticket, listener); + + assertNull(listener.getError()); + assertEquals(5, listener.getPutNextCount()); + assertEquals(5, flushCount.get()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + verify(root).close(); + } + + public void testGetStream_WithSlowClientTimeout() throws Exception { + final VectorSchemaRoot root = mock(VectorSchemaRoot.class); + + when(streamManager.removeStreamProducer(any(FlightStreamTicket.class))).thenReturn( + Optional.of(FlightStreamManager.StreamProducerHolder.create(streamProducer, allocator)) + ); + when(streamProducer.createJob(any(BufferAllocator.class))).thenReturn(batchedJob); + when(streamProducer.createRoot(any(BufferAllocator.class))).thenReturn(root); + + AtomicInteger flushCount = new AtomicInteger(0); + TestServerStreamListener listener = new TestServerStreamListener(); + doAnswer(invocation -> { + StreamProducer.FlushSignal flushSignal = invocation.getArgument(1); + for (int i = 0; i < 5; i++) { + Thread clientThread = new Thread(() -> { + try { + listener.setReady(false); + Thread.sleep(400); + listener.setReady(true); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + listener.setReady(false); + clientThread.start(); + flushSignal.awaitConsumption(TimeValue.timeValueMillis(100)); // waiting for consumption for less than client thread sleep + assertTrue(listener.getDataConsumed()); + flushCount.incrementAndGet(); + listener.resetConsumptionLatch(); + } + return null; + }).when(batchedJob).run(any(), any()); + + assertThrows(RuntimeException.class, () -> baseFlightProducer.getStream(mock(FlightProducer.CallContext.class), ticket, listener)); + + assertNotNull(listener.getError()); + assertEquals("Stream deadline exceeded for consumption", listener.getError().getMessage()); + assertEquals(0, listener.getPutNextCount()); + assertEquals(0, flushCount.get()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + verify(root).close(); + } + + public void testGetStream_WithClientCancel() throws Exception { + final VectorSchemaRoot root = mock(VectorSchemaRoot.class); + + when(streamManager.removeStreamProducer(any(FlightStreamTicket.class))).thenReturn( + Optional.of(FlightStreamManager.StreamProducerHolder.create(streamProducer, allocator)) + ); + when(streamProducer.createJob(any(BufferAllocator.class))).thenReturn(batchedJob); + when(streamProducer.createRoot(any(BufferAllocator.class))).thenReturn(root); + + AtomicInteger flushCount = new AtomicInteger(0); + TestServerStreamListener listener = new TestServerStreamListener(); + doAnswer(invocation -> { + StreamProducer.FlushSignal flushSignal = invocation.getArgument(1); + for (int i = 0; i < 5; i++) { + int finalI = i; + Thread clientThread = new Thread(() -> { + if (finalI == 4) { + listener.cancel(); + } else { + listener.setReady(false); + listener.setReady(true); + } + }); + listener.setReady(false); + clientThread.start(); + flushSignal.awaitConsumption(TimeValue.timeValueMillis(100)); // waiting for consumption for less than client thread sleep + assertTrue(listener.getDataConsumed()); + flushCount.incrementAndGet(); + listener.resetConsumptionLatch(); + } + return null; + }).when(batchedJob).run(any(), any()); + + assertThrows(RuntimeException.class, () -> baseFlightProducer.getStream(mock(FlightProducer.CallContext.class), ticket, listener)); + assertNotNull(listener.getError()); + assertEquals("Stream cancelled by client", listener.getError().getMessage()); + assertEquals(4, listener.getPutNextCount()); + assertEquals(4, flushCount.get()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + verify(root).close(); + } + + public void testGetStream_WithUnresponsiveClient() throws Exception { + final VectorSchemaRoot root = mock(VectorSchemaRoot.class); + + when(streamManager.removeStreamProducer(any(FlightStreamTicket.class))).thenReturn( + Optional.of(FlightStreamManager.StreamProducerHolder.create(streamProducer, allocator)) + ); + when(streamProducer.createJob(any(BufferAllocator.class))).thenReturn(batchedJob); + when(streamProducer.createRoot(any(BufferAllocator.class))).thenReturn(root); + + AtomicInteger flushCount = new AtomicInteger(0); + TestServerStreamListener listener = new TestServerStreamListener(); + doAnswer(invocation -> { + StreamProducer.FlushSignal flushSignal = invocation.getArgument(1); + for (int i = 0; i < 5; i++) { + Thread clientThread = new Thread(() -> { + listener.setReady(false); + // not setting ready to simulate unresponsive behaviour + }); + listener.setReady(false); + clientThread.start(); + flushSignal.awaitConsumption(TimeValue.timeValueMillis(100)); // waiting for consumption for less than client thread sleep + assertTrue(listener.getDataConsumed()); + flushCount.incrementAndGet(); + listener.resetConsumptionLatch(); + } + return null; + }).when(batchedJob).run(any(), any()); + + assertThrows(RuntimeException.class, () -> baseFlightProducer.getStream(mock(FlightProducer.CallContext.class), ticket, listener)); + + assertNotNull(listener.getError()); + assertEquals("Stream deadline exceeded for consumption", listener.getError().getMessage()); + assertEquals(0, listener.getPutNextCount()); + assertEquals(0, flushCount.get()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + verify(root).close(); + } + + public void testGetStream_WithServerBackpressure() throws Exception { + final VectorSchemaRoot root = mock(VectorSchemaRoot.class); + + when(streamManager.removeStreamProducer(any(FlightStreamTicket.class))).thenReturn( + Optional.of(FlightStreamManager.StreamProducerHolder.create(streamProducer, allocator)) + ); + when(streamProducer.createJob(any(BufferAllocator.class))).thenReturn(batchedJob); + when(streamProducer.createRoot(any(BufferAllocator.class))).thenReturn(root); + + TestServerStreamListener listener = new TestServerStreamListener(); + AtomicInteger flushCount = new AtomicInteger(0); + doAnswer(invocation -> { + StreamProducer.FlushSignal flushSignal = invocation.getArgument(1); + for (int i = 0; i < 5; i++) { + Thread clientThread = new Thread(() -> { + listener.setReady(false); + listener.setReady(true); + }); + listener.setReady(false); + clientThread.start(); + Thread.sleep(100); // simulating writer backpressure + flushSignal.awaitConsumption(TimeValue.timeValueMillis(100)); + assertTrue(listener.getDataConsumed()); + flushCount.incrementAndGet(); + listener.resetConsumptionLatch(); + } + return null; + }).when(batchedJob).run(any(VectorSchemaRoot.class), any(StreamProducer.FlushSignal.class)); + + baseFlightProducer.getStream(mock(FlightProducer.CallContext.class), ticket, listener); + + assertNull(listener.getError()); + assertEquals(5, listener.getPutNextCount()); + assertEquals(5, flushCount.get()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + verify(root).close(); + } + + public void testGetStream_WithServerError() throws Exception { + final VectorSchemaRoot root = mock(VectorSchemaRoot.class); + + when(streamManager.removeStreamProducer(any(FlightStreamTicket.class))).thenReturn( + Optional.of(FlightStreamManager.StreamProducerHolder.create(streamProducer, allocator)) + ); + when(streamProducer.createJob(any(BufferAllocator.class))).thenReturn(batchedJob); + when(streamProducer.createRoot(any(BufferAllocator.class))).thenReturn(root); + + TestServerStreamListener listener = new TestServerStreamListener(); + AtomicInteger flushCount = new AtomicInteger(0); + doAnswer(invocation -> { + StreamProducer.FlushSignal flushSignal = invocation.getArgument(1); + for (int i = 0; i < 5; i++) { + Thread clientThread = new Thread(() -> { + listener.setReady(false); + listener.setReady(true); + }); + listener.setReady(false); + clientThread.start(); + if (i == 4) { + throw new RuntimeException("Server error"); + } + flushSignal.awaitConsumption(TimeValue.timeValueMillis(100)); + assertTrue(listener.getDataConsumed()); + flushCount.incrementAndGet(); + listener.resetConsumptionLatch(); + } + return null; + }).when(batchedJob).run(any(VectorSchemaRoot.class), any(StreamProducer.FlushSignal.class)); + + assertThrows(RuntimeException.class, () -> baseFlightProducer.getStream(mock(FlightProducer.CallContext.class), ticket, listener)); + + assertNotNull(listener.getError()); + assertEquals("Server error", listener.getError().getMessage()); + assertEquals(4, listener.getPutNextCount()); + assertEquals(4, flushCount.get()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + verify(root).close(); + } + + public void testGetStream_StreamNotFound() throws Exception { + + when(streamManager.getStreamProducer(any(FlightStreamTicket.class))).thenReturn(null); + + TestServerStreamListener listener = new TestServerStreamListener(); + + baseFlightProducer.getStream(null, ticket, listener); + + assertNotNull(listener.getError()); + assertTrue(listener.getError().getMessage().contains("Stream not found")); + assertEquals(0, listener.getPutNextCount()); + + verify(streamManager).removeStreamProducer(any(FlightStreamTicket.class)); + } + + public void testProxyStreamProviderCreationWithDifferentNodeIDs() { + // TODO: proxy stream provider coverage + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamManagerTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamManagerTests.java new file mode 100644 index 0000000000000..1bc686fc446c7 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamManagerTests.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.flight.OSFlightClient; +import org.apache.arrow.flight.Ticket; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.Schema; +import org.opensearch.arrow.flight.bootstrap.FlightClientManager; +import org.opensearch.arrow.spi.StreamReader; +import org.opensearch.arrow.spi.StreamTicket; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.Collections; +import java.util.Optional; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class FlightStreamManagerTests extends OpenSearchTestCase { + + private OSFlightClient flightClient; + private FlightStreamManager flightStreamManager; + private static final String NODE_ID = "testNodeId"; + private static final String TICKET_ID = "testTicketId"; + + @Override + public void setUp() throws Exception { + super.setUp(); + flightClient = mock(OSFlightClient.class); + FlightClientManager clientManager = mock(FlightClientManager.class); + when(clientManager.getFlightClient(NODE_ID)).thenReturn(Optional.of(flightClient)); + BufferAllocator allocator = mock(BufferAllocator.class); + flightStreamManager = new FlightStreamManager(() -> allocator); + flightStreamManager.setClientManager(clientManager); + } + + public void testGetStreamReader() throws Exception { + StreamTicket ticket = new FlightStreamTicket(TICKET_ID, NODE_ID); + FlightStream mockFlightStream = mock(FlightStream.class); + VectorSchemaRoot mockRoot = mock(VectorSchemaRoot.class); + when(flightClient.getStream(new Ticket(ticket.toBytes()))).thenReturn(mockFlightStream); + when(mockFlightStream.getRoot()).thenReturn(mockRoot); + when(mockRoot.getSchema()).thenReturn(new Schema(Collections.emptyList())); + + StreamReader streamReader = flightStreamManager.getStreamReader(ticket); + + assertNotNull(streamReader); + assertNotNull(streamReader.getRoot()); + assertEquals(new Schema(Collections.emptyList()), streamReader.getRoot().getSchema()); + verify(flightClient).getStream(new Ticket(ticket.toBytes())); + } + + public void testGetVectorSchemaRootWithException() { + StreamTicket ticket = new FlightStreamTicket(TICKET_ID, NODE_ID); + when(flightClient.getStream(new Ticket(ticket.toBytes()))).thenThrow(new RuntimeException("Test exception")); + + expectThrows(RuntimeException.class, () -> flightStreamManager.getStreamReader(ticket)); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamReaderTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamReaderTests.java new file mode 100644 index 0000000000000..20e112dc730f6 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamReaderTests.java @@ -0,0 +1,89 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.opensearch.arrow.flight.bootstrap.ServerConfig; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.test.FeatureFlagSetter; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.List; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class FlightStreamReaderTests extends OpenSearchTestCase { + + private FlightStream mockFlightStream; + + private FlightStreamReader iterator; + private VectorSchemaRoot root; + private BufferAllocator allocator; + + @Override + public void setUp() throws Exception { + super.setUp(); + FeatureFlagSetter.set(FeatureFlags.ARROW_STREAMS_SETTING.getKey()); + ServerConfig.init(Settings.EMPTY); + mockFlightStream = mock(FlightStream.class); + allocator = new RootAllocator(100000); + Field field = new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null); + Schema schema = new Schema(List.of(field)); + root = VectorSchemaRoot.create(schema, allocator); + when(mockFlightStream.getRoot()).thenReturn(root); + iterator = new FlightStreamReader(mockFlightStream); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + root.close(); + allocator.close(); + } + + public void testNext_ReturnsTrue_WhenFlightStreamHasNext() throws Exception { + when(mockFlightStream.next()).thenReturn(true); + assertTrue(iterator.next()); + assert(mockFlightStream).next(); + } + + public void testNext_ReturnsFalse_WhenFlightStreamHasNoNext() throws Exception { + when(mockFlightStream.next()).thenReturn(false); + assertFalse(iterator.next()); + verify(mockFlightStream).next(); + } + + public void testGetRoot_ReturnsRootFromFlightStream() throws Exception { + VectorSchemaRoot returnedRoot = iterator.getRoot(); + assertEquals(root, returnedRoot); + verify(mockFlightStream).getRoot(); + } + + public void testClose_CallsCloseOnFlightStream() throws Exception { + iterator.close(); + verify(mockFlightStream).close(); + } + + public void testClose_WrapsExceptionInRuntimeException() throws Exception { + doThrow(new Exception("Test exception")).when(mockFlightStream).close(); + assertThrows(RuntimeException.class, () -> iterator.close()); + verify(mockFlightStream).close(); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamTicketTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamTicketTests.java new file mode 100644 index 0000000000000..819da2826c173 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/FlightStreamTicketTests.java @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.opensearch.arrow.spi.StreamTicket; +import org.opensearch.test.OpenSearchTestCase; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class FlightStreamTicketTests extends OpenSearchTestCase { + + public void testConstructorAndGetters() { + String ticketID = "ticket123"; + String nodeID = "node456"; + StreamTicket ticket = new FlightStreamTicket(ticketID, nodeID); + + assertEquals(ticketID, ticket.getTicketId()); + assertEquals(nodeID, ticket.getNodeId()); + } + + public void testToBytes() { + StreamTicket ticket = new FlightStreamTicket("ticket123", "node456"); + byte[] bytes = ticket.toBytes(); + + assertNotNull(bytes); + assertTrue(bytes.length > 0); + + // Decode the Base64 and check the structure + byte[] decoded = Base64.getDecoder().decode(bytes); + assertEquals(2 + 9 + 2 + 7, decoded.length); // 2 shorts + "ticket123" + "node456" + } + + public void testFromBytes() { + StreamTicket original = new FlightStreamTicket("ticket123", "node456"); + byte[] bytes = original.toBytes(); + + StreamTicket reconstructed = FlightStreamTicket.fromBytes(bytes); + + assertEquals(original.getTicketId(), reconstructed.getTicketId()); + assertEquals(original.getNodeId(), reconstructed.getNodeId()); + } + + public void testToBytesWithLongStrings() { + String longString = randomAlphaOfLength(Short.MAX_VALUE + 1); + StreamTicket ticket = new FlightStreamTicket(longString, "node456"); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, ticket::toBytes); + assertEquals("Field lengths exceed the maximum allowed size.", exception.getMessage()); + } + + public void testNullInput() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> FlightStreamTicket.fromBytes(null)); + assertEquals("Invalid byte array input.", e.getMessage()); + } + + public void testEmptyInput() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> FlightStreamTicket.fromBytes(new byte[0])); + assertEquals("Invalid byte array input.", e.getMessage()); + } + + public void testMalformedBase64() { + byte[] invalidBase64 = "Invalid Base64!@#$".getBytes(StandardCharsets.UTF_8); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> FlightStreamTicket.fromBytes(invalidBase64)); + assertEquals("Illegal base64 character 20", e.getMessage()); + } + + public void testModifiedLengthFields() { + StreamTicket original = new FlightStreamTicket("ticket123", "node456"); + byte[] bytes = original.toBytes(); + byte[] decoded = Base64.getDecoder().decode(bytes); + + // Modify the length field to be larger than actual data + decoded[0] = (byte) 0xFF; + decoded[1] = (byte) 0xFF; + + byte[] modified = Base64.getEncoder().encode(decoded); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> FlightStreamTicket.fromBytes(modified)); + assertEquals("Invalid ticketID length: -1", e.getMessage()); + } + + public void testEquals() { + StreamTicket ticket1 = new FlightStreamTicket("ticket123", "node456"); + StreamTicket ticket2 = new FlightStreamTicket("ticket123", "node456"); + StreamTicket ticket3 = new FlightStreamTicket("ticket789", "node456"); + + assertEquals(ticket1, ticket2); + assertNotEquals(ticket1, ticket3); + assertNotEquals(null, ticket1); + assertNotEquals("Not a StreamTicket", ticket1); + } + + public void testHashCode() { + StreamTicket ticket1 = new FlightStreamTicket("ticket123", "node456"); + StreamTicket ticket2 = new FlightStreamTicket("ticket123", "node456"); + + assertEquals(ticket1.hashCode(), ticket2.hashCode()); + } + + public void testToString() { + StreamTicket ticket = new FlightStreamTicket("ticket123", "node456"); + String expected = "FlightStreamTicket{ticketID='ticket123', nodeID='node456'}"; + assertEquals(expected, ticket.toString()); + } +} diff --git a/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/ProxyStreamProducerTests.java b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/ProxyStreamProducerTests.java new file mode 100644 index 0000000000000..6a6273a601f21 --- /dev/null +++ b/plugins/arrow-flight-rpc/src/test/java/org/opensearch/arrow/flight/impl/ProxyStreamProducerTests.java @@ -0,0 +1,112 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.arrow.flight.impl; + +import org.apache.arrow.flight.FlightStream; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.opensearch.arrow.spi.StreamProducer; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.After; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class ProxyStreamProducerTests extends OpenSearchTestCase { + + private FlightStream mockRemoteStream; + private BufferAllocator mockAllocator; + private ProxyStreamProducer proxyStreamProducer; + + @Override + public void setUp() throws Exception { + super.setUp(); + mockRemoteStream = mock(FlightStream.class); + mockAllocator = mock(BufferAllocator.class); + proxyStreamProducer = new ProxyStreamProducer(new FlightStreamReader(mockRemoteStream)); + } + + public void testCreateRoot() throws Exception { + VectorSchemaRoot mockRoot = mock(VectorSchemaRoot.class); + when(mockRemoteStream.getRoot()).thenReturn(mockRoot); + + VectorSchemaRoot result = proxyStreamProducer.createRoot(mockAllocator); + + assertEquals(mockRoot, result); + verify(mockRemoteStream).getRoot(); + } + + public void testDefaults() { + assertEquals("", proxyStreamProducer.getAction()); + assertEquals(-1, proxyStreamProducer.estimatedRowCount()); + } + + public void testCreateJob() { + StreamProducer.BatchedJob job = proxyStreamProducer.createJob(mockAllocator); + + assertNotNull(job); + assertTrue(job instanceof ProxyStreamProducer.ProxyBatchedJob); + } + + public void testProxyBatchedJob() throws Exception { + StreamProducer.BatchedJob job = proxyStreamProducer.createJob(mockAllocator); + VectorSchemaRoot mockRoot = mock(VectorSchemaRoot.class); + StreamProducer.FlushSignal mockFlushSignal = mock(StreamProducer.FlushSignal.class); + + when(mockRemoteStream.next()).thenReturn(true, true, false); + + job.run(mockRoot, mockFlushSignal); + + verify(mockRemoteStream, times(3)).next(); + verify(mockFlushSignal, times(2)).awaitConsumption(TimeValue.timeValueMillis(1000)); + } + + public void testProxyBatchedJobWithException() throws Exception { + StreamProducer.BatchedJob job = proxyStreamProducer.createJob(mockAllocator); + VectorSchemaRoot mockRoot = mock(VectorSchemaRoot.class); + StreamProducer.FlushSignal mockFlushSignal = mock(StreamProducer.FlushSignal.class); + + doThrow(new RuntimeException("Test exception")).when(mockRemoteStream).next(); + + try { + job.run(mockRoot, mockFlushSignal); + fail("Expected RuntimeException"); + } catch (RuntimeException e) { + assertEquals("Test exception", e.getMessage()); + } + + verify(mockRemoteStream, times(1)).next(); + } + + public void testProxyBatchedJobOnCancel() throws Exception { + StreamProducer.BatchedJob job = proxyStreamProducer.createJob(mockAllocator); + VectorSchemaRoot mockRoot = mock(VectorSchemaRoot.class); + StreamProducer.FlushSignal mockFlushSignal = mock(StreamProducer.FlushSignal.class); + when(mockRemoteStream.next()).thenReturn(true, true, false); + + // cancel the job + job.onCancel(); + job.run(mockRoot, mockFlushSignal); + verify(mockRemoteStream, times(0)).next(); + verify(mockFlushSignal, times(0)).awaitConsumption(TimeValue.timeValueMillis(1000)); + assertTrue(job.isCancelled()); + } + + @After + public void tearDown() throws Exception { + if (proxyStreamProducer != null) { + proxyStreamProducer.close(); + } + super.tearDown(); + } +} diff --git a/server/build.gradle b/server/build.gradle index 74a9d1a59922d..2aa3eee1c89fe 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -69,6 +69,7 @@ dependencies { api project(":libs:opensearch-geo") api project(":libs:opensearch-telemetry") api project(":libs:opensearch-task-commons") + implementation project(':libs:opensearch-arrow-spi') compileOnly project(':libs:opensearch-plugin-classloader') testRuntimeOnly project(':libs:opensearch-plugin-classloader') diff --git a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java index 59d999798868e..6753bb8eac083 100644 --- a/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/FeatureFlagSettings.java @@ -39,6 +39,7 @@ protected FeatureFlagSettings( FeatureFlags.STAR_TREE_INDEX_SETTING, FeatureFlags.APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, FeatureFlags.READER_WRITER_SPLIT_EXPERIMENTAL_SETTING, - FeatureFlags.TERM_VERSION_PRECOMMIT_ENABLE_SETTING + FeatureFlags.TERM_VERSION_PRECOMMIT_ENABLE_SETTING, + FeatureFlags.ARROW_STREAMS_SETTING ); } diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java index 6df68013a8119..4be45aed70023 100644 --- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java +++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java @@ -128,6 +128,9 @@ public class FeatureFlags { Property.NodeScope ); + public static final String ARROW_STREAMS = "opensearch.experimental.feature.arrow.streams.enabled"; + public static final Setting ARROW_STREAMS_SETTING = Setting.boolSetting(ARROW_STREAMS, false, Property.NodeScope); + private static final List> ALL_FEATURE_FLAG_SETTINGS = List.of( REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING, EXTENSIONS_SETTING, @@ -138,7 +141,8 @@ public class FeatureFlags { STAR_TREE_INDEX_SETTING, APPLICATION_BASED_CONFIGURATION_TEMPLATES_SETTING, READER_WRITER_SPLIT_EXPERIMENTAL_SETTING, - TERM_VERSION_PRECOMMIT_ENABLE_SETTING + TERM_VERSION_PRECOMMIT_ENABLE_SETTING, + ARROW_STREAMS_SETTING ); /** diff --git a/server/src/main/java/org/opensearch/plugins/DefaultSecureTransportParameters.java b/server/src/main/java/org/opensearch/plugins/DefaultSecureTransportParameters.java index e3771f224a7db..3265c582dba76 100644 --- a/server/src/main/java/org/opensearch/plugins/DefaultSecureTransportParameters.java +++ b/server/src/main/java/org/opensearch/plugins/DefaultSecureTransportParameters.java @@ -11,6 +11,13 @@ import org.opensearch.common.network.NetworkModule; import org.opensearch.common.settings.Settings; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.TrustManagerFactory; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; + /** * Default implementation of {@link SecureTransportSettingsProvider.SecureTransportParameters}. */ @@ -25,4 +32,34 @@ class DefaultSecureTransportParameters implements SecureTransportSettingsProvide public boolean dualModeEnabled() { return NetworkModule.TRANSPORT_SSL_DUAL_MODE_ENABLED.get(settings); } + + @Override + public Optional keyManagerFactory() { + return Optional.empty(); + } + + @Override + public Optional sslProvider() { + return Optional.empty(); + } + + @Override + public Optional clientAuth() { + return Optional.empty(); + } + + @Override + public Collection protocols() { + return List.of(); + } + + @Override + public Collection cipherSuites() { + return List.of(); + } + + @Override + public Optional trustManagerFactory() { + return Optional.empty(); + } } diff --git a/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java b/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java index 5f9e1a952b6e8..f4cf64c16cbd2 100644 --- a/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java +++ b/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java @@ -13,8 +13,10 @@ import org.opensearch.transport.Transport; import org.opensearch.transport.TransportAdapterProvider; +import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLException; +import javax.net.ssl.TrustManagerFactory; import java.util.Collection; import java.util.Collections; @@ -52,6 +54,18 @@ default Optional parameters(Settings settings) { @ExperimentalApi interface SecureTransportParameters { boolean dualModeEnabled(); + + Optional keyManagerFactory(); + + Optional sslProvider(); + + Optional clientAuth(); + + Collection protocols(); + + Collection cipherSuites(); + + Optional trustManagerFactory(); } /** diff --git a/server/src/main/java/org/opensearch/plugins/StreamManagerPlugin.java b/server/src/main/java/org/opensearch/plugins/StreamManagerPlugin.java new file mode 100644 index 0000000000000..60bdb789b3750 --- /dev/null +++ b/server/src/main/java/org/opensearch/plugins/StreamManagerPlugin.java @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugins; + +import org.opensearch.arrow.spi.StreamManager; + +import java.util.function.Supplier; + +/** + * An interface for OpenSearch plugins to implement to provide a StreamManager. + * Plugins can implement this interface to provide custom StreamManager implementation. + * @see StreamManager + */ +public interface StreamManagerPlugin { + /** + * Returns the StreamManager instance for this plugin. + * + * @return The StreamManager instance + */ + Supplier getStreamManager(); +} diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java index 052b1a4e52eb9..0bd5d8afda91e 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java @@ -1768,7 +1768,7 @@ public static String getPortRange() { return getBasePort() + "-" + (getBasePort() + 99); // upper bound is inclusive } - protected static int getBasePort() { + protected static int getBasePort(int start) { // some tests use MockTransportService to do network based testing. Yet, we run tests in multiple JVMs that means // concurrent tests could claim port that another JVM just released and if that test tries to simulate a disconnect it might // be smart enough to re-connect depending on what is tested. To reduce the risk, since this is very hard to debug we use @@ -1792,7 +1792,11 @@ protected static int getBasePort() { startAt = (int) Math.floorMod(workerId - 1, 223L) + 1; } assert startAt >= 0 : "Unexpected test worker Id, resulting port range would be negative"; - return 10300 + (startAt * 100); + return start + (startAt * 100); + } + + protected static int getBasePort() { + return getBasePort(10300); } protected static InetAddress randomIp(boolean v4) {