-
Notifications
You must be signed in to change notification settings - Fork 1.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Refactor Compressors from CompressorFactory to CompressorRegistry for…
… extensibility This commit refactors the CompressorFactory static singleton class and CompressorType enum to a formal CompressorRegistry and enables downstream implementations to register their own compression implementations for use in compressing Blob stores and MediaType data. This is different from Lucene's Codec compression extension points which expose different compression implementations for Lucene's Stored Fields. Signed-off-by: Nicholas Walter Knize <nknize@apache.org>
- Loading branch information
Showing
40 changed files
with
452 additions
and
243 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* The OpenSearch Contributors require contributions made to | ||
* this file be licensed under the Apache-2.0 license or a | ||
* compatible open source license. | ||
* | ||
* Modifications Copyright OpenSearch Contributors. See | ||
* GitHub history for details. | ||
*/ | ||
|
||
apply plugin: 'opensearch.build' | ||
apply plugin: 'opensearch.publish' | ||
|
||
base { | ||
archivesName = 'opensearch-common' | ||
} | ||
|
||
dependencies { | ||
api project(':libs:opensearch-common') | ||
api project(':libs:opensearch-core') | ||
|
||
//zstd | ||
api "com.github.luben:zstd-jni:${versions.zstd}" | ||
|
||
testImplementation(project(":test:framework")) { | ||
// tests use the locally compiled version of server | ||
exclude group: 'org.opensearch', module: 'opensearch-compress' | ||
} | ||
} | ||
|
||
tasks.named('forbiddenApisMain').configure { | ||
// :libs:opensearch-compress does not depend on server | ||
// TODO: Need to decide how we want to handle for forbidden signatures with the changes to server | ||
replaceSignatureFiles 'jdk-signatures' | ||
} | ||
|
||
jarHell.enabled = false |
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
29 changes: 29 additions & 0 deletions
29
libs/compress/src/main/java/org/opensearch/compress/spi/CompressionProvider.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* The OpenSearch Contributors require contributions made to | ||
* this file be licensed under the Apache-2.0 license or a | ||
* compatible open source license. | ||
*/ | ||
|
||
package org.opensearch.compress.spi; | ||
|
||
import org.opensearch.compress.ZstdCompressor; | ||
import org.opensearch.core.compress.CompressorRegistry; | ||
import org.opensearch.core.compress.spi.CompressorProvider; | ||
|
||
import java.util.List; | ||
|
||
/** | ||
* Additional "optional" compressor implementations provided by the opensearch compress library | ||
* | ||
* @opensearch.internal | ||
*/ | ||
public class CompressionProvider implements CompressorProvider { | ||
|
||
/** Returns the concrete {@link org.opensearch.core.common.compress.Compressor}s provided by the compress library */ | ||
@Override | ||
public List<CompressorRegistry.Entry> getCompressors() { | ||
return List.of(new CompressorRegistry.Entry(ZstdCompressor.NAME, new ZstdCompressor())); | ||
} | ||
} |
9 changes: 9 additions & 0 deletions
9
.../src/main/resources/META-INF/services/org.opensearch.core.compress.spi.CompressorProvider
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
# | ||
# SPDX-License-Identifier: Apache-2.0 | ||
# | ||
# The OpenSearch Contributors require contributions made to | ||
# this file be licensed under the Apache-2.0 license or a | ||
# compatible open source license. | ||
# | ||
|
||
org.opensearch.compress.spi.CompressionProvider |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
141 changes: 141 additions & 0 deletions
141
libs/core/src/main/java/org/opensearch/core/compress/CompressorRegistry.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,141 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* The OpenSearch Contributors require contributions made to | ||
* this file be licensed under the Apache-2.0 license or a | ||
* compatible open source license. | ||
*/ | ||
|
||
package org.opensearch.core.compress; | ||
|
||
import org.opensearch.common.Nullable; | ||
import org.opensearch.core.common.bytes.BytesReference; | ||
import org.opensearch.core.common.compress.Compressor; | ||
import org.opensearch.core.common.compress.NotCompressedException; | ||
import org.opensearch.core.common.compress.NotXContentException; | ||
import org.opensearch.core.compress.spi.CompressorProvider; | ||
import org.opensearch.core.xcontent.MediaTypeRegistry; | ||
|
||
import java.io.IOException; | ||
import java.util.ArrayList; | ||
import java.util.Map; | ||
import java.util.Objects; | ||
import java.util.ServiceLoader; | ||
import java.util.stream.Collectors; | ||
|
||
/** | ||
* A registry that wraps a static Map singleton which holds a mapping of unique String names (typically the | ||
* compressor header as a string) to registerd {@link Compressor} implementations. | ||
* | ||
* This enables plugins, modules, extensions to register their own compression implementations through SPI | ||
* | ||
* @opensearch.internal | ||
*/ | ||
public final class CompressorRegistry { | ||
/** No compression singleton - we still register so users can specify NONE in the API*/ | ||
public static final Compressor NONE; | ||
|
||
// the backing registry map | ||
private static final Map<String, Compressor> registeredCompressors; | ||
|
||
// no instance: | ||
private CompressorRegistry() {} | ||
|
||
static { | ||
ArrayList<Entry> compressors = new ArrayList<>(); | ||
for (CompressorProvider provider : ServiceLoader.load(CompressorProvider.class, CompressorProvider.class.getClassLoader())) { | ||
compressors.addAll(provider.getCompressors()); | ||
} | ||
registeredCompressors = Map.copyOf(compressors.stream().collect(Collectors.toMap(Entry::getName, Entry::getCompressor))); | ||
NONE = registeredCompressors.get(NoneCompressor.NAME); | ||
} | ||
|
||
public static class Entry { | ||
/** a unique key name to identify the compressor; this is typically the Compressor's Header as a string */ | ||
private String name; | ||
/** the compressor to register */ | ||
private Compressor compressor; | ||
|
||
public Entry(final String name, final Compressor compressor) { | ||
this.name = name; | ||
this.compressor = compressor; | ||
} | ||
|
||
public String getName() { | ||
return this.name; | ||
} | ||
|
||
public Compressor getCompressor() { | ||
return compressor; | ||
} | ||
} | ||
|
||
/** | ||
* Returns the default compressor | ||
*/ | ||
public static Compressor defaultCompressor() { | ||
return registeredCompressors.get("DEFLATE"); | ||
} | ||
|
||
public static Compressor none() { | ||
return registeredCompressors.get("NONE"); | ||
} | ||
|
||
public static boolean isCompressed(BytesReference bytes) { | ||
return compressor(bytes) != null; | ||
} | ||
|
||
@Nullable | ||
public static Compressor compressor(final BytesReference bytes) { | ||
for (Compressor compressor : registeredCompressors.values()) { | ||
if (compressor.isCompressed(bytes) == true) { | ||
// bytes should be either detected as compressed or as xcontent, | ||
// if we have bytes that can be either detected as compressed or | ||
// as a xcontent, we have a problem | ||
assert MediaTypeRegistry.xContentType(bytes) == null; | ||
return compressor; | ||
} | ||
} | ||
|
||
if (MediaTypeRegistry.xContentType(bytes) == null) { | ||
throw new NotXContentException("Compressor detection can only be called on some xcontent bytes or compressed xcontent bytes"); | ||
} | ||
|
||
return null; | ||
} | ||
|
||
/** Decompress the provided {@link BytesReference}. */ | ||
public static BytesReference uncompress(BytesReference bytes) throws IOException { | ||
Compressor compressor = compressor(bytes); | ||
if (compressor == null) { | ||
throw new NotCompressedException(); | ||
} | ||
return compressor.uncompress(bytes); | ||
} | ||
|
||
/** | ||
* Uncompress the provided data, data can be detected as compressed using {@link #isCompressed(BytesReference)}. | ||
*/ | ||
public static BytesReference uncompressIfNeeded(BytesReference bytes) throws IOException { | ||
Compressor compressor = compressor(Objects.requireNonNull(bytes, "the BytesReference must not be null")); | ||
return compressor == null ? bytes : compressor.uncompress(bytes); | ||
} | ||
|
||
/** Returns a registered compressor by its registered name */ | ||
public static Compressor getCompressor(final String name) { | ||
if (registeredCompressors.containsKey(name)) { | ||
return registeredCompressors.get(name); | ||
} | ||
throw new IllegalArgumentException("No registered compressor found by name [" + name + "]"); | ||
} | ||
|
||
/** | ||
* Returns the registered compressors as an Immutable collection | ||
* | ||
* note: used for testing | ||
*/ | ||
public static Map<String, Compressor> registeredCompressors() { | ||
// no destructive danger as backing map is immutable | ||
return registeredCompressors; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
30 changes: 30 additions & 0 deletions
30
libs/core/src/main/java/org/opensearch/core/compress/spi/CompressorProvider.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* The OpenSearch Contributors require contributions made to | ||
* this file be licensed under the Apache-2.0 license or a | ||
* compatible open source license. | ||
*/ | ||
|
||
package org.opensearch.core.compress.spi; | ||
|
||
import org.opensearch.core.common.compress.Compressor; | ||
import org.opensearch.core.compress.CompressorRegistry; | ||
|
||
import java.util.List; | ||
|
||
/** | ||
* Service Provider Interface for plugins, modules, extensions providing custom | ||
* compression algorithms | ||
* | ||
* see {@link Compressor} for implementing methods | ||
* and {@link org.opensearch.core.compress.CompressorRegistry} for the registration of custom | ||
* Compressors | ||
* | ||
* @opensearch.experimental | ||
* @opensearch.api | ||
*/ | ||
public interface CompressorProvider { | ||
/** Extensions that implement their own concrete {@link Compressor}s provide them through this interface method*/ | ||
List<CompressorRegistry.Entry> getCompressors(); | ||
} |
28 changes: 28 additions & 0 deletions
28
libs/core/src/main/java/org/opensearch/core/compress/spi/DefaultCompressorProvider.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* The OpenSearch Contributors require contributions made to | ||
* this file be licensed under the Apache-2.0 license or a | ||
* compatible open source license. | ||
*/ | ||
|
||
package org.opensearch.core.compress.spi; | ||
|
||
import org.opensearch.core.compress.CompressorRegistry; | ||
import org.opensearch.core.compress.NoneCompressor; | ||
|
||
import java.util.List; | ||
|
||
/** | ||
* Default {@link org.opensearch.core.common.compress.Compressor} implementations provided by the | ||
* opensearch core library | ||
* | ||
* @opensearch.internal | ||
*/ | ||
public class DefaultCompressorProvider implements CompressorProvider { | ||
/** Returns the default {@link org.opensearch.core.common.compress.Compressor}s provided by the core library */ | ||
@Override | ||
public List<CompressorRegistry.Entry> getCompressors() { | ||
return List.of(new CompressorRegistry.Entry(NoneCompressor.NAME, new NoneCompressor())); | ||
} | ||
} |
Oops, something went wrong.