From 960d0abad8375d66663c8450f44b4b8b5b6032d3 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 10 Oct 2024 10:20:57 -0400 Subject: [PATCH 1/4] Update codecs to Apache Lucene 9.12.0 Signed-off-by: Andriy Redko --- .github/workflows/check.yml | 4 +- .../codec/rest/CreateIndexWithCodecIT.java | 7 +- .../codec/customcodecs/CustomCodecPlugin.java | 16 +- .../customcodecs/CustomCodecService.java | 28 +-- ...omCodec.java => Lucene912CustomCodec.java} | 28 +-- .../Lucene912CustomStoredFieldsFormat.java | 140 ++++++++++++ .../codec/customcodecs/Lucene912QatCodec.java | 145 ++++++++++++ .../Lucene912QatStoredFieldsFormat.java | 179 +++++++++++++++ .../customcodecs/QatCompressionMode.java | 19 +- ...te99Codec.java => QatDeflate912Codec.java} | 14 +- ...QatLz499Codec.java => QatLz4912Codec.java} | 14 +- .../{Zstd99Codec.java => Zstd912Codec.java} | 10 +- .../customcodecs/ZstdCompressionMode.java | 6 +- ...ct99Codec.java => ZstdNoDict912Codec.java} | 10 +- .../ZstdNoDictCompressionMode.java | 6 +- .../lucene99/Lucene99CustomCodec.java | 100 +++++++++ .../Lucene99CustomStoredFieldsFormat.java | 5 +- .../lucene99}/Lucene99QatCodec.java | 44 +--- .../Lucene99QatStoredFieldsFormat.java | 2 +- .../lucene99/QatCompressionMode.java | 206 ++++++++++++++++++ .../lucene99/QatDeflate99Codec.java | 70 ++++++ .../lucene99/QatLz499Codec.java | 70 ++++++ .../backward_codecs/lucene99/Zstd99Codec.java | 53 +++++ .../lucene99/ZstdNoDict99Codec.java | 50 +++++ .../services/org.apache.lucene.codecs.Codec | 12 +- .../codec/customcodecs/CustomCodecTests.java | 57 ++--- ...ucene912CustomStoredFieldsFormatTests.java | 59 +++++ .../Lucene912QatStoredFieldsFormatTests.java | 67 ++++++ .../codec/customcodecs/QatCodecTests.java | 48 ++-- .../QatDeflateCompressorTests.java | 4 +- .../customcodecs/QatLz4CompressorTests.java | 4 +- ...Lucene95CustomStoredFieldsFormatTests.java | 0 ...Lucene99CustomStoredFieldsFormatTests.java | 4 +- .../Lucene99QatStoredFieldsFormatTests.java | 3 +- 34 files changed, 1306 insertions(+), 178 deletions(-) rename src/main/java/org/opensearch/index/codec/customcodecs/{Lucene99CustomCodec.java => Lucene912CustomCodec.java} (72%) create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormat.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatCodec.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java rename src/main/java/org/opensearch/index/codec/customcodecs/{QatDeflate99Codec.java => QatDeflate912Codec.java} (79%) rename src/main/java/org/opensearch/index/codec/customcodecs/{QatLz499Codec.java => QatLz4912Codec.java} (80%) rename src/main/java/org/opensearch/index/codec/customcodecs/{Zstd99Codec.java => Zstd912Codec.java} (80%) rename src/main/java/org/opensearch/index/codec/customcodecs/{ZstdNoDict99Codec.java => ZstdNoDict912Codec.java} (79%) create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomCodec.java rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene99}/Lucene99CustomStoredFieldsFormat.java (96%) rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene99}/Lucene99QatCodec.java (68%) rename src/main/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene99}/Lucene99QatStoredFieldsFormat.java (98%) create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatCompressionMode.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatDeflate99Codec.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatLz499Codec.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Zstd99Codec.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/ZstdNoDict99Codec.java create mode 100644 src/test/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormatTests.java create mode 100644 src/test/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormatTests.java rename src/test/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs}/Lucene95CustomStoredFieldsFormatTests.java (100%) rename src/test/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene99}/Lucene99CustomStoredFieldsFormatTests.java (92%) rename src/test/java/org/opensearch/index/codec/customcodecs/{ => backward_codecs/lucene99}/Lucene99QatStoredFieldsFormatTests.java (95%) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5c61305..a8d350b 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -29,7 +29,7 @@ jobs: run: | # https://github.com/opensearch-project/opensearch-build/issues/4191 chown -R ci-runner:ci-runner `pwd` - su ci-runner -c "source /etc/profile.d/java_home.sh && ./gradlew check -Dorg.gradle.java.home=/opt/java/openjdk-${{ matrix.java }}" + su ci-runner -c "source /etc/profile.d/java_home.sh && ./gradlew test check -Dorg.gradle.java.home=/opt/java/openjdk-${{ matrix.java }}" - name: Run Gradle (assemble) run: | # https://github.com/opensearch-project/opensearch-build/issues/4191 @@ -53,7 +53,7 @@ jobs: cache: gradle - name: Run Gradle (check) run: | - ./gradlew check + ./gradlew test check - name: Run Gradle (assemble) run: | ./gradlew assemble diff --git a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java index 958a3b3..e9ecec1 100644 --- a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java +++ b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java @@ -23,7 +23,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.Strings; -import org.opensearch.index.codec.customcodecs.Lucene99QatCodec; +import org.opensearch.index.codec.customcodecs.Lucene912QatCodec; import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -100,7 +100,10 @@ public void testCreateIndexWithQatSPICodecWithQatHardwareUnavailable() throws IO Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put("index.codec", randomFrom(Lucene99QatCodec.Mode.QAT_LZ4.getCodec(), Lucene99QatCodec.Mode.QAT_DEFLATE.getCodec())) + .put( + "index.codec", + randomFrom(Lucene912QatCodec.Mode.QAT_LZ4.getCodec(), Lucene912QatCodec.Mode.QAT_DEFLATE.getCodec()) + ) .put("index.codec.compression_level", randomIntBetween(1, 6)) .build() ) diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java index 0e45729..ec4cda3 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -11,6 +11,7 @@ import org.opensearch.common.settings.Setting; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecServiceFactory; +import org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec; import org.opensearch.index.engine.EngineConfig; import org.opensearch.plugins.EnginePlugin; import org.opensearch.plugins.Plugin; @@ -49,12 +50,19 @@ public Optional getCustomCodecServiceFactory(final IndexSet || codecName.equals(CustomCodecService.QAT_DEFLATE_CODEC)) { return Optional.of(new CustomCodecServiceFactory()); } else { - if (!QatZipperFactory.isQatAvailable() - && (codecName.equals(Lucene99QatCodec.Mode.QAT_LZ4.getCodec()) - || codecName.equals(Lucene99QatCodec.Mode.QAT_DEFLATE.getCodec()))) { - throw new IllegalArgumentException("QAT codecs are not supported. Please create indices with a different codec."); + if (codecName.equals(Lucene99QatCodec.Mode.QAT_LZ4.getCodec()) + || codecName.equals(Lucene99QatCodec.Mode.QAT_DEFLATE.getCodec())) { + if (!QatZipperFactory.isQatAvailable()) { + throw new IllegalArgumentException("QAT codecs are not supported. Please create indices with a different codec."); + } } + if (codecName.equals(Lucene912QatCodec.Mode.QAT_LZ4.getCodec()) + || codecName.equals(Lucene912QatCodec.Mode.QAT_DEFLATE.getCodec())) { + if (!QatZipperFactory.isQatAvailable()) { + throw new IllegalArgumentException("QAT codecs are not supported. Please create indices with a different codec."); + } + } } return Optional.empty(); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java index 93448c2..f953505 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java @@ -19,6 +19,7 @@ import java.util.Map; import java.util.stream.Stream; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING; import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; /** CustomCodecService provides ZSTD, ZSTD_NO_DICT, QAT_LZ4, and QAT_DEFLATE compression codecs. */ @@ -49,25 +50,26 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); final MapBuilder codecs = MapBuilder.newMapBuilder(); if (mapperService == null) { - codecs.put(ZSTD_CODEC, new Zstd99Codec(compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(compressionLevel)); + codecs.put(ZSTD_CODEC, new Zstd912Codec(compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict912Codec(compressionLevel)); if (QatZipperFactory.isQatAvailable()) { - codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(compressionLevel, () -> { - return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); - })); - codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(compressionLevel, () -> { - return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); + codecs.put( + QAT_LZ4_CODEC, + new QatLz4912Codec(compressionLevel, () -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); }) + ); + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate912Codec(compressionLevel, () -> { + return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })); } } else { - codecs.put(ZSTD_CODEC, new Zstd99Codec(mapperService, logger, compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_CODEC, new Zstd912Codec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict912Codec(mapperService, logger, compressionLevel)); if (QatZipperFactory.isQatAvailable()) { - codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(mapperService, logger, compressionLevel, () -> { - return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); + codecs.put(QAT_LZ4_CODEC, new QatLz4912Codec(mapperService, logger, compressionLevel, () -> { + return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })); - codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(mapperService, logger, compressionLevel, () -> { - return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate912Codec(mapperService, logger, compressionLevel, () -> { + return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); })); } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java similarity index 72% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java index 6b31167..0d925c9 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java @@ -11,14 +11,13 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; -import org.opensearch.common.settings.Settings; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; import org.opensearch.index.mapper.MapperService; import java.util.Set; -import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; /** * @@ -28,21 +27,18 @@ * * @opensearch.internal */ -public abstract class Lucene99CustomCodec extends FilterCodec { - - /** Default compression level used for compression */ - public static final int DEFAULT_COMPRESSION_LEVEL = INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getDefault(Settings.EMPTY); +public abstract class Lucene912CustomCodec extends FilterCodec { /** Each mode represents a compression algorithm. */ public enum Mode { /** * ZStandard mode with dictionary */ - ZSTD("ZSTD99", Set.of("zstd")), + ZSTD("ZSTD912", Set.of("zstd")), /** * ZStandard mode without dictionary */ - ZSTD_NO_DICT("ZSTDNODICT99", Set.of("zstd_no_dict")); + ZSTD_NO_DICT("ZSTDNODICT912", Set.of("zstd_no_dict")); private final String codec; private final Set aliases; @@ -74,7 +70,7 @@ public Set getAliases() { * * @param mode The compression codec (ZSTD or ZSTDNODICT). */ - public Lucene99CustomCodec(Mode mode) { + public Lucene912CustomCodec(Mode mode) { this(mode, DEFAULT_COMPRESSION_LEVEL); } @@ -86,9 +82,9 @@ public Lucene99CustomCodec(Mode mode) { * @param mode The compression codec (ZSTD or ZSTDNODICT). * @param compressionLevel The compression level. */ - public Lucene99CustomCodec(Mode mode, int compressionLevel) { - super(mode.getCodec(), new Lucene99Codec()); - this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel); + public Lucene912CustomCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene912Codec()); + this.storedFieldsFormat = new Lucene912CustomStoredFieldsFormat(mode, compressionLevel); } /** @@ -101,9 +97,9 @@ public Lucene99CustomCodec(Mode mode, int compressionLevel) { * @param mapperService The mapper service. * @param logger The logger. */ - public Lucene99CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); - this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel); + public Lucene912CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene912CustomStoredFieldsFormat(mode, compressionLevel); } @Override diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormat.java new file mode 100644 index 0000000..86005d5 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormat.java @@ -0,0 +1,140 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.StoredFieldsReader; +import org.apache.lucene.codecs.StoredFieldsWriter; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +import java.io.IOException; +import java.util.Objects; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + +/** Stored field format used by pluggable codec */ +public class Lucene912CustomStoredFieldsFormat extends StoredFieldsFormat { + + /** A key that we use to map to a mode */ + public static final String MODE_KEY = Lucene912CustomStoredFieldsFormat.class.getSimpleName() + ".mode"; + + protected static final int ZSTD_BLOCK_LENGTH = 10 * 48 * 1024; + protected static final int ZSTD_MAX_DOCS_PER_BLOCK = 4096; + protected static final int ZSTD_BLOCK_SHIFT = 10; + + private final CompressionMode zstdCompressionMode; + private final CompressionMode zstdNoDictCompressionMode; + + private final Lucene912CustomCodec.Mode mode; + private final int compressionLevel; + + /** default constructor */ + public Lucene912CustomStoredFieldsFormat() { + this(Lucene912CustomCodec.Mode.ZSTD, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents ZSTD or ZSTDNODICT + */ + public Lucene912CustomStoredFieldsFormat(Lucene912CustomCodec.Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents ZSTD or ZSTDNODICT + * @param compressionLevel The compression level for the mode. + */ + public Lucene912CustomStoredFieldsFormat(Lucene912CustomCodec.Mode mode, int compressionLevel) { + this.mode = Objects.requireNonNull(mode); + this.compressionLevel = compressionLevel; + zstdCompressionMode = new ZstdCompressionMode(compressionLevel); + zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); + } + + /** + * Returns a {@link StoredFieldsReader} to load stored fields. + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param fn The fieldInfos. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { + if (si.getAttribute(MODE_KEY) != null) { + String value = si.getAttribute(MODE_KEY); + Lucene912CustomCodec.Mode mode = Lucene912CustomCodec.Mode.valueOf(value); + return impl(mode).fieldsReader(directory, si, fn, context); + } else { + throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name); + } + } + + /** + * Returns a {@link StoredFieldsReader} to write stored fields. + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { + String previous = si.putAttribute(MODE_KEY, mode.name()); + if (previous != null && previous.equals(mode.name()) == false) { + throw new IllegalStateException( + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + ); + } + return impl(mode).fieldsWriter(directory, si, context); + } + + StoredFieldsFormat impl(Lucene912CustomCodec.Mode mode) { + switch (mode) { + case ZSTD: + return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstd", this.zstdCompressionMode); + case ZSTD_NO_DICT: + return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstdNoDict", this.zstdNoDictCompressionMode); + default: + throw new IllegalStateException("Unsupported compression mode: " + mode); + } + } + + private StoredFieldsFormat getCustomCompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode) { + return new Lucene90CompressingStoredFieldsFormat( + formatName, + compressionMode, + ZSTD_BLOCK_LENGTH, + ZSTD_MAX_DOCS_PER_BLOCK, + ZSTD_BLOCK_SHIFT + ); + } + + public Lucene912CustomCodec.Mode getMode() { + return mode; + } + + /** + * Returns the compression level. + */ + public int getCompressionLevel() { + return compressionLevel; + } + + public CompressionMode getCompressionMode() { + return mode == Lucene912CustomCodec.Mode.ZSTD_NO_DICT ? zstdNoDictCompressionMode : zstdCompressionMode; + } + +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatCodec.java new file mode 100644 index 0000000..4010741 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatCodec.java @@ -0,0 +1,145 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; +import org.opensearch.index.mapper.MapperService; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + +/** + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. + * + * @opensearch.internal + */ +public abstract class Lucene912QatCodec extends FilterCodec { + /** Each mode represents a compression algorithm. */ + public enum Mode { + /** QAT lz4 mode. */ + QAT_LZ4("QATLZ4912", Set.of("qat_lz4")), + + /** QAT deflate mode. */ + QAT_DEFLATE("QATDEFLATE912", Set.of("qat_deflate")); + + private final String codec; + private final Set aliases; + + Mode(String codec, Set aliases) { + this.codec = codec; + this.aliases = aliases; + } + + /** Returns the Codec that is registered with Lucene */ + public String getCodec() { + return codec; + } + + /** Returns the aliases of the Codec */ + public Set getAliases() { + return aliases; + } + } + + /** The default compression mode. */ + public static final Mode DEFAULT_COMPRESSION_MODE = Mode.QAT_LZ4; + + private final StoredFieldsFormat storedFieldsFormat; + + /** + * Creates a new compression codec with the default compression level. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + */ + public Lucene912QatCodec(Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + * @param compressionLevel The compression level. + */ + public Lucene912QatCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene912Codec()); + this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + * @param compressionLevel The compression level. + * @param supplier supplier for QAT mode. + */ + public Lucene912QatCodec(Mode mode, int compressionLevel, Supplier supplier) { + super(mode.getCodec(), new Lucene912Codec()); + this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel, supplier); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + * @param compressionLevel The compression level. + * @param mapperService The mapper service. + * @param logger The logger. + */ + public Lucene912QatCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). + * @param compressionLevel The compression level. + * @param mapperService The mapper service. + * @param logger The logger. + * @param supplier supplier for QAT mode. + */ + public Lucene912QatCodec( + Mode mode, + int compressionLevel, + MapperService mapperService, + Logger logger, + Supplier supplier + ) { + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene912QatStoredFieldsFormat(mode, compressionLevel, supplier); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java new file mode 100644 index 0000000..cf4ff21 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java @@ -0,0 +1,179 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.StoredFieldsReader; +import org.apache.lucene.codecs.StoredFieldsWriter; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +import java.io.IOException; +import java.util.Objects; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_QAT_MODE; + +/** Stored field format used by pluggable codec */ +public class Lucene912QatStoredFieldsFormat extends StoredFieldsFormat { + + /** A key that we use to map to a mode */ + public static final String MODE_KEY = Lucene912QatStoredFieldsFormat.class.getSimpleName() + ".mode"; + + private static final int QAT_DEFLATE_BLOCK_LENGTH = 10 * 48 * 1024; + private static final int QAT_DEFLATE_MAX_DOCS_PER_BLOCK = 4096; + private static final int QAT_DEFLATE_BLOCK_SHIFT = 10; + + private static final int QAT_LZ4_BLOCK_LENGTH = 10 * 8 * 1024; + private static final int QAT_LZ4_MAX_DOCS_PER_BLOCK = 4096; + private static final int QAT_LZ4_BLOCK_SHIFT = 10; + + private final QatCompressionMode qatCompressionMode; + private final Lucene912QatCodec.Mode mode; + + /** default constructor */ + public Lucene912QatStoredFieldsFormat() { + this(Lucene912QatCodec.DEFAULT_COMPRESSION_MODE, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + */ + public Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + * @param compressionLevel The compression level for the mode. + */ + public Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode mode, int compressionLevel) { + this(mode, compressionLevel, () -> { return DEFAULT_QAT_MODE; }); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + * @param supplier a supplier for QAT acceleration mode. + */ + public Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode mode, Supplier supplier) { + this(mode, DEFAULT_COMPRESSION_LEVEL, supplier); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE + * @param compressionLevel The compression level for the mode. + * @param supplier a supplier for QAT acceleration mode. + */ + public Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode mode, int compressionLevel, Supplier supplier) { + this.mode = Objects.requireNonNull(mode); + qatCompressionMode = new QatCompressionMode(mode, compressionLevel, supplier); + } + + /** + * Returns a {@link StoredFieldsReader} to load stored fields. + * + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param fn The fieldInfos. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { + if (si.getAttribute(MODE_KEY) != null) { + String value = si.getAttribute(MODE_KEY); + Lucene912QatCodec.Mode mode = Lucene912QatCodec.Mode.valueOf(value); + return impl(mode).fieldsReader(directory, si, fn, context); + } else { + throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name); + } + } + + /** + * Returns a {@link StoredFieldsReader} to write stored fields. + * + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { + String previous = si.putAttribute(MODE_KEY, mode.name()); + if (previous != null && previous.equals(mode.name()) == false) { + throw new IllegalStateException( + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + ); + } + return impl(mode).fieldsWriter(directory, si, context); + } + + private StoredFieldsFormat impl(Lucene912QatCodec.Mode mode) { + switch (mode) { + case QAT_LZ4: + return getQatCompressingStoredFieldsFormat( + "QatStoredFieldsLz4", + qatCompressionMode, + QAT_LZ4_BLOCK_LENGTH, + QAT_LZ4_MAX_DOCS_PER_BLOCK, + QAT_LZ4_BLOCK_SHIFT + ); + case QAT_DEFLATE: + return getQatCompressingStoredFieldsFormat( + "QatStoredFieldsDeflate", + qatCompressionMode, + QAT_DEFLATE_BLOCK_LENGTH, + QAT_DEFLATE_MAX_DOCS_PER_BLOCK, + QAT_DEFLATE_BLOCK_SHIFT + ); + default: + throw new IllegalStateException("Unsupported compression mode: " + mode); + } + } + + private StoredFieldsFormat getQatCompressingStoredFieldsFormat( + String formatName, + CompressionMode compressionMode, + int blockSize, + int maxDocs, + int blockShift + ) { + return new Lucene90CompressingStoredFieldsFormat(formatName, compressionMode, blockSize, maxDocs, blockShift); + } + + /** + * Gets the mode of compression. + * + * @return either QAT_LZ4 or QAT_DEFLATE + */ + public Lucene912QatCodec.Mode getMode() { + return mode; + } + + /** + * + * @return the CompressionMode instance. + */ + public QatCompressionMode getCompressionMode() { + return qatCompressionMode; + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java index 193c825..96800c5 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java @@ -22,6 +22,9 @@ import com.intel.qat.QatZipper; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_QAT_MODE; + /** QatCompressionMode offers QAT_LZ4 and QAT_DEFLATE compressors. */ public class QatCompressionMode extends CompressionMode { @@ -33,9 +36,7 @@ public class QatCompressionMode extends CompressionMode { /** default constructor */ protected QatCompressionMode() { - this(Lucene99QatCodec.DEFAULT_COMPRESSION_MODE, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, () -> { - return Lucene99QatCodec.DEFAULT_QAT_MODE; - }); + this(Lucene912QatCodec.DEFAULT_COMPRESSION_MODE, DEFAULT_COMPRESSION_LEVEL, () -> { return DEFAULT_QAT_MODE; }); } /** @@ -43,8 +44,8 @@ protected QatCompressionMode() { * * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) */ - protected QatCompressionMode(Lucene99QatCodec.Mode mode) { - this(mode, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }); + protected QatCompressionMode(Lucene912QatCodec.Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL, () -> { return DEFAULT_QAT_MODE; }); } /** @@ -53,8 +54,8 @@ protected QatCompressionMode(Lucene99QatCodec.Mode mode) { * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) * @param compressionLevel The compression level to use. */ - protected QatCompressionMode(Lucene99QatCodec.Mode mode, int compressionLevel) { - this(mode, compressionLevel, () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }); + protected QatCompressionMode(Lucene912QatCodec.Mode mode, int compressionLevel) { + this(mode, compressionLevel, () -> { return DEFAULT_QAT_MODE; }); } /** @@ -64,8 +65,8 @@ protected QatCompressionMode(Lucene99QatCodec.Mode mode, int compressionLevel) { * @param compressionLevel The compression level to use. * @param supplier a supplier for QAT acceleration mode. */ - protected QatCompressionMode(Lucene99QatCodec.Mode mode, int compressionLevel, Supplier supplier) { - this.algorithm = mode == Lucene99QatCodec.Mode.QAT_LZ4 ? QatZipper.Algorithm.LZ4 : QatZipper.Algorithm.DEFLATE; + protected QatCompressionMode(Lucene912QatCodec.Mode mode, int compressionLevel, Supplier supplier) { + this.algorithm = mode == Lucene912QatCodec.Mode.QAT_LZ4 ? QatZipper.Algorithm.LZ4 : QatZipper.Algorithm.DEFLATE; this.compressionLevel = compressionLevel; this.supplier = supplier; } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java similarity index 79% rename from src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java index 3690c2c..565928d 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java @@ -20,14 +20,16 @@ import com.intel.qat.QatZipper; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + /** * QatDeflate99Codec provides a DEFLATE compressor using the qat-java library. */ -public class QatDeflate99Codec extends Lucene99QatCodec implements CodecSettings, CodecAliases { +public class QatDeflate912Codec extends Lucene912QatCodec implements CodecSettings, CodecAliases { /** Creates a new QatDeflate99Codec instance with the default compression level. */ - public QatDeflate99Codec() { + public QatDeflate912Codec() { this(DEFAULT_COMPRESSION_LEVEL); } @@ -36,7 +38,7 @@ public QatDeflate99Codec() { * * @param compressionLevel The compression level. */ - public QatDeflate99Codec(int compressionLevel) { + public QatDeflate912Codec(int compressionLevel) { super(Mode.QAT_DEFLATE, compressionLevel); } @@ -46,7 +48,7 @@ public QatDeflate99Codec(int compressionLevel) { * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. */ - public QatDeflate99Codec(int compressionLevel, Supplier supplier) { + public QatDeflate912Codec(int compressionLevel, Supplier supplier) { super(Mode.QAT_DEFLATE, compressionLevel, supplier); } @@ -57,7 +59,7 @@ public QatDeflate99Codec(int compressionLevel, Supplier supplier * @param logger The logger. * @param compressionLevel The compression level. */ - public QatDeflate99Codec(MapperService mapperService, Logger logger, int compressionLevel) { + public QatDeflate912Codec(MapperService mapperService, Logger logger, int compressionLevel) { super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger); } @@ -69,7 +71,7 @@ public QatDeflate99Codec(MapperService mapperService, Logger logger, int compres * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. */ - public QatDeflate99Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { + public QatDeflate912Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger, supplier); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java similarity index 80% rename from src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java index 90c06fd..39267e2 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java @@ -20,14 +20,16 @@ import com.intel.qat.QatZipper; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; + /** * QatLz499Codec provides an LZ4 compressor using the qat-java library. */ -public class QatLz499Codec extends Lucene99QatCodec implements CodecSettings, CodecAliases { +public class QatLz4912Codec extends Lucene912QatCodec implements CodecSettings, CodecAliases { /** Creates a new QatLz499Codec instance with the default compression level. */ - public QatLz499Codec() { + public QatLz4912Codec() { this(DEFAULT_COMPRESSION_LEVEL); } @@ -36,7 +38,7 @@ public QatLz499Codec() { * * @param compressionLevel The compression level. */ - public QatLz499Codec(int compressionLevel) { + public QatLz4912Codec(int compressionLevel) { super(Mode.QAT_LZ4, compressionLevel); } @@ -46,7 +48,7 @@ public QatLz499Codec(int compressionLevel) { * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. */ - public QatLz499Codec(int compressionLevel, Supplier supplier) { + public QatLz4912Codec(int compressionLevel, Supplier supplier) { super(Mode.QAT_LZ4, compressionLevel, supplier); } @@ -57,7 +59,7 @@ public QatLz499Codec(int compressionLevel, Supplier supplier) { * @param logger The logger. * @param compressionLevel The compression level. */ - public QatLz499Codec(MapperService mapperService, Logger logger, int compressionLevel) { + public QatLz4912Codec(MapperService mapperService, Logger logger, int compressionLevel) { super(Mode.QAT_LZ4, compressionLevel, mapperService, logger); } @@ -69,7 +71,7 @@ public QatLz499Codec(MapperService mapperService, Logger logger, int compression * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. */ - public QatLz499Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { + public QatLz4912Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { super(Mode.QAT_LZ4, compressionLevel, mapperService, logger, supplier); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd912Codec.java similarity index 80% rename from src/main/java/org/opensearch/index/codec/customcodecs/Zstd99Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/Zstd912Codec.java index 29b5f4b..6a643e7 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd99Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd912Codec.java @@ -17,14 +17,16 @@ import java.util.Set; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + /** * ZstdCodec provides ZSTD compressor using the zstd-jni library. */ -public class Zstd99Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { +public class Zstd912Codec extends Lucene912CustomCodec implements CodecSettings, CodecAliases { /** Creates a new ZstdCodec instance with the default compression level. */ - public Zstd99Codec() { + public Zstd912Codec() { this(DEFAULT_COMPRESSION_LEVEL); } @@ -33,7 +35,7 @@ public Zstd99Codec() { * * @param compressionLevel The compression level. */ - public Zstd99Codec(int compressionLevel) { + public Zstd912Codec(int compressionLevel) { super(Mode.ZSTD, compressionLevel); } @@ -44,7 +46,7 @@ public Zstd99Codec(int compressionLevel) { * @param logger The logger. * @param compressionLevel The compression level. */ - public Zstd99Codec(MapperService mapperService, Logger logger, int compressionLevel) { + public Zstd912Codec(MapperService mapperService, Logger logger, int compressionLevel) { super(Mode.ZSTD, compressionLevel, mapperService, logger); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java index 67476ed..14bd7de 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java @@ -25,6 +25,8 @@ import java.io.IOException; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + /** Zstandard Compression Mode */ public class ZstdCompressionMode extends CompressionMode { @@ -35,7 +37,7 @@ public class ZstdCompressionMode extends CompressionMode { /** default constructor */ protected ZstdCompressionMode() { - this.compressionLevel = Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + this.compressionLevel = DEFAULT_COMPRESSION_LEVEL; } /** @@ -43,7 +45,7 @@ protected ZstdCompressionMode() { * * @param compressionLevel The compression level to use. */ - protected ZstdCompressionMode(int compressionLevel) { + public ZstdCompressionMode(int compressionLevel) { this.compressionLevel = compressionLevel; } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict912Codec.java similarity index 79% rename from src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict99Codec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict912Codec.java index 76c3ce6..b256e4d 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict99Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict912Codec.java @@ -17,11 +17,13 @@ import java.util.Set; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + /** ZstdNoDictCodec provides ZSTD compressor without a dictionary support. */ -public class ZstdNoDict99Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { +public class ZstdNoDict912Codec extends Lucene912CustomCodec implements CodecSettings, CodecAliases { /** Creates a new ZstdNoDictCodec instance with the default compression level. */ - public ZstdNoDict99Codec() { + public ZstdNoDict912Codec() { this(DEFAULT_COMPRESSION_LEVEL); } @@ -30,7 +32,7 @@ public ZstdNoDict99Codec() { * * @param compressionLevel The compression level. */ - public ZstdNoDict99Codec(int compressionLevel) { + public ZstdNoDict912Codec(int compressionLevel) { super(Mode.ZSTD_NO_DICT, compressionLevel); } @@ -41,7 +43,7 @@ public ZstdNoDict99Codec(int compressionLevel) { * @param logger The logger. * @param compressionLevel The compression level. */ - public ZstdNoDict99Codec(MapperService mapperService, Logger logger, int compressionLevel) { + public ZstdNoDict912Codec(MapperService mapperService, Logger logger, int compressionLevel) { super(Mode.ZSTD_NO_DICT, compressionLevel, mapperService, logger); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java index 2a44581..c918355 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java @@ -21,6 +21,8 @@ import java.io.IOException; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + /** ZSTD Compression Mode (without a dictionary support). */ public class ZstdNoDictCompressionMode extends CompressionMode { @@ -30,7 +32,7 @@ public class ZstdNoDictCompressionMode extends CompressionMode { /** default constructor */ protected ZstdNoDictCompressionMode() { - this.compressionLevel = Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; + this.compressionLevel = DEFAULT_COMPRESSION_LEVEL; } /** @@ -38,7 +40,7 @@ protected ZstdNoDictCompressionMode() { * * @param compressionLevel The compression level. */ - protected ZstdNoDictCompressionMode(int compressionLevel) { + public ZstdNoDictCompressionMode(int compressionLevel) { this.compressionLevel = compressionLevel; } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomCodec.java new file mode 100644 index 0000000..0dddcf7 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomCodec.java @@ -0,0 +1,100 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; + +import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.opensearch.common.settings.Settings; + +import java.util.Set; + +import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; + +/** + * + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. + * Supports two modes zstd and zstd_no_dict. + * Uses Lucene99 as the delegate codec + * + * @opensearch.internal + */ +public abstract class Lucene99CustomCodec extends FilterCodec { + + /** Default compression level used for compression */ + public static final int DEFAULT_COMPRESSION_LEVEL = INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getDefault(Settings.EMPTY); + + /** Each mode represents a compression algorithm. */ + public enum Mode { + /** + * ZStandard mode with dictionary + */ + ZSTD("ZSTD99", Set.of("zstd")), + /** + * ZStandard mode without dictionary + */ + ZSTD_NO_DICT("ZSTDNODICT99", Set.of("zstd_no_dict")); + + private final String codec; + private final Set aliases; + + Mode(String codec, Set aliases) { + this.codec = codec; + this.aliases = aliases; + } + + /** + * Returns the Codec that is registered with Lucene + */ + public String getCodec() { + return codec; + } + + /** + * Returns the aliases of the Codec + */ + public Set getAliases() { + return aliases; + } + } + + private final StoredFieldsFormat storedFieldsFormat; + + /** + * Creates a new compression codec with the default compression level. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + */ + public Lucene99CustomCodec(Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new compression codec with the given compression level. We use + * lowercase letters when registering the codec so that we remain consistent with + * the other compression codecs: default, lucene_default, and best_compression. + * + * @param mode The compression codec (ZSTD or ZSTDNODICT). + * @param compressionLevel The compression level. + */ + public Lucene99CustomCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene99Codec()); + this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormat.java similarity index 96% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormat.java index 27cb5c8..8a689cb 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormat.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.StoredFieldsReader; @@ -17,7 +17,8 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.opensearch.index.codec.customcodecs.backward_codecs.Lucene95CustomCodec; +import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; import java.io.IOException; import java.util.Objects; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatCodec.java similarity index 68% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatCodec.java index 20f265d..b22054f 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatCodec.java @@ -6,17 +6,14 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; -import org.apache.logging.log4j.Logger; +import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; -import org.opensearch.index.mapper.MapperService; import java.util.Set; import java.util.function.Supplier; @@ -118,43 +115,6 @@ public Lucene99QatCodec(Mode mode, int compressionLevel, Supplier supplier - ) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); - this.storedFieldsFormat = new Lucene99QatStoredFieldsFormat(mode, compressionLevel, supplier); - } - @Override public StoredFieldsFormat storedFieldsFormat() { return storedFieldsFormat; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormat.java similarity index 98% rename from src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java rename to src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormat.java index 9ea9dd9..3782bee 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormat.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.StoredFieldsReader; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatCompressionMode.java new file mode 100644 index 0000000..9bdd0e3 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatCompressionMode.java @@ -0,0 +1,206 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; + +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.compressing.Compressor; +import org.apache.lucene.codecs.compressing.Decompressor; +import org.apache.lucene.store.ByteBuffersDataInput; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.DataOutput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; + +import java.io.IOException; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +/** QatCompressionMode offers QAT_LZ4 and QAT_DEFLATE compressors. */ +public class QatCompressionMode extends CompressionMode { + + private static final int NUM_SUB_BLOCKS = 10; + + private final QatZipper.Algorithm algorithm; + private final int compressionLevel; + private final Supplier supplier; + + /** default constructor */ + protected QatCompressionMode() { + this(Lucene99QatCodec.DEFAULT_COMPRESSION_MODE, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, () -> { + return Lucene99QatCodec.DEFAULT_QAT_MODE; + }); + } + + /** + * Creates a new instance. + * + * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) + */ + protected QatCompressionMode(Lucene99QatCodec.Mode mode) { + this(mode, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }); + } + + /** + * Creates a new instance. + * + * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) + * @param compressionLevel The compression level to use. + */ + protected QatCompressionMode(Lucene99QatCodec.Mode mode, int compressionLevel) { + this(mode, compressionLevel, () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }); + } + + /** + * Creates a new instance. + * + * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) + * @param compressionLevel The compression level to use. + * @param supplier a supplier for QAT acceleration mode. + */ + protected QatCompressionMode(Lucene99QatCodec.Mode mode, int compressionLevel, Supplier supplier) { + this.algorithm = mode == Lucene99QatCodec.Mode.QAT_LZ4 ? QatZipper.Algorithm.LZ4 : QatZipper.Algorithm.DEFLATE; + this.compressionLevel = compressionLevel; + this.supplier = supplier; + } + + @Override + public Compressor newCompressor() { + return new QatCompressor(algorithm, compressionLevel, supplier.get()); + } + + @Override + public Decompressor newDecompressor() { + return new QatDecompressor(algorithm, supplier.get()); + } + + public int getCompressionLevel() { + return compressionLevel; + } + + /** The QatCompressor. */ + private static final class QatCompressor extends Compressor { + + private byte[] compressedBuffer; + private final QatZipper qatZipper; + + /** compressor with a given compresion level */ + public QatCompressor(QatZipper.Algorithm algorithm, int compressionLevel, QatZipper.Mode qatMode) { + compressedBuffer = BytesRef.EMPTY_BYTES; + qatZipper = QatZipperFactory.createInstance(algorithm, compressionLevel, qatMode, QatZipper.PollingMode.PERIODICAL); + } + + private void compress(byte[] bytes, int offset, int length, DataOutput out) throws IOException { + assert offset >= 0 : "Offset value must be greater than 0."; + + int blockLength = (length + NUM_SUB_BLOCKS - 1) / NUM_SUB_BLOCKS; + out.writeVInt(blockLength); + + final int end = offset + length; + assert end >= 0 : "Buffer read size must be greater than 0."; + + for (int start = offset; start < end; start += blockLength) { + int l = Math.min(blockLength, end - start); + + if (l == 0) { + out.writeVInt(0); + return; + } + + final int maxCompressedLength = qatZipper.maxCompressedLength(l); + compressedBuffer = ArrayUtil.grow(compressedBuffer, maxCompressedLength); + + int compressedSize = qatZipper.compress(bytes, start, l, compressedBuffer, 0, compressedBuffer.length); + out.writeVInt(compressedSize); + out.writeBytes(compressedBuffer, compressedSize); + } + } + + @Override + public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException { + final int length = (int) buffersInput.size(); + byte[] bytes = new byte[length]; + buffersInput.readBytes(bytes, 0, length); + compress(bytes, 0, length, out); + } + + @Override + public void close() throws IOException {} + } + + /** QAT_DEFLATE decompressor */ + private static final class QatDecompressor extends Decompressor { + + private byte[] compressed; + private final QatZipper qatZipper; + private final QatZipper.Mode qatMode; + private final QatZipper.Algorithm algorithm; + + /** default decompressor */ + public QatDecompressor(QatZipper.Algorithm algorithm, QatZipper.Mode qatMode) { + this.algorithm = algorithm; + this.qatMode = qatMode; + compressed = BytesRef.EMPTY_BYTES; + qatZipper = QatZipperFactory.createInstance(algorithm, qatMode, QatZipper.PollingMode.PERIODICAL); + } + + /*resuable decompress function*/ + @Override + public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException { + assert offset + length <= originalLength : "Buffer read size must be within limit."; + + if (length == 0) { + bytes.length = 0; + return; + } + + final int blockLength = in.readVInt(); + bytes.offset = bytes.length = 0; + int offsetInBlock = 0; + int offsetInBytesRef = offset; + + // Skip unneeded blocks + while (offsetInBlock + blockLength < offset) { + final int compressedLength = in.readVInt(); + in.skipBytes(compressedLength); + offsetInBlock += blockLength; + offsetInBytesRef -= blockLength; + } + + // Read blocks that intersect with the interval we need + while (offsetInBlock < offset + length) { + final int compressedLength = in.readVInt(); + if (compressedLength == 0) { + return; + } + compressed = ArrayUtil.grow(compressed, compressedLength); + in.readBytes(compressed, 0, compressedLength); + + int l = Math.min(blockLength, originalLength - offsetInBlock); + bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + l); + + final int uncompressed = qatZipper.decompress(compressed, 0, compressedLength, bytes.bytes, bytes.length, l); + + bytes.length += uncompressed; + offsetInBlock += blockLength; + } + + bytes.offset = offsetInBytesRef; + bytes.length = length; + + assert bytes.isValid() : "Decompression output is corrupted."; + } + + @Override + public Decompressor clone() { + return new QatDecompressor(algorithm, qatMode); + } + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatDeflate99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatDeflate99Codec.java new file mode 100644 index 0000000..6da8dba --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatDeflate99Codec.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; + +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +/** + * QatDeflate99Codec provides a DEFLATE compressor using the qat-java library. + */ +public class QatDeflate99Codec extends Lucene99QatCodec implements CodecSettings, CodecAliases { + + /** Creates a new QatDeflate99Codec instance with the default compression level. */ + public QatDeflate99Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new QatDeflate99Codec instance. + * + * @param compressionLevel The compression level. + */ + public QatDeflate99Codec(int compressionLevel) { + super(Mode.QAT_DEFLATE, compressionLevel); + } + + /** + * Creates a new QatDeflate99Codec instance with the default compression level. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatDeflate99Codec(int compressionLevel, Supplier supplier) { + super(Mode.QAT_DEFLATE, compressionLevel, supplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + if (!QatZipperFactory.isQatAvailable()) { + return Set.of(); + } + return Mode.QAT_DEFLATE.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatLz499Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatLz499Codec.java new file mode 100644 index 0000000..4271ee6 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatLz499Codec.java @@ -0,0 +1,70 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; + +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +/** + * QatLz499Codec provides an LZ4 compressor using the qat-java library. + */ +public class QatLz499Codec extends Lucene99QatCodec implements CodecSettings, CodecAliases { + + /** Creates a new QatLz499Codec instance with the default compression level. */ + public QatLz499Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new QatLz499Codec instance. + * + * @param compressionLevel The compression level. + */ + public QatLz499Codec(int compressionLevel) { + super(Mode.QAT_LZ4, compressionLevel); + } + + /** + * Creates a new QatLz499Codec instance with the default compression level. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatLz499Codec(int compressionLevel, Supplier supplier) { + super(Mode.QAT_LZ4, compressionLevel, supplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + if (!QatZipperFactory.isQatAvailable()) { + return Set.of(); + } + return Mode.QAT_LZ4.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Zstd99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Zstd99Codec.java new file mode 100644 index 0000000..53eb8b4 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Zstd99Codec.java @@ -0,0 +1,53 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; + +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; + +/** + * ZstdCodec provides ZSTD compressor using the zstd-jni library. + */ +public class Zstd99Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { + + /** Creates a new ZstdCodec instance with the default compression level. */ + public Zstd99Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new ZstdCodec instance. + * + * @param compressionLevel The compression level. + */ + public Zstd99Codec(int compressionLevel) { + super(Mode.ZSTD, compressionLevel); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + return Mode.ZSTD.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/ZstdNoDict99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/ZstdNoDict99Codec.java new file mode 100644 index 0000000..3bf42a4 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/ZstdNoDict99Codec.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; + +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; + +import java.util.Set; + +/** ZstdNoDictCodec provides ZSTD compressor without a dictionary support. */ +public class ZstdNoDict99Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { + + /** Creates a new ZstdNoDictCodec instance with the default compression level. */ + public ZstdNoDict99Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new ZstdNoDictCodec instance. + * + * @param compressionLevel The compression level. + */ + public ZstdNoDict99Codec(int compressionLevel) { + super(Mode.ZSTD_NO_DICT, compressionLevel); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + return Mode.ZSTD_NO_DICT.getAliases(); + } +} diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 4e25917..4979cdf 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -1,7 +1,11 @@ org.opensearch.index.codec.customcodecs.backward_codecs.Zstd95Codec org.opensearch.index.codec.customcodecs.backward_codecs.ZstdNoDict95Codec org.opensearch.index.codec.customcodecs.backward_codecs.Zstd95DeprecatedCodec -org.opensearch.index.codec.customcodecs.Zstd99Codec -org.opensearch.index.codec.customcodecs.ZstdNoDict99Codec -org.opensearch.index.codec.customcodecs.QatDeflate99Codec -org.opensearch.index.codec.customcodecs.QatLz499Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Zstd99Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.ZstdNoDict99Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.QatDeflate99Codec +org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.QatLz499Codec +org.opensearch.index.codec.customcodecs.Zstd912Codec +org.opensearch.index.codec.customcodecs.ZstdNoDict912Codec +org.opensearch.index.codec.customcodecs.QatDeflate912Codec +org.opensearch.index.codec.customcodecs.QatLz4912Codec diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java index e995ead..692cbca 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java @@ -35,7 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene912.Lucene912Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -69,6 +69,7 @@ import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_LZ4_CODEC; import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_CODEC; import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_NO_DICT_CODEC; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; @SuppressCodecs("*") // we test against default codec so never get a random one here! @@ -83,16 +84,16 @@ public void setup() { public void testZstd() throws Exception { Codec codec = createCodecService(false).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD, codec); - Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD, codec); + Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDict() throws Exception { Codec codec = createCodecService(false).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdDeprecatedCodec() { @@ -106,16 +107,16 @@ public void testZstdDeprecatedCodec() { public void testZstdWithCompressionLevel() throws Exception { int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "zstd").codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD, codec); - Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD, codec); + Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictWithCompressionLevel() throws Exception { int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "zstd_no_dict").codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); } @@ -156,26 +157,26 @@ public void testZstandardCompressionLevelSupport() throws Exception { public void testDefaultMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("default"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_SPEED, codec); } public void testBestCompressionMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene912Codec.Mode.BEST_COMPRESSION, codec); } public void testZstdMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD, codec); - Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD, codec); + Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testZstdNoDictMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene99CustomStoredFieldsFormat storedFieldsFormat = (Lucene99CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene912CustomStoredFieldsFormat storedFieldsFormat = (Lucene912CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); } public void testQatCodecsNotAvailable() throws IOException { @@ -183,11 +184,11 @@ public void testQatCodecsNotAvailable() throws IOException { assertThrows(IllegalArgumentException.class, () -> createCodecService(false).codec("qat_lz4")); assertThrows(IllegalArgumentException.class, () -> createCodecService(false).codec("qat_deflate")); - QatLz499Codec qatLz499Codec = new QatLz499Codec(); - assertTrue(qatLz499Codec.aliases().isEmpty()); + QatLz4912Codec qatLz4912Codec = new QatLz4912Codec(); + assertTrue(qatLz4912Codec.aliases().isEmpty()); - QatDeflate99Codec qatDeflate99Codec = new QatDeflate99Codec(); - assertTrue(qatDeflate99Codec.aliases().isEmpty()); + QatDeflate912Codec qatDeflate912Codec = new QatDeflate912Codec(); + assertTrue(qatDeflate912Codec.aliases().isEmpty()); } } @@ -203,18 +204,18 @@ public void testCodecServiceFactoryQatUnavailable() throws IOException { } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene99Codec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene912Codec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene99Codec.Mode.valueOf(v)); + assertEquals(expected, Lucene912Codec.Mode.valueOf(v)); } - private void assertStoredFieldsCompressionEquals(Lucene99CustomCodec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene912CustomCodec.Mode expected, Codec actual) throws Exception { SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene99CustomStoredFieldsFormat.MODE_KEY); + String v = sr.getSegmentInfo().info.getAttribute(Lucene912CustomStoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene99CustomCodec.Mode.valueOf(v)); + assertEquals(expected, Lucene912CustomCodec.Mode.valueOf(v)); } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormatTests.java new file mode 100644 index 0000000..1118cd1 --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormatTests.java @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.test.OpenSearchTestCase; + +public class Lucene912CustomStoredFieldsFormatTests extends OpenSearchTestCase { + + public void testDefaultLucene912CustomCodecMode() { + Lucene912CustomStoredFieldsFormat lucene912CustomStoredFieldsFormat = new Lucene912CustomStoredFieldsFormat(); + assertEquals(Lucene912CustomCodec.Mode.ZSTD, lucene912CustomStoredFieldsFormat.getMode()); + } + + public void testZstdNoDictLucene912CustomCodecMode() { + Lucene912CustomStoredFieldsFormat lucene912CustomStoredFieldsFormat = new Lucene912CustomStoredFieldsFormat( + Lucene912CustomCodec.Mode.ZSTD_NO_DICT + ); + assertEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, lucene912CustomStoredFieldsFormat.getMode()); + } + + public void testZstdModeWithCompressionLevel() { + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene912CustomStoredFieldsFormat lucene912CustomStoredFieldsFormat = new Lucene912CustomStoredFieldsFormat( + Lucene912CustomCodec.Mode.ZSTD, + randomCompressionLevel + ); + assertEquals(Lucene912CustomCodec.Mode.ZSTD, lucene912CustomStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene912CustomStoredFieldsFormat.getCompressionLevel()); + } + + public void testZstdNoDictLucene912CustomCodecModeWithCompressionLevel() { + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene912CustomStoredFieldsFormat lucene912CustomStoredFieldsFormat = new Lucene912CustomStoredFieldsFormat( + Lucene912CustomCodec.Mode.ZSTD_NO_DICT, + randomCompressionLevel + ); + assertEquals(Lucene912CustomCodec.Mode.ZSTD_NO_DICT, lucene912CustomStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene912CustomStoredFieldsFormat.getCompressionLevel()); + } + + public void testCompressionModes() { + Lucene912CustomStoredFieldsFormat lucene912CustomStoredFieldsFormat = new Lucene912CustomStoredFieldsFormat(); + assertTrue(lucene912CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdCompressionMode); + } + + public void testZstdNoDictCompressionModes() { + Lucene912CustomStoredFieldsFormat lucene912CustomStoredFieldsFormat = new Lucene912CustomStoredFieldsFormat( + Lucene912CustomCodec.Mode.ZSTD_NO_DICT + ); + assertTrue(lucene912CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdNoDictCompressionMode); + } + +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormatTests.java new file mode 100644 index 0000000..936e712 --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormatTests.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.test.OpenSearchTestCase; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assume.assumeThat; + +public class Lucene912QatStoredFieldsFormatTests extends OpenSearchTestCase { + + public void testLz4Lucene912QatCodecMode() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene912QatStoredFieldsFormat lucene912QatStoredFieldsFormat = new Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode.QAT_LZ4); + assertEquals(Lucene912QatCodec.Mode.QAT_LZ4, lucene912QatStoredFieldsFormat.getMode()); + } + + public void testDeflateLucene912QatCodecMode() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene912QatStoredFieldsFormat lucene912QatStoredFieldsFormat = new Lucene912QatStoredFieldsFormat( + Lucene912QatCodec.Mode.QAT_DEFLATE + ); + assertEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, lucene912QatStoredFieldsFormat.getMode()); + } + + public void testLz4Lucene912QatCodecModeWithCompressionLevel() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene912QatStoredFieldsFormat lucene912QatStoredFieldsFormat = new Lucene912QatStoredFieldsFormat( + Lucene912QatCodec.Mode.QAT_LZ4, + randomCompressionLevel + ); + assertEquals(Lucene912QatCodec.Mode.QAT_LZ4, lucene912QatStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene912QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); + } + + public void testDeflateLucene912QatCodecModeWithCompressionLevel() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene912QatStoredFieldsFormat lucene912QatStoredFieldsFormat = new Lucene912QatStoredFieldsFormat( + Lucene912QatCodec.Mode.QAT_DEFLATE, + randomCompressionLevel + ); + assertEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, lucene912QatStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene912QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); + } + + public void testLz4CompressionModes() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene912QatStoredFieldsFormat lucene912QatStoredFieldsFormat = new Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode.QAT_LZ4); + assertTrue(lucene912QatStoredFieldsFormat.getCompressionMode() instanceof QatCompressionMode); + } + + public void testDeflateCompressionModes() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + Lucene912QatStoredFieldsFormat lucene912QatStoredFieldsFormat = new Lucene912QatStoredFieldsFormat( + Lucene912QatCodec.Mode.QAT_DEFLATE + ); + assertTrue(lucene912QatStoredFieldsFormat.getCompressionMode() instanceof QatCompressionMode); + } +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java index e4220d3..2eb9a06 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java @@ -61,6 +61,8 @@ import java.util.Collections; import java.util.Optional; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; +import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING; import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; import static org.hamcrest.Matchers.is; import static org.junit.Assume.assumeThat; @@ -78,25 +80,25 @@ public void setup() { public void testQatLz4() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(false).codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); - Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_LZ4, codec); + Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatDeflate() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(false).codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); - Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, codec); + Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatLz4WithCompressionLevel() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "qat_lz4").codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); - Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_LZ4, codec); + Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } @@ -104,8 +106,8 @@ public void testQatDeflateWithCompressionLevel() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "qat_deflate").codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); - Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, codec); + Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } @@ -121,25 +123,25 @@ public void testQatCompressionLevelSupport() throws Exception { public void testQatLz4MapperServiceNull() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(true).codec("qat_lz4"); - assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); - Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_LZ4, codec); + Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatDeflateMapperServiceNull() throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(true).codec("qat_deflate"); - assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); - Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); + assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode.QAT_DEFLATE, codec); + Lucene912QatStoredFieldsFormat storedFieldsFormat = (Lucene912QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } - private void assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene912QatCodec.Mode expected, Codec actual) throws Exception { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene99QatStoredFieldsFormat.MODE_KEY); + String v = sr.getSegmentInfo().info.getAttribute(Lucene912QatStoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene99QatCodec.Mode.valueOf(v)); + assertEquals(expected, Lucene912QatCodec.Mode.valueOf(v)); } private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { @@ -147,7 +149,7 @@ private CodecService createCodecService(boolean isMapperServiceNull) throws IOEx if (isMapperServiceNull) { return new CustomCodecService( null, - IndexSettingsModule.newIndexSettings("_na", nodeSettings, Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING), + IndexSettingsModule.newIndexSettings("_na", nodeSettings, INDEX_CODEC_QAT_MODE_SETTING), LogManager.getLogger("test") ); } @@ -164,11 +166,7 @@ private CodecService createCodecService(int randomCompressionLevel, String codec } private CodecService buildCodecService(Settings nodeSettings) throws IOException { - IndexSettings indexSettings = IndexSettingsModule.newIndexSettings( - "_na", - nodeSettings, - Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING - ); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings, INDEX_CODEC_QAT_MODE_SETTING); SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap()); IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, nodeSettings).indexAnalyzers; MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java index 1994f69..edecb0a 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java @@ -21,13 +21,13 @@ public class QatDeflateCompressorTests extends AbstractCompressorTests { @Override Compressor compressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatCompressionMode(Lucene99QatCodec.Mode.QAT_DEFLATE).newCompressor(); + return new QatCompressionMode(Lucene912QatCodec.Mode.QAT_DEFLATE).newCompressor(); } @Override Decompressor decompressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatCompressionMode(Lucene99QatCodec.Mode.QAT_DEFLATE).newDecompressor(); + return new QatCompressionMode(Lucene912QatCodec.Mode.QAT_DEFLATE).newDecompressor(); } @Override diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java index 905773f..b11db58 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java @@ -21,13 +21,13 @@ public class QatLz4CompressorTests extends AbstractCompressorTests { @Override Compressor compressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatCompressionMode(Lucene99QatCodec.Mode.QAT_LZ4).newCompressor(); + return new QatCompressionMode(Lucene912QatCodec.Mode.QAT_LZ4).newCompressor(); } @Override Decompressor decompressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatCompressionMode(Lucene99QatCodec.Mode.QAT_LZ4).newDecompressor(); + return new QatCompressionMode(Lucene912QatCodec.Mode.QAT_LZ4).newDecompressor(); } @Override diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormatTests.java similarity index 100% rename from src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java rename to src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormatTests.java diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormatTests.java similarity index 92% rename from src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java rename to src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormatTests.java index c966901..3a6ae20 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormatTests.java @@ -6,8 +6,10 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; +import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; import org.opensearch.test.OpenSearchTestCase; public class Lucene99CustomStoredFieldsFormatTests extends OpenSearchTestCase { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormatTests.java similarity index 95% rename from src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java rename to src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormatTests.java index de3722b..a70dc81 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormatTests.java @@ -6,8 +6,9 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.is; From ef154c88515dbcc936c802710d1d3b2dbe397f2b Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Tue, 15 Oct 2024 10:45:40 -0400 Subject: [PATCH 2/4] Address code review comments Signed-off-by: Andriy Redko --- .../codec/customcodecs/CustomCodecPlugin.java | 23 +++++++++---------- .../customcodecs/ZstdCompressionMode.java | 2 +- .../ZstdNoDictCompressionMode.java | 2 +- .../Lucene99CustomStoredFieldsFormat.java | 6 +++-- ...Lucene95CustomStoredFieldsFormatTests.java | 6 ++--- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java index ec4cda3..7015704 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -50,18 +50,10 @@ public Optional getCustomCodecServiceFactory(final IndexSet || codecName.equals(CustomCodecService.QAT_DEFLATE_CODEC)) { return Optional.of(new CustomCodecServiceFactory()); } else { - if (codecName.equals(Lucene99QatCodec.Mode.QAT_LZ4.getCodec()) - || codecName.equals(Lucene99QatCodec.Mode.QAT_DEFLATE.getCodec())) { - if (!QatZipperFactory.isQatAvailable()) { - throw new IllegalArgumentException("QAT codecs are not supported. Please create indices with a different codec."); - } - } - - if (codecName.equals(Lucene912QatCodec.Mode.QAT_LZ4.getCodec()) - || codecName.equals(Lucene912QatCodec.Mode.QAT_DEFLATE.getCodec())) { - if (!QatZipperFactory.isQatAvailable()) { - throw new IllegalArgumentException("QAT codecs are not supported. Please create indices with a different codec."); - } + if (!QatZipperFactory.isQatAvailable() && isQatCodec(codecName)) { + throw new IllegalArgumentException( + "QAT codecs are not supported (QAT is not available). Please create indices with a different codec." + ); } } return Optional.empty(); @@ -71,4 +63,11 @@ public Optional getCustomCodecServiceFactory(final IndexSet public List> getSettings() { return Arrays.asList(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); } + + private static boolean isQatCodec(String codecName) { + return codecName.equals(Lucene99QatCodec.Mode.QAT_LZ4.getCodec()) + || codecName.equals(Lucene99QatCodec.Mode.QAT_DEFLATE.getCodec()) + || codecName.equals(Lucene912QatCodec.Mode.QAT_LZ4.getCodec()) + || codecName.equals(Lucene912QatCodec.Mode.QAT_DEFLATE.getCodec()); + } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java index 14bd7de..4fb53b7 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java @@ -45,7 +45,7 @@ protected ZstdCompressionMode() { * * @param compressionLevel The compression level to use. */ - public ZstdCompressionMode(int compressionLevel) { + protected ZstdCompressionMode(int compressionLevel) { this.compressionLevel = compressionLevel; } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java index c918355..fdee56d 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java @@ -40,7 +40,7 @@ protected ZstdNoDictCompressionMode() { * * @param compressionLevel The compression level. */ - public ZstdNoDictCompressionMode(int compressionLevel) { + protected ZstdNoDictCompressionMode(int compressionLevel) { this.compressionLevel = compressionLevel; } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormat.java index 8a689cb..df8ed60 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99CustomStoredFieldsFormat.java @@ -62,8 +62,10 @@ public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode) { public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode, int compressionLevel) { this.mode = Objects.requireNonNull(mode); this.compressionLevel = compressionLevel; - zstdCompressionMode = new ZstdCompressionMode(compressionLevel); - zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); + zstdCompressionMode = new ZstdCompressionMode(compressionLevel) { + }; + zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel) { + }; } /** diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormatTests.java index f420d56..be694df 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormatTests.java @@ -6,10 +6,10 @@ * compatible open source license. */ -package org.opensearch.index.codec.customcodecs; +package org.opensearch.index.codec.customcodecs.backward_codecs; -import org.opensearch.index.codec.customcodecs.backward_codecs.Lucene95CustomCodec; -import org.opensearch.index.codec.customcodecs.backward_codecs.Lucene95CustomStoredFieldsFormat; +import org.opensearch.index.codec.customcodecs.ZstdCompressionMode; +import org.opensearch.index.codec.customcodecs.ZstdNoDictCompressionMode; import org.opensearch.test.OpenSearchTestCase; public class Lucene95CustomStoredFieldsFormatTests extends OpenSearchTestCase { From 39082230b384275b6a308b95578eaecd8d33332f Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 16 Oct 2024 08:47:19 -0400 Subject: [PATCH 3/4] Address code review comments Signed-off-by: Andriy Redko --- .../Lucene912QatStoredFieldsFormat.java | 11 +- .../customcodecs/QatCompressionMode.java | 20 +- .../Lucene99QatStoredFieldsFormat.java | 13 +- .../lucene99/QatCompressionMode.java | 206 ------------------ .../QatDeflateCompressorTests.java | 6 +- .../customcodecs/QatLz4CompressorTests.java | 6 +- .../Lucene99QatStoredFieldsFormatTests.java | 1 + 7 files changed, 41 insertions(+), 222 deletions(-) delete mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatCompressionMode.java diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java index cf4ff21..819c8e6 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912QatStoredFieldsFormat.java @@ -87,7 +87,7 @@ public Lucene912QatStoredFieldsFormat(Lucene912QatCodec.Mode mode, Supplier supplier) { this.mode = Objects.requireNonNull(mode); - qatCompressionMode = new QatCompressionMode(mode, compressionLevel, supplier); + qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier); } /** @@ -176,4 +176,13 @@ public Lucene912QatCodec.Mode getMode() { public QatCompressionMode getCompressionMode() { return qatCompressionMode; } + + /** + * Returns {@link QatZipper.Algorithm} instance that corresponds codec's {@link Lucene912QatCodec.Mode mode} + * @param mode codec's {@link Lucene912QatCodec.Mode mode} + * @return the {@link QatZipper.Algorithm} instance that corresponds codec's {@link Lucene912QatCodec.Mode mode} + */ + private static QatZipper.Algorithm getAlgorithm(Lucene912QatCodec.Mode mode) { + return (mode == Lucene912QatCodec.Mode.QAT_LZ4) ? QatZipper.Algorithm.LZ4 : QatZipper.Algorithm.DEFLATE; + } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java index 96800c5..86046c6 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java @@ -36,37 +36,37 @@ public class QatCompressionMode extends CompressionMode { /** default constructor */ protected QatCompressionMode() { - this(Lucene912QatCodec.DEFAULT_COMPRESSION_MODE, DEFAULT_COMPRESSION_LEVEL, () -> { return DEFAULT_QAT_MODE; }); + this(QatZipper.Algorithm.LZ4, DEFAULT_COMPRESSION_LEVEL, () -> { return DEFAULT_QAT_MODE; }); } /** * Creates a new instance. * - * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) + * @param algorithm The compression algorithm (LZ4 or DEFLATE) */ - protected QatCompressionMode(Lucene912QatCodec.Mode mode) { - this(mode, DEFAULT_COMPRESSION_LEVEL, () -> { return DEFAULT_QAT_MODE; }); + protected QatCompressionMode(QatZipper.Algorithm algorithm) { + this(algorithm, DEFAULT_COMPRESSION_LEVEL, () -> { return DEFAULT_QAT_MODE; }); } /** * Creates a new instance. * - * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) + * @param algorithm The compression algorithm (LZ4 or DEFLATE) * @param compressionLevel The compression level to use. */ - protected QatCompressionMode(Lucene912QatCodec.Mode mode, int compressionLevel) { - this(mode, compressionLevel, () -> { return DEFAULT_QAT_MODE; }); + protected QatCompressionMode(QatZipper.Algorithm algorithm, int compressionLevel) { + this(algorithm, compressionLevel, () -> { return DEFAULT_QAT_MODE; }); } /** * Creates a new instance. * - * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) + * @param algorithm The compression algorithm (LZ4 or DEFLATE) * @param compressionLevel The compression level to use. * @param supplier a supplier for QAT acceleration mode. */ - protected QatCompressionMode(Lucene912QatCodec.Mode mode, int compressionLevel, Supplier supplier) { - this.algorithm = mode == Lucene912QatCodec.Mode.QAT_LZ4 ? QatZipper.Algorithm.LZ4 : QatZipper.Algorithm.DEFLATE; + protected QatCompressionMode(QatZipper.Algorithm algorithm, int compressionLevel, Supplier supplier) { + this.algorithm = algorithm; this.compressionLevel = compressionLevel; this.supplier = supplier; } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormat.java index 3782bee..d2f4103 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormat.java @@ -17,6 +17,7 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.opensearch.index.codec.customcodecs.QatCompressionMode; import java.io.IOException; import java.util.Objects; @@ -84,7 +85,8 @@ public Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode mode, Supplier supplier) { this.mode = Objects.requireNonNull(mode); - qatCompressionMode = new QatCompressionMode(mode, compressionLevel, supplier); + qatCompressionMode = new QatCompressionMode(getAlgorithm(mode), compressionLevel, supplier) { + }; } /** @@ -173,4 +175,13 @@ public Lucene99QatCodec.Mode getMode() { public QatCompressionMode getCompressionMode() { return qatCompressionMode; } + + /** + * Returns {@link QatZipper.Algorithm} instance that corresponds codec's {@link Lucene99QatCodec.Mode mode} + * @param mode codec's {@link Lucene99QatCodec.Mode mode} + * @return the {@link QatZipper.Algorithm} instance that corresponds codec's {@link Lucene99QatCodec.Mode mode} + */ + private static QatZipper.Algorithm getAlgorithm(Lucene99QatCodec.Mode mode) { + return (mode == Lucene99QatCodec.Mode.QAT_LZ4) ? QatZipper.Algorithm.LZ4 : QatZipper.Algorithm.DEFLATE; + } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatCompressionMode.java deleted file mode 100644 index 9bdd0e3..0000000 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/QatCompressionMode.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; - -import org.apache.lucene.codecs.compressing.CompressionMode; -import org.apache.lucene.codecs.compressing.Compressor; -import org.apache.lucene.codecs.compressing.Decompressor; -import org.apache.lucene.store.ByteBuffersDataInput; -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.DataOutput; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; -import org.opensearch.index.codec.customcodecs.QatZipperFactory; - -import java.io.IOException; -import java.util.function.Supplier; - -import com.intel.qat.QatZipper; - -/** QatCompressionMode offers QAT_LZ4 and QAT_DEFLATE compressors. */ -public class QatCompressionMode extends CompressionMode { - - private static final int NUM_SUB_BLOCKS = 10; - - private final QatZipper.Algorithm algorithm; - private final int compressionLevel; - private final Supplier supplier; - - /** default constructor */ - protected QatCompressionMode() { - this(Lucene99QatCodec.DEFAULT_COMPRESSION_MODE, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, () -> { - return Lucene99QatCodec.DEFAULT_QAT_MODE; - }); - } - - /** - * Creates a new instance. - * - * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) - */ - protected QatCompressionMode(Lucene99QatCodec.Mode mode) { - this(mode, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }); - } - - /** - * Creates a new instance. - * - * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) - * @param compressionLevel The compression level to use. - */ - protected QatCompressionMode(Lucene99QatCodec.Mode mode, int compressionLevel) { - this(mode, compressionLevel, () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }); - } - - /** - * Creates a new instance. - * - * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) - * @param compressionLevel The compression level to use. - * @param supplier a supplier for QAT acceleration mode. - */ - protected QatCompressionMode(Lucene99QatCodec.Mode mode, int compressionLevel, Supplier supplier) { - this.algorithm = mode == Lucene99QatCodec.Mode.QAT_LZ4 ? QatZipper.Algorithm.LZ4 : QatZipper.Algorithm.DEFLATE; - this.compressionLevel = compressionLevel; - this.supplier = supplier; - } - - @Override - public Compressor newCompressor() { - return new QatCompressor(algorithm, compressionLevel, supplier.get()); - } - - @Override - public Decompressor newDecompressor() { - return new QatDecompressor(algorithm, supplier.get()); - } - - public int getCompressionLevel() { - return compressionLevel; - } - - /** The QatCompressor. */ - private static final class QatCompressor extends Compressor { - - private byte[] compressedBuffer; - private final QatZipper qatZipper; - - /** compressor with a given compresion level */ - public QatCompressor(QatZipper.Algorithm algorithm, int compressionLevel, QatZipper.Mode qatMode) { - compressedBuffer = BytesRef.EMPTY_BYTES; - qatZipper = QatZipperFactory.createInstance(algorithm, compressionLevel, qatMode, QatZipper.PollingMode.PERIODICAL); - } - - private void compress(byte[] bytes, int offset, int length, DataOutput out) throws IOException { - assert offset >= 0 : "Offset value must be greater than 0."; - - int blockLength = (length + NUM_SUB_BLOCKS - 1) / NUM_SUB_BLOCKS; - out.writeVInt(blockLength); - - final int end = offset + length; - assert end >= 0 : "Buffer read size must be greater than 0."; - - for (int start = offset; start < end; start += blockLength) { - int l = Math.min(blockLength, end - start); - - if (l == 0) { - out.writeVInt(0); - return; - } - - final int maxCompressedLength = qatZipper.maxCompressedLength(l); - compressedBuffer = ArrayUtil.grow(compressedBuffer, maxCompressedLength); - - int compressedSize = qatZipper.compress(bytes, start, l, compressedBuffer, 0, compressedBuffer.length); - out.writeVInt(compressedSize); - out.writeBytes(compressedBuffer, compressedSize); - } - } - - @Override - public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException { - final int length = (int) buffersInput.size(); - byte[] bytes = new byte[length]; - buffersInput.readBytes(bytes, 0, length); - compress(bytes, 0, length, out); - } - - @Override - public void close() throws IOException {} - } - - /** QAT_DEFLATE decompressor */ - private static final class QatDecompressor extends Decompressor { - - private byte[] compressed; - private final QatZipper qatZipper; - private final QatZipper.Mode qatMode; - private final QatZipper.Algorithm algorithm; - - /** default decompressor */ - public QatDecompressor(QatZipper.Algorithm algorithm, QatZipper.Mode qatMode) { - this.algorithm = algorithm; - this.qatMode = qatMode; - compressed = BytesRef.EMPTY_BYTES; - qatZipper = QatZipperFactory.createInstance(algorithm, qatMode, QatZipper.PollingMode.PERIODICAL); - } - - /*resuable decompress function*/ - @Override - public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException { - assert offset + length <= originalLength : "Buffer read size must be within limit."; - - if (length == 0) { - bytes.length = 0; - return; - } - - final int blockLength = in.readVInt(); - bytes.offset = bytes.length = 0; - int offsetInBlock = 0; - int offsetInBytesRef = offset; - - // Skip unneeded blocks - while (offsetInBlock + blockLength < offset) { - final int compressedLength = in.readVInt(); - in.skipBytes(compressedLength); - offsetInBlock += blockLength; - offsetInBytesRef -= blockLength; - } - - // Read blocks that intersect with the interval we need - while (offsetInBlock < offset + length) { - final int compressedLength = in.readVInt(); - if (compressedLength == 0) { - return; - } - compressed = ArrayUtil.grow(compressed, compressedLength); - in.readBytes(compressed, 0, compressedLength); - - int l = Math.min(blockLength, originalLength - offsetInBlock); - bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + l); - - final int uncompressed = qatZipper.decompress(compressed, 0, compressedLength, bytes.bytes, bytes.length, l); - - bytes.length += uncompressed; - offsetInBlock += blockLength; - } - - bytes.offset = offsetInBytesRef; - bytes.length = length; - - assert bytes.isValid() : "Decompression output is corrupted."; - } - - @Override - public Decompressor clone() { - return new QatDecompressor(algorithm, qatMode); - } - } -} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java index edecb0a..1cf21ed 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java @@ -10,6 +10,8 @@ import org.apache.lucene.codecs.compressing.Compressor; import org.apache.lucene.codecs.compressing.Decompressor; +import com.intel.qat.QatZipper; + import java.io.IOException; import static org.hamcrest.Matchers.is; @@ -21,13 +23,13 @@ public class QatDeflateCompressorTests extends AbstractCompressorTests { @Override Compressor compressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatCompressionMode(Lucene912QatCodec.Mode.QAT_DEFLATE).newCompressor(); + return new QatCompressionMode(QatZipper.Algorithm.DEFLATE).newCompressor(); } @Override Decompressor decompressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatCompressionMode(Lucene912QatCodec.Mode.QAT_DEFLATE).newDecompressor(); + return new QatCompressionMode(QatZipper.Algorithm.DEFLATE).newDecompressor(); } @Override diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java index b11db58..2fc4044 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java @@ -10,6 +10,8 @@ import org.apache.lucene.codecs.compressing.Compressor; import org.apache.lucene.codecs.compressing.Decompressor; +import com.intel.qat.QatZipper; + import java.io.IOException; import static org.hamcrest.Matchers.is; @@ -21,13 +23,13 @@ public class QatLz4CompressorTests extends AbstractCompressorTests { @Override Compressor compressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatCompressionMode(Lucene912QatCodec.Mode.QAT_LZ4).newCompressor(); + return new QatCompressionMode(QatZipper.Algorithm.LZ4).newCompressor(); } @Override Decompressor decompressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatCompressionMode(Lucene912QatCodec.Mode.QAT_LZ4).newDecompressor(); + return new QatCompressionMode(QatZipper.Algorithm.LZ4).newDecompressor(); } @Override diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormatTests.java index a70dc81..0f0697a 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/backward_codecs/lucene99/Lucene99QatStoredFieldsFormatTests.java @@ -8,6 +8,7 @@ package org.opensearch.index.codec.customcodecs.backward_codecs.lucene99; +import org.opensearch.index.codec.customcodecs.QatCompressionMode; import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.test.OpenSearchTestCase; From 502bab93092c5264437a2bb41a7420b6ebc4d544 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 16 Oct 2024 12:03:45 -0400 Subject: [PATCH 4/4] Fix javadoc comments Signed-off-by: Andriy Redko --- .../codec/customcodecs/Lucene912CustomCodec.java | 2 +- .../index/codec/customcodecs/QatDeflate912Codec.java | 12 ++++++------ .../index/codec/customcodecs/QatLz4912Codec.java | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java index 0d925c9..692360e 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java @@ -23,7 +23,7 @@ * * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. * Supports two modes zstd and zstd_no_dict. - * Uses Lucene99 as the delegate codec + * Uses Lucene912 as the delegate codec * * @opensearch.internal */ diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java index 565928d..62365ad 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate912Codec.java @@ -23,18 +23,18 @@ import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; /** - * QatDeflate99Codec provides a DEFLATE compressor using the qat-java library. */ public class QatDeflate912Codec extends Lucene912QatCodec implements CodecSettings, CodecAliases { - /** Creates a new QatDeflate99Codec instance with the default compression level. */ + /** Creates a new QatDeflate912Codec instance with the default compression level. */ public QatDeflate912Codec() { this(DEFAULT_COMPRESSION_LEVEL); } /** - * Creates a new QatDeflate99Codec instance. + * Creates a new QatDeflate912Codec instance. * * @param compressionLevel The compression level. */ @@ -43,7 +43,7 @@ public QatDeflate912Codec(int compressionLevel) { } /** - * Creates a new QatDeflate99Codec instance with the default compression level. + * Creates a new QatDeflate912Codec instance with the default compression level. * * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. @@ -53,7 +53,7 @@ public QatDeflate912Codec(int compressionLevel, Supplier supplie } /** - * Creates a new QatDeflate99Codec instance. + * Creates a new QatDeflate912Codec instance. * * @param mapperService The mapper service. * @param logger The logger. @@ -64,7 +64,7 @@ public QatDeflate912Codec(MapperService mapperService, Logger logger, int compre } /** - * Creates a new QatDeflate99Codec instance. + * Creates a new QatDeflate912Codec instance. * * @param mapperService The mapper service. * @param logger The logger. diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java index 39267e2..5a85aa4 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4912Codec.java @@ -23,18 +23,18 @@ import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; /** - * QatLz499Codec provides an LZ4 compressor using the qat-java library. */ public class QatLz4912Codec extends Lucene912QatCodec implements CodecSettings, CodecAliases { - /** Creates a new QatLz499Codec instance with the default compression level. */ + /** Creates a new QatLz4912Codec instance with the default compression level. */ public QatLz4912Codec() { this(DEFAULT_COMPRESSION_LEVEL); } /** - * Creates a new QatLz499Codec instance. + * Creates a new QatLz4912Codec instance. * * @param compressionLevel The compression level. */ @@ -43,7 +43,7 @@ public QatLz4912Codec(int compressionLevel) { } /** - * Creates a new QatLz499Codec instance with the default compression level. + * Creates a new QatLz4912Codec instance with the default compression level. * * @param compressionLevel The compression level. * @param supplier supplier for QAT acceleration mode. @@ -53,7 +53,7 @@ public QatLz4912Codec(int compressionLevel, Supplier supplier) { } /** - * Creates a new QatLz499Codec instance. + * Creates a new QatLz4912Codec instance. * * @param mapperService The mapper service. * @param logger The logger. @@ -64,7 +64,7 @@ public QatLz4912Codec(MapperService mapperService, Logger logger, int compressio } /** - * Creates a new QatLz499Codec instance. + * Creates a new QatLz4912Codec instance. * * @param mapperService The mapper service. * @param logger The logger.