Skip to content

Commit

Permalink
Revert "Update Apache Lucene to 9.12.0 (opensearch-project#15333)"
Browse files Browse the repository at this point in the history
This reverts commit 146b0f7.
  • Loading branch information
finnegancarroll committed Oct 8, 2024
1 parent 958f8c7 commit 6524bdd
Show file tree
Hide file tree
Showing 84 changed files with 148 additions and 231 deletions.
1 change: 0 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041))
- Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042))
- Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137))
- Bump Apache lucene from 9.11.1 to 9.12.0 ([#15333](https://github.com/opensearch-project/OpenSearch/pull/15333))
- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133))
- Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135))
- Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182))
Expand Down
2 changes: 1 addition & 1 deletion buildSrc/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
opensearch = 3.0.0
lucene = 9.12.0
lucene = 9.12.0-snapshot-847316d

bundled_jdk_vendor = adoptium
bundled_jdk = 21.0.4+7
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
51ff4940eb1024184bbaa5dae39695d2392c5bab
1 change: 0 additions & 1 deletion libs/core/licenses/lucene-core-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b866103bbaca4141c152deca9252bd137026dafc

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
04436942995a4952ce5654126dfb767d6335674e

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
85918e24fc3bf63fcd953807ab2eb3fa55c987c2

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
15e425e9cc0ab9d65fac3c919199a24dfa3631eb

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3d16c18348e7d4a00cb83100c43f3e21239d224e

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2ef6d9dffc6816d3cd04a54fe1ee43e13f850a37

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e72b2262f5393d9ff255fb901297d4e6790e9102

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
416ac44b2e76592c9e85338798cae93c3cf5475e

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

package org.opensearch.plugin.correlation.core.index.codec;

import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene99.Lucene99Codec;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.plugin.correlation.core.index.codec.correlation990.CorrelationCodec;
import org.opensearch.plugin.correlation.core.index.codec.correlation990.PerFieldCorrelationVectorsFormat;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7e282aab7388efc911348f1eacd90e661580dda7
1 change: 0 additions & 1 deletion server/licenses/lucene-analysis-common-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
69e59ba4bed4c58836d2727d72b7f0095d2dcb92
1 change: 0 additions & 1 deletion server/licenses/lucene-backward-codecs-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
51ff4940eb1024184bbaa5dae39695d2392c5bab
1 change: 0 additions & 1 deletion server/licenses/lucene-core-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5847a7d47f13ecb7f039fb9adf6f3b8e4bddde77
1 change: 0 additions & 1 deletion server/licenses/lucene-grouping-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7cc0a26777a479f06fbcfae7abc23e784e1a00dc
1 change: 0 additions & 1 deletion server/licenses/lucene-highlighter-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
9cd99401c826d910da3c2beab8e42f1af8be6ea4
1 change: 0 additions & 1 deletion server/licenses/lucene-join-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
cfee136ecbc3df7adc38b38e020dca5e61c22773
1 change: 0 additions & 1 deletion server/licenses/lucene-memory-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
afbc5adf93d4eb1a1b109ad828d1968bf16ef292
1 change: 0 additions & 1 deletion server/licenses/lucene-misc-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
16907c36f6adb8ba8f260e05738c66afb37c72d3
1 change: 0 additions & 1 deletion server/licenses/lucene-queries-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
72baa9bddcf2efb71ffb695f1e9f548699ec13a0
1 change: 0 additions & 1 deletion server/licenses/lucene-queryparser-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
dd3c63066f583d90b563ebaa6fbe61c603403acb
1 change: 0 additions & 1 deletion server/licenses/lucene-sandbox-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
69b99530e0b05251c12863bee6a9325cafd5fdaa
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial-extras-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a67d193b4b08790169db7cf005a2429991260287
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial3d-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7a1625ae39071ccbfb3af11df5a74291758f4b47
1 change: 0 additions & 1 deletion server/licenses/lucene-suggest-9.12.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ public void testConcurrentSearchTaskTracking() {
Map<Long, List<ThreadResourceInfo>> threadStats = getThreadStats(SearchAction.NAME + "[*]", taskInfo.getTaskId());
// Concurrent search forks each slice of 5 segments to different thread (see please
// https://github.com/apache/lucene/issues/12498)
assertEquals((int) Math.ceil(getSegmentCount(INDEX_NAME) / 5.0), threadStats.size());
assertEquals((int) Math.ceil(getSegmentCount(INDEX_NAME) / 5.0) + 1, threadStats.size());

// assert that all task descriptions have non-zero length
MatcherAssert.assertThat(taskInfo.getDescription().length(), greaterThan(0));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

import static org.opensearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_CHUNK_SIZE_SETTING;
Expand Down Expand Up @@ -185,7 +184,7 @@ public void testCancelRecoveryAndResume() throws Exception {
)
.get();

latch.await(5, TimeUnit.MINUTES);
latch.await();

// at this point we got some truncated left overs on the replica on the unlucky node
// now we are allowing the recovery to allocate again and finish to see if we wipe the truncated files
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ public LegacyBM25Similarity(float k1, float b) {
* within the range {@code [0..1]}
*/
public LegacyBM25Similarity(float k1, float b, boolean discountOverlaps) {
super(discountOverlaps);
this.bm25Similarity = new BM25Similarity(k1, b, discountOverlaps);
}

Expand Down Expand Up @@ -102,6 +101,15 @@ public final float getB() {
return bm25Similarity.getB();
}

/**
* Returns true if overlap tokens are discounted from the document's length.
*
* @see #LegacyBM25Similarity(float, float, boolean)
*/
public boolean getDiscountOverlaps() {
return bm25Similarity.getDiscountOverlaps();
}

@Override
public String toString() {
return bm25Similarity.toString();
Expand Down
26 changes: 9 additions & 17 deletions server/src/main/java/org/apache/lucene/util/CombinedBitSet.java
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,15 @@ public int prevSetBit(int index) {

@Override
public int nextSetBit(int index) {
return nextSetBit(index, length() - 1);
assert index >= 0 && index < length : "index=" + index + " numBits=" + length();
int next = first.nextSetBit(index);
while (next != DocIdSetIterator.NO_MORE_DOCS && second.get(next) == false) {
if (next == length() - 1) {
return DocIdSetIterator.NO_MORE_DOCS;
}
next = first.nextSetBit(next + 1);
}
return next;
}

@Override
Expand Down Expand Up @@ -124,20 +132,4 @@ public void clear(int startIndex, int endIndex) {
public boolean getAndSet(int i) {
throw new UnsupportedOperationException("not implemented");
}

@Override
public int nextSetBit(int start, int end) {
assert start >= 0 && start < length() : "start=" + start + " numBits=" + length();
assert end >= start && end < length() : "end=" + end + " numBits=" + length();

int next = first.nextSetBit(start);
while (next != DocIdSetIterator.NO_MORE_DOCS && second.get(next) == false) {
if (next >= end) {
return DocIdSetIterator.NO_MORE_DOCS;
}
next = first.nextSetBit(next + 1);
}
return next;

}
}
4 changes: 2 additions & 2 deletions server/src/main/java/org/opensearch/common/lucene/Lucene.java
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@
* @opensearch.internal
*/
public class Lucene {
public static final String LATEST_CODEC = "Lucene912";
public static final String LATEST_CODEC = "Lucene99";

public static final String SOFT_DELETES_FIELD = "__soft_deletes";

Expand Down Expand Up @@ -272,7 +272,7 @@ public static void checkSegmentInfoIntegrity(final Directory directory) throws I

@Override
protected Object doBody(String segmentFileName) throws IOException {
try (IndexInput input = directory.openInput(segmentFileName, IOContext.READONCE)) {
try (IndexInput input = directory.openInput(segmentFileName, IOContext.READ)) {
CodecUtil.checksumEntireFile(input);
}
return null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene912.Lucene912Codec;
import org.apache.lucene.codecs.lucene912.Lucene912Codec.Mode;
import org.apache.lucene.codecs.lucene99.Lucene99Codec;
import org.apache.lucene.codecs.lucene99.Lucene99Codec.Mode;
import org.opensearch.common.Nullable;
import org.opensearch.common.collect.MapBuilder;
import org.opensearch.index.IndexSettings;
Expand Down Expand Up @@ -70,10 +70,10 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe
final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
assert null != indexSettings;
if (mapperService == null) {
codecs.put(DEFAULT_CODEC, new Lucene912Codec());
codecs.put(LZ4, new Lucene912Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene912Codec(Mode.BEST_COMPRESSION));
codecs.put(ZLIB, new Lucene912Codec(Mode.BEST_COMPRESSION));
codecs.put(DEFAULT_CODEC, new Lucene99Codec());
codecs.put(LZ4, new Lucene99Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene99Codec(Mode.BEST_COMPRESSION));
codecs.put(ZLIB, new Lucene99Codec(Mode.BEST_COMPRESSION));
} else {
// CompositeCodec still delegates to PerFieldMappingPostingFormatCodec
// We can still support all the compression codecs when composite index is present
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene912.Lucene912Codec;
import org.apache.lucene.codecs.lucene99.Lucene99Codec;
import org.opensearch.common.lucene.Lucene;
import org.opensearch.index.codec.fuzzy.FuzzyFilterPostingsFormat;
import org.opensearch.index.codec.fuzzy.FuzzySetFactory;
Expand All @@ -59,7 +59,7 @@
*
* @opensearch.internal
*/
public class PerFieldMappingPostingFormatCodec extends Lucene912Codec {
public class PerFieldMappingPostingFormatCodec extends Lucene99Codec {
private final Logger logger;
private final MapperService mapperService;
private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,8 @@

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene912.Lucene912Codec;
import org.apache.lucene.codecs.lucene99.Lucene99Codec;
import org.opensearch.common.annotation.ExperimentalApi;
import org.opensearch.index.codec.composite.composite912.Composite912Codec;
import org.opensearch.index.codec.composite.composite99.Composite99Codec;
import org.opensearch.index.mapper.MapperService;

Expand All @@ -39,10 +38,10 @@ public CompositeCodecFactory() {}

public Map<String, Codec> getCompositeIndexCodecs(MapperService mapperService, Logger logger) {
Map<String, Codec> codecs = new HashMap<>();
codecs.put(DEFAULT_CODEC, new Composite912Codec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger));
codecs.put(LZ4, new Composite912Codec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger));
codecs.put(BEST_COMPRESSION_CODEC, new Composite912Codec(Lucene912Codec.Mode.BEST_COMPRESSION, mapperService, logger));
codecs.put(ZLIB, new Composite912Codec(Lucene912Codec.Mode.BEST_COMPRESSION, mapperService, logger));
codecs.put(DEFAULT_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger));
codecs.put(LZ4, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger));
codecs.put(BEST_COMPRESSION_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger));
codecs.put(ZLIB, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger));
return codecs;
}
}

This file was deleted.

Loading

0 comments on commit 6524bdd

Please sign in to comment.