Skip to content

Commit

Permalink
Allow validation for non knn index only after 2.17.0 (#2315)
Browse files Browse the repository at this point in the history 
* Allow validation for non knn index only after 2.17.0

Before 2.17.0, non knn index are allowed to create knn_vector fields with model id
and method params, however this is not allowed from 2.17.0.
Hence, to maintain backward compatibility we will not add validation
for any indices created before 2.17.0.

Signed-off-by: Vijayan Balasubramanian <[email protected]>

* Add bwc test for model id

This is applicable for index created before 2.17

Signed-off-by: Vijayan Balasubramanian <[email protected]>

---------

Signed-off-by: Vijayan Balasubramanian <[email protected]>
  • Loading branch information
@VijayanB
VijayanB authored Dec 17, 2024
1 parent f50a8f5 commit d57fdea
Show file tree
Hide file tree
Showing 9 changed files with 243 additions and 9 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Optimizes lucene query execution to prevent unnecessary rewrites (#2305)[https://github.com/opensearch-project/k-NN/pull/2305]
### Bug Fixes
* Fixing the bug when a segment has no vector field present for disk based vector search (#2282)[https://github.com/opensearch-project/k-NN/pull/2282]
* Allow validation for non knn index only after 2.17.0 (#2315)[https://github.com/opensearch-project/k-NN/pull/2315]
### Infrastructure
* Updated C++ version in JNI from c++11 to c++17 [#2259](https://github.com/opensearch-project/k-NN/pull/2259)
* Upgrade bytebuddy and objenesis version to match OpenSearch core and, update github ci runner for macos [#2279](https://github.com/opensearch-project/k-NN/pull/2279)
Expand Down
29 changes: 29 additions & 0 deletions qa/restart-upgrade/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,20 @@ testClusters {
excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexLuceneForceMerge"
}
}
if (!(knn_bwc_version.startsWith("2.13.") ||
knn_bwc_version.startsWith("2.14.") ||
knn_bwc_version.startsWith("2.15.") ||
knn_bwc_version.startsWith("2.16."))) {
filter {
excludeTestsMatching "org.opensearch.knn.bwc.ModelIT.testNonKNNIndex_withModelId"
excludeTestsMatching "org.opensearch.knn.bwc.PainlessScriptScoringIT.testNonKNNIndex_withMethodParams_withFaissEngine"
excludeTestsMatching "org.opensearch.knn.bwc.PainlessScriptScoringIT.testNonKNNIndex_withMethodParams_withNmslibEngine"
excludeTestsMatching "org.opensearch.knn.bwc.PainlessScriptScoringIT.testNonKNNIndex_withMethodParams_withLuceneEngine"
excludeTestsMatching "org.opensearch.knn.bwc.ScriptScoringIT.testNonKNNIndex_withMethodParams_withFaissEngine"
excludeTestsMatching "org.opensearch.knn.bwc.ScriptScoringIT.testNonKNNIndex_withMethodParams_withNmslibEngine"
excludeTestsMatching "org.opensearch.knn.bwc.ScriptScoringIT.testNonKNNIndex_withMethodParams_withLuceneEngine"
}
}

nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
Expand Down Expand Up @@ -201,6 +215,21 @@ testClusters {
}
}

if (!(knn_bwc_version.startsWith("2.13.") ||
knn_bwc_version.startsWith("2.14.") ||
knn_bwc_version.startsWith("2.15.") ||
knn_bwc_version.startsWith("2.16."))) {
filter {
excludeTestsMatching "org.opensearch.knn.bwc.ModelIT.testNonKNNIndex_withModelId"
excludeTestsMatching "org.opensearch.knn.bwc.PainlessScriptScoringIT.testNonKNNIndex_withMethodParams_withFaissEngine"
excludeTestsMatching "org.opensearch.knn.bwc.PainlessScriptScoringIT.testNonKNNIndex_withMethodParams_withNmslibEngine"
excludeTestsMatching "org.opensearch.knn.bwc.PainlessScriptScoringIT.testNonKNNIndex_withMethodParams_withLuceneEngine"
excludeTestsMatching "org.opensearch.knn.bwc.ScriptScoringIT.testNonKNNIndex_withMethodParams_withFaissEngine"
excludeTestsMatching "org.opensearch.knn.bwc.ScriptScoringIT.testNonKNNIndex_withMethodParams_withNmslibEngine"
excludeTestsMatching "org.opensearch.knn.bwc.ScriptScoringIT.testNonKNNIndex_withMethodParams_withLuceneEngine"
}
}

nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
systemProperty 'tests.security.manager', 'false'
Expand Down
9 changes: 6 additions & 3 deletions qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRestartUpgradeTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,12 @@ public abstract class AbstractRestartUpgradeTestCase extends KNNRestTestCase {

@Before
protected void setIndex() {
// Creating index name by concatenating "knn-bwc-" prefix with test method name
// for all the tests in this sub-project
testIndex = KNN_BWC_PREFIX + getTestName().toLowerCase(Locale.ROOT);
// Creating index name by concatenating "knn-bwc-" prefix with test class name and then with method name
// for all the tests in this sub-project to generate unique index name
testIndex = new StringBuilder().append(KNN_BWC_PREFIX)
.append(getTestClass().getName().toLowerCase(Locale.ROOT))
.append(getTestName().toLowerCase(Locale.ROOT))
.toString();
}

@Override
Expand Down
33 changes: 31 additions & 2 deletions qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ModelIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,24 +43,27 @@ public class ModelIT extends AbstractRestartUpgradeTestCase {
private static final String TEST_MODEL_INDEX_DEFAULT = KNN_BWC_PREFIX + "test-model-index-default";
private static final String TRAINING_INDEX = KNN_BWC_PREFIX + "train-index";
private static final String TRAINING_INDEX_DEFAULT = KNN_BWC_PREFIX + "train-index-default";
private static final String TRAINING_INDEX_FOR_NON_KNN_INDEX = KNN_BWC_PREFIX + "train-index-for-non-knn-index";
private static final String TRAINING_FIELD = "train-field";
private static final String TEST_FIELD = "test-field";
private static final int DIMENSIONS = 5;
private static int DOC_ID = 0;
private static int DOC_ID_TEST_MODEL_INDEX = 0;
private static int DOC_ID_TEST_MODEL_INDEX_DEFAULT = 0;
private static final int DELAY_MILLI_SEC = 1000;
private static final int EXP_NUM_OF_MODELS = 2;
private static final int MIN_NUM_OF_MODELS = 2;
private static final int K = 5;
private static final int NUM_DOCS = 10;
private static final int NUM_DOCS_TEST_MODEL_INDEX = 100;
private static final int NUM_DOCS_TEST_MODEL_INDEX_DEFAULT = 100;
private static final int NUM_DOCS_TEST_MODEL_INDEX_FOR_NON_KNN_INDEX = 100;
private static final int NUM_OF_ATTEMPTS = 30;
private static int QUERY_COUNT = 0;
private static int QUERY_COUNT_TEST_MODEL_INDEX = 0;
private static int QUERY_COUNT_TEST_MODEL_INDEX_DEFAULT = 0;
private static final String TEST_MODEL_ID = "test-model-id";
private static final String TEST_MODEL_ID_DEFAULT = "test-model-id-default";
private static final String TEST_MODEL_ID_FOR_NON_KNN_INDEX = "test-model-id-for-non-knn-index";
private static final String MODEL_DESCRIPTION = "Description for train model test";

// KNN model test
Expand Down Expand Up @@ -135,6 +138,32 @@ public void testKNNModelDefault() throws Exception {
}
}

public void testNonKNNIndex_withModelId() throws Exception {
if (isRunningAgainstOldCluster()) {

// Create a training index and randomly ingest data into it
createBasicKnnIndex(TRAINING_INDEX_FOR_NON_KNN_INDEX, TRAINING_FIELD, DIMENSIONS);
bulkIngestRandomVectors(TRAINING_INDEX_FOR_NON_KNN_INDEX, TRAINING_FIELD, NUM_DOCS, DIMENSIONS);

trainKNNModel(TEST_MODEL_ID_FOR_NON_KNN_INDEX, TRAINING_INDEX_FOR_NON_KNN_INDEX, TRAINING_FIELD, DIMENSIONS, MODEL_DESCRIPTION);
validateModelCreated(TEST_MODEL_ID_FOR_NON_KNN_INDEX);

createKnnIndex(
testIndex,
createKNNDefaultScriptScoreSettings(),
modelIndexMapping(TEST_FIELD, TEST_MODEL_ID_FOR_NON_KNN_INDEX)
);
addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
} else {
Thread.sleep(1000);
DOC_ID = NUM_DOCS;
addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
deleteKNNIndex(testIndex);
deleteKNNIndex(TRAINING_INDEX_FOR_NON_KNN_INDEX);
deleteKNNModel(TEST_MODEL_ID_FOR_NON_KNN_INDEX);
}
}

// Delete Models and ".opensearch-knn-models" index to clear cluster metadata
@AfterClass
public static void wipeAllModels() throws IOException {
Expand Down Expand Up @@ -168,7 +197,7 @@ public void searchKNNModel(String testModelID) throws Exception {
XContentParser parser = createParser(MediaTypeRegistry.getDefaultMediaType().xContent(), responseBody);
SearchResponse searchResponse = SearchResponse.fromXContent(parser);
assertNotNull(searchResponse);
assertEquals(EXP_NUM_OF_MODELS, searchResponse.getHits().getHits().length);
assertTrue(MIN_NUM_OF_MODELS <= searchResponse.getHits().getHits().length);

for (SearchHit hit : searchResponse.getHits().getHits()) {
assertTrue(hit.getId().startsWith(testModelID));
Expand Down
71 changes: 71 additions & 0 deletions qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/PainlessScriptScoringIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@

package org.opensearch.knn.bwc;

import org.opensearch.knn.index.SpaceType;
import org.opensearch.knn.index.engine.KNNEngine;

public class PainlessScriptScoringIT extends AbstractRestartUpgradeTestCase {
private static final String TEST_FIELD = "test-field";
private static final int DIMENSIONS = 5;
Expand Down Expand Up @@ -53,4 +56,72 @@ public void testKNNL1PainlessScriptScore() throws Exception {
}
}

public void testNonKNNIndex_withMethodParams_withFaissEngine() throws Exception {
if (isRunningAgainstOldCluster()) {
createKnnIndex(
testIndex,
createKNNDefaultScriptScoreSettings(),
createKnnIndexMapping(TEST_FIELD, DIMENSIONS, "hnsw", KNNEngine.FAISS.getName(), SpaceType.DEFAULT.getValue(), false)
);
addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
} else {
DOC_ID = NUM_DOCS;
QUERY_COUNT = NUM_DOCS;
String source = createL1PainlessScriptSource(TEST_FIELD, DIMENSIONS, QUERY_COUNT);
validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
QUERY_COUNT = QUERY_COUNT + NUM_DOCS;
source = createL1PainlessScriptSource(TEST_FIELD, DIMENSIONS, QUERY_COUNT);
validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
forceMergeKnnIndex(testIndex, 1);
validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
deleteKNNIndex(testIndex);
}
}

public void testNonKNNIndex_withMethodParams_withNmslibEngine() throws Exception {
if (isRunningAgainstOldCluster()) {
createKnnIndex(
testIndex,
createKNNDefaultScriptScoreSettings(),
createKnnIndexMapping(TEST_FIELD, DIMENSIONS, "hnsw", KNNEngine.NMSLIB.getName(), SpaceType.DEFAULT.getValue(), false)
);
addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
} else {
DOC_ID = NUM_DOCS;
QUERY_COUNT = NUM_DOCS;
String source = createL1PainlessScriptSource(TEST_FIELD, DIMENSIONS, QUERY_COUNT);
validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
QUERY_COUNT = QUERY_COUNT + NUM_DOCS;
source = createL1PainlessScriptSource(TEST_FIELD, DIMENSIONS, QUERY_COUNT);
validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
forceMergeKnnIndex(testIndex, 1);
validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
deleteKNNIndex(testIndex);
}
}

public void testNonKNNIndex_withMethodParams_withLuceneEngine() throws Exception {
if (isRunningAgainstOldCluster()) {
createKnnIndex(
testIndex,
createKNNDefaultScriptScoreSettings(),
createKnnIndexMapping(TEST_FIELD, DIMENSIONS, "hnsw", KNNEngine.LUCENE.getName(), SpaceType.DEFAULT.getValue(), false)
);
addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
} else {
DOC_ID = NUM_DOCS;
QUERY_COUNT = NUM_DOCS;
String source = createL1PainlessScriptSource(TEST_FIELD, DIMENSIONS, QUERY_COUNT);
validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
QUERY_COUNT = QUERY_COUNT + NUM_DOCS;
source = createL1PainlessScriptSource(TEST_FIELD, DIMENSIONS, QUERY_COUNT);
validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
forceMergeKnnIndex(testIndex, 1);
validateKNNPainlessScriptScoreSearch(testIndex, TEST_FIELD, source, QUERY_COUNT, K);
deleteKNNIndex(testIndex);
}
}
}
59 changes: 58 additions & 1 deletion qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/ScriptScoringIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.opensearch.knn.KNNResult;
import org.opensearch.knn.index.SpaceType;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.knn.index.engine.KNNEngine;

import java.util.Collections;
import java.util.HashMap;
Expand Down Expand Up @@ -96,6 +97,63 @@ public void testKNNInnerProductScriptScore() throws Exception {
}
}

public void testNonKNNIndex_withMethodParams_withFaissEngine() throws Exception {
if (isRunningAgainstOldCluster()) {
createKnnIndex(
testIndex,
createKNNDefaultScriptScoreSettings(),
createKnnIndexMapping(TEST_FIELD, DIMENSIONS, "hnsw", KNNEngine.FAISS.getName(), SpaceType.DEFAULT.getValue(), false)
);
addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
} else {
QUERY_COUNT = NUM_DOCS;
DOC_ID = NUM_DOCS;
validateKNNScriptScoreSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K, SpaceType.L2);
addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
QUERY_COUNT = QUERY_COUNT + NUM_DOCS;
validateKNNScriptScoreSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K, SpaceType.L2);
deleteKNNIndex(testIndex);
}
}

public void testNonKNNIndex_withMethodParams_withNmslibEngine() throws Exception {
if (isRunningAgainstOldCluster()) {
createKnnIndex(
testIndex,
createKNNDefaultScriptScoreSettings(),
createKnnIndexMapping(TEST_FIELD, DIMENSIONS, "hnsw", KNNEngine.NMSLIB.getName(), SpaceType.DEFAULT.getValue(), false)
);
addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
} else {
QUERY_COUNT = NUM_DOCS;
DOC_ID = NUM_DOCS;
validateKNNScriptScoreSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K, SpaceType.L2);
addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
QUERY_COUNT = QUERY_COUNT + NUM_DOCS;
validateKNNScriptScoreSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K, SpaceType.L2);
deleteKNNIndex(testIndex);
}
}

public void testNonKNNIndex_withMethodParams_withLuceneEngine() throws Exception {
if (isRunningAgainstOldCluster()) {
createKnnIndex(
testIndex,
createKNNDefaultScriptScoreSettings(),
createKnnIndexMapping(TEST_FIELD, DIMENSIONS, "hnsw", KNNEngine.LUCENE.getName(), SpaceType.DEFAULT.getValue(), false)
);
addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
} else {
QUERY_COUNT = NUM_DOCS;
DOC_ID = NUM_DOCS;
validateKNNScriptScoreSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K, SpaceType.L2);
addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, DOC_ID, NUM_DOCS);
QUERY_COUNT = QUERY_COUNT + NUM_DOCS;
validateKNNScriptScoreSearch(testIndex, TEST_FIELD, DIMENSIONS, QUERY_COUNT, K, SpaceType.L2);
deleteKNNIndex(testIndex);
}
}

// Validate Script score search for space_type : "inner_product"
private void validateKNNInnerProductScriptScoreSearch(String testIndex, String testField, int dimension, int numDocs, int k)
throws Exception {
Expand All @@ -121,5 +179,4 @@ private void validateKNNInnerProductScriptScoreSearch(String testIndex, String t
assertEquals(expDocID, actualDocID);
}
}

}
9 changes: 6 additions & 3 deletions src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,10 @@ public KNNVectorFieldMapper build(BuilderContext context) {
);
}

if (originalParameters.getResolvedKnnMethodContext() == null) {
// return FlatVectorFieldMapper only for indices that are created on or after 2.17.0, for others, use either LuceneFieldMapper
// or
// MethodFieldMapper to maintain backwards compatibility
if (originalParameters.getResolvedKnnMethodContext() == null && context.indexCreatedVersion().onOrAfter(Version.V_2_17_0)) {
return FlatVectorFieldMapper.createFieldMapper(
buildFullName(context),
name,
Expand Down Expand Up @@ -375,8 +378,8 @@ public Mapper.Builder<?> parse(String name, Map<String, Object> node, ParserCont
);
}

// Check for flat configuration
if (isKNNDisabled(parserContext.getSettings())) {
// Check for flat configuration and validate only if index is created after 2.17
if (isKNNDisabled(parserContext.getSettings()) && parserContext.indexVersionCreated().onOrAfter(Version.V_2_17_0)) {
validateFromFlat(builder);
} else if (builder.modelId.get() != null) {
validateFromModel(builder);
Expand Down
24 changes: 24 additions & 0 deletions src/test/java/org/opensearch/knn/index/OpenSearchIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -814,6 +814,30 @@ public void testKNNIndex_whenBuildVectorDataStructureIsLessThanDocCount_thenBuil
deleteKNNIndex(indexName);
}

public void testCreateNonKNNIndex_withKNNModelID_throwsException() throws Exception {
Settings settings = Settings.builder().put(createKNNDefaultScriptScoreSettings()).build();
ResponseException ex = expectThrows(
ResponseException.class,
() -> createKnnIndex(INDEX_NAME, settings, createKnnIndexMapping(FIELD_NAME, "random-model-id"))
);
String expMessage = "Cannot set modelId or method parameters when index.knn setting is false";
assertThat(EntityUtils.toString(ex.getResponse().getEntity()), containsString(expMessage));
}

public void testCreateNonKNNIndex_withKNNMethodParams_throwsException() throws Exception {
Settings settings = Settings.builder().put(createKNNDefaultScriptScoreSettings()).build();
ResponseException ex = expectThrows(
ResponseException.class,
() -> createKnnIndex(
INDEX_NAME,
settings,
createKnnIndexMapping(FIELD_NAME, 2, "hnsw", KNNEngine.FAISS.getName(), SpaceType.DEFAULT.getValue(), false)
)
);
String expMessage = "Cannot set modelId or method parameters when index.knn setting is false";
assertThat(EntityUtils.toString(ex.getResponse().getEntity()), containsString(expMessage));
}

/*
For this testcase, we will create index with setting build_vector_data_structure_threshold as -1, then index few documents, perform knn search,
then, confirm hits because of exact search though there are no graph. In next step, update setting to 0, force merge segment to 1, perform knn search and confirm expected
Expand Down
Loading

0 comments on commit d57fdea

Please sign in to comment.