Skip to content

Commit

Permalink
Code refactor to split classes
Browse files Browse the repository at this point in the history
  • Loading branch information
rishabhmaurya committed Mar 15, 2024
1 parent edd6a68 commit dc79099
Show file tree
Hide file tree
Showing 5 changed files with 353 additions and 257 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,34 +8,15 @@

package org.opensearch.index.mapper;

import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.KeywordField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.Query;
import org.opensearch.common.Nullable;
import org.opensearch.common.geo.ShapeRelation;
import org.opensearch.common.lucene.Lucene;
import org.opensearch.common.time.DateMathParser;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.query.DerivedFieldQuery;
import org.opensearch.index.query.DerivedFieldScript;
import org.opensearch.index.query.QueryShardContext;
import org.opensearch.script.Script;
import org.opensearch.search.lookup.SearchLookup;
import org.opensearch.search.lookup.SourceLookup;

import java.io.IOException;
import java.time.ZoneId;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.Function;

/**
Expand Down Expand Up @@ -94,216 +75,23 @@ public Builder(String name) {
super(name);
}


@Override
protected List<Parameter<?>> getParameters() {
return Arrays.asList(type, script);
}

@Override
public DerivedFieldMapper build(BuilderContext context) {
FieldMapper typeFieldMapper;
Function<Object, IndexableField> fieldFunction;
switch (type.getValue()) {
// TODO: add logic all supported type in derived fields
// TODO: should we support mapping settings exposed by a given field type from derived fields too?
// for example, support `format` for date type?
case KeywordFieldMapper.CONTENT_TYPE:
FieldType dummyFieldType = new FieldType();
dummyFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
KeywordFieldMapper.Builder keywordBuilder = new KeywordFieldMapper.Builder(name());
KeywordFieldMapper.KeywordFieldType keywordFieldType = keywordBuilder.buildFieldType(context, dummyFieldType);
keywordFieldType.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
typeFieldMapper = new KeywordFieldMapper(
name(), dummyFieldType, keywordFieldType,
keywordBuilder.multiFieldsBuilder.build(this, context),
keywordBuilder.copyTo.build(),
keywordBuilder
);
fieldFunction = o -> new KeywordField(name(), (String) o, Field.Store.NO);
break;
case "long":
// ignoreMalformed?
NumberFieldMapper.Builder longBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.LONG, false, false);
typeFieldMapper = longBuilder.build(context);
fieldFunction = o -> new LongField(name(), Long.parseLong(o.toString()), Field.Store.NO);
break;
case "double":
// ignoreMalformed?
NumberFieldMapper.Builder doubleBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.DOUBLE, false, false);
typeFieldMapper = doubleBuilder.build(context);
fieldFunction = o -> new DoubleField(name(), Double.parseDouble(o.toString()), Field.Store.NO);
break;
default:
throw new IllegalArgumentException("Field [" + name() + "] of type [" + type + "] isn't supported " +
"in Derived field context.");

}
MappedFieldType ft = new DerivedFieldType(buildFullName(context), type.getValue(), script.getValue(), typeFieldMapper, fieldFunction);
FieldMapper fieldMapper = DerivedFieldSupportedTypes.getFieldMapperFromType(type.getValue(), name, context);
Function<Object, IndexableField> fieldFunction =
DerivedFieldSupportedTypes.getIndexableFieldGeneratorType(type.getValue(), name);
DerivedFieldType ft = new DerivedFieldType(buildFullName(context), type.getValue(), script.getValue(), fieldMapper, fieldFunction);
return new DerivedFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this);
}
}


public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));

/**
* Field type for derived field mapper
*
* @opensearch.internal
*/
public static final class DerivedFieldType extends MappedFieldType {
private final String type;

private final Script script;

FieldMapper typeFieldMapper;

private final Function<Object, IndexableField> fieldFunction;

public DerivedFieldType(
String name,
String type,
Script script,
boolean isIndexed,
boolean isStored,
boolean hasDocValues,
Map<String, String> meta,
FieldMapper typeFieldMapper,
Function<Object, IndexableField> fieldFunction
) {
super(name, isIndexed, isStored, hasDocValues, typeFieldMapper.fieldType().getTextSearchInfo(), meta);
this.type = type;
this.script = script;
this.typeFieldMapper = typeFieldMapper;
this.fieldFunction = fieldFunction;
}

public DerivedFieldType(String name, String type, Script script, FieldMapper typeFieldMapper, Function<Object, IndexableField> fieldFunction) {
this(name, type, script, false, false, false, Collections.emptyMap(), typeFieldMapper, fieldFunction);
}

@Override
public String typeName() {
return CONTENT_TYPE;
}

@Override
public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) {
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
DerivedFieldScript.LeafFactory derivedFieldScriptFactory = factory.newFactory(script.getParams(), context.lookup());

return new SourceValueFetcher(name(), context) {
DerivedFieldScript derivedFieldScript;

@Override
public List<Object> fetchValues(SourceLookup lookup) {
derivedFieldScript.setDocument(lookup.docId());
// TODO: remove List.of() when derivedFieldScript.execute() returns list of objects.
return List.of(derivedFieldScript.execute());
}

@Override
protected Object parseSourceValue(Object value) {
return value;
}

public void setNextReader(LeafReaderContext context) {
try {
derivedFieldScript = derivedFieldScriptFactory.newInstance(context);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
};
}

@Override
public Query termQuery(Object value, @Nullable QueryShardContext context) {
Query query = typeFieldMapper.mappedFieldType.termQuery(value, context);
DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
}

@Override
public Query prefixQuery(
String value,
@Nullable MultiTermQuery.RewriteMethod method,
boolean caseInsensitive,
QueryShardContext context
) {
Query query = typeFieldMapper.mappedFieldType.prefixQuery(value, method, caseInsensitive, context);
DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
}

@Override
public Query wildcardQuery(
String value,
@Nullable MultiTermQuery.RewriteMethod method,
boolean caseInsensitive,
QueryShardContext context
) {
Query query = typeFieldMapper.mappedFieldType.wildcardQuery(value, method, caseInsensitive, context);
DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
}

@Override
public Query regexpQuery(
String value,
int syntaxFlags,
int matchFlags,
int maxDeterminizedStates,
@Nullable MultiTermQuery.RewriteMethod method,
QueryShardContext context
) {
Query query = typeFieldMapper.mappedFieldType.regexpQuery(value, syntaxFlags, matchFlags,
maxDeterminizedStates, method, context);
DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
}

// TODO: Override all types of queries which can be supported by all types supported within derived fields.
@Override
public Query rangeQuery(
Object lowerTerm,
Object upperTerm,
boolean includeLower,
boolean includeUpper,
ShapeRelation relation,
ZoneId timeZone,
DateMathParser parser,
QueryShardContext context
) {
Query query = typeFieldMapper.mappedFieldType.rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper,
relation, timeZone, parser, context);
DerivedFieldScript.Factory factory = context.compile(script, DerivedFieldScript.CONTEXT);
DerivedFieldScript.LeafFactory derivedFieldScript = factory.newFactory(script.getParams(), context.lookup());
return new DerivedFieldQuery(query, derivedFieldScript, typeFieldMapper.mappedFieldType.indexAnalyzer(), fieldFunction, context);
}

@Override
public Query existsQuery(QueryShardContext context) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support exist queries");
}

@Override
public boolean isAggregatable() {
return false;
}
}


private final String type;

private final Script script;

protected DerivedFieldMapper(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.index.mapper;

import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.KeywordField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.opensearch.common.lucene.Lucene;

import java.util.Arrays;
import java.util.Map;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.stream.Collectors;

public enum DerivedFieldSupportedTypes {
KEYWORD (
"keyword",
(name, context) -> {
FieldType dummyFieldType = new FieldType();
dummyFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
KeywordFieldMapper.Builder keywordBuilder = new KeywordFieldMapper.Builder(name);
KeywordFieldMapper.KeywordFieldType keywordFieldType = keywordBuilder.buildFieldType(context, dummyFieldType);
keywordFieldType.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
return new KeywordFieldMapper(
name, dummyFieldType, keywordFieldType,
keywordBuilder.multiFieldsBuilder.build(keywordBuilder, context),
keywordBuilder.copyTo.build(),
keywordBuilder
);
},
name -> o -> new KeywordField(name, (String) o, Field.Store.NO)
),
LONG (
"long",
(name, context) -> {
NumberFieldMapper.Builder longBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.LONG, false, false);
return longBuilder.build(context);
},
name -> o -> new LongField(name, Long.parseLong(o.toString()), Field.Store.NO)
),
DOUBLE (
"double",
(name, context) -> {
NumberFieldMapper.Builder doubleBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.DOUBLE, false, false);
return doubleBuilder.build(context);
},
name -> o -> new DoubleField(name, Double.parseDouble(o.toString()), Field.Store.NO)
);
// TODO: add logic all supported type in derived fields
// TODO: should we support mapping settings exposed by a given field type from derived fields too?
// for example, support `format` for date type?
final String name;
private final BiFunction<String, Mapper.BuilderContext, FieldMapper> builder;

private final Function<String, Function<Object, IndexableField>> indexableFieldBuilder;

DerivedFieldSupportedTypes(String name, BiFunction<String, Mapper.BuilderContext, FieldMapper> builder,
Function<String, Function<Object, IndexableField>> indexableFieldBuilder){
this.name = name;
this.builder = builder;
this.indexableFieldBuilder = indexableFieldBuilder;
}

public String getName() {
return name;
}
private FieldMapper getFieldMapper(String name, Mapper.BuilderContext context) {
return builder.apply(name, context);
}

private Function<Object, IndexableField> getIndexableFieldGenerator(String name) {
return indexableFieldBuilder.apply(name);
}
private static final Map<String, DerivedFieldSupportedTypes> enumMap = Arrays.stream(DerivedFieldSupportedTypes.values())
.collect(Collectors.toMap(DerivedFieldSupportedTypes::getName, enumValue -> enumValue));

public static FieldMapper getFieldMapperFromType(String type, String name, Mapper.BuilderContext context) {
if (!enumMap.containsKey(type)) {
throw new IllegalArgumentException("Type [" + type + "] isn't supported in Derived field context.");
}
return enumMap.get(type).getFieldMapper(name, context);
}

public static Function<Object, IndexableField> getIndexableFieldGeneratorType(String type, String name) {
if (!enumMap.containsKey(type)) {
throw new IllegalArgumentException("Type [" + type + "] isn't supported in Derived field context.");
}
return enumMap.get(type).getIndexableFieldGenerator(name);
}
}
Loading

0 comments on commit dc79099

Please sign in to comment.