Skip to content

Commit

Permalink
commit make elastic search useable
Browse files Browse the repository at this point in the history
  • Loading branch information
@baisui1981
baisui1981 committed Nov 1, 2022
1 parent 240f5b2 commit f5229c2
Show file tree
Hide file tree
Showing 11 changed files with 268 additions and 171 deletions.
23 changes: 19 additions & 4 deletions tis-console/src/main/java/com/qlangtech/tis/coredefine/module/action/DataxAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
import com.qlangtech.tis.runtime.module.misc.IControlMsgHandler;
import com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler;
import com.qlangtech.tis.runtime.module.misc.impl.DelegateControl4JsonPostMsgHandler;
import com.qlangtech.tis.solrdao.ISchema;
import com.qlangtech.tis.util.*;
import com.qlangtech.tis.workflow.pojo.WorkFlowBuildHistory;
import com.qlangtech.tis.workflow.pojo.WorkFlowBuildHistoryCriteria;
Expand Down Expand Up @@ -805,16 +806,30 @@ public void doGotoEsAppCreateConfirm(Context context) throws Exception {
// 这里只做schema的校验
CreateIndexConfirmModel confiemModel = parseJsonPost(CreateIndexConfirmModel.class);
String schemaContent = null;
ISchema schema = null;
ISearchEngineTypeTransfer typeTransfer = ISearchEngineTypeTransfer.load(this, confiemModel.getDataxName());
if (confiemModel.isExpertModel()) {
schemaContent = confiemModel.getExpert().getXml();
CreateIndexConfirmModel.ExpertEditorModel expect = confiemModel.getExpert();
schemaContent = expect.getXml();
schema = typeTransfer.projectionFromExpertModel(expect.asJson());
} else {

ISearchEngineTypeTransfer typeTransfer = ISearchEngineTypeTransfer.load(this, confiemModel.getDataxName());
schemaContent = typeTransfer.mergeFromStupidModel(confiemModel.getStupid().getModel()
schema = confiemModel.getStupid().getModel();
schemaContent = typeTransfer.mergeFromStupidModel(schema
, ISearchEngineTypeTransfer.getOriginExpertSchema(null)).toJSONString();
}

if (!schema.isValid()) {
for (String err : schema.getErrors()) {
this.addErrorMessage(context, err);
}
return;
}

DataxProcessor.DataXCreateProcessMeta processMeta = DataxProcessor.getDataXCreateProcessMeta(this, confiemModel.getDataxName());
List<ISelectedTab> selectedTabs = processMeta.getReader().getSelectedTabs();
ESTableAlias esTableAlias = new ESTableAlias();
esTableAlias.setFrom(selectedTabs.stream().findFirst().get().getName());
esTableAlias.setTo(((ISearchEngineTypeTransfer) processMeta.getWriter()).getIndexName());
esTableAlias.setSchemaContent(schemaContent);

this.saveTableMapper(this, confiemModel.getDataxName(), Collections.singletonList(esTableAlias));
Expand Down
36 changes: 21 additions & 15 deletions ...onsole/src/main/java/com/qlangtech/tis/runtime/module/action/CreateIndexConfirmModel.java
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,26 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.qlangtech.tis.runtime.module.action;

import com.alibaba.fastjson.JSONObject;
import com.qlangtech.tis.coredefine.module.control.SelectableServer;
import com.qlangtech.tis.coredefine.module.control.SelectableServer.CoreNode;
import com.qlangtech.tis.datax.ISearchEngineTypeTransfer;
import com.qlangtech.tis.fullbuild.indexbuild.LuceneVersion;
import com.qlangtech.tis.runtime.module.action.AddAppAction.ExtendApp;
import com.qlangtech.tis.runtime.module.action.SchemaAction.UploadSchemaWithRawContentForm;
Expand Down Expand Up @@ -150,6 +152,10 @@ public String getXml() {
return this.xml;
}

public JSONObject asJson() {
return ISearchEngineTypeTransfer.getOriginExpertSchema(getXml());
}

public void setXml(String xml) {
this.xml = xml;
}
Expand Down
69 changes: 30 additions & 39 deletions tis-console/src/main/java/com/qlangtech/tis/runtime/module/action/SchemaAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,8 @@ public class SchemaAction extends BasicModule {
protected static final String FIELD_Recept = "recept";
protected static final String FIELD_DptId = "dptId";
private static final long serialVersionUID = 1L;

protected static final String INDEX_PREFIX = "search4";

//private static Logger log = LoggerFactory.getLogger(SchemaAction.class);

/**
* DataX 创建流程中取得es的默认字段
*
Expand All @@ -112,31 +109,45 @@ public void doGetEsTplFields(Context context) throws Exception {

ISearchEngineTypeTransfer typeTransfer = ISearchEngineTypeTransfer.load(this, dataxName);
DataxReader dataxReader = DataxReader.load(this, dataxName);
ISelectedTab esTab = null;
for (ISelectedTab tab : dataxReader.getSelectedTabs()) {
esTab = tab;
break;
}
Objects.requireNonNull(esTab, "esTab can not be null");
// ESField field = null;
if (stepType.update) {
DataxProcessor dataxProcessor = DataxProcessor.load(this, dataxName);

Optional<TableAlias> f = dataxProcessor.getTabAlias().findFirst();
if (f.isPresent()) {
writerStructFields(context, f.get(), typeTransfer);
return;
TableAlias tabMapper = f.get();
if (StringUtils.equals(tabMapper.getFrom(), esTab.getName())) {
writerStructFields(context, tabMapper, typeTransfer);
return;
} else {
// 更新时，源表改变了重新初始化
writeStructFields(context, typeTransfer, esTab);
return;
}
}
// for (Map.Entry<String, TableAlias> e : dataxProcessor.getTabAlias().entrySet()) {
// writerStructFields(context, e.getValue(), typeTransfer);
// return;
// }

} else {
for (ISelectedTab tab : dataxReader.getSelectedTabs()) {
// ESSchema parseResult = new ESSchema();
SchemaMetaContent tplSchema = typeTransfer.initSchemaMetaContent(tab);
this.setBizResult(context, tplSchema.toJSON());
return;
}
// for (ISelectedTab tab : dataxReader.getSelectedTabs()) {
// ESSchema parseResult = new ESSchema();
writeStructFields(context, typeTransfer, esTab);
return;
//}
}

throw new IllegalStateException("have not find any tab in DataXReader");
}

private void writeStructFields(Context context, ISearchEngineTypeTransfer typeTransfer, ISelectedTab esTab) {
SchemaMetaContent tplSchema = typeTransfer.initSchemaMetaContent(esTab);
this.setBizResult(context, tplSchema.toJSON());
}

/**
* 创建新索引流程取得通过workflow反射Schema 生成索引
*
Expand Down Expand Up @@ -369,51 +380,39 @@ private boolean validateStupidContent(Context context, UploadSchemaWithRawConten
this.addErrorMessage(context, "字段名称不能为空");
hasBlankError = true;
}
FieldErrorInfo err = fieldsErrors.getFieldErrorInfo(f.getId());// new FieldErrorInfo(f.getId());
FieldErrorInfo err = fieldsErrors.getFieldErrorInfo(f.getId());
err.setFieldNameError(true);
} else {
if (!duplicate.add(f.getName())) {
// if (!hasDuplicateError) {
this.addErrorMessage(context, "字段名‘" + f.getName() + "’不能重复");
// }
// 有重复
hasDuplicateError = true;
FieldErrorInfo err = fieldsErrors.getFieldErrorInfo(f.getId());
err.setFieldNameError(true);
// fieldsErrors.add(err);
} else {
if (!PATTERN_FIELD.matcher(f.getName()).matches() && !f.isDynamic()) {
// if (!hasNamePatternError) {
this.addErrorMessage(context, "字段名‘" + f.getName() + "’开通必须以[a-z]作为开头且中间不能有除数字、下划线、大小写字母以外的字符出现");
// }
// hasNamePatternError = true;
FieldErrorInfo err = fieldsErrors.getFieldErrorInfo(f.getId());
err.setFieldNameError(true);

} else if (StringUtils.isBlank(f.getFieldtype())) {
this.addErrorMessage(context, "请为字段‘" + f.getName() + "’选择类型");
hasFieldTypeError = true;
FieldErrorInfo err = fieldsErrors.getFieldErrorInfo(f.getId());
err.setFieldTypeError(true);
//fieldsErrors.add(err);
} else if ("string".equalsIgnoreCase(f.getFieldtype()) //
&& (StringUtils.isBlank(f.getTokenizerType())
|| "-1".equalsIgnoreCase(f.getTokenizerType()))) {
// if (!hasFieldTypeError) {
this.addErrorMessage(context, "请为字段‘" + f.getName() + "’选择分词类型");
hasFieldTypeError = true;
// }
FieldErrorInfo err = fieldsErrors.getFieldErrorInfo(f.getId());
err.setFieldTypeError(true);
// fieldsErrors.add(err);
}
}

if (!f.getSortable() && !f.isIndexed() && !f.isStored()) {
this.addErrorMessage(context, SolrFieldsParser.getFieldPropRequiredErr(f.getName()));
this.addErrorMessage(context, ISchema.getFieldPropRequiredErr(f.getName()));
FieldErrorInfo err = fieldsErrors.getFieldErrorInfo(f.getId());
err.setFieldPropRequiredError(true);
// fieldsErrors.add(err);
}

}
Expand Down Expand Up @@ -444,23 +443,17 @@ public void doToggleEsStupidModel(Context context) throws Exception {
// 整段xml文本
com.alibaba.fastjson.JSONObject body = this.parseJsonPost();

ISearchEngineTypeTransfer typeTransfer = ISearchEngineTypeTransfer.load(this, body.getString(DataxUtils.DATAX_NAME));

// final String content = body.getString("content");
//byte[] schemaContent = body.getString("content").getBytes(TisUTF8.get());
//JSONArray fields = JSON.parseArray(content);
ISearchEngineTypeTransfer typeTransfer
= ISearchEngineTypeTransfer.load(this, body.getString(DataxUtils.DATAX_NAME));
writerStructFields(context, body, typeTransfer);
}

private void writerStructFields(Context context, TableAlias tableAlias, ISearchEngineTypeTransfer typeTransfer) {

SchemaMetaContent schemaContent = new SchemaMetaContent();
//schemaContent.content = content.getBytes(TisUTF8.get());
schemaContent.parseResult = typeTransfer.projectionFromExpertModel(tableAlias, (content) -> {
schemaContent.content = content;
});
// this.getStructSchema(context, ISchemaPluginContext.NULL, schemaContent);

this.setBizResult(context, schemaContent.toJSON());
}

Expand All @@ -469,8 +462,6 @@ private void writerStructFields(Context context, JSONObject body, ISearchEngineT
SchemaMetaContent schemaContent = new SchemaMetaContent();
schemaContent.content = content.getBytes(TisUTF8.get());
schemaContent.parseResult = typeTransfer.projectionFromExpertModel(body);
// this.getStructSchema(context, ISchemaPluginContext.NULL, schemaContent);

this.setBizResult(context, schemaContent.toJSON());
}

Expand Down
5 changes: 0 additions & 5 deletions tis-console/src/main/java/com/qlangtech/tis/runtime/module/misc/SchemaField.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ public class SchemaField implements ISchemaField {

private boolean multiValue = false;

//private boolean sortable;

// 如果选择了string分词
private String textAnalysis;

Expand All @@ -58,8 +56,6 @@ public class SchemaField implements ISchemaField {

// 是否开通docvalue
private boolean docval;


public int getId() {
return id;
}
Expand All @@ -73,7 +69,6 @@ public boolean isMultiValue() {
return this.multiValue;
}


@Override
public boolean isDynamic() {
return StringUtils.indexOf(this.name, "*") > -1;
Expand Down
62 changes: 38 additions & 24 deletions tis-console/src/main/java/com/qlangtech/tis/runtime/module/misc/UploadSchemaForm.java
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,29 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.qlangtech.tis.runtime.module.misc;

import com.alibaba.fastjson.JSONArray;
import com.qlangtech.tis.solrdao.ISchema;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

/**
Expand All @@ -31,21 +33,15 @@
* @date 2017年5月8日
*/
public class UploadSchemaForm implements ISchema {

private final List<SchemaField> fields = new ArrayList<SchemaField>();

// private final Set<String> fieldKeys = new HashSet<String>();
@Override
public List<SchemaField> getSchemaFields() {
return fields;
// List<ISchemaField> result = new ArrayList<>();
// for (SchemaField f : fields) {
// result.add(f);
// }
// return result;
return this.fields;
}

public List<SchemaField> getFields() {
return this.fields;
return this.getSchemaFields();
}

public boolean containsField(String name) {
Expand Down Expand Up @@ -103,4 +99,22 @@ public JSONArray serialTypes() {
public void clearFields() {
this.fields.clear();
}


private List<String> errlist = new ArrayList<String>();

@Override
public boolean isValid() {
if (!this.errlist.isEmpty()) {
return false;
}
return CollectionUtils.isEmpty(this.errlist = ISchema.validateSchema(this.fields));
}

@Override
public List<String> getErrors() {
return Collections.unmodifiableList(this.errlist);
}


}
Loading

0 comments on commit f5229c2

Please sign in to comment.