diff --git a/be/src/pipeline/exec/table_function_operator.cpp b/be/src/pipeline/exec/table_function_operator.cpp index 13951076c8182f..9256d1deb2b072 100644 --- a/be/src/pipeline/exec/table_function_operator.cpp +++ b/be/src/pipeline/exec/table_function_operator.cpp @@ -140,6 +140,7 @@ bool TableFunctionLocalState::_roll_table_functions(int last_eos_idx) { bool TableFunctionLocalState::_is_inner_and_empty() { for (int i = 0; i < _parent->cast()._fn_num; i++) { // if any table function is not outer and has empty result, go to next child row + // if it's outer function, will be insert into one row NULL if (!_fns[i]->is_outer() && _fns[i]->current_empty()) { return true; } diff --git a/be/src/vec/exec/vtable_function_node.cpp b/be/src/vec/exec/vtable_function_node.cpp index 395cee36acd035..8a04be3b6bb4b0 100644 --- a/be/src/vec/exec/vtable_function_node.cpp +++ b/be/src/vec/exec/vtable_function_node.cpp @@ -92,6 +92,7 @@ Status VTableFunctionNode::_prepare_output_slot_ids(const TPlanNode& tnode) { bool VTableFunctionNode::_is_inner_and_empty() { for (int i = 0; i < _fn_num; i++) { // if any table function is not outer and has empty result, go to next child row + // if it's outer function, will be insert into NULL if (!_fns[i]->is_outer() && _fns[i]->current_empty()) { return true; } diff --git a/be/src/vec/exprs/table_function/table_function.h b/be/src/vec/exprs/table_function/table_function.h index 98d811364399f5..c817067470a22a 100644 --- a/be/src/vec/exprs/table_function/table_function.h +++ b/be/src/vec/exprs/table_function/table_function.h @@ -58,6 +58,7 @@ class TableFunction { virtual int get_value(MutableColumnPtr& column, int max_step) { max_step = std::max(1, std::min(max_step, (int)(_cur_size - _cur_offset))); int i = 0; + // TODO: this for loop maybe could refactor, and call once get_value function, it's could insert into max_step value once for (; i < max_step && !eos(); i++) { get_value(column); forward(); diff --git a/be/src/vec/exprs/table_function/table_function_factory.cpp b/be/src/vec/exprs/table_function/table_function_factory.cpp index b6662e1215d2f5..29b201b59479d5 100644 --- a/be/src/vec/exprs/table_function/table_function_factory.cpp +++ b/be/src/vec/exprs/table_function/table_function_factory.cpp @@ -70,13 +70,16 @@ const std::unordered_map {}}}; Status TableFunctionFactory::get_fn(const TFunction& t_fn, ObjectPool* pool, TableFunction** fn) { - const std::string fn_name_raw = t_fn.name.function_name; + bool is_outer = match_suffix(t_fn.name.function_name, COMBINATOR_SUFFIX_OUTER); if (t_fn.binary_type == TFunctionBinaryType::JAVA_UDF) { *fn = pool->add(UDFTableFunction::create_unique(t_fn).release()); + if (is_outer) { + (*fn)->set_outer(); + } return Status::OK(); } else { - bool is_outer = match_suffix(t_fn.name.function_name, COMBINATOR_SUFFIX_OUTER); - std::string fn_name_real = + const std::string& fn_name_raw = t_fn.name.function_name; + const std::string& fn_name_real = is_outer ? remove_suffix(fn_name_raw, COMBINATOR_SUFFIX_OUTER) : fn_name_raw; auto fn_iterator = _function_map.find(fn_name_real); @@ -89,7 +92,7 @@ Status TableFunctionFactory::get_fn(const TFunction& t_fn, ObjectPool* pool, Tab return Status::OK(); } } - return Status::NotSupported("Table function {} is not support", fn_name_raw); + return Status::NotSupported("Table function {} is not support", t_fn.name.function_name); } } // namespace doris::vectorized diff --git a/be/src/vec/exprs/table_function/udf_table_function.cpp b/be/src/vec/exprs/table_function/udf_table_function.cpp index cf5d7401fc1aa4..9136cd7d4b1c35 100644 --- a/be/src/vec/exprs/table_function/udf_table_function.cpp +++ b/be/src/vec/exprs/table_function/udf_table_function.cpp @@ -19,21 +19,11 @@ #include -#include -#include -#include -#include -#include - -#include "common/status.h" #include "runtime/user_function_cache.h" -#include "vec/columns/column.h" #include "vec/columns/column_array.h" #include "vec/columns/column_nullable.h" -#include "vec/columns/column_string.h" #include "vec/common/assert_cast.h" #include "vec/core/block.h" -#include "vec/core/column_with_type_and_name.h" #include "vec/data_types/data_type_array.h" #include "vec/data_types/data_type_factory.hpp" #include "vec/exec/jni_connector.h" @@ -49,7 +39,9 @@ UDFTableFunction::UDFTableFunction(const TFunction& t_fn) : TableFunction(), _t_ _fn_name = _t_fn.name.function_name; _return_type = DataTypeFactory::instance().create_data_type( TypeDescriptor::from_thrift(t_fn.ret_type)); - _return_type = std::make_shared(make_nullable(_return_type)); + // as the java-utdf function in java code is eg: ArrayList + // so we need a array column to save the execute result, and make_nullable could help deal with nullmap + _return_type = make_nullable(std::make_shared(make_nullable(_return_type))); } Status UDFTableFunction::open() { @@ -133,6 +125,8 @@ Status UDFTableFunction::process_init(Block* block, RuntimeState* state) { env->DeleteLocalRef(input_map); env->DeleteLocalRef(output_map); RETURN_IF_ERROR(JniConnector::fill_block(block, {_result_column_idx}, output_address)); + LOG(INFO)<<"block dump: "<dump_structure(); + LOG(INFO)<dump_data(); block->erase(_result_column_idx); if (!extract_column_array_info(*_array_result_column, _array_column_detail)) { return Status::NotSupported("column type {} not supported now", @@ -148,6 +142,10 @@ void UDFTableFunction::process_row(size_t row_idx) { _array_offset = (*_array_column_detail.offsets_ptr)[row_idx - 1]; _cur_size = (*_array_column_detail.offsets_ptr)[row_idx] - _array_offset; } + // so when it's NULL of row_idx, will not update _cur_size + // it's will be _cur_size == 0, and means current_empty. + // if the fn is outer, will be continue insert_default + // if the fn is not outer function, will be not insert any value. } void UDFTableFunction::process_close() { diff --git a/be/src/vec/exprs/table_function/udf_table_function.h b/be/src/vec/exprs/table_function/udf_table_function.h index 3739f98e02ea1f..b268a39845913a 100644 --- a/be/src/vec/exprs/table_function/udf_table_function.h +++ b/be/src/vec/exprs/table_function/udf_table_function.h @@ -17,26 +17,13 @@ #pragma once -#include -#include - -#include -#include - #include "common/status.h" #include "jni.h" #include "util/jni-util.h" #include "vec/columns/column.h" -#include "vec/common/string_ref.h" #include "vec/data_types/data_type.h" #include "vec/exprs/table_function/table_function.h" #include "vec/functions/array/function_array_utils.h" -namespace doris { -namespace vectorized { -class Block; -class ColumnString; -} // namespace vectorized -} // namespace doris namespace doris::vectorized { @@ -82,7 +69,6 @@ class UDFTableFunction final : public TableFunction { if (is_closed) { return Status::OK(); } - VLOG_DEBUG << "Free resources for JniContext"; JNIEnv* env = nullptr; Status status = JniUtil::GetJNIEnv(&env); if (!status.ok() || env == nullptr) { diff --git a/be/src/vec/exprs/vectorized_fn_call.cpp b/be/src/vec/exprs/vectorized_fn_call.cpp index 9cb173f0b75408..b84499653ebf9b 100644 --- a/be/src/vec/exprs/vectorized_fn_call.cpp +++ b/be/src/vec/exprs/vectorized_fn_call.cpp @@ -73,7 +73,8 @@ Status VectorizedFnCall::prepare(RuntimeState* state, const RowDescriptor& desc, } else if (_fn.binary_type == TFunctionBinaryType::JAVA_UDF) { if (config::enable_java_support) { if (_fn.is_udtf_function) { - _function = FakeJavaUDTF::create(_fn, argument_template, _data_type); + // fake function. it's no use and can't execute. + _function = FunctionFake::create(); } else { _function = JavaFunctionCall::create(_fn, argument_template, _data_type); } diff --git a/be/src/vec/functions/function_fake.h b/be/src/vec/functions/function_fake.h index d7eb5c33957fcc..b4891ed9392576 100644 --- a/be/src/vec/functions/function_fake.h +++ b/be/src/vec/functions/function_fake.h @@ -65,35 +65,10 @@ class FunctionFake : public IFunction { struct UDTFImpl { static DataTypePtr get_return_type_impl(const DataTypes& arguments) { - DCHECK(false) << "get_return_type_impl not supported"; + DCHECK(false) << "get_return_type_impl not supported, shouldn't into here."; return nullptr; } static std::string get_error_msg() { return "Fake function do not support execute"; } }; -class FakeJavaUDTF : public FunctionFake { -public: - FakeJavaUDTF(const TFunction& fn, const DataTypes& argument_types, - const DataTypePtr& return_type) - : _fn(fn), _argument_types(argument_types), _return_type(return_type) {} - - static FunctionPtr create(const TFunction& fn, const ColumnsWithTypeAndName& argument_types, - const DataTypePtr& return_type) { - DataTypes data_types(argument_types.size()); - for (size_t i = 0; i < argument_types.size(); ++i) { - data_types[i] = argument_types[i].type; - } - return std::make_shared(fn, data_types, return_type); - } - String get_name() const override { return _fn.name.function_name; } - DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { - return _return_type; - } - -private: - const TFunction& _fn; - const DataTypes _argument_types; - const DataTypePtr _return_type; -}; - } // namespace doris::vectorized diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java index b000190d42f033..bb8a47622ac5a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java @@ -330,6 +330,8 @@ private void analyzeTableFunction() throws AnalysisException { function.setChecksum(checksum); function.setNullableMode(returnNullMode); function.setUDTFunction(true); + // Todo: maybe in create tables function, need register two function, one is + // normal and one is outer as those have different result when result is NULL. } private void analyzeUda() throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index d64aa47215b521..9cfce9e67de8ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1673,7 +1673,7 @@ && collectChildReturnTypes()[0].isDecimalV3()) { fn = getTableFunction(fnName.getFunction(), matchFuncChildTypes, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); if (fn == null) { - throw new AnalysisException(getFunctionNotFoundError(argTypes)); + throw new AnalysisException(getFunctionNotFoundError(argTypes) + " in table function"); } // set param child types fn.setReturnType(((ArrayType) childTypes[0]).getItemType()); @@ -1687,7 +1687,7 @@ && collectChildReturnTypes()[0].isDecimalV3()) { if (fn != null) { FunctionUtil.checkEnableJavaUdf(); if (!fn.isUDTFunction()) { - throw new AnalysisException(getFunctionNotFoundError(argTypes)); + throw new AnalysisException(getFunctionNotFoundError(argTypes) + " in table function"); } } } diff --git a/regression-test/data/javaudf_p0/test_javaudtf_arrayint.out b/regression-test/data/javaudf_p0/test_javaudtf_arrayint.out new file mode 100644 index 00000000000000..c7cf111f2e682e --- /dev/null +++ b/regression-test/data/javaudf_p0/test_javaudtf_arrayint.out @@ -0,0 +1,25 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +1 2 2022-01-01 2022-01-01T11:11:11 a1b +2 4 2022-01-01 2022-01-01T11:11:11 a2b +3 6 2022-01-01 2022-01-01T11:11:11 a3b +4 8 2022-01-01 2022-01-01T11:11:11 a4b +5 10 2022-01-01 2022-01-01T11:11:11 a5b +6 12 2022-01-01 2022-01-01T11:11:11 a6b +7 14 2022-01-01 2022-01-01T11:11:11 a7b +8 16 2022-01-01 2022-01-01T11:11:11 a8b +9 18 2022-01-01 2022-01-01T11:11:11 a9b +10 20 2022-06-06 2022-01-01T12:12:12 a10b + +-- !select1 -- +1 a1b 1 +2 a2b 2 +3 a3b 3 +4 a4b 4 +5 a5b 5 +6 a6b 6 +7 a7b 7 +8 a8b 8 +9 a9b 9 +10 a10b 10 + diff --git a/regression-test/data/javaudf_p0/test_javaudtf_decimal.out b/regression-test/data/javaudf_p0/test_javaudtf_decimal.out new file mode 100644 index 00000000000000..4a0fcd24181601 --- /dev/null +++ b/regression-test/data/javaudf_p0/test_javaudtf_decimal.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +111 11111.111110000 222222.333333300 +112 1234556.111110000 222222.333333300 +113 87654321.111110000 \N + +-- !select1 -- +111 11111.111110000 22222.222220000 +112 1234556.111110000 2469112.222220000 +113 87654321.111110000 175308642.222220000 + +-- !select2 -- +111 222222.333333300 444444.666666600 +112 222222.333333300 444444.666666600 + diff --git a/regression-test/data/javaudf_p0/test_javaudtf_double.out b/regression-test/data/javaudf_p0/test_javaudtf_double.out new file mode 100644 index 00000000000000..0ce1c152d0e62e --- /dev/null +++ b/regression-test/data/javaudf_p0/test_javaudtf_double.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +111 11111.111 222222.33 1.234567834455677E7 1111112.0 +112 1234556.1 222222.33 2.2222222233333334E8 4.444444444444556E12 +113 8.765432E7 \N 6.666666666666667E9 \N + +-- !select1 -- +111 1.234567834455677E7 1.234567834455677E8 +112 2.2222222233333334E8 2.2222222233333335E9 +113 6.666666666666667E9 6.666666666666667E10 + +-- !select2 -- +111 1111112.0 1.111112E7 +112 4.444444444444556E12 4.4444444444445555E13 + diff --git a/regression-test/data/javaudf_p0/test_javaudtf_int.out b/regression-test/data/javaudf_p0/test_javaudtf_int.out new file mode 100644 index 00000000000000..2fa8be4ac94002 --- /dev/null +++ b/regression-test/data/javaudf_p0/test_javaudtf_int.out @@ -0,0 +1,32 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +1 1 abc,defg poiuytre,abcdefg +2 2 abc,defg poiuytre,abcdefg +0 3 abc,defg poiuytre,abcdefg +1 4 abc,defg poiuytre,abcdefg +2 5 abc,defg poiuytre,abcdefg +0 6 abc,defg poiuytre,abcdefg +1 7 abc,defg poiuytre,abcdefg +2 8 abc,defg poiuytre,abcdefg +9 9 ab,cdefg poiuytreabcde,fg + +-- !select1 -- +1 abc,defg 1 +1 abc,defg 1 +1 abc,defg 1 +2 abc,defg 2 +2 abc,defg 2 +2 abc,defg 2 +2 abc,defg 2 +2 abc,defg 2 +2 abc,defg 2 +9 ab,cdefg 9 +9 ab,cdefg 9 +9 ab,cdefg 9 +9 ab,cdefg 9 +9 ab,cdefg 9 +9 ab,cdefg 9 +9 ab,cdefg 9 +9 ab,cdefg 9 +9 ab,cdefg 9 + diff --git a/regression-test/data/javaudf_p0/test_javaudtf_string.out b/regression-test/data/javaudf_p0/test_javaudtf_string.out new file mode 100644 index 00000000000000..7fc0036c5bfbab --- /dev/null +++ b/regression-test/data/javaudf_p0/test_javaudtf_string.out @@ -0,0 +1,34 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_default -- +1 1 abc,defg poiuytre,abcdefg +2 2 abc,defg poiuytre,abcdefg +0 3 abc,defg poiuytre,abcdefg +1 4 abc,defg poiuytre,abcdefg +2 5 abc,defg poiuytre,abcdefg +0 6 abc,defg poiuytre,abcdefg +1 7 abc,defg poiuytre,abcdefg +2 8 abc,defg poiuytre,abcdefg +9 9 ab,cdefg poiuytreabcde,fg + +-- !select1 -- +0 abc,defg abc +0 abc,defg defg +0 abc,defg abc +0 abc,defg defg +1 abc,defg abc +1 abc,defg defg +1 abc,defg abc +1 abc,defg defg +1 abc,defg abc +1 abc,defg defg +2 abc,defg abc +2 abc,defg defg +2 abc,defg abc +2 abc,defg defg +2 abc,defg abc +2 abc,defg defg +9 ab,cdefg ab +9 ab,cdefg cdefg + +-- !select2 -- + diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFArrayIntTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFArrayIntTest.java new file mode 100644 index 00000000000000..00fb948c826c2f --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFArrayIntTest.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import java.util.ArrayList; + +public class UDTFArrayIntTest { + public ArrayList evaluate(ArrayList val) { + if (val == null) return null; + ArrayList result = new ArrayList<>(); + for (int i = 0; i < val.size(); i = i + 2) { + result.add(val.get(i)); + } + return val; + } +} diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFDecimalTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFDecimalTest.java new file mode 100644 index 00000000000000..9f8d3c07ddc359 --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFDecimalTest.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import java.math.BigDecimal; +import java.util.ArrayList; + +public class UDTFDecimalTest { + public ArrayList evaluate(BigDecimal val) { + if (val == null) return null; + ArrayList result = new ArrayList<>(); + BigDecimal sum = val.add(val); + result.add(sum); + return result; + } +} diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFDoubleTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFDoubleTest.java new file mode 100644 index 00000000000000..359df11ebbdae4 --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFDoubleTest.java @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import java.util.ArrayList; + +public class UDTFDoubleTest { + public ArrayList evaluate(Double val) { + if (val == null) return null; + ArrayList result = new ArrayList<>(); + result.add(val * 10); + return result; + } +} diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFIntTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFIntTest.java new file mode 100644 index 00000000000000..27b435727bef3c --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFIntTest.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import java.util.ArrayList; + +public class UDTFIntTest { + public ArrayList evaluate(int count) { + ArrayList result = new ArrayList<>(); + for (int i = 0; i < count; i++) { + result.add(count); + } + return result; + } +} diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFNullTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFNullTest.java new file mode 100644 index 00000000000000..dc6ac9be90c029 --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFNullTest.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import java.util.ArrayList; + +public class UDTFNullTest { + public ArrayList evaluate(String value, String separator) { + return null; + } +} diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFStringTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFStringTest.java new file mode 100644 index 00000000000000..cb2eb45c9c1f92 --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFStringTest.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import java.util.ArrayList; +import java.util.Arrays; + +public class UDTFStringTest { + public ArrayList evaluate(String value, String separator) { + if (value == null || separator == null) { + return null; + } else { + return new ArrayList<>(Arrays.asList(value.split(separator))); + } + } +} diff --git a/regression-test/suites/javaudf_p0/test_javaudtf_arrayint.groovy b/regression-test/suites/javaudf_p0/test_javaudtf_arrayint.groovy new file mode 100644 index 00000000000000..670080a5299a73 --- /dev/null +++ b/regression-test/suites/javaudf_p0/test_javaudtf_arrayint.groovy @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_javaudtf_arrayint") { + def tableName = "test_javaudtf_arrayint" + def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" + + log.info("Jar path: ${jarPath}".toString()) + try { + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` INT NOT NULL COMMENT "", + `tinyint_col` TINYINT NOT NULL COMMENT "", + `datev2_col` datev2 NOT NULL COMMENT "", + `datetimev2_col` datetimev2 NOT NULL COMMENT "", + `string_col` STRING NOT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + StringBuilder sb = new StringBuilder() + int i = 1 + for (; i < 10; i ++) { + sb.append(""" + (${i},${i}*2,'2022-01-01','2022-01-01 11:11:11','a${i}b'), + """) + } + sb.append(""" + (${i},${i}*2,'2022-06-06','2022-01-01 12:12:12','a${i}b') + """) + sql """ INSERT INTO ${tableName} VALUES + ${sb.toString()} + """ + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + + File path = new File(jarPath) + if (!path.exists()) { + throw new IllegalStateException("""${jarPath} doesn't exist! """) + } + + sql """ CREATE TABLES FUNCTION udtf_arrayint(array) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFArrayIntTest", + "always_nullable"="true", + "type"="JAVA_UDF" + ); """ + + qt_select1 """ SELECT user_id, string_col, e1 FROM ${tableName} lateral view udtf_arrayint(array(user_id)) temp as e1 order by user_id; """ + + } finally { + try_sql("DROP FUNCTION IF EXISTS udtf_arrayint(array);") + try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/javaudf_p0/test_javaudtf_decimal.groovy b/regression-test/suites/javaudf_p0/test_javaudtf_decimal.groovy new file mode 100644 index 00000000000000..fdb5ea04e6e3db --- /dev/null +++ b/regression-test/suites/javaudf_p0/test_javaudtf_decimal.groovy @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_javaudtf_decimal") { + def tableName = "test_javaudtf_decimal" + def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" + + log.info("Jar path: ${jarPath}".toString()) + try { + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` INT NOT NULL COMMENT "", + `cost_1` decimal(27,9) NOT NULL COMMENT "", + `cost_2` decimal(27,9) COMMENT "" + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + + + sql """ INSERT INTO ${tableName} (`user_id`,`cost_1`,`cost_2`) VALUES + (111,11111.11111,222222.3333333), + (112,1234556.11111,222222.3333333), + (113,87654321.11111,null) + """ + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + + File path = new File(jarPath) + if (!path.exists()) { + throw new IllegalStateException("""${jarPath} doesn't exist! """) + } + + sql """ CREATE TABLES FUNCTION udtf_decimal(decimal(27,9)) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFDecimalTest", + "always_nullable"="true", + "type"="JAVA_UDF" + ); """ + + qt_select1 """ SELECT user_id, cost_1, e1 FROM ${tableName} lateral view udtf_decimal(cost_1) temp as e1 order by user_id; """ + qt_select2 """ SELECT user_id, cost_2, e1 FROM ${tableName} lateral view udtf_decimal(cost_2) temp as e1 order by user_id; """ + + } finally { + // try_sql("DROP FUNCTION IF EXISTS udtf_decimal(decimal);") + // try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/javaudf_p0/test_javaudtf_double.groovy b/regression-test/suites/javaudf_p0/test_javaudtf_double.groovy new file mode 100644 index 00000000000000..af50de49d050d6 --- /dev/null +++ b/regression-test/suites/javaudf_p0/test_javaudtf_double.groovy @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_javaudtf_double") { + def tableName = "test_javaudtf_double" + def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" + + log.info("Jar path: ${jarPath}".toString()) + try { + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` INT NOT NULL COMMENT "", + `float_1` FLOAT NOT NULL COMMENT "", + `float_2` FLOAT COMMENT "", + `double_1` DOUBLE NOT NULL COMMENT "", + `double_2` DOUBLE COMMENT "" + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + + + sql """ INSERT INTO ${tableName} (`user_id`,`float_1`,`float_2`,double_1,double_2) VALUES + (111,11111.11111,222222.3333333,12345678.34455677,1111111.999999999999), + (112,1234556.11111,222222.3333333,222222222.3333333333333,4444444444444.555555555555), + (113,87654321.11111,null,6666666666.6666666666,null) + """ + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """ + + File path = new File(jarPath) + if (!path.exists()) { + throw new IllegalStateException("""${jarPath} doesn't exist! """) + } + + sql """ CREATE TABLES FUNCTION udtf_double(double) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFDoubleTest", + "always_nullable"="true", + "type"="JAVA_UDF" + ); """ + + qt_select1 """ SELECT user_id, double_1, e1 FROM ${tableName} lateral view udtf_double(double_1) temp as e1 order by user_id; """ + qt_select2 """ SELECT user_id, double_2, e1 FROM ${tableName} lateral view udtf_double(double_2) temp as e1 order by user_id; """ + + } finally { + // try_sql("DROP FUNCTION IF EXISTS udtf_double(double);") + // try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/javaudf_p0/test_javaudtf_int.groovy b/regression-test/suites/javaudf_p0/test_javaudtf_int.groovy new file mode 100644 index 00000000000000..ad3d6983200398 --- /dev/null +++ b/regression-test/suites/javaudf_p0/test_javaudtf_int.groovy @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_javaudtf_int") { + def tableName = "test_javaudtf_int" + def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" + + log.info("Jar path: ${jarPath}".toString()) + try { + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` INT NOT NULL COMMENT "用户id", + `char_col` CHAR NOT NULL COMMENT "", + `varchar_col` VARCHAR(10) NOT NULL COMMENT "", + `string_col` STRING NOT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + StringBuilder sb = new StringBuilder() + int i = 1 + for (; i < 9; i ++) { + sb.append(""" + (${i % 3}, '${i}','abc,defg','poiuytre,abcdefg'), + """) + } + sb.append(""" + (${i}, '${i}','ab,cdefg','poiuytreabcde,fg') + """) + sql """ INSERT INTO ${tableName} VALUES + ${sb.toString()} + """ + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY char_col; """ + + File path = new File(jarPath) + if (!path.exists()) { + throw new IllegalStateException("""${jarPath} doesn't exist! """) + } + + sql """ CREATE TABLES FUNCTION udtf_int(int) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFIntTest", + "always_nullable"="true", + "type"="JAVA_UDF" + ); """ + + qt_select1 """ SELECT user_id, varchar_col, e1 FROM ${tableName} lateral view udtf_int(user_id) temp as e1 order by user_id; """ + + } finally { + try_sql("DROP FUNCTION IF EXISTS udtf_int(int);") + try_sql("DROP TABLE IF EXISTS ${tableName}") + } +} diff --git a/regression-test/suites/javaudf_p0/test_javaudtf_string.groovy b/regression-test/suites/javaudf_p0/test_javaudtf_string.groovy new file mode 100644 index 00000000000000..cfd4fa3d0d0a19 --- /dev/null +++ b/regression-test/suites/javaudf_p0/test_javaudtf_string.groovy @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_javaudtf_string") { + def tableName = "test_javaudtf_string" + def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" + + log.info("Jar path: ${jarPath}".toString()) + try { + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `user_id` INT NOT NULL COMMENT "用户id", + `char_col` CHAR NOT NULL COMMENT "", + `varchar_col` VARCHAR(10) NOT NULL COMMENT "", + `string_col` STRING NOT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1"); + """ + StringBuilder sb = new StringBuilder() + int i = 1 + for (; i < 9; i ++) { + sb.append(""" + (${i % 3}, '${i}','abc,defg','poiuytre,abcdefg'), + """) + } + sb.append(""" + (${i}, '${i}','ab,cdefg','poiuytreabcde,fg') + """) + sql """ INSERT INTO ${tableName} VALUES + ${sb.toString()} + """ + qt_select_default """ SELECT * FROM ${tableName} t ORDER BY char_col; """ + + File path = new File(jarPath) + if (!path.exists()) { + throw new IllegalStateException("""${jarPath} doesn't exist! """) + } + + sql """ CREATE TABLES FUNCTION udtf_string_split(string, string) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFStringTest", + "always_nullable"="true", + "type"="JAVA_UDF" + ); """ + + qt_select1 """ SELECT user_id, varchar_col, e1 FROM ${tableName} lateral view udtf_string_split(varchar_col, ",") temp as e1 order by user_id; """ + + sql """ CREATE TABLES FUNCTION udtf_null(string, string) RETURNS array PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.UDTFNullTest", + "always_nullable"="true", + "type"="JAVA_UDF" + ); """ + + qt_select2 """ SELECT user_id, varchar_col, e1 FROM ${tableName} lateral view udtf_null(varchar_col, ",") temp as e1 order by user_id; """ + + } finally { + try_sql("DROP FUNCTION IF EXISTS udtf_string_split(string, string);") + try_sql("DROP FUNCTION IF EXISTS udtf_null(string, string);") + try_sql("DROP TABLE IF EXISTS ${tableName}") + } +}