From 6c32c7b8094b93fb9a3f236b3e20a16cdab72e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E5=8B=87?= Date: Fri, 19 Apr 2024 17:31:21 +0800 Subject: [PATCH] =?UTF-8?q?=E5=85=BC=E5=AE=B9=E6=80=A7=E6=8F=90=E5=8D=87?= =?UTF-8?q?=20(#238)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: support datetime(n) * feat: support find_in_set * feat: field hit range weight * feat: 兼容mysql索引语法 * feat: 补齐时间类函数 * feat: 兼容mysql建表语法 * feat: add FLAGS_max_select_region_count for select sql * feat: support change column * feat: default charset utf8 * feat: support multiple alter_specifications for add column * feat: store请求部分错误码不重试 * feat: index.short_name保留大小写 * feat: support drop index name on table * feat: 补齐字符串类函数 * fix: for code review * fix: for code review * fix: for code review --------- Co-authored-by: yuzhong.chen --- include/common/datetime.h | 3 +- include/common/expr_value.h | 44 +- include/common/type_utils.h | 14 +- include/expr/internal_functions.h | 33 ++ include/logical_plan/ddl_planner.h | 3 +- include/meta_server/table_manager.h | 9 +- include/sqlparser/ddl.h | 1 + include/sqlparser/sql_lex.l | 16 +- include/sqlparser/sql_parse.y | 309 +++++++++-- src/common/datetime.cpp | 31 +- src/common/information_schema.cpp | 3 +- src/common/schema_factory.cpp | 5 +- src/engine/transaction_pool.cpp | 25 +- src/exec/dml_node.cpp | 5 +- src/exec/insert_manager_node.cpp | 6 +- src/exec/load_node.cpp | 2 +- src/exec/select_manager_node.cpp | 10 + src/exec/update_manager_node.cpp | 4 +- src/expr/fn_manager.cpp | 38 +- src/expr/internal_functions.cpp | 756 +++++++++++++++++++++++++-- src/logical_plan/ddl_planner.cpp | 181 +++++-- src/logical_plan/insert_planner.cpp | 8 +- src/logical_plan/logical_planner.cpp | 4 +- src/logical_plan/update_planner.cpp | 4 +- src/meta_server/table_manager.cpp | 20 +- src/physical_plan/index_selector.cpp | 56 +- src/protocol/show_helper.cpp | 9 +- src/protocol/state_machine.cpp | 10 +- 28 files changed, 1402 insertions(+), 207 deletions(-) diff --git a/include/common/datetime.h b/include/common/datetime.h index 5fd696e3..c917069a 100755 --- a/include/common/datetime.h +++ b/include/common/datetime.h @@ -22,7 +22,7 @@ extern time_t str_to_timestamp(const char* str_time); // encode DATETIME to string format // ref: https://dev.mysql.com/doc/internals/en/date-and-time-data-type-representation.html -extern std::string datetime_to_str(uint64_t datetime); +extern std::string datetime_to_str(uint64_t datetime, int precision_len = 0); extern uint64_t str_to_datetime(const char* str_time, bool* is_full_datetime = nullptr); extern time_t datetime_to_timestamp(uint64_t datetime); @@ -41,6 +41,7 @@ extern int32_t datetime_to_time(uint64_t datetime); extern uint64_t time_to_datetime(int32_t time); extern std::string time_to_str(int32_t time); extern int32_t str_to_time(const char* str_time); +extern int32_t time_to_seconds(int32_t time); extern int32_t seconds_to_time(int32_t seconds); struct DateTime; extern uint64_t bin_date_to_datetime(DateTime time_struct); diff --git a/include/common/expr_value.h b/include/common/expr_value.h index bd2a6f22..707a1780 100644 --- a/include/common/expr_value.h +++ b/include/common/expr_value.h @@ -31,7 +31,7 @@ namespace baikaldb { struct ExprValue { pb::PrimitiveType type; - int32_t float_precision_len; + int32_t float_precision_len = -1; // -1代表未定义精度，保留原值的最大精度。0代表不保留小数点部分 union { bool bool_val; int8_t int8_val; @@ -50,7 +50,7 @@ struct ExprValue { explicit ExprValue(pb::PrimitiveType type_ = pb::NULL_TYPE) : type(type_) { _u.int64_val = 0; - float_precision_len = 0; + float_precision_len = -1; if (type_ == pb::BITMAP) { _u.bitmap = new(std::nothrow) Roaring(); } else if (type_ == pb::TDIGEST) { @@ -124,7 +124,7 @@ struct ExprValue { } explicit ExprValue(const pb::ExprValue& value) { type = value.type(); - float_precision_len = 0; + float_precision_len = -1; switch (type) { case pb::BOOL: _u.bool_val = value.bool_val(); @@ -187,7 +187,7 @@ struct ExprValue { explicit ExprValue(pb::PrimitiveType primitive_type, const std::string& value_str) { type = pb::STRING; - float_precision_len = 0; + float_precision_len = -1; str_val = value_str; if (primitive_type == pb::STRING || primitive_type == pb::HEX @@ -624,7 +624,7 @@ struct ExprValue { case pb::TDIGEST: return str_val; case pb::DATETIME: - return datetime_to_str(_u.uint64_val); + return datetime_to_str(_u.uint64_val, float_precision_len); case pb::TIME: return time_to_str(_u.int32_val); case pb::TIMESTAMP: @@ -860,14 +860,18 @@ struct ExprValue { return ret; } // 默认不带us - static ExprValue Now(int precision = 0) { + static ExprValue Now(int32_t precision = 0) { ExprValue tmp(pb::TIMESTAMP); tmp._u.uint32_val = time(NULL); tmp.cast_to(pb::DATETIME); - if (precision == 6) { + + if (precision > 0 and precision <= 6) { timeval tv; gettimeofday(&tv, NULL); tmp._u.uint64_val |= tv.tv_usec; + tmp.set_precision_len(precision); + } else { + tmp.set_precision_len(0); } return tmp; } @@ -883,7 +887,7 @@ struct ExprValue { ExprValue ret(pb::UINT64); return ret; } - static ExprValue UTC_TIMESTAMP() { + static ExprValue UTC_TIMESTAMP(int32_t precision = 0) { // static int UTC_OFFSET = 8 * 60 * 60; time_t current_time; struct tm timeinfo; @@ -896,6 +900,11 @@ struct ExprValue { timeval tv; gettimeofday(&tv, NULL); tmp._u.uint64_val |= tv.tv_usec; + if (precision >=0 && precision <= 6) { + tmp.set_precision_len(precision); + } else { + tmp.set_precision_len(0); + } return tmp; } // For string, end-self @@ -924,6 +933,25 @@ struct ExprValue { return ev._u.uint64_val; } }; + + void dt_cast_len() { + if (float_precision_len >= 0 and float_precision_len <= 6) { + uint64_t carry = std::pow(10, (6 - float_precision_len)); + uint64_t oldmic = _u.uint64_val & 0xffffff; + uint64_t newmic = ::round(oldmic / carry) * carry; + _u.uint64_val = (_u.uint64_val & ~0xffffff) | newmic; + } + } + + void set_precision_len(int32_t precision) { + if (float_precision_len == precision) { + return; + } + float_precision_len = precision; + if (type == pb::DATETIME) { + dt_cast_len(); + } + } }; using ExprValueFlatSet = butil::FlatSet; diff --git a/include/common/type_utils.h b/include/common/type_utils.h index 9a59db25..8e7c17d6 100644 --- a/include/common/type_utils.h +++ b/include/common/type_utils.h @@ -392,7 +392,8 @@ inline std::string to_mysql_type_string(pb::PrimitiveType type) { } } -inline std::string to_mysql_type_full_string(pb::PrimitiveType type) { +inline std::string to_mysql_type_full_string(pb::PrimitiveType type, + int32_t float_total_len = -1, int32_t float_precision_len = -1) { switch (type) { case pb::BOOL: return "tinyint(3)"; @@ -413,12 +414,23 @@ inline std::string to_mysql_type_full_string(pb::PrimitiveType type) { case pb::UINT64: return "bigint(21) unsigned"; case pb::FLOAT: + if (float_total_len != -1 && float_precision_len != -1) { + return "float(" + std::to_string(float_total_len) + "," + std::to_string(float_precision_len) + ")"; + } return "float"; case pb::DOUBLE: + if (float_total_len != -1 && float_precision_len != -1) { + return "double(" + std::to_string(float_total_len) + "," + std::to_string(float_precision_len) + ")"; + } return "double"; case pb::STRING: return "varchar(1024)"; case pb::DATETIME: + if (float_precision_len == -1) { + return "datetime(6)"; + } else if (float_precision_len > 0 && float_precision_len <= 6) { + return "datetime(" + std::to_string(float_precision_len) + ")"; + } return "datetime"; case pb::DATE: return "date"; diff --git a/include/expr/internal_functions.h b/include/expr/internal_functions.h index 3ad677da..95d869c4 100644 --- a/include/expr/internal_functions.h +++ b/include/expr/internal_functions.h @@ -66,18 +66,45 @@ ExprValue lpad(const std::vector& input); ExprValue rpad(const std::vector& input); ExprValue instr(const std::vector& input); ExprValue json_extract(const std::vector& input); +ExprValue export_set(const std::vector& input); +ExprValue to_base64(const std::vector& input); +ExprValue from_base64(const std::vector& input); +ExprValue make_set(const std::vector& input); +ExprValue oct(const std::vector& input); +ExprValue hex(const std::vector& input); +ExprValue unhex(const std::vector& input); +ExprValue bin(const std::vector& input); +ExprValue space(const std::vector& input); +ExprValue elt(const std::vector& input); +ExprValue char_length(const std::vector& input); +ExprValue format(const std::vector& input); +ExprValue field(const std::vector& input); +ExprValue quote(const std::vector& input); +ExprValue func_char(const std::vector& input); +ExprValue soundex(const std::vector& input); + // datetime functions ExprValue unix_timestamp(const std::vector& input); ExprValue from_unixtime(const std::vector& input); ExprValue now(const std::vector& input); ExprValue utc_timestamp(const std::vector& input); +ExprValue utc_date(const std::vector& input); +ExprValue utc_time(const std::vector& input); +ExprValue minute(const std::vector& input); +ExprValue second(const std::vector& input); +ExprValue microsecond(const std::vector& input); +ExprValue func_time(const std::vector& input); +ExprValue func_quarter(const std::vector& input); +ExprValue period_diff(const std::vector& input); +ExprValue period_add(const std::vector& input); ExprValue date_format(const std::vector& input); ExprValue str_to_date(const std::vector& input); ExprValue time_format(const std::vector& input); ExprValue convert_tz(const std::vector& input); ExprValue timediff(const std::vector& input); ExprValue timestampdiff(const std::vector& input); +ExprValue timestampadd(const std::vector& input); ExprValue curdate(const std::vector& input); ExprValue current_date(const std::vector& input); ExprValue curtime(const std::vector& input); @@ -106,6 +133,11 @@ ExprValue weekday(const std::vector& input); ExprValue extract(const std::vector& input); ExprValue tso_to_timestamp(const std::vector& input); ExprValue timestamp_to_tso(const std::vector& input); +ExprValue to_days(const std::vector& input); +ExprValue to_seconds(const std::vector& input); +ExprValue addtime(const std::vector& input); +ExprValue subtime(const std::vector& input); + // hll functions ExprValue hll_add(const std::vector& input); ExprValue hll_merge(const std::vector& input); @@ -161,6 +193,7 @@ ExprValue tdigest_location(const std::vector& input); // other ExprValue version(const std::vector& input); ExprValue last_insert_id(const std::vector& input); +ExprValue find_in_set(const std::vector& input); //transfer (latitude A, longitude A), (latitude B, longitude B) to distance of A to B (m) ExprValue point_distance(const std::vector& input); ExprValue cast_to_date(const std::vector& inpt); diff --git a/include/logical_plan/ddl_planner.h b/include/logical_plan/ddl_planner.h index fb943648..442a32ad 100644 --- a/include/logical_plan/ddl_planner.h +++ b/include/logical_plan/ddl_planner.h @@ -42,7 +42,8 @@ class DDLPlanner : public LogicalPlanner { int parse_alter_table(pb::MetaManagerRequest& alter_request); int check_partition_key_constraint(pb::SchemaInfo& table, const std::string& field_name); - int add_column_def(pb::SchemaInfo& table, parser::ColumnDef* column, bool is_unique_indicator); + int add_column_def(pb::SchemaInfo& table, parser::ColumnDef* column, + bool is_unique_indicator, const std::string& old_field_name = ""); int parse_create_namespace(pb::NameSpaceInfo& namespace_info); int parse_drop_namespace(pb::NameSpaceInfo& namespace_info); int parse_alter_namespace(pb::NameSpaceInfo& namespace_info); diff --git a/include/meta_server/table_manager.h b/include/meta_server/table_manager.h index 7902fda9..377aa76f 100644 --- a/include/meta_server/table_manager.h +++ b/include/meta_server/table_manager.h @@ -980,8 +980,15 @@ class TableManager { return false; } - bool check_field_is_compatible_type(pb::PrimitiveType src_type, pb::PrimitiveType target_type) { + bool check_field_is_compatible_type(const pb::FieldInfo& src_field, const pb::FieldInfo& target_field) { + auto src_type = src_field.mysql_type(); + auto target_type = target_field.mysql_type(); if (src_type == target_type) { +// if (src_type == pb::DATETIME || src_type == pb::FLOAT || src_type == pb::DOUBLE) { +// if (src_field.float_precision_len() > target_field.float_precision_len()) { +// return false; +// } +// } return true; } switch (src_type) { diff --git a/include/sqlparser/ddl.h b/include/sqlparser/ddl.h index c9810fe8..50c211ac 100644 --- a/include/sqlparser/ddl.h +++ b/include/sqlparser/ddl.h @@ -171,6 +171,7 @@ enum AlterSpecType : unsigned char { ALTER_SPEC_ADD_LEARNER, ALTER_SPEC_DROP_LEARNER, ALTER_SPEC_MODIFY_COLUMN, + ALTER_SPEC_CHANGE_COLUMN, ALTER_SPEC_SWAP_TABLE, ALTER_SPEC_ADD_PARTITION, diff --git a/include/sqlparser/sql_lex.l b/include/sqlparser/sql_lex.l index d17ff07d..7ca3a386 100644 --- a/include/sqlparser/sql_lex.l +++ b/include/sqlparser/sql_lex.l @@ -163,7 +163,6 @@ REAL { return REAL; } REFERENCES { return REFERENCES; } REGEXP { return REGEXP; } RENAME { return RENAME; } -REPEAT { return REPEAT; } REPLACE { return REPLACE; } MERGE { return MERGE; } RESTRICT { return RESTRICT; } @@ -199,8 +198,6 @@ UPDATE { return UPDATE; } USAGE { return USAGE; } USE { return USE; } USING { return USING; } -UTC_DATE { return UTC_DATE; } -UTC_TIME { return UTC_TIME; } VALUES { return VALUES; } LONG { return LONG; } VARCHAR { return VARCHAR; } @@ -276,6 +273,7 @@ EXECUTE { un_reserved_keyword(yylval, yyscanner, parser); return EXECUTE; } FIELDS { un_reserved_keyword(yylval, yyscanner, parser); return FIELDS; } FILE { un_reserved_keyword(yylval, yyscanner, parser); return FILE_T; } FIRST { un_reserved_keyword(yylval, yyscanner, parser); return FIRST; } +LAST { un_reserved_keyword(yylval, yyscanner, parser); return LAST; } FIXED { un_reserved_keyword(yylval, yyscanner, parser); return FIXED; } FLUSH { un_reserved_keyword(yylval, yyscanner, parser); return FLUSH; } FORMAT { un_reserved_keyword(yylval, yyscanner, parser); return FORMAT; } @@ -307,6 +305,12 @@ MAX_UPDATES_PER_HOUR { un_reserved_keyword(yylval, yyscanner, parser); return MA MAX_USER_CONNECTIONS { un_reserved_keyword(yylval, yyscanner, parser); return MAX_USER_CONNECTIONS; } MERGE { un_reserved_keyword(yylval, yyscanner, parser); return MERGE; } MIN_ROWS { un_reserved_keyword(yylval, yyscanner, parser); return MIN_ROWS; } +ENCRYPTION { un_reserved_keyword(yylval, yyscanner, parser); return ENCRYPTION; } +ENGINE_ATTRIBUTE { un_reserved_keyword(yylval, yyscanner, parser); return ENGINE_ATTRIBUTE; } +SECONDARY_ENGINE_ATTRIBUTE {un_reserved_keyword(yylval, yyscanner, parser); +return SECONDARY_ENGINE_ATTRIBUTE; } +INSERT_METHOD { un_reserved_keyword(yylval, yyscanner, parser); return INSERT_METHOD; } +AUTOEXTEND_SIZE { un_reserved_keyword(yylval, yyscanner, parser); return AUTOEXTEND_SIZE; } NAMES { un_reserved_keyword(yylval, yyscanner, parser); return NAMES; } NATIONAL { un_reserved_keyword(yylval, yyscanner, parser); return NATIONAL; } NO { un_reserved_keyword(yylval, yyscanner, parser); return NO; } @@ -332,12 +336,12 @@ REDUNDANT { un_reserved_keyword(yylval, yyscanner, parser); return REDUNDANT; } RELOAD { un_reserved_keyword(yylval, yyscanner, parser); return RELOAD; } REPEATABLE { un_reserved_keyword(yylval, yyscanner, parser); return REPEATABLE; } REPLICATION { un_reserved_keyword(yylval, yyscanner, parser); return REPLICATION; } -REVERSE { un_reserved_keyword(yylval, yyscanner, parser); return REVERSE; } ROLLBACK { un_reserved_keyword(yylval, yyscanner, parser); return ROLLBACK; } ROUTINE { un_reserved_keyword(yylval, yyscanner, parser); return ROUTINE; } ROW { un_reserved_keyword(yylval, yyscanner, parser); return ROW; } ROW_COUNT { un_reserved_keyword(yylval, yyscanner, parser); return ROW_COUNT; } ROW_FORMAT { un_reserved_keyword(yylval, yyscanner, parser); return ROW_FORMAT; } +STATS_AUTO_RECALC { un_reserved_keyword(yylval, yyscanner, parser); return STATS_AUTO_RECALC; } SECOND { un_reserved_keyword(yylval, yyscanner, parser); return SECOND; } SECURITY { un_reserved_keyword(yylval, yyscanner, parser); return SECURITY; } SEPARATOR { un_reserved_keyword(yylval, yyscanner, parser); return SEPARATOR; } @@ -345,6 +349,9 @@ SERIALIZABLE { un_reserved_keyword(yylval, yyscanner, parser); return SERIALIZAB SESSION { un_reserved_keyword(yylval, yyscanner, parser); return SESSION; } SHARE { un_reserved_keyword(yylval, yyscanner, parser); return SHARE; } SHARED { un_reserved_keyword(yylval, yyscanner, parser); return SHARED; } +INPLACE { un_reserved_keyword(yylval, yyscanner, parser); return INPLACE; } +INSTANT { un_reserved_keyword(yylval, yyscanner, parser); return INSTANT; } +COPY { un_reserved_keyword(yylval, yyscanner, parser); return COPY; } SHUTDOWN { un_reserved_keyword(yylval, yyscanner, parser); return SHUTDOWN; } SIGNED { un_reserved_keyword(yylval, yyscanner, parser); return SIGNED; } SLAVE { un_reserved_keyword(yylval, yyscanner, parser); return SLAVE; } @@ -353,6 +360,7 @@ SQL_CACHE { un_reserved_keyword(yylval, yyscanner, parser); return SQL_CACHE; } SQL_NO_CACHE { un_reserved_keyword(yylval, yyscanner, parser); return SQL_NO_CACHE; } START { un_reserved_keyword(yylval, yyscanner, parser); return START; } STATS_PERSISTENT { un_reserved_keyword(yylval, yyscanner, parser); return STATS_PERSISTENT; } +STATS_SAMPLE_PAGES { un_reserved_keyword(yylval, yyscanner, parser); return STATS_SAMPLE_PAGES; } STATUS { un_reserved_keyword(yylval, yyscanner, parser); return STATUS; } SUPER { un_reserved_keyword(yylval, yyscanner, parser); return SUPER; } SOME { un_reserved_keyword(yylval, yyscanner, parser); return SOME; } diff --git a/include/sqlparser/sql_parse.y b/include/sqlparser/sql_parse.y index 665464f2..83a2e00c 100644 --- a/include/sqlparser/sql_parse.y +++ b/include/sqlparser/sql_parse.y @@ -191,7 +191,6 @@ extern int sql_error(YYLTYPE* yylloc, yyscan_t yyscanner, SqlParser* parser, con REFERENCES REGEXP RENAME - REPEAT REPLACE MERGE RESTRICT @@ -225,8 +224,6 @@ extern int sql_error(YYLTYPE* yylloc, yyscan_t yyscanner, SqlParser* parser, con USAGE USE USING - UTC_DATE - UTC_TIME VALUES LONG VARCHAR @@ -272,6 +269,7 @@ extern int sql_error(YYLTYPE* yylloc, yyscan_t yyscanner, SqlParser* parser, con ANY ASCII AUTO_INCREMENT + AUTOEXTEND_SIZE AVG_ROW_LENGTH AVG BEGINX @@ -298,6 +296,7 @@ extern int sql_error(YYLTYPE* yylloc, yyscan_t yyscanner, SqlParser* parser, con COMPRESSION CONNECTION CONSISTENT + COPY DAY DATA DATE @@ -310,9 +309,11 @@ extern int sql_error(YYLTYPE* yylloc, yyscan_t yyscanner, SqlParser* parser, con DUPLICATE DYNAMIC ENABLE + ENCRYPTION END ENGINE ENGINES + ENGINE_ATTRIBUTE ENUM EVENT EVENTS @@ -321,6 +322,7 @@ extern int sql_error(YYLTYPE* yylloc, yyscan_t yyscanner, SqlParser* parser, con EXECUTE FIELDS FIRST + LAST FIXED FILE_T FLUSH @@ -333,6 +335,9 @@ extern int sql_error(YYLTYPE* yylloc, yyscan_t yyscanner, SqlParser* parser, con IDENTIFIED ISOLATION INDEXES + INPLACE + INSERT_METHOD + INSTANT INVOKER JSON KEY_BLOCK_SIZE @@ -379,13 +384,13 @@ extern int sql_error(YYLTYPE* yylloc, yyscan_t yyscanner, SqlParser* parser, con RELOAD REPEATABLE REPLICATION - REVERSE ROLLBACK ROUTINE ROW ROW_COUNT ROW_FORMAT SECOND + SECONDARY_ENGINE_ATTRIBUTE SECURITY SEPARATOR SERIALIZABLE @@ -400,6 +405,8 @@ extern int sql_error(YYLTYPE* yylloc, yyscan_t yyscanner, SqlParser* parser, con SHUTDOWN START STATS_PERSISTENT + STATS_AUTO_RECALC + STATS_SAMPLE_PAGES STATUS SUPER SOME @@ -511,6 +518,7 @@ extern int sql_error(YYLTYPE* yylloc, yyscan_t yyscanner, SqlParser* parser, con RowExprList RowExpr ColumnName + IndexColumnField ExprList Expr ElseOpt @@ -544,6 +552,7 @@ extern int sql_error(YYLTYPE* yylloc, yyscan_t yyscanner, SqlParser* parser, con %type ColumnNameListOpt ColumnNameList + IndexColumnList TableName AssignmentList ByList @@ -587,8 +596,16 @@ extern int sql_error(YYLTYPE* yylloc, yyscan_t yyscanner, SqlParser* parser, con ColumnDefList IndexOptionList IndexOption + DropIndexOpt IndexType TablePartitionOpt + IndexTypeOpt + RowFormatOpt + AlgorithmVal + AlgorighmOpt + IndexLockVal + IndexLockOpt + InsertMethodOpt PartitionRange PartitionRangeList PartitionRangeListOpt @@ -1094,6 +1111,31 @@ ColumnName: } ; +IndexColumnField: + ColumnName { + $$ = $1; + } + | ColumnName '(' INTEGER_LIT ')' { + $$ = $1; + } + | IndexColumnField Order { + $$ = $1; + } + ; + +IndexColumnList: + IndexColumnField { + Node* list = new_node(Node); + list->children.reserve(10, parser->arena); + list->children.push_back($1, parser->arena); + $$ = list; + } + | IndexColumnList ',' IndexColumnField { + $1->children.push_back($3, parser->arena); + $$ = $1; + } + ; + TableName: AllIdent { TableName* name = new_node(TableName); @@ -1979,7 +2021,7 @@ FunctionNameDateArith: DATE_ADD | DATE_SUB ; FunctionNameSubstring: - SUBSTR | SUBSTRING + SUBSTR | SUBSTRING | MID ; TimestampUnit: @@ -2142,6 +2184,36 @@ FunctionCallNonKeyword: fun->children.push_back($3, parser->arena); $$ = fun; } + | MINUTE '(' Expr ')' { + FuncExpr* fun = new_node(FuncExpr); + fun->fn_name = $1; + fun->children.push_back($3, parser->arena); + $$ = fun; + } + | SECOND '(' Expr ')' { + FuncExpr* fun = new_node(FuncExpr); + fun->fn_name = $1; + fun->children.push_back($3, parser->arena); + $$ = fun; + } + | TIME '(' Expr ')' { + FuncExpr* fun = new_node(FuncExpr); + fun->fn_name = $1; + fun->children.push_back($3, parser->arena); + $$ = fun; + } + | QUARTER '(' Expr ')' { + FuncExpr* fun = new_node(FuncExpr); + fun->fn_name = $1; + fun->children.push_back($3, parser->arena); + $$ = fun; + } + | MICROSECOND '(' Expr ')' { + FuncExpr* fun = new_node(FuncExpr); + fun->fn_name = $1; + fun->children.push_back($3, parser->arena); + $$ = fun; + } | SYSDATE '(' FuncDatetimePrecListOpt ')' { FuncExpr* fun = new_node(FuncExpr); fun->fn_name = $1; @@ -2166,6 +2238,12 @@ FunctionCallNonKeyword: fun->fn_name = $1; $$ = fun; } + | UTC_TIMESTAMP '(' INTEGER_LIT ')' { + FuncExpr* fun = new_node(FuncExpr); + fun->fn_name = $1; + fun->children.push_back($3, parser->arena); + $$ = fun; + } | TIMESTAMP '(' ExprList ')' { FuncExpr* fun = new_node(FuncExpr); fun->fn_name = $1; @@ -2377,7 +2455,9 @@ FunctionCallNonKeyword: $$ = fun; } | TIMESTAMPADD '(' TimestampUnit ',' Expr ',' Expr ')' { - $$ = nullptr; + FuncExpr* fun = FuncExpr::new_ternary_op_node(FT_COMMON, $3, $5, $7, parser->arena); + fun->fn_name = $1; + $$ = fun; } | TIMESTAMPDIFF '(' TimestampUnit ',' Expr ',' Expr ')' { FuncExpr* fun = FuncExpr::new_ternary_op_node(FT_COMMON, $3, $5, $7, parser->arena); @@ -2420,6 +2500,34 @@ FunctionCallNonKeyword: fun->children.push_back($5, parser->arena); $$ = fun; } + | FORMAT '(' Expr ',' Expr ')' { + FuncExpr* fun = new_node(FuncExpr); + fun->fn_name = "format"; + fun->children.push_back($3, parser->arena); + fun->children.push_back($5, parser->arena); + $$ = fun; + } + | INSERT '(' Expr ',' Expr ',' Expr ',' Expr ')' { + FuncExpr* fun = new_node(FuncExpr); + fun->fn_name = "insert"; + fun->children.push_back($3, parser->arena); + fun->children.push_back($5, parser->arena); + fun->children.push_back($7, parser->arena); + fun->children.push_back($9, parser->arena); + $$ = fun; + } + | ASCII '(' Expr ')' { + FuncExpr* fun = new_node(FuncExpr); + fun->fn_name = "ascii"; + fun->children.push_back($3, parser->arena); + $$ = fun; + } + | CHAR '(' ExprList ')' { + FuncExpr* fun = new_node(FuncExpr); + fun->fn_name = "char"; + fun->children = $3->children; + $$ = fun; + } ; FunctionCallKeyword: VALUES '(' ColumnName ')' { @@ -2692,6 +2800,7 @@ AllIdent: | ANY | ASCII | AUTO_INCREMENT + | AUTOEXTEND_SIZE | AVG_ROW_LENGTH | AVG | BEGINX @@ -2718,6 +2827,7 @@ AllIdent: | COMPRESSION | CONNECTION | CONSISTENT + | COPY | DAY | DATA | DATE @@ -2730,9 +2840,11 @@ AllIdent: | DUPLICATE | DYNAMIC | ENABLE + | ENCRYPTION | END | ENGINE | ENGINES + | ENGINE_ATTRIBUTE | ENUM | EVENT | EVENTS @@ -2741,6 +2853,7 @@ AllIdent: | EXECUTE | FIELDS | FIRST + | LAST | FIXED | FILE_T | FLUSH @@ -2753,7 +2866,10 @@ AllIdent: | IDENTIFIED | ISOLATION | INDEXES + | INPLACE + | INSERT_METHOD | INVOKER + | INSTANT | JSON | KEY_BLOCK_SIZE | DYNAMIC_PARTITION_ATTR @@ -2799,13 +2915,13 @@ AllIdent: | RELOAD | REPEATABLE | REPLICATION - | REVERSE | ROLLBACK | ROUTINE | ROW | ROW_COUNT | ROW_FORMAT | SECOND + | SECONDARY_ENGINE_ATTRIBUTE | SECURITY | SEPARATOR | SERIALIZABLE @@ -2820,6 +2936,8 @@ AllIdent: | SHUTDOWN | START | STATS_PERSISTENT + | STATS_AUTO_RECALC + | STATS_SAMPLE_PAGES | STATUS | SUPER | SOME @@ -3475,8 +3593,10 @@ DefaultValue: FunctionCallCurTimestamp: NOW '(' ')' + | NOW '(' NumLiteral ')' | FunctionNameCurTimestamp | FunctionNameCurTimestamp '(' ')' + | FunctionNameCurTimestamp '(' NumLiteral ')' ; FunctionNameCurTimestamp: CURRENT_TIMESTAMP @@ -3519,17 +3639,17 @@ ConstraintKeywordOpt: ; ConstraintElem: - PRIMARY KEY '(' ColumnNameList ')' IndexOptionList + PRIMARY KEY IndexTypeOpt '(' IndexColumnList ')' IndexOptionList { Constraint* item = new_node(Constraint); item->type = CONSTRAINT_PRIMARY; - for (int idx = 0; idx < $4->children.size(); ++idx) { - item->columns.push_back((ColumnName*)($4->children[idx]), parser->arena); + for (int idx = 0; idx < $5->children.size(); ++idx) { + item->columns.push_back((ColumnName*)($5->children[idx]), parser->arena); } - item->index_option = (IndexOption*)$6; + item->index_option = (IndexOption*)$7; $$ = item; } - | FULLTEXT KeyOrIndexOpt IndexName '(' ColumnNameList ')' IndexOptionList + | FULLTEXT KeyOrIndexOpt IndexName '(' IndexColumnList ')' IndexOptionList { Constraint* item = new_node(Constraint); item->type = CONSTRAINT_FULLTEXT; @@ -3551,50 +3671,50 @@ ConstraintElem: item->index_option = (IndexOption*)$7; $$ = item; } - | KeyOrIndex IndexName '(' ColumnNameList ')' IndexOptionList + | KeyOrIndex IndexName IndexTypeOpt '(' IndexColumnList ')' IndexOptionList { Constraint* item = new_node(Constraint); item->type = CONSTRAINT_INDEX; item->name = $2; - for (int idx = 0; idx < $4->children.size(); ++idx) { - item->columns.push_back((ColumnName*)($4->children[idx]), parser->arena); + for (int idx = 0; idx < $5->children.size(); ++idx) { + item->columns.push_back((ColumnName*)($5->children[idx]), parser->arena); } - item->index_option = (IndexOption*)$6; + item->index_option = (IndexOption*)$7; $$ = item; } - | UNIQUE KeyOrIndexOpt IndexName '(' ColumnNameList ')' IndexOptionList + | UNIQUE KeyOrIndexOpt IndexName IndexTypeOpt '(' IndexColumnList ')' IndexOptionList { Constraint* item = new_node(Constraint); item->type = CONSTRAINT_UNIQ; item->name = $3; - for (int idx = 0; idx < $5->children.size(); ++idx) { - item->columns.push_back((ColumnName*)($5->children[idx]), parser->arena); + for (int idx = 0; idx < $6->children.size(); ++idx) { + item->columns.push_back((ColumnName*)($6->children[idx]), parser->arena); } - item->index_option = (IndexOption*)$7; + item->index_option = (IndexOption*)$8; $$ = item; } - | KeyOrIndex GlobalOrLocal IndexName '(' ColumnNameList ')' IndexOptionList + | KeyOrIndex GlobalOrLocal IndexName IndexTypeOpt '(' IndexColumnList ')' IndexOptionList { Constraint* item = new_node(Constraint); item->type = CONSTRAINT_INDEX; item->index_dist = static_cast($2); item->name = $3; - for (int idx = 0; idx < $5->children.size(); ++idx) { - item->columns.push_back((ColumnName*)($5->children[idx]), parser->arena); + for (int idx = 0; idx < $6->children.size(); ++idx) { + item->columns.push_back((ColumnName*)($6->children[idx]), parser->arena); } - item->index_option = (IndexOption*)$7; + item->index_option = (IndexOption*)$8; $$ = item; } - | UNIQUE KeyOrIndexOpt GlobalOrLocal IndexName '(' ColumnNameList ')' IndexOptionList + | UNIQUE KeyOrIndexOpt GlobalOrLocal IndexName IndexTypeOpt '(' IndexColumnList ')' IndexOptionList { Constraint* item = new_node(Constraint); item->type = CONSTRAINT_UNIQ; item->index_dist = static_cast($3); item->name = $4; - for (int idx = 0; idx < $6->children.size(); ++idx) { - item->columns.push_back((ColumnName*)($6->children[idx]), parser->arena); + for (int idx = 0; idx < $7->children.size(); ++idx) { + item->columns.push_back((ColumnName*)($7->children[idx]), parser->arena); } - item->index_option = (IndexOption*)$8; + item->index_option = (IndexOption*)$9; $$ = item; } ; @@ -3640,6 +3760,31 @@ IndexOptionList: } ; +AlgorighmOpt: + ALGORITHM EqOpt AlgorithmVal { + + $$ = nullptr; + } + ; + +IndexLockOpt: + LOCK EqOpt IndexLockVal { + $$ = nullptr; + } + ; + +DropIndexOpt: + { + $$ = nullptr; + } + | AlgorighmOpt { + $$ = nullptr; + } + | IndexLockOpt { + + } + ; + IndexOption: KEY_BLOCK_SIZE EqOpt INTEGER_LIT { $$ = nullptr; @@ -3652,6 +3797,13 @@ IndexOption: op->comment = ((LiteralExpr*)$2)->_u.str_val; $$ = op; } + | AlgorighmOpt { + + $$ = nullptr; + } + | IndexLockOpt { + $$ = nullptr; + } ; IndexType: @@ -3663,6 +3815,14 @@ IndexType: } ; +IndexTypeOpt: + { + $$ = nullptr; + } + | IndexType { + $$ = nullptr; + }; + /*************************************Type Begin***************************************/ Type: NumericType @@ -4067,6 +4227,11 @@ DateAndTimeType: // TODO: fractional seconds precision FieldType* field_type = new_node(FieldType); field_type->type = MYSQL_TYPE_DATETIME; + if ($2 >= 1 && $2 <= 6) { + field_type->float_len = $2; + } else { + field_type->float_len = 0; + } $$ = field_type; } | TIMESTAMP OptFieldLen @@ -4254,12 +4419,16 @@ TableOptionList: } | TableOptionList TableOption { - $1->children.push_back($2, parser->arena); + if ($2 != nullptr) { + $1->children.push_back($2, parser->arena); + } $$ = $1; } | TableOptionList ',' TableOption { - $1->children.push_back($3, parser->arena); + if ($3 != nullptr) { + $1->children.push_back($3, parser->arena); + } $$ = $1; } ; @@ -4326,6 +4495,10 @@ TableOption: option->type = TABLE_OPT_PARTITION; $$ = option; } + | IgnoreTableOption + { + $$ = nullptr; + } ; PartitionOption: @@ -4429,6 +4602,43 @@ PartitionRangeListOpt: $$ = $1; } ; +InsertMethodOpt: + NO | FIRST | LAST + ; + +RowFormatOpt: + DEFAULT | DYNAMIC | FIXED | COMPRESSED | REDUNDANT | COMPACT + ; + +AlgorithmVal: + DEFAULT | INSTANT | INPLACE | COPY + ; + +IndexLockVal: + DEFAULT | NONE | SHARED | EXCLUSIVE + ; + +IgnoreTableOption: + ROW_FORMAT EqOpt RowFormatOpt + | STATS_PERSISTENT EqOpt NumLiteral + | STATS_PERSISTENT EqOpt DEFAULT + | STATS_AUTO_RECALC EqOpt NumLiteral + | STATS_AUTO_RECALC EqOpt DEFAULT + | STATS_SAMPLE_PAGES EqOpt NumLiteral + | CHECKSUM EqOpt NumLiteral + | MAX_ROWS EqOpt NumLiteral + | MIN_ROWS EqOpt NumLiteral + | PACK_KEYS EqOpt NumLiteral + | PACK_KEYS EqOpt DEFAULT + | ENCRYPTION EqOpt STRING_LIT + | ENGINE_ATTRIBUTE EqOpt STRING_LIT + | SECONDARY_ENGINE_ATTRIBUTE EqOpt STRING_LIT + | INSERT_METHOD EqOpt InsertMethodOpt + | DELAY_KEY_WRITE EqOpt NumLiteral + | CONNECTION EqOpt STRING_LIT + | COMPRESSION EqOpt STRING_LIT + | AUTOEXTEND_SIZE EqOpt NumLiteral + ; TablePartitionOpt: RANGE '(' Expr ')' '(' PartitionRangeListOpt ')' @@ -4937,42 +5147,53 @@ AlterTableStmt: } $$ = stmt; } - | CREATE KeyOrIndex GlobalOrLocalOpt IndexName ON TableName '(' ColumnNameList ')' IndexOptionList + | CREATE KeyOrIndex GlobalOrLocalOpt IndexName IndexTypeOpt ON TableName '(' IndexColumnList ')' IndexOptionList { AlterTableStmt* stmt = new_node(AlterTableStmt); - stmt->table_name = (TableName*)$6; + stmt->table_name = (TableName*)$7; Constraint* item = new_node(Constraint); item->type = CONSTRAINT_INDEX; item->index_dist = static_cast($3); item->name = $4; - for (int idx = 0; idx < $8->children.size(); ++idx) { - item->columns.push_back((ColumnName*)($8->children[idx]), parser->arena); + for (int idx = 0; idx < $9->children.size(); ++idx) { + item->columns.push_back((ColumnName*)($9->children[idx]), parser->arena); } - item->index_option = (IndexOption*)$10; + item->index_option = (IndexOption*)$11; AlterTableSpec* spec = new_node(AlterTableSpec); spec->spec_type = ALTER_SPEC_ADD_INDEX; spec->new_constraints.push_back((Constraint*)item, parser->arena); stmt->alter_specs.push_back((AlterTableSpec*)spec, parser->arena); $$ = stmt; } - | CREATE UNIQUE KeyOrIndexOpt GlobalOrLocalOpt IndexName ON TableName '(' ColumnNameList ')' IndexOptionList + | CREATE UNIQUE KeyOrIndexOpt GlobalOrLocalOpt IndexName IndexTypeOpt ON TableName '(' IndexColumnList ')' IndexOptionList { AlterTableStmt* stmt = new_node(AlterTableStmt); - stmt->table_name = (TableName*)$7; + stmt->table_name = (TableName*)$8; Constraint* item = new_node(Constraint); item->type = CONSTRAINT_UNIQ; item->index_dist = static_cast($4); item->name = $5; - for (int idx = 0; idx < $9->children.size(); ++idx) { - item->columns.push_back((ColumnName*)($9->children[idx]), parser->arena); + for (int idx = 0; idx < $10->children.size(); ++idx) { + item->columns.push_back((ColumnName*)($10->children[idx]), parser->arena); } - item->index_option = (IndexOption*)$11; + item->index_option = (IndexOption*)$12; AlterTableSpec* spec = new_node(AlterTableSpec); spec->spec_type = ALTER_SPEC_ADD_INDEX; spec->new_constraints.push_back((Constraint*)item, parser->arena); stmt->alter_specs.push_back((AlterTableSpec*)spec, parser->arena); $$ = stmt; } + | DROP INDEX IndexName ON TableName ForceOrNot DropIndexOpt + { + AlterTableStmt* stmt = new_node(AlterTableStmt); + stmt->table_name = (TableName*)$5; + AlterTableSpec* spec = new_node(AlterTableSpec); + spec->spec_type = ALTER_SPEC_DROP_INDEX; + spec->index_name = $3; + spec->force = $6; + stmt->alter_specs.push_back((AlterTableSpec*)spec, parser->arena); + $$ = stmt; + } ; AlterSpecList: @@ -5173,6 +5394,14 @@ AlterSpec: } $$ = spec; } + | CHANGE ColumnKwdOpt AllIdent ColumnDef ColumnPosOpt + { + AlterTableSpec* spec = new_node(AlterTableSpec); + spec->spec_type = ALTER_SPEC_CHANGE_COLUMN; + spec->column_name = $3; + spec->new_columns.push_back((ColumnDef*)$4, parser->arena); + $$ = spec; + } ; // Prepare Statement diff --git a/src/common/datetime.cpp b/src/common/datetime.cpp index 1f006571..6622604f 100755 --- a/src/common/datetime.cpp +++ b/src/common/datetime.cpp @@ -40,7 +40,7 @@ std::string timestamp_to_str(time_t timestamp, bool is_utc) { } // encode DATETIME to string format // ref: https://dev.mysql.com/doc/internals/en/date-and-time-data-type-representation.html -std::string datetime_to_str(uint64_t datetime) { +std::string datetime_to_str(uint64_t datetime, int precision_len) { int year_month = ((datetime >> 46) & 0x1FFFF); int year = year_month / 13; int month = year_month % 13; @@ -51,12 +51,14 @@ std::string datetime_to_str(uint64_t datetime) { int macrosec = (datetime & 0xFFFFFF); char buf[30] = {0}; - if (macrosec > 0) { - snprintf(buf, sizeof(buf), "%04d-%02d-%02d %02d:%02d:%02d.%06d", - year, month, day, hour, minute, second, macrosec); + snprintf(buf, sizeof(buf), "%04d-%02d-%02d %02d:%02d:%02d.%06d", + year, month, day, hour, minute, second, macrosec); + if (precision_len > 0 and precision_len <=6) { + buf[20 + precision_len] = '\0'; + } else if (precision_len == 0) { + buf[19] = '\0'; } else { - snprintf(buf, sizeof(buf), "%04d-%02d-%02d %02d:%02d:%02d", - year, month, day, hour, minute, second); + buf[26] = '\0'; } return std::string(buf); } @@ -354,6 +356,23 @@ std::string time_to_str(int32_t time) { snprintf(buf, sizeof(buf), "%s%02d:%02d:%02d", OP_STR[minus], hour, min, sec); return std::string(buf); } + +int32_t time_to_seconds(int32_t time) { + bool minus = false; + if (time < 0) { + minus = true; + time = -time; + } + int hour = (time >> 12) & 0x3FF; + int min = (time >> 6) & 0x3F; + int sec = time & 0x3F; + int32_t res = sec + min * 60 + hour * 3600; + if (minus) { + res = res * -1; + } + return res; +} + int32_t str_to_time(const char* str_time) { while (*str_time == ' ') { str_time++; diff --git a/src/common/information_schema.cpp b/src/common/information_schema.cpp index 45832bac..a5d39fab 100644 --- a/src/common/information_schema.cpp +++ b/src/common/information_schema.cpp @@ -909,7 +909,8 @@ void InformationSchema::init_columns() { } record->set_string(record->get_field_by_name("CHARACTER_SET_NAME"), "utf8"); record->set_string(record->get_field_by_name("COLLATION_NAME"), "utf8_general_ci"); - record->set_string(record->get_field_by_name("COLUMN_TYPE"), to_mysql_type_full_string(field.type)); + record->set_string(record->get_field_by_name("COLUMN_TYPE"), + to_mysql_type_full_string(field.type, field.float_total_len, field.float_precision_len)); std::vector extra_vec; if (field_index.count(field.id) == 0) { record->set_string(record->get_field_by_name("COLUMN_KEY"), " "); diff --git a/src/common/schema_factory.cpp b/src/common/schema_factory.cpp index 1e94dae3..7b7cf5a8 100644 --- a/src/common/schema_factory.cpp +++ b/src/common/schema_factory.cpp @@ -767,7 +767,8 @@ void SchemaFactory::update_index(TableInfo& table_info, const pb::IndexInfo& ind std::transform(lower_index_name.begin(), lower_index_name.end(), lower_index_name.begin(), ::tolower); idx_info.name = table_info.name + "." + lower_index_name; - idx_info.short_name = lower_index_name; + // idx_info.short_name = lower_index_name; + idx_info.short_name = index.index_name(); idx_info.type = index.index_type(); idx_info.state = index.state(); idx_info.max_field_id = table_info.max_field_id; @@ -3137,7 +3138,7 @@ int SchemaFactory::fill_default_value(SmartRecord record, FieldInfo& field) { } ExprValue default_value = field.default_expr_value; if (field.default_value == "(current_timestamp())") { - default_value = ExprValue::Now(); + default_value = ExprValue::Now(field.float_precision_len); default_value.cast_to(field.type); } // mysql非strict mode，不填not null字段会补充空串/0等 diff --git a/src/engine/transaction_pool.cpp b/src/engine/transaction_pool.cpp index 78c4733b..1a67cfcc 100644 --- a/src/engine/transaction_pool.cpp +++ b/src/engine/transaction_pool.cpp @@ -196,7 +196,7 @@ void TransactionPool::txn_query_primary_region(uint64_t txn_id, Region* region, request.ShortDebugString().c_str(), response.ShortDebugString().c_str()); bthread_usleep(retry_times * FLAGS_retry_interval_us); - break; + break; } case pb::VERSION_OLD: { for (auto r : response.regions()) { @@ -223,6 +223,12 @@ void TransactionPool::txn_query_primary_region(uint64_t txn_id, Region* region, success = true; break; } + case pb::EXEC_FAIL: { + DB_WARNING("send txn query failed , request:%s response: %s", + request.ShortDebugString().c_str(), + response.ShortDebugString().c_str()); + return; + } default: { DB_WARNING("send txn query failed , request:%s response: %s", request.ShortDebugString().c_str(), @@ -389,6 +395,11 @@ void TransactionPool::txn_commit_through_raft(uint64_t txn_id, bthread_usleep(retry_times * FLAGS_retry_interval_us); break; } + case pb::EXEC_FAIL: + DB_WARNING("send txn commit failed , request:%s response: %s", + request.ShortDebugString().c_str(), + response.ShortDebugString().c_str()); + return; default: { DB_WARNING("send txn commit failed , request:%s response: %s", request.ShortDebugString().c_str(), @@ -397,9 +408,7 @@ void TransactionPool::txn_commit_through_raft(uint64_t txn_id, break; } } - if (retry_times < 5) { - retry_times++; - } + retry_times++; } while (!success && retry_times < 5); } @@ -475,6 +484,13 @@ void TransactionPool::update_primary_timestamp(const pb::TransactionInfo& txn_in response.ShortDebugString().c_str()); break; } + case pb::EXEC_FAIL: { + DB_WARNING("send update primary timestamp failed , request:%s response: %s", + request.ShortDebugString().c_str(), + response.ShortDebugString().c_str()); + return; + + } default: { DB_WARNING("send update primary timestamp failed , request:%s response: %s", request.ShortDebugString().c_str(), @@ -485,7 +501,6 @@ void TransactionPool::update_primary_timestamp(const pb::TransactionInfo& txn_in } retry_times++; } while (!success && retry_times < 3); - }; Bthread bth; bth.run(update_fun); diff --git a/src/exec/dml_node.cpp b/src/exec/dml_node.cpp index 0b00f544..f93c0d78 100644 --- a/src/exec/dml_node.cpp +++ b/src/exec/dml_node.cpp @@ -716,10 +716,11 @@ int DMLNode::update_row(RuntimeState* state, SmartRecord record, MemRow* row) { for (size_t i = 0; i < _update_exprs.size(); i++) { auto& slot = _update_slots[i]; auto expr = _update_exprs[i]; + auto field = _update_fields[slot.field_id()]; - if (field->type == pb::FLOAT || field->type == pb::DOUBLE) { + if (field->type == pb::FLOAT || field->type == pb::DOUBLE || field->type == pb::DATETIME) { auto& expr_value = expr->get_value(row).cast_to(slot.slot_type()); - expr_value.float_precision_len = field->float_precision_len; + expr_value.set_precision_len(field->float_precision_len); record->set_value(record->get_field_by_tag(slot.field_id()), expr_value); } else { record->set_value(record->get_field_by_tag(slot.field_id()), diff --git a/src/exec/insert_manager_node.cpp b/src/exec/insert_manager_node.cpp index b47f75be..6b43add4 100644 --- a/src/exec/insert_manager_node.cpp +++ b/src/exec/insert_manager_node.cpp @@ -207,7 +207,7 @@ int InsertManagerNode::subquery_open(RuntimeState* state) { } // 20190101101112 这种转换现在只支持string类型 pb::PrimitiveType field_type = table_field_map[_selected_field_ids[i]]->type; - result.float_precision_len = table_field_map[_selected_field_ids[i]]->float_precision_len; + result.set_precision_len(table_field_map[_selected_field_ids[i]]->float_precision_len); if (is_datetime_specic(field_type) && result.is_numberic()) { result.cast_to(pb::STRING).cast_to(field_type); } else { @@ -655,9 +655,9 @@ void InsertManagerNode::update_record(const SmartRecord& record, const SmartReco auto& slot = _update_slots[i]; auto expr = _update_exprs[i]; auto field = _update_fields[slot.field_id()]; - if (field->type == pb::FLOAT || field->type == pb::DOUBLE) { + if (field->type == pb::FLOAT || field->type == pb::DOUBLE || field->type == pb::DATETIME) { auto& expr_value = expr->get_value(row).cast_to(slot.slot_type()); - expr_value.float_precision_len = field->float_precision_len; + expr_value.set_precision_len(field->float_precision_len); record->set_value(record->get_field_by_tag(slot.field_id()), expr_value); } else { record->set_value(record->get_field_by_tag(slot.field_id()), diff --git a/src/exec/load_node.cpp b/src/exec/load_node.cpp index 107622ce..07c51461 100755 --- a/src/exec/load_node.cpp +++ b/src/exec/load_node.cpp @@ -313,7 +313,7 @@ int LoadNode::fill_field_value(SmartRecord record, FieldInfo& field, ExprValue& } ExprValue default_value = field.default_expr_value; if (field.default_value == "(current_timestamp())") { - default_value = ExprValue::Now(); + default_value = ExprValue::Now(field.float_precision_len); default_value.cast_to(field.type); } if (0 != record->set_value(record->get_field_by_tag(field.id), default_value)) { diff --git a/src/exec/select_manager_node.cpp b/src/exec/select_manager_node.cpp index e946d22b..9dd39f7f 100755 --- a/src/exec/select_manager_node.cpp +++ b/src/exec/select_manager_node.cpp @@ -22,6 +22,7 @@ namespace baikaldb { DEFINE_bool(global_index_read_consistent, true, "double check for global and primary region consistency"); +DEFINE_int32(max_select_region_count, -1, "max select sql region count limit, default:-1 means no limit"); int SelectManagerNode::open(RuntimeState* state) { START_LOCAL_TRACE(get_trace(), state->get_trace_cost(), OPEN_TRACE, ([state](TraceLocalNode& local_node) { local_node.set_scan_rows(state->num_scan_rows()); @@ -220,6 +221,15 @@ int SelectManagerNode::fetcher_store_run(RuntimeState* state, ExecNode* exec_nod if (main_scan_index == nullptr) { return -1; } + auto ctx = state->client_conn()->query_ctx; + if (ctx->is_select && !scan_node->has_index() && FLAGS_max_select_region_count > 0 + && main_scan_index->region_infos.size() > FLAGS_max_select_region_count + && scan_node->get_limit() == -1) { + ctx->stat_info.error_code = ER_SQL_REFUSE; + ctx->stat_info.error_msg << "sql is forbid, reason is not use index and region count=" + << main_scan_index->region_infos.size(); + return -1; + }; if (backup_scan_index != nullptr && dynamic_timeout_ms > 0 && state->txn_id == 0) { // 非事务情况下才进行全局二级索引降级，txn_id != 0 情况下state中会有修改，无法多个请求并发使用state diff --git a/src/exec/update_manager_node.cpp b/src/exec/update_manager_node.cpp index 05bf9a8e..50bb2b42 100644 --- a/src/exec/update_manager_node.cpp +++ b/src/exec/update_manager_node.cpp @@ -246,9 +246,9 @@ void UpdateManagerNode::update_record(RuntimeState* state, SmartRecord record) { auto& slot = _update_slots[i]; auto expr = _update_exprs[i]; auto field = _update_fields[slot.field_id()]; - if (field->type == pb::FLOAT || field->type == pb::DOUBLE) { + if (field->type == pb::FLOAT || field->type == pb::DOUBLE || field->type == pb::DATETIME) { auto& expr_value = expr->get_value(row).cast_to(slot.slot_type()); - expr_value.float_precision_len = field->float_precision_len; + expr_value.set_precision_len(field->float_precision_len); record->set_value(record->get_field_by_tag(slot.field_id()), expr_value); } else { record->set_value(record->get_field_by_tag(slot.field_id()), diff --git a/src/expr/fn_manager.cpp b/src/expr/fn_manager.cpp index 0e2cda3f..b9bd0e61 100644 --- a/src/expr/fn_manager.cpp +++ b/src/expr/fn_manager.cpp @@ -135,6 +135,7 @@ void FunctionManager::register_operators() { register_object_ret("lcase", lower, pb::STRING); register_object_ret("concat", concat, pb::STRING); register_object_ret("substr", substr, pb::STRING); + register_object_ret("mid", substr, pb::STRING); register_object_ret("left", left, pb::STRING); register_object_ret("right", right, pb::STRING); register_object_ret("trim", trim, pb::STRING); @@ -153,6 +154,23 @@ void FunctionManager::register_operators() { register_object_ret("rpad", rpad, pb::STRING); register_object_ret("instr", instr, pb::INT32); register_object_ret("json_extract", json_extract, pb::STRING); + register_object_ret("export_set", export_set, pb::STRING); + register_object_ret("to_base64", to_base64, pb::STRING); + register_object_ret("from_base64", from_base64, pb::STRING); + register_object_ret("make_set", make_set, pb::STRING); + register_object_ret("oct", oct, pb::STRING); + register_object_ret("hex", hex, pb::STRING); + register_object_ret("unhex", unhex, pb::STRING); + register_object_ret("bin", bin, pb::STRING); + register_object_ret("space", space, pb::STRING); + register_object_ret("elt", elt, pb::STRING); + register_object_ret("char_length", char_length, pb::INT32); + register_object_ret("format", format, pb::STRING); + register_object_ret("field", field, pb::INT32); + register_object_ret("quote", quote, pb::STRING); + register_object_ret("char", func_char, pb::STRING); + register_object_ret("soundex", soundex, pb::STRING); + // date funcs register_object_ret("unix_timestamp", unix_timestamp, pb::INT64); @@ -160,7 +178,20 @@ void FunctionManager::register_operators() { register_object_ret("now", now, pb::DATETIME); register_object_ret("sysdate", now, pb::DATETIME); register_object_ret("utc_timestamp", utc_timestamp, pb::DATETIME); + register_object_ret("utc_date", utc_date, pb::DATE); + register_object_ret("utc_time", utc_time, pb::TIME); register_object_ret("date_format", date_format, pb::STRING); + register_object_ret("period_diff", period_diff, pb::INT32); + register_object_ret("period_add", period_add, pb::STRING); + register_object_ret("minute", minute, pb::INT32); + register_object_ret("second", second, pb::INT32); + register_object_ret("time", func_time, pb::TIME); + register_object_ret("quarter", func_quarter, pb::INT32); + register_object_ret("microsecond", microsecond, pb::INT32); + register_object_ret("timestampadd", timestampadd, pb::TIMESTAMP); + register_object_ret("adddate", date_add, pb::DATETIME); + register_object_ret("addtime", addtime, pb::STRING); + register_object_ret("subtime", subtime, pb::STRING); /* str_to_date实现较为复杂，需要满足任意格式的string转换为标准形式的DATETIME，现在为了方便确保str_to_date可以使用， 默认string是标准形式的date，故其实现内容和date_format函数一致 @@ -174,7 +205,7 @@ void FunctionManager::register_operators() { register_object_ret("current_date", current_date, pb::DATE); register_object_ret("curtime", curtime, pb::TIME); register_object_ret("current_time", current_time, pb::TIME); - register_object_ret("current_timestamp", current_timestamp, pb::TIMESTAMP); + register_object_ret("current_timestamp", current_timestamp, pb::DATETIME); register_object_ret("timestamp", timestamp, pb::TIMESTAMP); register_object_ret("date", date, pb::DATE); register_object_ret("hour", hour, pb::UINT32); @@ -192,12 +223,14 @@ void FunctionManager::register_operators() { register_object_ret("time_to_sec", time_to_sec, pb::UINT32); register_object_ret("sec_to_time", sec_to_time, pb::TIME); register_object_ret("weekday", weekday, pb::UINT32); - register_object_ret("datediff", datediff, pb::UINT32); + register_object_ret("datediff", datediff, pb::INT32); register_object_ret("date_add", date_add, pb::DATETIME); register_object_ret("date_sub", date_sub, pb::DATETIME); register_object_ret("extract", extract, pb::UINT32); register_object_ret("tso_to_timestamp", tso_to_timestamp, pb::DATETIME); register_object_ret("timestamp_to_tso", timestamp_to_tso, pb::INT64); + register_object_ret("to_days", to_days, pb::INT64); + register_object_ret("to_seconds", to_seconds, pb::INT64); // hll funcs register_object_ret("hll_add", hll_add, pb::HLL); register_object_ret("hll_merge", hll_merge, pb::HLL); @@ -210,6 +243,7 @@ void FunctionManager::register_operators() { register_object_ret("ifnull", ifnull, pb::STRING); register_object_ret("nullif", nullif, pb::STRING); register_object_ret("isnull", isnull, pb::BOOL); + register_object_ret("find_in_set", find_in_set, pb::INT64); // MurmurHash sign register_object_ret("murmur_hash", murmur_hash, pb::UINT64); register_object_ret("md5", md5, pb::STRING); diff --git a/src/expr/internal_functions.cpp b/src/expr/internal_functions.cpp index 5f49aef3..fe6776ab 100644 --- a/src/expr/internal_functions.cpp +++ b/src/expr/internal_functions.cpp @@ -19,6 +19,7 @@ #include "hll_common.h" #include "datetime.h" #include +#include #include #include #include @@ -29,6 +30,12 @@ namespace baikaldb { #else DEFINE_string(db_version, "5.7.16-BaikalDB", "db version"); #endif +#define INPUT_CHECK_NULL \ +for (auto& s: input) { \ + if (s.is_null()) { \ + return ExprValue::Null(); \ + } \ +} static const int32_t DATE_FORMAT_LENGTH = 128; static const std::vector day_names = { "Sunday", "Monday", "Tuesday", "Wednesday", @@ -582,12 +589,9 @@ ExprValue ascii(const std::vector& input) { } ExprValue tmp(pb::INT32); - - if (input[0].str_val.empty()) { - tmp._u.int32_val = 0; - } else { - tmp._u.int32_val = static_cast(input[0].str_val[0]); - } + ExprValue in = input[0]; + in.cast_to(pb::STRING); + tmp._u.int32_val = static_cast(in.str_val[0]); return tmp; } @@ -623,19 +627,22 @@ ExprValue insert(const std::vector& input) { } int pos = input[1].get_numberic(); - if (pos < 0) { + if (pos < 1) { return input[0]; } + pos -= 1; int len = input[2].get_numberic(); if (len <= 0) { return input[0]; } - ExprValue tmp(pb::STRING); tmp.str_val = input[0].str_val; - tmp.str_val.replace(pos, len, input[2].str_val); - + if (pos >= input[0].str_val.size()) { + return tmp; + } + tmp.str_val.replace(pos, len, input[3].str_val); + return tmp; } @@ -912,14 +919,175 @@ ExprValue now(const std::vector& input) { } ExprValue current_timestamp(const std::vector& input) { + if (input.size() > 0) { + return ExprValue::Now(input[0].get_numberic()); + } return ExprValue::Now(0); } ExprValue utc_timestamp(const std::vector& input) { - return ExprValue::UTC_TIMESTAMP(); + if (input.size() > 0) { + return ExprValue::UTC_TIMESTAMP(input[0].get_numberic()); + } + return ExprValue::UTC_TIMESTAMP(0); +} + +ExprValue utc_date(const std::vector& input) { + return ExprValue::UTC_TIMESTAMP(0).cast_to(pb::DATE); +} + +ExprValue utc_time(const std::vector& input) { + return ExprValue::UTC_TIMESTAMP(0).cast_to(pb::TIME); +} + +ExprValue period_add(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 2) { + return ExprValue::Null(); + } + ExprValue tmp = input[0]; + tmp.cast_to(pb::INT64); + int year = tmp._u.int64_val / 100; + int month = tmp._u.int64_val % 100; + ExprValue num = input[1]; + num.cast_to(pb::INT32); + month += num._u.int32_val + year * 12 - 1; + year = month / 12; + month = month % 12 + 1; + char buf[30] = {0}; + snprintf(buf, sizeof(buf), "%d%02d", year, month); + ExprValue res(pb::STRING); + res.str_val = buf; + return res; +} + +ExprValue period_diff(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 2) { + return ExprValue::Null(); + } + ExprValue dt1 = input[0]; + dt1.cast_to(pb::INT64); + + ExprValue dt2 = input[0]; + dt2.cast_to(pb::INT64); + + int month1 = dt1._u.int64_val / 100 * 12 + dt1._u.int64_val % 100; + int month2 = dt2._u.int64_val / 100 * 12 + dt2._u.int64_val % 100; + int diff = month1 - month2; + ExprValue res(pb::INT32); + res._u.int32_val = diff; + return res; +} + +ExprValue minute(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue tmp = input[0]; + tmp.cast_to(pb::TIME); + ExprValue res(pb::INT32); + res._u.int32_val = (tmp._u.int32_val >> 6 ) & 0x3F; + return res; +} + +ExprValue func_time(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue res = input[0]; + res.cast_to(pb::TIME); + return res; +} + +ExprValue func_quarter(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue tmp = input[0]; + tmp.cast_to(pb::DATE); + int year_month = ((tmp._u.int32_val >> 5) & 0x1FFFF); + int month = year_month % 13; + int quarter = (month - 1) / 3; + ExprValue res(pb::INT32); + res._u.int32_val = quarter + 1; + return res; +} + +ExprValue microsecond(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue tmp = input[0]; + tmp.cast_to(pb::STRING); + int idx = 0; + while (idx < tmp.str_val.length()) { + if (tmp.str_val[idx] == '.') { + break; + } + idx ++; + } + idx ++; + int micro = 0; + int cnt = 0; + while (idx < tmp.str_val.length()) { + if (tmp.str_val[idx] >= '0' && tmp.str_val[idx] <= '9') { + micro = micro * 10 + tmp.str_val[idx] - '0'; + cnt ++; + idx ++; + continue; + } + break; + } + if (cnt == 0 || cnt > 6) { + micro = 0; + } + ExprValue res(pb::INT32) ; + res._u.int32_val = micro; + return res; +} + +ExprValue second(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue tmp = input[0]; + tmp.cast_to(pb::TIME); + ExprValue res(pb::INT32); + res._u.int32_val = tmp._u.int32_val & 0x3F; + return res; +} + +ExprValue timestampadd(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 3) { + return ExprValue::Null(); + } + ExprValue ret = input[2]; + ret.cast_to(pb::TIMESTAMP); + ExprValue num = input[1]; + num.cast_to(pb::INT32); + if (input[0].str_val == "second") { + ret._u.uint32_val += num._u.int32_val; + } else if (input[0].str_val == "minute") { + ret._u.uint32_val += num._u.int32_val * 60; + } else if (input[0].str_val == "hour") { + ret._u.uint32_val += num._u.int32_val * 3600; + } else if (input[0].str_val == "day") { + ret._u.uint32_val += num._u.int32_val * 24 * 3600; + } else { + return ExprValue::Null(); + } + return ret; } ExprValue timestamp(const std::vector& input) { + INPUT_CHECK_NULL; if (input.size() == 0 || input.size() > 2) { return ExprValue::Null(); } @@ -932,14 +1100,10 @@ ExprValue timestamp(const std::vector& input) { } ExprValue date_format(const std::vector& input) { + INPUT_CHECK_NULL; if (input.size() != 2) { return ExprValue::Null(); } - for (auto& s : input) { - if (s.is_null()) { - return ExprValue::Null(); - } - } ExprValue tmp = input[0]; time_t t = tmp.cast_to(pb::TIMESTAMP)._u.uint32_val; struct tm t_result; @@ -951,27 +1115,19 @@ ExprValue date_format(const std::vector& input) { return format_result; } ExprValue str_to_date(const std::vector& input) { + INPUT_CHECK_NULL; if (input.size() != 2) { return ExprValue::Null(); } - for (auto& s : input) { - if (s.is_null()) { - return ExprValue::Null(); - } - } ExprValue tmp = input[0]; return tmp.cast_to(pb::DATETIME); } ExprValue time_format(const std::vector& input) { + INPUT_CHECK_NULL; if (input.size() != 2) { return ExprValue::Null(); } - for (auto& s : input) { - if (s.is_null()) { - return ExprValue::Null(); - } - } ExprValue tmp = input[0]; struct tm t_result; uint32_t second = tmp.cast_to(pb::TIME)._u.int32_val; @@ -986,15 +1142,36 @@ ExprValue time_format(const std::vector& input) { return format_result; } +ExprValue to_days(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue tmp = input[0]; + tmp.cast_to(pb::TIMESTAMP); + tmp.cast_to(pb::INT64); + tmp._u.int64_val = (tmp._u.int64_val + 62167248000) / 86400; + return tmp; +} + +ExprValue to_seconds(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue tmp = input[0]; + tmp.cast_to(pb::TIMESTAMP); + tmp.cast_to(pb::INT64); + tmp._u.int64_val += 62167248000; + return tmp; +} + + ExprValue convert_tz(const std::vector& input) { + INPUT_CHECK_NULL; if (input.size() != 3){ return ExprValue::Null(); } - for (auto& s : input) { - if (s.is_null()) { - return ExprValue::Null(); - } - } ExprValue time = input[0]; ExprValue from_tz = input[1]; ExprValue to_tz = input[2]; @@ -1013,14 +1190,10 @@ ExprValue convert_tz(const std::vector& input) { } ExprValue timediff(const std::vector& input) { + INPUT_CHECK_NULL; if (input.size() < 2) { return ExprValue::Null(); } - for (auto& s : input) { - if (s.is_null()) { - return ExprValue::Null(); - } - } ExprValue arg1 = input[0]; ExprValue arg2 = input[1]; int32_t seconds = arg1.cast_to(pb::TIMESTAMP)._u.uint32_val - @@ -1030,15 +1203,10 @@ ExprValue timediff(const std::vector& input) { return ret; } ExprValue timestampdiff(const std::vector& input) { + INPUT_CHECK_NULL; if (input.size() < 3) { return ExprValue::Null(); } - for (auto& s : input) { - if (s.is_null()) { - return ExprValue::Null(); - } - } - ExprValue arg2 = input[1]; ExprValue arg3 = input[2]; int32_t seconds = arg3.cast_to(pb::TIMESTAMP)._u.uint32_val - @@ -1494,8 +1662,10 @@ ExprValue datediff(const std::vector& input) { if (right.type == pb::INT64) { right.cast_to(pb::STRING); } - time_t t1 = left.cast_to(pb::TIMESTAMP)._u.uint32_val; - time_t t2 = right.cast_to(pb::TIMESTAMP)._u.uint32_val; + left.cast_to(pb::DATE); + right.cast_to(pb::DATE); + int64_t t1 = left.cast_to(pb::TIMESTAMP)._u.uint32_val; + int64_t t2 = right.cast_to(pb::TIMESTAMP)._u.uint32_val; ExprValue tmp(pb::INT32); tmp._u.int32_val = (t1 - t2) / (3600 * 24); return tmp; @@ -1567,6 +1737,56 @@ ExprValue date_sub(const std::vector& input) { return ret; } +ExprValue addtime(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 2) { + return ExprValue::Null(); + } + ExprValue arg2 = input[1]; + if (arg2.cast_to(pb::TIME)._u.int32_val == 0) { + return input[0]; + } + ExprValue arg1 = input[0]; + int seconds = time_to_seconds(arg2._u.int32_val); + if (arg1.cast_to(pb::DATETIME)._u.int64_val != 0) { + arg1.cast_to(pb::TIMESTAMP); + arg1._u.int32_val += seconds; + return arg1.cast_to(pb::STRING); + } else { + arg1 = input[0]; + if (arg1.cast_to(pb::TIME)._u.int32_val != 0) { + arg1._u.int32_val = seconds_to_time(time_to_seconds(arg1._u.int32_val) + seconds); + return arg1.cast_to(pb::STRING); + } + } + return ExprValue::Null(); +} + +ExprValue subtime(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 2) { + return ExprValue::Null(); + } + ExprValue arg2 = input[1]; + if (arg2.cast_to(pb::TIME)._u.int32_val == 0) { + return input[0]; + } + ExprValue arg1 = input[0]; + int seconds = time_to_seconds(arg2._u.int32_val); + if (arg1.cast_to(pb::DATETIME)._u.int64_val != 0) { + arg1.cast_to(pb::TIMESTAMP); + arg1._u.int32_val -= seconds; + return arg1.cast_to(pb::STRING); + } else { + arg1 = input[0]; + if (arg1.cast_to(pb::TIME)._u.int32_val != 0) { + arg1._u.int32_val = seconds_to_time(time_to_seconds(arg1._u.int32_val) - seconds); + return arg1.cast_to(pb::STRING); + } + } + return ExprValue::Null(); +} + ExprValue extract(const std::vector& input) { if (input.size() != 2 || input[0].is_null() || input[1].is_null()) { return ExprValue::Null(); @@ -2243,7 +2463,9 @@ ExprValue cast_to_datetime(const std::vector& input) { return ExprValue::Null(); } ExprValue tmp = input[0]; - return tmp.cast_to(pb::DATETIME); + tmp.cast_to(pb::DATETIME); + tmp.set_precision_len(0); + return tmp; } ExprValue cast_to_time(const std::vector& input) { @@ -2286,6 +2508,450 @@ ExprValue cast_to_double(const std::vector& input) { return tmp.cast_to(pb::DOUBLE); } +ExprValue find_in_set(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 2) { + return ExprValue::Null(); + } + std::string target = ExprValue(input[0]).cast_to(pb::STRING).str_val; + std::string set_str = ExprValue(input[1]).cast_to(pb::STRING).str_val; + std::vectorset; + boost::split(set, set_str, boost::is_any_of(",")); + ExprValue res(pb::INT64); + for (int i = 0; i < set.size(); i ++) { + if (set[i] == target) { + res._u.int64_val = i + 1; + break; + } + } + return res; +} + +ExprValue export_set(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 4 && input.size() != 5) { + return ExprValue::Null(); + } + ExprValue num = input[0]; + num.cast_to(pb::INT64); + ExprValue on = input[1]; + on.cast_to(pb::STRING); + ExprValue off = input[2]; + off.cast_to(pb::STRING); + ExprValue sep = input[3]; + sep.cast_to(pb::STRING); + ExprValue len(pb::INT64); + len._u.int64_val = 64; + if (input.size() == 5) { + len = input[4]; + len.cast_to(pb::INT64); + } + if (len._u.int64_val > 64 || len._u.int64_val < 1) { + len._u.int64_val = 64; + } + ExprValue res(pb::STRING); + for (int i = 0; i < len._u.int64_val; i ++) { + if (i != 0) { + res.str_val += sep.str_val; + } + if (num._u.int64_val & 1) { + res.str_val += on.str_val; + } else { + res.str_val += off.str_val; + } + num._u.int64_val >>= 1; + } + return res; +} + +ExprValue to_base64(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue tmp = input[0]; + tmp.cast_to(pb::STRING); + ExprValue res(pb::STRING); + butil::StringPiece sp(tmp.str_val); + butil::Base64Encode(sp, &res.str_val); + return res; +} + +ExprValue from_base64(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue tmp = input[0]; + tmp.cast_to(pb::STRING); + ExprValue res(pb::STRING); + butil::StringPiece sp(tmp.str_val); + butil::Base64Decode(sp, &res.str_val); + return res; +} + +ExprValue make_set(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() < 2) { + return ExprValue::Null(); + } + ExprValue num = input[0]; + num.cast_to(pb::INT64); + ExprValue res(pb::STRING); + bool flag = false; + ExprValue t; + for (size_t i = 0; i < input.size() - 1; ++ i) { + if (num._u.int64_val & 1) { + if (flag) { + res.str_val += ","; + } + t = input[i+1]; + t.cast_to(pb::STRING); + res.str_val += t.str_val; + flag = true; + } + num._u.int64_val >>= 1; + if (num._u.int64_val == 0) { + break; + } + } + return res; +} + +ExprValue oct(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue num = input[0]; + num.cast_to(pb::INT64); + ExprValue res(pb::STRING); + std::stringstream ss; + ss << std::oct << num._u.int64_val; + res.str_val = ss.str(); + return res; +} + +ExprValue hex(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue num = input[0]; + ExprValue res(pb::STRING); + if (num.is_numberic() ){ + num.cast_to(pb::INT64); + std::stringstream ss; + ss << std::hex << num._u.int64_val; + res.str_val = ss.str(); + } else { + num.cast_to(pb::STRING); + char * c = (char *)malloc(2 * num.str_val.size() + 1); + int idx = 0; + for (char ch : num.str_val) { + sprintf(c + idx, "%02X", static_cast(ch)); + idx += 2; + } + c[2 * num.str_val.size()] = 0; + res.str_val = c; + delete c; + } + std::transform(res.str_val.begin(), res.str_val.end(), res.str_val.begin(), ::toupper); + return res; +} + +ExprValue bin(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue num = input[0]; + num.cast_to(pb::INT64); + ExprValue res(pb::STRING); + uint64_t u = static_cast(num._u.int64_val); + if (u == 0) { + res.str_val = "0"; + return res; + } + while (u) { + res.str_val += (u & 1) + '0'; + u >>= 1; + } + std::reverse(res.str_val.begin(), res.str_val.end()); + return res; +} + +ExprValue space(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue arg1 = input[0]; + arg1.cast_to(pb::INT32); + int num = arg1._u.int32_val; + if (num < 1 || num > 1000000) { + return ExprValue::Null(); + } + ExprValue res(pb::STRING); + for (int i = 0; i < num; i ++ ){ + res.str_val += " "; + } + return res; +} + +ExprValue unhex(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue argv = input[0]; + argv.cast_to(pb::STRING); + ExprValue result(pb::STRING); + int num = 0; + bool flag = argv.str_val.size() % 2 == 0; + for (char ch : argv.str_val) { + flag = !flag; + if (ch >= '0' && ch <= '9') { + num = (num << 4) + (ch - '0'); + } else if (ch >= 'A' && ch <= 'F') { + num = (num << 4) + (ch - 'A' + 10); + } else if (ch >= 'a' && ch <= 'f') { + num = (num << 4) + (ch - 'a' + 10); + } else { + return ExprValue::Null(); + } + if (flag) { + result.str_val += static_cast(num); + num = 0; + } + } + if (num != 0) { + result.str_val += static_cast(num); + } + return result; +} + +ExprValue elt(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() <= 1) { + return ExprValue::Null(); + } + ExprValue idx = input[0]; + idx.cast_to(pb::INT64); + int64_t index = idx._u.int64_val; + if (index < 1 || index > input.size() -1) { + return ExprValue::Null(); + } + ExprValue res = input[index]; + return res.cast_to(pb::STRING); +} + +ExprValue char_length(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue argv = input[0]; + argv.cast_to(pb::STRING); + int length = 0; + int bytes = 0; + for (char ch : argv.str_val) { + if ((ch & 0b10000000) == 0b00000000) { // 1 byte + ++length; + bytes = 0; + } else if ((ch & 0b11100000) == 0b11000000) { // 2 bytes + ++length; + bytes = 1; + } else if ((ch & 0b11110000) == 0b11100000) { // 3 bytes + bytes = 2; + } else if ((ch & 0b11111000) == 0b11110000) { // 4 bytes + bytes = 3; + } else if ((ch & 0b11111100) == 0b11111000) { // 5 bytes + bytes = 4; + } else if ((ch & 0b11111110) == 0b11111100) { // 6 bytes + bytes = 5; + } + if (bytes == 0) { + bytes = 1; + } else { + --bytes; + } + } + ExprValue res(pb::INT32); + res._u.int32_val = length; + return res; +} + +ExprValue format(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 2) { + return ExprValue::Null(); + } + ExprValue num = input[0]; + num.cast_to(pb::DOUBLE); + ExprValue pre = input[1]; + pre.cast_to(pb::INT32); + std::ostringstream stream; + stream << std::fixed << std::setprecision(pre._u.int32_val) << num._u.double_val; + std::string str = stream.str(); + int len = str.size(); + int pos = str.find('.'); + if (pos == std::string::npos) { + pos = len; + } + for (int i = pos - 3; i > 0; i -= 3) { + str.insert(i, ","); + } + ExprValue res(pb::STRING); + res.str_val = str; + return res; +} + +ExprValue field(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() < 2) { + return ExprValue::Null(); + } + ExprValue target = input[0]; + target.cast_to(pb::STRING); + ExprValue res(pb::INT32); + for (size_t i = 1; i < input.size(); i ++) { + ExprValue tmp = input[i]; + if (tmp.cast_to(pb::STRING).str_val == target.str_val) { + res._u.int32_val = i; + break; + } + } + return res; +} + +ExprValue quote(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue in = input[0]; + in.cast_to(pb::STRING); + std::string result = "'"; + for (char c : in.str_val) { + if (c == '\'') { + result += "\\'"; + } else if (c == '\\') { + result += "\\\\"; + } else { + result += c; + } + } + result += "'"; + ExprValue res(pb::STRING); + res.str_val = result; + return res; +} + +ExprValue func_char(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() < 1) { + return ExprValue::Null(); + } + ExprValue res(pb::STRING); + for (size_t i = 0; i < input.size(); ++ i) { + ExprValue tmp = input[i]; + if (tmp.is_null()) { + continue; + } + tmp.cast_to(pb::UINT32); + std::string cur; + while (tmp._u.uint32_val) { + cur += static_cast(tmp._u.uint32_val & 255); + tmp._u.uint32_val >>= 8; + } + std::reverse(cur.begin(), cur.end()); + res.str_val += cur; + } + return res; +} + +ExprValue soundex(const std::vector& input) { + INPUT_CHECK_NULL; + if (input.size() != 1) { + return ExprValue::Null(); + } + ExprValue argv = input[0]; + argv.cast_to(pb::STRING); + std::string result; + for (char ch : argv.str_val) { + result += std::tolower(ch); + } + + int len = argv.str_val.length(); + int i = 0; + int size = 0; + char last_char = 0; + std::string code; + while (i < len && size < 3) { + unsigned char c = argv.str_val[i]; + int bytes = 1; + if (c >= 0xc0 && c <= 0xdf) { + bytes = 2; + } else if (c >= 0xe0 && c <= 0xef) { + bytes = 3; + } else if (c >= 0xf0 && c <= 0xf7) { + bytes = 4; + } else if (c >= 0xf8 && c <= 0xfb) { + bytes = 5; + } else if (c >= 0xfc && c <= 0xfd) { + bytes = 6; + } + if (code.empty()) { + if (bytes == 1 && std::tolower(c) >= 'a' and std::tolower(c) <= 'z') { + code = std::toupper(c); + } else if (bytes > 1) { + if (i + bytes < len) { + code = argv.str_val.substr(i, bytes); + } + } + } else if (bytes == 1) { + char low = std::tolower(c); + char cur_c = '0'; + switch (low) { + case 'b': case 'f': case 'p': case 'v': + cur_c = '1'; + break; + case 'c': case 'g': case 'j': case 'k': case 'q': case 's': case 'x': case 'z': + cur_c= '2'; + break; + case 'd': case 't': + cur_c = '3'; + break; + case 'l': + cur_c = '4'; + break; + case 'm': case 'n': + cur_c = '5'; + break; + case 'r': + cur_c = '6'; + break; + } + if (cur_c != '0' && cur_c != last_char) { + size += 1; + code += cur_c; + last_char = cur_c; + } + } + i += bytes; + } + if (!code.empty()) { + while (size < 3) { + code += '0'; + size ++; + } + } + ExprValue res(pb::STRING); + res.str_val = code; + return res; +} + } /* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/src/logical_plan/ddl_planner.cpp b/src/logical_plan/ddl_planner.cpp index 03e33969..bfb5b1c4 100644 --- a/src/logical_plan/ddl_planner.cpp +++ b/src/logical_plan/ddl_planner.cpp @@ -219,13 +219,18 @@ int DDLPlanner::plan() { return 0; } -int DDLPlanner::add_column_def(pb::SchemaInfo& table, parser::ColumnDef* column, bool is_unique_indicator) { +int DDLPlanner::add_column_def(pb::SchemaInfo& table, parser::ColumnDef* column, + bool is_unique_indicator, const std::string& old_field_name) { pb::FieldInfo* field = table.add_fields(); if (column->name == nullptr || column->name->name.empty()) { DB_WARNING("column_name is empty"); return -1; } field->set_field_name(column->name->name.value); + if (old_field_name != "") { + field->set_field_name(old_field_name); + field->set_new_field_name(column->name->name.value); + } if (column->type == nullptr) { DB_WARNING("data_type is empty for column: %s", column->name->name.value); return -1; @@ -236,7 +241,7 @@ int DDLPlanner::add_column_def(pb::SchemaInfo& table, parser::ColumnDef* column, DB_WARNING("data_type is unsupported: %s", column->name->name.value); return -1; } - if (data_type == pb::FLOAT || data_type == pb::DOUBLE) { + if (data_type == pb::FLOAT || data_type == pb::DOUBLE || pb::DATETIME) { if (column->type->total_len != -1) { field->set_float_total_len((int8_t)column->type->total_len); } @@ -269,7 +274,7 @@ int DDLPlanner::add_column_def(pb::SchemaInfo& table, parser::ColumnDef* column, } else if (col_option->type == parser::COLUMN_OPT_DEFAULT_VAL && col_option->expr != nullptr) { if (col_option->expr->to_string() == "(current_timestamp())") { if (is_current_timestamp_specic(data_type)) { - field->set_default_literal(ExprValue::Now().get_string()); + field->set_default_literal(ExprValue::Now(field->float_precision_len()).get_string()); field->set_default_value("(current_timestamp())"); continue; } else { @@ -820,43 +825,44 @@ int DDLPlanner::parse_create_table(pb::SchemaInfo& table) { DB_WARNING("parse create table json comments error [code:%d][%s]", code, value); return -1; - } - auto iter = root.FindMember("segment_type"); - if (iter != root.MemberEnd()) { - std::string segment_type = iter->value.GetString(); - pb::SegmentType pb_segment_type = pb::S_DEFAULT; - SegmentType_Parse(segment_type, &pb_segment_type); - index->set_segment_type(pb_segment_type); - } - iter = root.FindMember("storage_type"); - pb::StorageType pb_storage_type = pb::ST_ARROW; - if (iter != root.MemberEnd()) { - std::string storage_type = iter->value.GetString(); - StorageType_Parse(storage_type, &pb_storage_type); - } - if (!is_fulltext_type_constraint(pb_storage_type, has_arrow_fulltext, has_pb_fulltext)) { - DB_WARNING("fulltext has two types : pb&arrow"); - return -1; - } - index->set_storage_type(pb_storage_type); - iter = root.FindMember("vector_description"); - if (iter != root.MemberEnd()) { - index->set_vector_description(iter->value.GetString()); - } - iter = root.FindMember("dimension"); - if (iter != root.MemberEnd()) { - index->set_dimension(iter->value.GetInt()); - } - iter = root.FindMember("nprobe"); - if (iter != root.MemberEnd()) { - index->set_nprobe(iter->value.GetInt()); - } - iter = root.FindMember("metric_type"); - pb::MetricType metric_type = pb::METRIC_L2; - if (iter != root.MemberEnd()) { - std::string metric_type_str = iter->value.GetString(); - MetricType_Parse(metric_type_str, &metric_type); - index->set_metric_type(metric_type); + } else if (root.IsObject()) { + auto iter = root.FindMember("segment_type"); + if (iter != root.MemberEnd()) { + std::string segment_type = iter->value.GetString(); + pb::SegmentType pb_segment_type = pb::S_DEFAULT; + SegmentType_Parse(segment_type, &pb_segment_type); + index->set_segment_type(pb_segment_type); + } + iter = root.FindMember("storage_type"); + pb::StorageType pb_storage_type = pb::ST_ARROW; + if (iter != root.MemberEnd()) { + std::string storage_type = iter->value.GetString(); + StorageType_Parse(storage_type, &pb_storage_type); + } + if (!is_fulltext_type_constraint(pb_storage_type, has_arrow_fulltext, has_pb_fulltext)) { + DB_WARNING("fulltext has two types : pb&arrow"); + return -1; + } + index->set_storage_type(pb_storage_type); + iter = root.FindMember("vector_description"); + if (iter != root.MemberEnd()) { + index->set_vector_description(iter->value.GetString()); + } + iter = root.FindMember("dimension"); + if (iter != root.MemberEnd()) { + index->set_dimension(iter->value.GetInt()); + } + iter = root.FindMember("nprobe"); + if (iter != root.MemberEnd()) { + index->set_nprobe(iter->value.GetInt()); + } + iter = root.FindMember("metric_type"); + pb::MetricType metric_type = pb::METRIC_L2; + if (iter != root.MemberEnd()) { + std::string metric_type_str = iter->value.GetString(); + MetricType_Parse(metric_type_str, &metric_type); + index->set_metric_type(metric_type); + } } } catch (...) { DB_WARNING("parse create table json comments error [%s]", value); @@ -915,6 +921,12 @@ int DDLPlanner::parse_create_table(pb::SchemaInfo& table) { continue; // return -1; } + if (!root.IsObject()) { + // comment = '1' ，json会解析成功，但FindMember会出core + table.set_comment(option->str_value.value); + DB_WARNING("parse create table comment as mysql-format [%s]", option->str_value.value); + continue; + } auto json_iter = root.FindMember("resource_tag"); if (json_iter != root.MemberEnd()) { table_resource_tag = json_iter->value.GetString(); @@ -1353,7 +1365,19 @@ int DDLPlanner::parse_alter_table(pb::MetaManagerRequest& alter_request) { } table->set_table_name(stmt->table_name->table.value); table->set_namespace_name(_ctx->user_info->namespace_); - if (stmt->alter_specs.size() > 1 || stmt->alter_specs.size() == 0) { + + // 支持Alter Table Add Column Field1, Add Column Field2;用法 + bool has_same_spec_type = true; + if (stmt->alter_specs.size() > 1) { + for (int i = 0; i < stmt->alter_specs.size(); ++i) { + parser::AlterTableSpec* s = stmt->alter_specs[i]; + if (s == nullptr || s->spec_type != parser::ALTER_SPEC_ADD_COLUMN) { + has_same_spec_type = false; + break; + } + } + } + if ((stmt->alter_specs.size() > 1 && !has_same_spec_type) || stmt->alter_specs.size() == 0) { _ctx->stat_info.error_code = ER_ALTER_OPERATION_NOT_SUPPORTED;; _ctx->stat_info.error_msg << "Alter with multiple alter_specifications is not supported in this version"; return -1; @@ -1452,6 +1476,23 @@ int DDLPlanner::parse_alter_table(pb::MetaManagerRequest& alter_request) { return -1; } } + if (has_same_spec_type) { + for (int i = 1; i < stmt->alter_specs.size(); ++i) { + parser::AlterTableSpec* s = stmt->alter_specs[i]; + int column_len = s->new_columns.size(); + for (int idx = 0; idx < column_len; ++idx) { + parser::ColumnDef* column = s->new_columns[idx]; + if (column == nullptr) { + DB_WARNING("column is nullptr"); + return -1; + } + if (0 != add_column_def(*table, column, s->is_unique_indicator)) { + DB_WARNING("add column to table failed."); + return -1; + } + } + } + } if (table->indexs_size() != 0) { _ctx->stat_info.error_code = ER_ALTER_OPERATION_NOT_SUPPORTED; _ctx->stat_info.error_msg << "add table column with index is not supported in this version"; @@ -1490,6 +1531,31 @@ int DDLPlanner::parse_alter_table(pb::MetaManagerRequest& alter_request) { _ctx->stat_info.error_msg << "modify table column with index is not supported"; return -1; } + } else if (spec->spec_type == parser::ALTER_SPEC_CHANGE_COLUMN && spec->new_columns.size() > 0) { + alter_request.set_op_type(pb::OP_MODIFY_FIELD); + int column_len = spec->new_columns.size(); + if (column_len != 1) { + _ctx->stat_info.error_code = ER_ALTER_OPERATION_NOT_SUPPORTED;; + _ctx->stat_info.error_msg << "unsupported multi schema change"; + return -1; + } + std::string old_field_name = spec->column_name.value; + for (int idx = 0; idx < column_len; ++idx) { + parser::ColumnDef* column = spec->new_columns[idx]; + if (column == nullptr) { + DB_WARNING("column is nullptr"); + return -1; + } + if (0 != add_column_def(*table, column, spec->is_unique_indicator, old_field_name)) { + DB_WARNING("add column to table failed."); + return -1; + } + } + if (table->indexs_size() != 0) { + _ctx->stat_info.error_code = ER_ALTER_OPERATION_NOT_SUPPORTED;; + _ctx->stat_info.error_msg << "change table column with index is not supported"; + return -1; + } } else if (spec->spec_type == parser::ALTER_SPEC_RENAME_COLUMN) { alter_request.set_op_type(pb::OP_RENAME_FIELD); pb::FieldInfo* field = table->add_fields(); @@ -1913,23 +1979,24 @@ int DDLPlanner::add_constraint_def(pb::SchemaInfo& table, parser::Constraint* co rapidjson::ParseErrorCode code = root.GetParseError(); DB_WARNING("parse create table json comments error [code:%d][%s]", code, value); - return -1; + // return -1; + } else if (root.IsObject()){ + auto json_iter = root.FindMember("segment_type"); + if (json_iter != root.MemberEnd()) { + std::string segment_type = json_iter->value.GetString(); + pb::SegmentType pb_segment_type = pb::S_DEFAULT; + SegmentType_Parse(segment_type, &pb_segment_type); + index->set_segment_type(pb_segment_type); + } + + auto storage_type_iter = root.FindMember("storage_type"); + pb::StorageType pb_storage_type = pb::ST_ARROW; + if (storage_type_iter != root.MemberEnd()) { + std::string storage_type = storage_type_iter->value.GetString(); + StorageType_Parse(storage_type, &pb_storage_type); + } + index->set_storage_type(pb_storage_type); } - auto json_iter = root.FindMember("segment_type"); - if (json_iter != root.MemberEnd()) { - std::string segment_type = json_iter->value.GetString(); - pb::SegmentType pb_segment_type = pb::S_DEFAULT; - SegmentType_Parse(segment_type, &pb_segment_type); - index->set_segment_type(pb_segment_type); - } - - auto storage_type_iter = root.FindMember("storage_type"); - pb::StorageType pb_storage_type = pb::ST_ARROW; - if (storage_type_iter != root.MemberEnd()) { - std::string storage_type = storage_type_iter->value.GetString(); - StorageType_Parse(storage_type, &pb_storage_type); - } - index->set_storage_type(pb_storage_type); } catch (...) { DB_WARNING("parse create table json comments error [%s]", value); return -1; diff --git a/src/logical_plan/insert_planner.cpp b/src/logical_plan/insert_planner.cpp index b0b42ee2..8bd0bccf 100644 --- a/src/logical_plan/insert_planner.cpp +++ b/src/logical_plan/insert_planner.cpp @@ -164,7 +164,7 @@ int InsertPlanner::parse_kv_list() { DB_WARNING("invalid field name in: %s", full_name.c_str()); return -1; } - auto slot = get_scan_ref_slot(alias_name, + auto slot = get_scan_ref_slot(alias_name, field_info->table_id, field_info->id, field_info->type); _update_slots.emplace_back(slot); update_field_ids.insert(field_info->id); @@ -181,7 +181,7 @@ int InsertPlanner::parse_kv_list() { node->set_num_children(0); node->set_node_type(pb::STRING_LITERAL); node->set_col_type(pb::STRING); - node->mutable_derive_node()->set_string_val(ExprValue::Now().get_string()); + node->mutable_derive_node()->set_string_val(ExprValue::Now(field_info->float_precision_len).get_string()); } } _update_values.emplace_back(value_expr); @@ -196,7 +196,7 @@ int InsertPlanner::parse_kv_list() { node->set_num_children(0); node->set_node_type(pb::STRING_LITERAL); node->set_col_type(pb::STRING); - node->mutable_derive_node()->set_string_val(ExprValue::Now().get_string()); + node->mutable_derive_node()->set_string_val(ExprValue::Now(field.float_precision_len).get_string()); auto slot = get_scan_ref_slot(tbl_ptr->name, field.table_id, field.id, field.type); _update_slots.emplace_back(slot); _update_values.emplace_back(value_expr); @@ -389,7 +389,7 @@ int InsertPlanner::fill_record_field(const parser::ExprNode* parser_expr, SmartR return -1; } } - value.float_precision_len = field.float_precision_len; + value.set_precision_len(field.float_precision_len); if (0 != record->set_value(record->get_field_by_tag(field.id), value)) { DB_WARNING("fill insert value failed"); return -1; diff --git a/src/logical_plan/logical_planner.cpp b/src/logical_plan/logical_planner.cpp index debc6c2a..7a0b9655 100644 --- a/src/logical_plan/logical_planner.cpp +++ b/src/logical_plan/logical_planner.cpp @@ -2410,7 +2410,7 @@ ScanTupleInfo* LogicalPlanner::get_scan_tuple(const std::string& table_name, int } pb::SlotDescriptor& LogicalPlanner::get_scan_ref_slot( - const std::string& alias_name, int64_t table, + const std::string& alias_name, int64_t table, int32_t field, pb::PrimitiveType type) { ScanTupleInfo* tuple_info = get_scan_tuple(alias_name, table); _ctx->current_tuple_ids.emplace(tuple_info->tuple_id); @@ -2433,7 +2433,7 @@ pb::SlotDescriptor& LogicalPlanner::get_scan_ref_slot( return *slot_desc; } -pb::SlotDescriptor& LogicalPlanner::get_values_ref_slot(int64_t table, +pb::SlotDescriptor& LogicalPlanner::get_values_ref_slot(int64_t table, int32_t field, pb::PrimitiveType type) { auto& tuple_info = _values_tuple_info; if (tuple_info.tuple_id == -1) { diff --git a/src/logical_plan/update_planner.cpp b/src/logical_plan/update_planner.cpp index 451e9308..607bdc05 100644 --- a/src/logical_plan/update_planner.cpp +++ b/src/logical_plan/update_planner.cpp @@ -227,7 +227,7 @@ int UpdatePlanner::parse_kv_list() { node->set_num_children(0); node->set_node_type(pb::STRING_LITERAL); node->set_col_type(pb::STRING); - node->mutable_derive_node()->set_string_val(ExprValue::Now().get_string()); + node->mutable_derive_node()->set_string_val(ExprValue::Now(field_info->float_precision_len).get_string()); } } else if (value_expr.nodes(0).node_type() == pb::NULL_LITERAL && !field_info->can_null) { @@ -249,7 +249,7 @@ int UpdatePlanner::parse_kv_list() { node->set_num_children(0); node->set_node_type(pb::STRING_LITERAL); node->set_col_type(pb::STRING); - node->mutable_derive_node()->set_string_val(ExprValue::Now().get_string()); + node->mutable_derive_node()->set_string_val(ExprValue::Now(field.float_precision_len).get_string()); auto slot = get_scan_ref_slot(table_info.name, field.table_id, field.id, field.type); _update_slots.push_back(slot); _update_values.push_back(value_expr); diff --git a/src/meta_server/table_manager.cpp b/src/meta_server/table_manager.cpp index a31c6a0f..16c6e0bb 100644 --- a/src/meta_server/table_manager.cpp +++ b/src/meta_server/table_manager.cpp @@ -2052,7 +2052,9 @@ void TableManager::modify_field(const pb::MetaManagerRequest& request, return; } pb::SchemaInfo mem_schema_pb = table_mem.schema_pb; + std::unordered_map id_new_field_map; std::vector drop_field_names; + std::unordered_map add_field_id_map; for (auto& field : request.table_info().fields()) { std::string field_name = field.field_name(); if (_table_info_map[table_id].field_id_map.count(field_name) == 0) { @@ -2071,7 +2073,7 @@ void TableManager::modify_field(const pb::MetaManagerRequest& request, for (auto& mem_field : *mem_schema_pb.mutable_fields()) { if (mem_field.field_name() == field_name) { if (field.has_mysql_type()) { - if (!check_field_is_compatible_type(mem_field.mysql_type(), field.mysql_type())) { + if (!check_field_is_compatible_type(mem_field, field)) { // TODO 数据类型变更仅支持meta-only, 有损变更待支持 IF_DONE_SET_RESPONSE(done, pb::INPUT_PARAM_ERROR, "modify field data type unsupported lossy changes"); @@ -2115,7 +2117,19 @@ void TableManager::modify_field(const pb::MetaManagerRequest& request, //修改default value需谨慎，会导致存储null数据的值变成新默认值 mem_field.set_default_value(field.default_value()); } - } + } + if (field.has_float_total_len()) { + mem_field.set_float_total_len(field.float_total_len()); + } + if (field.has_float_precision_len()) { + mem_field.set_float_precision_len(field.float_precision_len()); + } + if (field.has_new_field_name()) { + mem_field.set_field_name(field.new_field_name()); + id_new_field_map[field_id] = field.new_field_name(); + add_field_id_map[field.new_field_name()] = field_id; + drop_field_names.push_back(field.field_name()); + } } } } @@ -2128,6 +2142,8 @@ void TableManager::modify_field(const pb::MetaManagerRequest& request, set_table_pb(mem_schema_pb); std::vector schema_infos{mem_schema_pb}; put_incremental_schemainfo(apply_index, schema_infos); + drop_field_mem(table_id, drop_field_names); + add_field_mem(table_id, add_field_id_map); IF_DONE_SET_RESPONSE(done, pb::SUCCESS, "success"); DB_NOTICE("modify field type success, request:%s", request.ShortDebugString().c_str()); } diff --git a/src/physical_plan/index_selector.cpp b/src/physical_plan/index_selector.cpp index c1bf3558..e2eac8ff 100644 --- a/src/physical_plan/index_selector.cpp +++ b/src/physical_plan/index_selector.cpp @@ -23,6 +23,20 @@ namespace baikaldb { using namespace range; + +int get_field_hit_type_weight(RangeType &ty) { + if (ty == EQ || ty == LIKE_EQ) { + return 10; + } + if (ty == IN) { + return 9; + } + if (ty == RANGE || ty == LIKE_PREFIX) { + return 5; + } + return 0; +} + int IndexSelector::analyze(QueryContext* ctx) { ExecNode* root = ctx->root; _ctx = ctx; @@ -156,6 +170,7 @@ void IndexSelector::hit_row_field_range(ExprNode* expr, for (auto& pair : slots) { field_ids.push_back(pair.second->field_id()); } + RangeType tmp_type; switch (expr->node_type()) { case pb::FUNCTION_CALL: { int32_t fn_op = static_cast(expr)->fn().fn_op(); @@ -182,6 +197,10 @@ void IndexSelector::hit_row_field_range(ExprNode* expr, if (field_range_map[field_id].left.size() > 0) { return; } + tmp_type = RANGE; + if (get_field_hit_type_weight(field_range_map[field_id].type) > get_field_hit_type_weight(tmp_type)) { + return; + } for (auto pair : values) { field_range_map[field_id].left.push_back(pair.second[0]); } @@ -201,6 +220,10 @@ void IndexSelector::hit_row_field_range(ExprNode* expr, if (field_range_map[field_id].right.size() > 0) { return; } + tmp_type = RANGE; + if (get_field_hit_type_weight(field_range_map[field_id].type) > get_field_hit_type_weight(tmp_type)) { + return; + } for (auto pair : values) { field_range_map[field_id].right.push_back(pair.second[0]); } @@ -425,6 +448,7 @@ void IndexSelector::hit_field_range(ExprNode* expr, *index_predicate_is_null = is_index_predicate(expr); return; } + RangeType tmp_type; switch (expr->node_type()) { case pb::FUNCTION_CALL: { int32_t fn_op = static_cast(expr)->fn().fn_op(); @@ -436,6 +460,10 @@ void IndexSelector::hit_field_range(ExprNode* expr, return; case parser::FT_GE: case parser::FT_GT: + tmp_type = RANGE; + if (get_field_hit_type_weight(field_range_map[field_id].type) > get_field_hit_type_weight(tmp_type)) { + return; + } if (field_range_map[field_id].left.size() > 0){ field_range_map[field_id].left.clear(); field_range_map[field_id].left_row_field_ids.clear(); @@ -448,6 +476,10 @@ void IndexSelector::hit_field_range(ExprNode* expr, return; case parser::FT_LE: case parser::FT_LT: + tmp_type = RANGE; + if (get_field_hit_type_weight(field_range_map[field_id].type) > get_field_hit_type_weight(tmp_type)) { + return; + } if(field_range_map[field_id].right.size() > 0){ field_range_map[field_id].right.clear(); field_range_map[field_id].right_row_field_ids.clear(); @@ -463,11 +495,15 @@ void IndexSelector::hit_field_range(ExprNode* expr, } } case pb::IN_PREDICATE: { - if (field_range_map[field_id].type != NONE && + if (field_range_map[field_id].type == IN && field_range_map[field_id].eq_in_values.size() <= values.size()) { // 1个字段对应多个in preds时 例如: a in ("a1", "a2") and (a, b) in (("a1","b1")),取数量少的pred return; } + tmp_type = IN; + if (get_field_hit_type_weight(field_range_map[field_id].type) > get_field_hit_type_weight(tmp_type)) { + return ; + } field_range_map[field_id].eq_in_values = values; field_range_map[field_id].conditions.clear(); field_range_map[field_id].conditions.insert(expr); @@ -479,6 +515,20 @@ void IndexSelector::hit_field_range(ExprNode* expr, return; } case pb::LIKE_PREDICATE: { + bool is_eq = false; + bool is_prefix = false; + ExprValue prefix_value(pb::STRING); + static_cast(expr)->hit_index(&is_eq, &is_prefix, &(prefix_value.str_val)); + if (is_eq) { + tmp_type = LIKE_EQ; + } else if (is_prefix) { + tmp_type = LIKE_PREFIX; + } else { + tmp_type = LIKE; + } + if (get_field_hit_type_weight(field_range_map[field_id].type) > get_field_hit_type_weight(tmp_type)) { + return; + } range::FieldRange fulltext_and_range; field_range_map[field_id].like_values.push_back(values[0]); fulltext_and_range.like_values.push_back(values[0]); @@ -488,10 +538,6 @@ void IndexSelector::hit_field_range(ExprNode* expr, } field_range_map[field_id].conditions.insert(expr); fulltext_and_range.conditions.insert(expr); - bool is_eq = false; - bool is_prefix = false; - ExprValue prefix_value(pb::STRING); - static_cast(expr)->hit_index(&is_eq, &is_prefix, &(prefix_value.str_val)); if (is_eq) { field_range_map[field_id].eq_in_values.push_back(prefix_value); fulltext_and_range.eq_in_values.push_back(prefix_value); diff --git a/src/protocol/show_helper.cpp b/src/protocol/show_helper.cpp index d6997e48..31eb773b 100644 --- a/src/protocol/show_helper.cpp +++ b/src/protocol/show_helper.cpp @@ -914,10 +914,7 @@ bool ShowHelper::_show_create_table(const SmartSocket& client, const std::vector continue; } oss << " " << "`" << field.short_name << "` "; - oss << to_mysql_type_full_string[field.type]; - if ((field.type == pb::FLOAT || field.type == pb::DOUBLE) && field.float_total_len != -1 && field.float_precision_len != -1) { - oss << "(" << field.float_total_len << "," << field.float_precision_len << ")"; - } + oss << to_mysql_type_full_string(field.type, field.float_total_len, field.float_precision_len); oss << " "; oss << (field.can_null ? "NULL " : "NOT NULL "); if (!field.default_expr_value.is_null()) { @@ -1736,7 +1733,7 @@ bool ShowHelper::_show_full_columns(const SmartSocket& client, const std::vector } } row.emplace_back(split_vec[split_vec.size() - 1]); - row.emplace_back(to_mysql_type_full_string(field.type)); + row.emplace_back(to_mysql_type_full_string(field.type, field.float_total_len, field.float_precision_len)); row.emplace_back("NULL"); row.emplace_back(field.can_null ? "YES" : "NO"); if (field_index.count(field.id) == 0) { @@ -1872,7 +1869,7 @@ bool ShowHelper::_show_columns(const SmartSocket& client, const std::vectorcharset_name = "utf8"; sock->charset_num = 33; } else { - DB_TRACE_CLIENT(sock, "unknown charset num: %u, charset will be set as gbk.", - charset_num); - sock->charset_name = "gbk"; - sock->charset_num = 28; + DB_TRACE_CLIENT(sock, "unknown charset num: %u, charset will be set as %s.", + FLAGS_default_charset_number, FLAGS_default_charset_name.c_str()); + sock->charset_name = FLAGS_default_charset_name; + sock->charset_num = FLAGS_default_charset_number; } off += 23;