兼容性提升 (#238)

* feat: support datetime(n) * feat: support find_in_set * feat: field hit range weight * feat: 兼容mysql索引语法 * feat: 补齐时间类函数 * feat: 兼容mysql建表语法 * feat: add FLAGS_max_select_region_count for select sql * feat: support change column * feat: default charset utf8 * feat: support multiple alter_specifications for add column * feat: store请求部分错误码不重试 * feat: index.short_name保留大小写 * feat: support drop index name on table * feat: 补齐字符串类函数 * fix: for code review * fix: for code review * fix: for code review --------- Co-authored-by: yuzhong.chen <[email protected]>
baidu · Apr 19, 2024 · 6c32c7b · 6c32c7b
1 parent 6d563e5
commit 6c32c7b
Show file tree

Hide file tree

Showing 28 changed files with 1,402 additions and 207 deletions.
diff --git a/include/common/datetime.h b/include/common/datetime.h
@@ -22,7 +22,7 @@ extern time_t str_to_timestamp(const char* str_time);
 
 // encode DATETIME to string format
 // ref: https://dev.mysql.com/doc/internals/en/date-and-time-data-type-representation.html
-extern std::string datetime_to_str(uint64_t datetime);
+extern std::string datetime_to_str(uint64_t datetime, int precision_len = 0);
 extern uint64_t str_to_datetime(const char* str_time, bool* is_full_datetime = nullptr);
 
 extern time_t datetime_to_timestamp(uint64_t datetime);
@@ -41,6 +41,7 @@ extern int32_t datetime_to_time(uint64_t datetime);
 extern uint64_t time_to_datetime(int32_t time);
 extern std::string time_to_str(int32_t time);
 extern int32_t str_to_time(const char* str_time);
+extern int32_t time_to_seconds(int32_t time);
 extern int32_t seconds_to_time(int32_t seconds);
 struct DateTime;
 extern uint64_t bin_date_to_datetime(DateTime time_struct);

diff --git a/include/common/expr_value.h b/include/common/expr_value.h
@@ -31,7 +31,7 @@ namespace baikaldb {
 
 struct ExprValue {
     pb::PrimitiveType type;
-    int32_t float_precision_len;
+    int32_t float_precision_len = -1; // -1代表未定义精度，保留原值的最大精度。0代表不保留小数点部分
     union {
         bool bool_val;
         int8_t int8_val;
@@ -50,7 +50,7 @@ struct ExprValue {
 
     explicit ExprValue(pb::PrimitiveType type_ = pb::NULL_TYPE) : type(type_) {
         _u.int64_val = 0;
-        float_precision_len = 0;
+        float_precision_len = -1;
         if (type_ == pb::BITMAP) {
             _u.bitmap = new(std::nothrow) Roaring();
         } else if (type_ == pb::TDIGEST) {
@@ -124,7 +124,7 @@ struct ExprValue {
     }
     explicit ExprValue(const pb::ExprValue& value) {
         type = value.type();
-        float_precision_len = 0;
+        float_precision_len = -1;
         switch (type) {
             case pb::BOOL:
                 _u.bool_val = value.bool_val();
@@ -187,7 +187,7 @@ struct ExprValue {
 
     explicit ExprValue(pb::PrimitiveType primitive_type, const std::string& value_str) {
         type = pb::STRING;
-        float_precision_len = 0;
+        float_precision_len = -1;
         str_val = value_str;
         if (primitive_type == pb::STRING 
             || primitive_type == pb::HEX 
@@ -624,7 +624,7 @@ struct ExprValue {
             case pb::TDIGEST:
                 return str_val;
             case pb::DATETIME:
-                return datetime_to_str(_u.uint64_val);
+                return datetime_to_str(_u.uint64_val, float_precision_len);
             case pb::TIME:
                 return time_to_str(_u.int32_val);
             case pb::TIMESTAMP:
@@ -860,14 +860,18 @@ struct ExprValue {
         return ret;
     }
     // 默认不带us
-    static ExprValue Now(int precision = 0) {
+    static ExprValue Now(int32_t precision = 0) {
         ExprValue tmp(pb::TIMESTAMP);
         tmp._u.uint32_val = time(NULL);
         tmp.cast_to(pb::DATETIME);
-        if (precision == 6) {
+
+        if (precision > 0 and precision <= 6) {
             timeval tv;
             gettimeofday(&tv, NULL);
             tmp._u.uint64_val |= tv.tv_usec;
+            tmp.set_precision_len(precision);
+        } else {
+            tmp.set_precision_len(0);
         }
         return tmp;
     }
@@ -883,7 +887,7 @@ struct ExprValue {
         ExprValue ret(pb::UINT64);
         return ret;
     }
-    static ExprValue UTC_TIMESTAMP() {
+    static ExprValue UTC_TIMESTAMP(int32_t precision = 0) {
         // static int UTC_OFFSET = 8 * 60 * 60;
         time_t current_time;
         struct tm timeinfo;
@@ -896,6 +900,11 @@ struct ExprValue {
         timeval tv;
         gettimeofday(&tv, NULL);
         tmp._u.uint64_val |= tv.tv_usec;
+        if (precision >=0 && precision <= 6) {
+            tmp.set_precision_len(precision);
+        } else {
+            tmp.set_precision_len(0);
+        }
         return tmp;
     }
     // For string, end-self
@@ -924,6 +933,25 @@ struct ExprValue {
             return ev._u.uint64_val;
         }
     };
+
+    void dt_cast_len() {
+         if (float_precision_len >= 0 and float_precision_len <= 6) {
+             uint64_t carry = std::pow(10, (6 - float_precision_len));
+             uint64_t oldmic = _u.uint64_val & 0xffffff;
+             uint64_t newmic = ::round(oldmic / carry) * carry;
+             _u.uint64_val = (_u.uint64_val & ~0xffffff) | newmic;
+         }
+    }
+
+    void set_precision_len(int32_t precision) {
+        if (float_precision_len == precision) {
+            return;
+        }
+        float_precision_len = precision;
+        if (type == pb::DATETIME) {
+            dt_cast_len();
+        }
+    }
 };
 
 using ExprValueFlatSet = butil::FlatSet<ExprValue, ExprValue::HashFunction>;

diff --git a/include/common/type_utils.h b/include/common/type_utils.h
@@ -392,7 +392,8 @@ inline std::string to_mysql_type_string(pb::PrimitiveType type) {
     }
 }
 
-inline std::string to_mysql_type_full_string(pb::PrimitiveType type) {
+inline std::string to_mysql_type_full_string(pb::PrimitiveType type,
+        int32_t float_total_len = -1, int32_t float_precision_len = -1) {
     switch (type) {
         case pb::BOOL:
             return "tinyint(3)";
@@ -413,12 +414,23 @@ inline std::string to_mysql_type_full_string(pb::PrimitiveType type) {
         case pb::UINT64:
             return "bigint(21) unsigned";
         case pb::FLOAT:
+            if (float_total_len != -1 && float_precision_len != -1) {
+                return "float(" + std::to_string(float_total_len) + "," + std::to_string(float_precision_len) + ")";
+            }
             return "float";
         case pb::DOUBLE:
+            if (float_total_len != -1 && float_precision_len != -1) {
+                return "double(" + std::to_string(float_total_len) + "," + std::to_string(float_precision_len) + ")";
+            }
             return "double";
         case pb::STRING:
             return "varchar(1024)";
         case pb::DATETIME:
+            if (float_precision_len == -1) {
+               return "datetime(6)";
+            } else if (float_precision_len > 0 && float_precision_len <= 6) {
+               return "datetime(" + std::to_string(float_precision_len) + ")";
+            }
             return "datetime";
         case pb::DATE:
             return "date";

diff --git a/include/expr/internal_functions.h b/include/expr/internal_functions.h
@@ -66,18 +66,45 @@ ExprValue lpad(const std::vector<ExprValue>& input);
 ExprValue rpad(const std::vector<ExprValue>& input);
 ExprValue instr(const std::vector<ExprValue>& input);
 ExprValue json_extract(const std::vector<ExprValue>& input);
+ExprValue export_set(const std::vector<ExprValue>& input);
+ExprValue to_base64(const std::vector<ExprValue>& input);
+ExprValue from_base64(const std::vector<ExprValue>& input);
+ExprValue make_set(const std::vector<ExprValue>& input);
+ExprValue oct(const std::vector<ExprValue>& input);
+ExprValue hex(const std::vector<ExprValue>& input);
+ExprValue unhex(const std::vector<ExprValue>& input);
+ExprValue bin(const std::vector<ExprValue>& input);
+ExprValue space(const std::vector<ExprValue>& input);
+ExprValue elt(const std::vector<ExprValue>& input);
+ExprValue char_length(const std::vector<ExprValue>& input);
+ExprValue format(const std::vector<ExprValue>& input);
+ExprValue field(const std::vector<ExprValue>& input);
+ExprValue quote(const std::vector<ExprValue>& input);
+ExprValue func_char(const std::vector<ExprValue>& input);
+ExprValue soundex(const std::vector<ExprValue>& input);
+
 
 // datetime functions
 ExprValue unix_timestamp(const std::vector<ExprValue>& input);
 ExprValue from_unixtime(const std::vector<ExprValue>& input);
 ExprValue now(const std::vector<ExprValue>& input);
 ExprValue utc_timestamp(const std::vector<ExprValue>& input);
+ExprValue utc_date(const std::vector<ExprValue>& input);
+ExprValue utc_time(const std::vector<ExprValue>& input);
+ExprValue minute(const std::vector<ExprValue>& input);
+ExprValue second(const std::vector<ExprValue>& input);
+ExprValue microsecond(const std::vector<ExprValue>& input);
+ExprValue func_time(const std::vector<ExprValue>& input);
+ExprValue func_quarter(const std::vector<ExprValue>& input);
+ExprValue period_diff(const std::vector<ExprValue>& input);
+ExprValue period_add(const std::vector<ExprValue>& input);
 ExprValue date_format(const std::vector<ExprValue>& input);
 ExprValue str_to_date(const std::vector<ExprValue>& input);
 ExprValue time_format(const std::vector<ExprValue>& input);
 ExprValue convert_tz(const std::vector<ExprValue>& input);
 ExprValue timediff(const std::vector<ExprValue>& input);
 ExprValue timestampdiff(const std::vector<ExprValue>& input);
+ExprValue timestampadd(const std::vector<ExprValue>& input);
 ExprValue curdate(const std::vector<ExprValue>& input);
 ExprValue current_date(const std::vector<ExprValue>& input);
 ExprValue curtime(const std::vector<ExprValue>& input);
@@ -106,6 +133,11 @@ ExprValue weekday(const std::vector<ExprValue>& input);
 ExprValue extract(const std::vector<ExprValue>& input);
 ExprValue tso_to_timestamp(const std::vector<ExprValue>& input);
 ExprValue timestamp_to_tso(const std::vector<ExprValue>& input);
+ExprValue to_days(const std::vector<ExprValue>& input);
+ExprValue to_seconds(const std::vector<ExprValue>& input);
+ExprValue addtime(const std::vector<ExprValue>& input);
+ExprValue subtime(const std::vector<ExprValue>& input);
+
 // hll functions
 ExprValue hll_add(const std::vector<ExprValue>& input);
 ExprValue hll_merge(const std::vector<ExprValue>& input);
@@ -161,6 +193,7 @@ ExprValue tdigest_location(const std::vector<ExprValue>& input);
 // other
 ExprValue version(const std::vector<ExprValue>& input);
 ExprValue last_insert_id(const std::vector<ExprValue>& input);
+ExprValue find_in_set(const std::vector<ExprValue>& input);
 //transfer (latitude A, longitude A), (latitude B, longitude B) to distance of A to B (m)
 ExprValue point_distance(const std::vector<ExprValue>& input);
 ExprValue cast_to_date(const std::vector<ExprValue>& inpt);

diff --git a/include/logical_plan/ddl_planner.h b/include/logical_plan/ddl_planner.h
@@ -42,7 +42,8 @@ class DDLPlanner : public LogicalPlanner {
     int parse_alter_table(pb::MetaManagerRequest& alter_request);
     int check_partition_key_constraint(pb::SchemaInfo& table, const std::string& field_name);
 
-    int add_column_def(pb::SchemaInfo& table, parser::ColumnDef* column, bool is_unique_indicator);
+    int add_column_def(pb::SchemaInfo& table, parser::ColumnDef* column,
+                       bool is_unique_indicator, const std::string& old_field_name = "");
     int parse_create_namespace(pb::NameSpaceInfo& namespace_info);
     int parse_drop_namespace(pb::NameSpaceInfo& namespace_info);
     int parse_alter_namespace(pb::NameSpaceInfo& namespace_info);

diff --git a/include/meta_server/table_manager.h b/include/meta_server/table_manager.h
@@ -980,8 +980,15 @@ class TableManager {
         return false;
     }
 
-    bool check_field_is_compatible_type(pb::PrimitiveType src_type, pb::PrimitiveType target_type) {
+    bool check_field_is_compatible_type(const pb::FieldInfo& src_field, const pb::FieldInfo& target_field) {
+        auto src_type = src_field.mysql_type();
+        auto target_type = target_field.mysql_type();
         if (src_type == target_type) {
+//            if (src_type == pb::DATETIME || src_type == pb::FLOAT || src_type == pb::DOUBLE) {
+//                if (src_field.float_precision_len() > target_field.float_precision_len()) {
+//                    return false;
+//                }
+//            }
             return true;
         }
         switch (src_type) {

diff --git a/include/sqlparser/ddl.h b/include/sqlparser/ddl.h
@@ -171,6 +171,7 @@ enum AlterSpecType : unsigned char {
     ALTER_SPEC_ADD_LEARNER,
     ALTER_SPEC_DROP_LEARNER,
     ALTER_SPEC_MODIFY_COLUMN,
+    ALTER_SPEC_CHANGE_COLUMN,
     ALTER_SPEC_SWAP_TABLE,
 
     ALTER_SPEC_ADD_PARTITION,

diff --git a/include/sqlparser/sql_lex.l b/include/sqlparser/sql_lex.l
@@ -163,7 +163,6 @@ REAL { return REAL; }
 REFERENCES { return REFERENCES; }
 REGEXP { return REGEXP; }
 RENAME { return RENAME; }
-REPEAT { return REPEAT; }
 REPLACE { return REPLACE; }
 MERGE { return MERGE; }
 RESTRICT { return RESTRICT; }
@@ -199,8 +198,6 @@ UPDATE { return UPDATE; }
 USAGE { return USAGE; }
 USE { return USE; }
 USING { return USING; }
-UTC_DATE { return UTC_DATE; }
-UTC_TIME { return UTC_TIME; }
 VALUES { return VALUES; }
 LONG { return LONG; }
 VARCHAR { return VARCHAR; }
@@ -276,6 +273,7 @@ EXECUTE { un_reserved_keyword(yylval, yyscanner, parser); return EXECUTE; }
 FIELDS { un_reserved_keyword(yylval, yyscanner, parser); return FIELDS; }
 FILE { un_reserved_keyword(yylval, yyscanner, parser); return FILE_T; }
 FIRST { un_reserved_keyword(yylval, yyscanner, parser); return FIRST; }
+LAST { un_reserved_keyword(yylval, yyscanner, parser); return LAST; }
 FIXED { un_reserved_keyword(yylval, yyscanner, parser); return FIXED; }
 FLUSH { un_reserved_keyword(yylval, yyscanner, parser); return FLUSH; }
 FORMAT { un_reserved_keyword(yylval, yyscanner, parser); return FORMAT; }
@@ -307,6 +305,12 @@ MAX_UPDATES_PER_HOUR { un_reserved_keyword(yylval, yyscanner, parser); return MA
 MAX_USER_CONNECTIONS { un_reserved_keyword(yylval, yyscanner, parser); return MAX_USER_CONNECTIONS; }
 MERGE { un_reserved_keyword(yylval, yyscanner, parser); return MERGE; }
 MIN_ROWS { un_reserved_keyword(yylval, yyscanner, parser); return MIN_ROWS; }
+ENCRYPTION { un_reserved_keyword(yylval, yyscanner, parser); return ENCRYPTION; }
+ENGINE_ATTRIBUTE { un_reserved_keyword(yylval, yyscanner, parser); return ENGINE_ATTRIBUTE; }
+SECONDARY_ENGINE_ATTRIBUTE {un_reserved_keyword(yylval, yyscanner, parser); 
+return SECONDARY_ENGINE_ATTRIBUTE; }
+INSERT_METHOD { un_reserved_keyword(yylval, yyscanner, parser); return INSERT_METHOD; }
+AUTOEXTEND_SIZE { un_reserved_keyword(yylval, yyscanner, parser); return AUTOEXTEND_SIZE; }
 NAMES { un_reserved_keyword(yylval, yyscanner, parser); return NAMES; }
 NATIONAL { un_reserved_keyword(yylval, yyscanner, parser); return NATIONAL; }
 NO { un_reserved_keyword(yylval, yyscanner, parser); return NO; }
@@ -332,19 +336,22 @@ REDUNDANT { un_reserved_keyword(yylval, yyscanner, parser); return REDUNDANT; }
 RELOAD { un_reserved_keyword(yylval, yyscanner, parser); return RELOAD; }
 REPEATABLE { un_reserved_keyword(yylval, yyscanner, parser); return REPEATABLE; }
 REPLICATION { un_reserved_keyword(yylval, yyscanner, parser); return REPLICATION; }
-REVERSE { un_reserved_keyword(yylval, yyscanner, parser); return REVERSE; }
 ROLLBACK { un_reserved_keyword(yylval, yyscanner, parser); return ROLLBACK; }
 ROUTINE { un_reserved_keyword(yylval, yyscanner, parser); return ROUTINE; }
 ROW { un_reserved_keyword(yylval, yyscanner, parser); return ROW; }
 ROW_COUNT { un_reserved_keyword(yylval, yyscanner, parser); return ROW_COUNT; }
 ROW_FORMAT { un_reserved_keyword(yylval, yyscanner, parser); return ROW_FORMAT; }
+STATS_AUTO_RECALC { un_reserved_keyword(yylval, yyscanner, parser); return STATS_AUTO_RECALC; }
 SECOND { un_reserved_keyword(yylval, yyscanner, parser); return SECOND; }
 SECURITY { un_reserved_keyword(yylval, yyscanner, parser); return SECURITY; }
 SEPARATOR { un_reserved_keyword(yylval, yyscanner, parser); return SEPARATOR; }
 SERIALIZABLE { un_reserved_keyword(yylval, yyscanner, parser); return SERIALIZABLE; }
 SESSION { un_reserved_keyword(yylval, yyscanner, parser); return SESSION; }
 SHARE { un_reserved_keyword(yylval, yyscanner, parser); return SHARE; }
 SHARED { un_reserved_keyword(yylval, yyscanner, parser); return SHARED; }
+INPLACE { un_reserved_keyword(yylval, yyscanner, parser); return INPLACE; }
+INSTANT { un_reserved_keyword(yylval, yyscanner, parser); return INSTANT; }
+COPY { un_reserved_keyword(yylval, yyscanner, parser); return COPY; }
 SHUTDOWN { un_reserved_keyword(yylval, yyscanner, parser); return SHUTDOWN; }
 SIGNED { un_reserved_keyword(yylval, yyscanner, parser); return SIGNED; }
 SLAVE { un_reserved_keyword(yylval, yyscanner, parser); return SLAVE; }
@@ -353,6 +360,7 @@ SQL_CACHE { un_reserved_keyword(yylval, yyscanner, parser); return SQL_CACHE; }
 SQL_NO_CACHE { un_reserved_keyword(yylval, yyscanner, parser); return SQL_NO_CACHE; }
 START { un_reserved_keyword(yylval, yyscanner, parser); return START; }
 STATS_PERSISTENT { un_reserved_keyword(yylval, yyscanner, parser); return STATS_PERSISTENT; }
+STATS_SAMPLE_PAGES { un_reserved_keyword(yylval, yyscanner, parser); return STATS_SAMPLE_PAGES; }
 STATUS { un_reserved_keyword(yylval, yyscanner, parser); return STATUS; }
 SUPER { un_reserved_keyword(yylval, yyscanner, parser); return SUPER; }
 SOME { un_reserved_keyword(yylval, yyscanner, parser); return SOME; }