From 21d25f9a50f8959fe067f373e2a904486d1b16f4 Mon Sep 17 00:00:00 2001 From: tomato <1083369179@qq.com> Date: Thu, 7 Sep 2023 16:07:57 +0800 Subject: [PATCH] repaire debug slow bug Signed-off-by: tomato <1083369179@qq.com> --- src/api/api_utils.py | 7 +- src/api/optimizer.py | 7 +- src/metadata/metadata_utils.py | 7 +- src/optimizer/mysql_engine.py | 5 +- src/optimizer/oceanbase_engine.py | 5 +- src/parser/mysql_parser/parser.py | 138 +++++++++------ src/parser/oceanbase_parser/parser.py | 14 +- src/parser/odps_parser/parser.py | 14 +- test/metadata/test_metadata_utils.py | 5 +- test/optimizer/test_formatter.py | 17 +- test/optimizer/test_pmd.py | 89 +++------- test/optimizer/test_rewrite.py | 52 ++---- test/parser/test_parser_batch.py | 2 +- test/parser/test_parser_ddl.py | 5 +- test/parser/test_parser_dml.py | 234 +++++++++++++++----------- test/parser/test_parser_dql.py | 158 +++++++---------- test/parser/test_parser_utils.py | 25 ++- 17 files changed, 382 insertions(+), 402 deletions(-) diff --git a/src/api/api_utils.py b/src/api/api_utils.py index 1f79d44..321476b 100644 --- a/src/api/api_utils.py +++ b/src/api/api_utils.py @@ -11,8 +11,7 @@ """ from src.optimizer.optimizer import Optimizer -from src.parser.mysql_parser.parser import parser as mysql_parser -from src.parser.mysql_parser.lexer import lexer as mysql_lexer +from src.parser.mysql_parser import parser as mysql_parser from src.parser.parser_utils import ParserUtils from src.common.utils import Utils @@ -30,9 +29,7 @@ def get_xml_log_details(sql_text, catalog_object): after_sql_rewrite, ) = optimizer.optimize(sql_text, catalog_object) - visitor = ParserUtils.format_statement( - mysql_parser.parse(sql_text, lexer=mysql_lexer) - ) + visitor = ParserUtils.format_statement(mysql_parser.parse(sql_text)) table_list = [] for _table in visitor.table_list: table_list.append(_table['table_name']) diff --git a/src/api/optimizer.py b/src/api/optimizer.py index 130eccf..fba3098 100644 --- a/src/api/optimizer.py +++ b/src/api/optimizer.py @@ -18,8 +18,7 @@ from src.common.utils import Utils from src.metadata.metadata_utils import MetaDataUtils from src.optimizer.optimizer import Optimizer as opt -from src.parser.mysql_parser.parser import parser as mysql_parser -from src.parser.mysql_parser.lexer import lexer as mysql_lexer +from src.parser.mysql_parser import parser as mysql_parser from src.parser.parser_utils import ParserUtils NOTHING_TO_DO = 'Current table index is so good , nothing to do' @@ -192,9 +191,7 @@ def get(self): description: parse result """ sql_text = Utils.remove_sql_text_affects_parser(self.sql_text) - visitor = ParserUtils.format_statement( - mysql_parser.parse(sql_text, lexer=mysql_lexer) - ) + visitor = ParserUtils.format_statement(mysql_parser.parse(sql_text)) table_list = visitor.table_list column_list = [] temp_table_list = [] diff --git a/src/metadata/metadata_utils.py b/src/metadata/metadata_utils.py index f945e99..f41657c 100644 --- a/src/metadata/metadata_utils.py +++ b/src/metadata/metadata_utils.py @@ -16,8 +16,7 @@ from src.metadata.catalog import Catalog, Statistics, Selectivity, Table, Index, Column from src.optimizer.optimizer_enum import IndexType, OptType -from src.parser.mysql_parser.parser import parser as mysql_parser -from src.parser.mysql_parser.lexer import lexer as mysql_lexer +from src.parser.mysql_parser import parser as mysql_parser class MetaDataUtils(object): @@ -30,7 +29,7 @@ def schema_sql_to_catalog_index(schema_sql): for ddl_sql in schema_sql.split(';'): if not ddl_sql: break - res = mysql_parser.parse(ddl_sql, lexer=mysql_lexer) + res = mysql_parser.parse(ddl_sql) table_name = res['table_name'] element_list = res['element_list'] index_list = res['index_list'] @@ -95,7 +94,7 @@ def json_to_catalog(catalog_json, schema_sql=None) -> Catalog: for ddl_sql in schema_sql.split(';'): if not ddl_sql: break - res = mysql_parser.parse(ddl_sql, lexer=mysql_lexer) + res = mysql_parser.parse(ddl_sql) table_name = res['table_name'] element_list = res['element_list'] index_list = res['index_list'] diff --git a/src/optimizer/mysql_engine.py b/src/optimizer/mysql_engine.py index e808631..e9ee52e 100644 --- a/src/optimizer/mysql_engine.py +++ b/src/optimizer/mysql_engine.py @@ -11,8 +11,7 @@ """ from src.optimizer.engine import Engine -from src.parser.mysql_parser.parser import parser as mysql_parser -from src.parser.mysql_parser.lexer import lexer as mysql_lexer +from src.parser.mysql_parser import parser as mysql_parser from .rewrite_rule import mysql_rules, common_rules @@ -27,7 +26,7 @@ def __new__(cls): return singleton def parse(self, sql, tracking=False): - return mysql_parser.parse(sql, lexer=mysql_lexer, tracking=tracking) + return mysql_parser.parse(sql, tracking=tracking) def rewrite(self, statement, catalog=None): common_rules.extend(mysql_rules) diff --git a/src/optimizer/oceanbase_engine.py b/src/optimizer/oceanbase_engine.py index 4073fb7..ca4cfb4 100644 --- a/src/optimizer/oceanbase_engine.py +++ b/src/optimizer/oceanbase_engine.py @@ -11,8 +11,7 @@ """ from src.optimizer.engine import Engine -from src.parser.oceanbase_parser.parser import parser as oceanbase_parser -from src.parser.oceanbase_parser.lexer import lexer as oceanbase_lexer +from src.parser.oceanbase_parser import parser as oceanbase_parser from .rewrite_rule import common_rules @@ -27,7 +26,7 @@ def __new__(cls): return singleton def parse(self, sql, tracking=False): - return oceanbase_parser.parse(sql, lexer=oceanbase_lexer, tracking=tracking) + return oceanbase_parser.parse(sql, tracking=tracking) def rewrite(self, statement, catalog=None): rule_explanation_list = [] diff --git a/src/parser/mysql_parser/parser.py b/src/parser/mysql_parser/parser.py index 0a7e3a9..4de3076 100644 --- a/src/parser/mysql_parser/parser.py +++ b/src/parser/mysql_parser/parser.py @@ -12,6 +12,7 @@ from __future__ import print_function +import threading import types from src.parser.tree.window import ( @@ -84,7 +85,7 @@ from ply import yacc from src.optimizer.optimizer_enum import IndexType -from src.parser.mysql_parser.lexer import tokens +from src.parser.mysql_parser.lexer import tokens, lexer tokens = tokens @@ -96,7 +97,7 @@ ('left', 'AND', 'ANDAND'), ('right', 'NOT'), ('left', 'BETWEEN', 'CASE', 'WHEN', 'THEN', 'ELSE'), - ('left', 'EQ', 'NULL_SAFE_EQ','NE', 'LT', 'LE', 'GT', 'GE', 'IS', 'LIKE', 'RLIKE', 'REGEXP', 'IN'), + ('left', 'EQ', 'NULL_SAFE_EQ', 'NE', 'LT', 'LE', 'GT', 'GE', 'IS', 'LIKE', 'RLIKE', 'REGEXP', 'IN'), ('left', 'BIT_OR'), ('left', 'BIT_AND'), ('left', 'BIT_MOVE_LEFT', 'BIT_MOVE_RIGHT'), @@ -312,6 +313,7 @@ def p_ignore(p): | empty """ + def p_delete(p): r"""delete : DELETE FROM relations where_opt order_by_opt limit_opt | DELETE FROM relations partition where_opt order_by_opt limit_opt @@ -320,39 +322,42 @@ def p_delete(p): | DELETE FROM table_name_list USING relations where_opt order_by_opt limit_opt | DELETE FROM table_name_list USING relations partition where_opt order_by_opt limit_opt """ - length=len(p) - p_limit = p[length-1] + length = len(p) + p_limit = p[length - 1] if p_limit is not None: - offset,limit = int(p_limit[0]),int(p_limit[1]) + offset, limit = int(p_limit[0]), int(p_limit[1]) else: - offset,limit=0,0 - if p.slice[3].type=="relations": - tables,table_refs=p[3],None - elif p.slice[2].type=="table_name_list": - tables,table_refs=p[4],p[2] + offset, limit = 0, 0 + if p.slice[3].type == "relations": + tables, table_refs = p[3], None + elif p.slice[2].type == "table_name_list": + tables, table_refs = p[4], p[2] else: - tables,table_refs=p[3],p[5] - p[0] = Delete(table=tables,table_refs=table_refs,where=p[length-3], order_by=p[length-2], limit=limit, offset=offset) + tables, table_refs = p[3], p[5] + p[0] = Delete(table=tables, table_refs=table_refs, where=p[length - 3], order_by=p[length - 2], limit=limit, + offset=offset) def p_table_name_list(p): r"""table_name_list : table_name_list COMMA table_name_opt_wild | table_name_opt_wild""" - if len(p)==4: + if len(p) == 4: p[1].append(p[3]) - p[0]=p[1] + p[0] = p[1] else: - p[0]=[p[1]] + p[0] = [p[1]] + def p_table_name_opt_wild(p): r"""table_name_opt_wild : identifier | identifier PERIOD identifier | identifier PERIOD ASTERISK | identifier PERIOD identifier PERIOD ASTERISK""" - if len(p)==6 or (len(p)==4 and p.slice[3]=='identifier'): - p[0]=QualifiedName(parts=[p[2],p[3]]) + if len(p) == 6 or (len(p) == 4 and p.slice[3] == 'identifier'): + p[0] = QualifiedName(parts=[p[2], p[3]]) else: - p[0]=QualifiedName(parts=[p[1]]) + p[0] = QualifiedName(parts=[p[1]]) + def p_opt_asterisk(p): r"""opt_asterisk : PERIOD ASTERISK @@ -667,7 +672,7 @@ def p_limit_stmt(p): else: p[0] = (p[2], p[4]) if p[3] == ',' else (p[4], p[2]) else: - p[0]=(0,p[3]) + p[0] = (0, p[3]) def p_parameterization(p): @@ -675,24 +680,28 @@ def p_parameterization(p): | QM """ if p.slice[1].type == "number": - p[0]=p[1].value + p[0] = p[1].value else: p[0] = p[1] + def p_first_or_next(p): r"""first_or_next : FIRST | NEXT""" - p[0]=p[1] + p[0] = p[1] + def p_fetch_first_opt(p): r"""fetch_first_opt : parameterization | empty""" - p[0]=p[1] if p[1] else 1 + p[0] = p[1] if p[1] else 1 + def p_row_or_rows(p): r"""row_or_rows : ROW | ROWS""" - p[0]=p[1] + p[0] = p[1] + def p_number(p): r"""number : NUMBER""" @@ -717,7 +726,7 @@ def p_set_operation_expressions(p): def _set_operation(line, pos, left, right, oper, distinctOrAll): - distinct = distinctOrAll is not None and distinctOrAll.upper() in {"DISTINCT","UNIQUE","DISTINCTROW"} + distinct = distinctOrAll is not None and distinctOrAll.upper() in {"DISTINCT", "UNIQUE", "DISTINCTROW"} all = distinctOrAll is not None and distinctOrAll.upper() == "ALL" oper = oper.upper() if oper == "UNION": @@ -789,6 +798,7 @@ def _item_list(p): else: p[0] = None + def p_query_spec(p): r"""query_spec : SELECT select_stmt_opts select_items table_expression_opt order_by_opt limit_opt window_clause_opt for_update_opt""" select_items = p[3] @@ -859,25 +869,29 @@ def p_having_opt(p): | empty""" p[0] = p[2] if p[1] else None + def p_set_quantifier_opt(p): r"""set_quantifier_opt : distinct_opt | empty """ - p[0]=p[1] + p[0] = p[1] + def p_select_stmt_opts(p): r"""select_stmt_opts : select_stmt_opt_list | empty""" - p[0]=p[1] + p[0] = p[1] + def p_select_stmt_opt_list(p): r"""select_stmt_opt_list : select_stmt_opt_list select_stmt_opt | select_stmt_opt""" - if len(p)==3: + if len(p) == 3: p[1].append(p[2]) - p[0]=p[1] + p[0] = p[1] else: - p[0]=[p[1]] + p[0] = [p[1]] + def p_select_stmt_opt(p): r"""select_stmt_opt : distinct_opt @@ -888,11 +902,13 @@ def p_select_stmt_opt(p): | SQL_NO_CACHE | SQL_CALC_FOUND_ROWS | STRAIGHT_JOIN""" - p[0]=p[1] + p[0] = p[1] + def p_priority(p): r"""priority : HIGH_PRIORITY""" - p[0]=p[1] + p[0] = p[1] + def p_distinct_opt(p): r"""distinct_opt : ALL @@ -900,6 +916,7 @@ def p_distinct_opt(p): | DISTINCTROW""" p[0] = p[1] + def p_select_items(p): r"""select_items : select_item | select_items COMMA select_item""" @@ -910,6 +927,7 @@ def p_select_item(p): r"""select_item : derived_column""" p[0] = p[1] + def p_derived_column(p): r"""derived_column : expression alias_opt | ASTERISK @@ -928,17 +946,19 @@ def p_derived_column(p): else: p[0] = SingleColumn(p.lineno(1), p.lexpos(1), alias=p[2], expression=p[1]) + def p_table_expression_opt(p): r"""table_expression_opt : FROM relations partition where_opt group_by_opt having_opt | FROM relations where_opt group_by_opt having_opt | empty""" if len(p) == 7: - p[0] = Node(p.lineno(1),p.lexpos(1),from_=p[2],partition=p[3],where=p[4],group_by=p[5],having=p[6]) + p[0] = Node(p.lineno(1), p.lexpos(1), from_=p[2], partition=p[3], where=p[4], group_by=p[5], having=p[6]) elif len(p) == 6: p[0] = Node(p.lineno(1), p.lexpos(1), from_=p[2], where=p[3], group_by=p[4], having=p[5]) else: p[0] = p[1] + def p_partition(p): r"""partition : PARTITION LPAREN identifiers RPAREN""" p[0] = Partition(p.lineno(1), p.lexpos(1), partition_list=p[3]) @@ -1060,6 +1080,7 @@ def p_aliased_relation(p): else: p[0] = rel + def p_index_hint_opt(p): r"""index_hint_opt : index_hint_list | empty""" @@ -1071,6 +1092,7 @@ def p_index_hint_list(p): | index_hint""" pass + def p_index_hint(p): r"""index_hint : use_index | force_or_ignore_index""" @@ -1104,11 +1126,13 @@ def p_index_or_key(p): | KEY""" pass + def p_index_name(p): r"""index_name : PRIMARY | identifiers""" pass + def p_derived_table(p): r"""derived_table : subquery alias_opt""" if p[2]: @@ -1256,7 +1280,7 @@ def p_sounds_predicate(p): def p_in_predicate(p): r"""in_predicate : value_expression IN in_value | value_expression NOT IN in_value""" - if len(p)==5: + if len(p) == 5: p[0] = InPredicate(p.lineno(1), p.lexpos(1), is_not=True, value=p[1], value_list=p[4]) else: p[0] = InPredicate(p.lineno(1), p.lexpos(1), is_not=False, value=p[1], value_list=p[3]) @@ -1313,7 +1337,6 @@ def p_string_lit(p): p[0] = StringLiteral(p.lineno(1), p.lexpos(1), value=p[1].value + p[2][1:-1]) - def p_in_value(p): r"""in_value : LPAREN call_list RPAREN | subquery""" @@ -1422,6 +1445,7 @@ def p_base_primary_expression(p): else: p[0] = p[1] + def p_define_variable(p): r"""define_variable : SINGLE_AT_IDENTIFIER | SINGLE_AT_IDENTIFIER PERIOD variables @@ -1429,21 +1453,23 @@ def p_define_variable(p): | DOUBLE_AT_IDENTIFIER PERIOD variables """ if len(p) == 4: - parts=[p[1]] + parts = [p[1]] parts.extend(p[3].parts) - p[0]=QualifiedName(parts=parts) + p[0] = QualifiedName(parts=parts) else: p[0] = QualifiedName(parts=[p[1]]) + def p_variables(p): r"""variables : variables PERIOD identifier | identifier""" if len(p) == 4: p[1].parts.append(p[3]) - p[0]=p[1] + p[0] = p[1] else: p[0] = QualifiedName(parts=[p[1]]) + def p_exists_func_call(p): r"""exists_func_call : EXISTS subquery""" p[0] = ExistsPredicate(p.lineno(1), p.lexpos(1), is_not=False, subquery=p[2]) @@ -1949,7 +1975,7 @@ def p_add_or_sub_date_func(p): | DATE_ADD LPAREN expression COMMA expression RPAREN | DATE_SUB LPAREN expression COMMA expression RPAREN """ - p[0] = FunctionCall(p.lineno(1), p.lexpos(1), name=p[1], arguments=[p[3],p[5]]) + p[0] = FunctionCall(p.lineno(1), p.lexpos(1), name=p[1], arguments=[p[3], p[5]]) def p_extract_func(p): @@ -1997,9 +2023,11 @@ def p_date_two_para_func(p): | TIME_FORMAT LPAREN expression COMMA expression RPAREN""" p[0] = FunctionCall(p.lineno(1), p.lexpos(1), name=p[1], arguments=[p[3], p[5]]) + def p_date_three_para_func(p): r"""date_three_para_func : CONVERT_TZ LPAREN expression COMMA expression COMMA expression RPAREN""" - p[0] = FunctionCall(p.lineno(1), p.lexpos(1), name=p[1], arguments=[p[3], p[5],p[7]]) + p[0] = FunctionCall(p.lineno(1), p.lexpos(1), name=p[1], arguments=[p[3], p[5], p[7]]) + def p_string_operator_func_call(p): r"""string_operator_func_call : ASCII LPAREN expression RPAREN @@ -2060,9 +2088,9 @@ def p_string_operator_func_call(p): if length == 2: p[0] = p[1] else: - arguments,call_list = [],[] + arguments, call_list = [], [] if p.slice[length - 2].type == "call_list": - call_list=p[length - 2] + call_list = p[length - 2] length = length - 2 for i in range(3, length, 2): arguments.append(p[i]) @@ -2129,6 +2157,7 @@ def p_remstr_position(p): | TRAILING""" p[0] = p[1] + def p_string_comparsion_func_call(p): r"""string_comparsion_func_call : STRCMP LPAREN expression COMMA expression RPAREN | REGEXP_INSTR LPAREN expression COMMA expression RPAREN @@ -2149,7 +2178,7 @@ def p_string_comparsion_func_call(p): | REGEXP_SUBSTR LPAREN expression COMMA expression COMMA expression COMMA expression COMMA expression RPAREN | REGEXP_SUBSTR LPAREN expression COMMA expression COMMA expression COMMA expression COMMA expression COMMA expression RPAREN """ - arguments,length=[],len(p) + arguments, length = [], len(p) for i in range(3, length, 2): arguments.append(p[i]) p[0] = FunctionCall(p.lineno(1), p.lexpos(1), name=p[1], arguments=arguments) @@ -2293,9 +2322,9 @@ def p_search_json_func_call(p): """ length = len(p) arguments = [] - call_list=[] + call_list = [] if p.slice[length - 2].type == "call_list": - call_list=p[length - 2] + call_list = p[length - 2] length -= 2 for i in range(3, length, 2): arguments.append(p[i]) @@ -2325,9 +2354,9 @@ def p_modify_json_func_call(p): | JSON_UNQUOTE LPAREN expression RPAREN """ length = len(p) - arguments,call_list= [],[] + arguments, call_list = [], [] if p.slice[length - 2].type == "call_list": - call_list=p[length - 2] + call_list = p[length - 2] length -= 2 for i in range(3, length, 2): arguments.append(p[i]) @@ -2663,8 +2692,8 @@ def p_field_len_opt(p): r"""field_len_opt : LPAREN NUMBER RPAREN | empty""" if len(p) == 4: - field_len=LongLiteral(p.lineno(1), p.lexpos(1),p[2]) - p[0] =field_len.value& 0xFFFFFFFF # convert to unsigned int + field_len = LongLiteral(p.lineno(1), p.lexpos(1), p[2]) + p[0] = field_len.value & 0xFFFFFFFF # convert to unsigned int p[0] = UNSPECIFIEDLENGTH @@ -2684,7 +2713,7 @@ def p_field_parameter(p): r"""field_parameter : number | base_data_type""" if p.slice[1].type == "number": - p[0]=p[1].value + p[0] = p[1].value else: p[0] = p[1] @@ -3385,4 +3414,13 @@ def _print_error(self): raise SyntaxError("The current version does not support this SQL") -parser = yacc.yacc(tabmodule="parser_table", start="command", debugfile="parser.out", optimize=True) +parser = None + + +def parse(sql=None, debug=False, tracking=False, tokenfunc=None): + global parser + if parser is None: + with threading.Lock(): + if parser is None: + parser = yacc.yacc(tabmodule="parser_table", start="command", debugfile="parser.out", optimize=True) + return parser.parse(input=sql, lexer=lexer, debug=debug, tracking=tracking, tokenfunc=tokenfunc) diff --git a/src/parser/oceanbase_parser/parser.py b/src/parser/oceanbase_parser/parser.py index fd1c1e5..a6afdcd 100644 --- a/src/parser/oceanbase_parser/parser.py +++ b/src/parser/oceanbase_parser/parser.py @@ -12,6 +12,7 @@ from __future__ import print_function +import threading import types from src.parser.tree.window import ( FrameBound, @@ -24,7 +25,7 @@ from ply import yacc from src.optimizer.optimizer_enum import IndexType -from src.parser.oceanbase_parser.lexer import tokens +from src.parser.oceanbase_parser.lexer import tokens, lexer from src.parser.tree.expression import ( AggregateFunc, ArithmeticBinaryExpression, @@ -3805,4 +3806,13 @@ def _print_error(self): raise SyntaxError("The current version does not support this SQL") -parser = yacc.yacc(tabmodule="parser_table", start="command", debugfile="parser.out",optimize=True) +parser = None + + +def parse(sql=None, debug=False, tracking=False, tokenfunc=None): + global parser + if parser is None: + with threading.Lock(): + if parser is None: + parser = yacc.yacc(tabmodule="parser_table", start="command", debugfile="parser.out", optimize=True) + return parser.parse(input=sql, lexer=lexer, debug=debug, tracking=tracking, tokenfunc=tokenfunc) diff --git a/src/parser/odps_parser/parser.py b/src/parser/odps_parser/parser.py index 2be0a5d..5e7dc85 100644 --- a/src/parser/odps_parser/parser.py +++ b/src/parser/odps_parser/parser.py @@ -12,6 +12,7 @@ from __future__ import print_function +import threading import types from src.parser.tree.window import ( FrameBound, @@ -24,7 +25,7 @@ from ply import yacc from src.optimizer.optimizer_enum import IndexType -from src.parser.odps_parser.lexer import tokens +from src.parser.odps_parser.lexer import tokens, lexer from src.parser.tree.expression import ( AggregateFunc, ArithmeticBinaryExpression, @@ -3815,4 +3816,13 @@ def _print_error(self): raise SyntaxError("The current version does not support this SQL") -parser = yacc.yacc(tabmodule="parser_table", start="command", debugfile="parser.out",optimize=True) +parser = None + + +def parse(sql=None, debug=False, tracking=False, tokenfunc=None): + global parser + if parser is None: + with threading.Lock(): + if parser is None: + parser = yacc.yacc(tabmodule="parser_table", start="command", debugfile="parser.out", optimize=True) + return parser.parse(input=sql, lexer=lexer, debug=debug, tracking=tracking, tokenfunc=tokenfunc) diff --git a/test/metadata/test_metadata_utils.py b/test/metadata/test_metadata_utils.py index 0259f9f..48f7b9e 100644 --- a/test/metadata/test_metadata_utils.py +++ b/test/metadata/test_metadata_utils.py @@ -17,8 +17,7 @@ from src.metadata.catalog import Catalog from src.metadata.metadata_utils import MetaDataUtils -from src.parser.mysql_parser.parser import parser -from src.parser.mysql_parser.lexer import lexer +from src.parser.mysql_parser import parser from src.parser.parser_utils import ParserUtils @@ -124,7 +123,7 @@ def setUpClass(self): self.catalog_object = MetaDataUtils.json_to_catalog( json.loads(self.catalog_json) ) - visitor = ParserUtils.format_statement(parser.parse(self.sql, lexer=lexer)) + visitor = ParserUtils.format_statement(parser.parse(self.sql)) self.table_list = visitor.table_list self.projection_column_list = visitor.projection_column_list self.order_list = visitor.order_list diff --git a/test/optimizer/test_formatter.py b/test/optimizer/test_formatter.py index 46b46e1..6d94b2e 100644 --- a/test/optimizer/test_formatter.py +++ b/test/optimizer/test_formatter.py @@ -1,8 +1,7 @@ import unittest from src.optimizer.formatter import format_sql -from src.parser.mysql_parser.parser import parser -from src.parser.mysql_parser.lexer import lexer +from src.parser.mysql_parser import parser class MyTestCase(unittest.TestCase): @@ -12,7 +11,6 @@ def test_union_all(self): SELECT * FROM T1 WHERE C1 < 20000 UNION ALL SELECT * FROM T1 WHERE C2 < 30 """, - lexer=lexer, ) after_sql_rewrite_format = format_sql(statement, 0) assert ( @@ -33,8 +31,6 @@ def test_union(self): SELECT * FROM T1 WHERE C1 < 20000 UNION SELECT * FROM T1 WHERE C2 < 30 """, - lexer=lexer, - debug=True, ) after_sql_rewrite_format = format_sql(statement, 0) assert ( @@ -50,7 +46,7 @@ def test_union(self): ) def test_as(self): - statement = parser.parse("""SELECT a.* FROM d1 as a""", lexer=lexer, debug=True) + statement = parser.parse("""SELECT a.* FROM d1 as a""") after_sql_rewrite_format = format_sql(statement, 0) assert ( after_sql_rewrite_format @@ -60,11 +56,11 @@ def test_as(self): ) def test_update(self): - statement = parser.parse("""update t set a = 1,b = 2 where c= 3""", lexer=lexer) + statement = parser.parse("""update t set a = 1,b = 2 where c= 3""") after_sql_rewrite_format = format_sql(statement, 0) assert after_sql_rewrite_format == """UPDATE t SET a = 1 , b = 2 WHERE c = 3""" statement = parser.parse( - """update t set a = 1,b = 2 where c= 3 order by c limit 1""", lexer=lexer + """update t set a = 1,b = 2 where c= 3 order by c limit 1""" ) after_sql_rewrite_format = format_sql(statement, 0) assert ( @@ -75,13 +71,12 @@ def test_update(self): ) def test_delete(self): - statement = parser.parse("""delete from t where c= 3 and a = 1""", lexer=lexer) + statement = parser.parse("""delete from t where c= 3 and a = 1""") after_sql_rewrite_format = format_sql(statement, 0) assert after_sql_rewrite_format == """DELETE FROM t WHERE c = 3 AND a = 1""" statement = parser.parse( """delete from t where c= 3 and a = 1 order by c limit 1""", - lexer=lexer, ) after_sql_rewrite_format = format_sql(statement, 0) assert ( @@ -94,7 +89,6 @@ def test_delete(self): def test_sql_1(self): statement = parser.parse( """select tnt_inst_id as tnt_inst_id,gmt_create as gmt_create,gmt_modified as gmt_modified,principal_id as principal_id,version as version from cu_version_control where (principal_id = 'TOKENREL|100100000003358587777|IPAY_HK' )""", - lexer=lexer, ) after_sql_rewrite_format = format_sql(statement, 0) assert ( @@ -115,7 +109,6 @@ def test_subquery_limit(self): """ SELECT COUNT(*) FROM ( SELECT * FROM customs_script_match_history LIMIT ? ) a """, - lexer=lexer, ) after_sql_rewrite_format = format_sql(statement, 0) assert ( diff --git a/test/optimizer/test_pmd.py b/test/optimizer/test_pmd.py index 4cf8208..42b9e08 100644 --- a/test/optimizer/test_pmd.py +++ b/test/optimizer/test_pmd.py @@ -10,79 +10,67 @@ PMDUpdateDeleteMultiTableRule, PMDMultiTableRule, ) -from src.parser.mysql_parser.parser import parser -from src.parser.mysql_parser.lexer import lexer +from src.parser.mysql_parser import parser class MyTestCase(unittest.TestCase): def test_pmd_select_all_rule_match(self): - statement = parser.parse( - "SELECT * FROM T1 WHERE C1 < 20000 OR C2 < 30", lexer=lexer - ) + statement = parser.parse("SELECT * FROM T1 WHERE C1 < 20000 OR C2 < 30") pmd_result = PMDSelectAllRule().match(statement) assert pmd_result is True - statement = parser.parse( - "SELECT a FROM T1 WHERE C1 < 20000 OR C2 < 30", lexer=lexer - ) + statement = parser.parse("SELECT a FROM T1 WHERE C1 < 20000 OR C2 < 30") pmd_result = PMDSelectAllRule().match(statement) assert pmd_result is False - statement = parser.parse( - "SELECT a.* FROM T1 a WHERE C1 < 20000 OR C2 < 30", lexer=lexer - ) + statement = parser.parse("SELECT a.* FROM T1 a WHERE C1 < 20000 OR C2 < 30") pmd_result = PMDSelectAllRule().match(statement) assert pmd_result is True statement = parser.parse( - "SELECT a.* , a.b FROM T1 a WHERE C1 < 20000 OR C2 < 30", lexer=lexer + "SELECT a.* , a.b FROM T1 a WHERE C1 < 20000 OR C2 < 30" ) pmd_result = PMDSelectAllRule().match(statement) assert pmd_result is True def test_pmd_select_all_rule(self): - statement = parser.parse( - "SELECT * FROM T1 WHERE C1 < 20000 OR C2 < 30", lexer=lexer - ) + statement = parser.parse("SELECT * FROM T1 WHERE C1 < 20000 OR C2 < 30") pmd_result = PMDSelectAllRule().match_action(statement) assert pmd_result is not None def test_pmd_full_scan_rule(self): - statement = parser.parse("select 1 from a", lexer=lexer) + statement = parser.parse("select 1 from a") match = PMDFullScanRule().match(statement) assert match - statement = parser.parse("select 1 from a where b != 1", lexer=lexer) + statement = parser.parse("select 1 from a where b != 1") match = PMDFullScanRule().match(statement) assert match - statement = parser.parse("select 1 from a where b <> 1", lexer=lexer) + statement = parser.parse("select 1 from a where b <> 1") match = PMDFullScanRule().match(statement) assert match - statement = parser.parse("select 1 from a where b not like '1%' ", lexer=lexer) + statement = parser.parse("select 1 from a where b not like '1%' ") match = PMDFullScanRule().match(statement) assert match - statement = parser.parse("select 1 from a where b not in (1) ", lexer=lexer) + statement = parser.parse("select 1 from a where b not in (1) ") match = PMDFullScanRule().match(statement) assert match - statement = parser.parse( - "select 1 from a where not exists (select 1 from a) ", lexer=lexer - ) + statement = parser.parse("select 1 from a where not exists (select 1 from a) ") match = PMDFullScanRule().match(statement) assert match statement = parser.parse( "select 1 from a where not exists (select 1 from a where c = 2) ", - lexer=lexer, ) match = PMDFullScanRule().match(statement) assert match - statement = parser.parse("select 1 from a where b like '%a' ", lexer=lexer) + statement = parser.parse("select 1 from a where b like '%a' ") match = PMDFullScanRule().match(statement) assert match - statement = parser.parse("select 1 from a where b like '%a%' ", lexer=lexer) + statement = parser.parse("select 1 from a where b like '%a%' ") match = PMDFullScanRule().match(statement) assert match @@ -91,13 +79,12 @@ def test_pmd_full_scan_rule(self): ON (product.id = product_details.id) AND product.amount=200 """, - lexer=lexer, ) match = PMDFullScanRule().match(statement) assert not match statement = parser.parse( - """select 1 from a where b like '%a%' and c BETWEEN 1 AND 20""", lexer=lexer + """select 1 from a where b like '%a%' and c BETWEEN 1 AND 20""" ) match = PMDFullScanRule().match(statement) assert not match @@ -108,7 +95,6 @@ def test_update_delete(self): sqless_base set nick=1231 where a = 1""", - lexer=lexer, ) match = PMDFullScanRule().match(statement) assert not match @@ -117,50 +103,37 @@ def test_update_delete(self): sqless_base where a = 1""", - lexer=lexer, ) match = PMDFullScanRule().match(statement) assert not match def test_is_null(self): - statement = parser.parse( - "select * from sqless_base where a is null", lexer=lexer - ) + statement = parser.parse("select * from sqless_base where a is null") match = PMDIsNullRule().match(statement) assert not match - statement = parser.parse( - "select * from sqless_base where a = null", lexer=lexer - ) + statement = parser.parse("select * from sqless_base where a = null") match = PMDIsNullRule().match(statement) assert match def test_count(self): - statement = parser.parse("select count(a) from sqless_base", lexer=lexer) + statement = parser.parse("select count(a) from sqless_base") match = PMDCountRule().match(statement) assert match - statement = parser.parse( - "select count(1) from sqless_base", debug=True, lexer=lexer - ) + statement = parser.parse("select count(1) from sqless_base") match = PMDCountRule().match(statement) assert match - statement = parser.parse( - "select count(DISTINCT a) from sqless_base", lexer=lexer - ) + statement = parser.parse("select count(DISTINCT a) from sqless_base") match = PMDCountRule().match(statement) assert match - statement = parser.parse("select count(*) from sqless_base", lexer=lexer) + statement = parser.parse("select count(*) from sqless_base") match = PMDCountRule().match(statement) assert not match def test_arithmetic_binary(self): - statement = parser.parse( - "select count(a) from sqless_base where a * 2 > 1", lexer=lexer - ) + statement = parser.parse("select count(a) from sqless_base where a * 2 > 1") match = PMDArithmeticRule().match(statement) assert match - statement = parser.parse( - "select count(1) from sqless_base where a > 1 * 2", lexer=lexer - ) + statement = parser.parse("select count(1) from sqless_base where a > 1 * 2") match = PMDArithmeticRule().match(statement) assert not match @@ -169,8 +142,7 @@ def test_update_delete_multi_table(self): """DELETE FROM Product P LEFT JOIN OrderItem I ON P.Id = I.ProductId - WHERE I.Id IS NULL""", - lexer=lexer, + WHERE I.Id IS NULL""" ) match = PMDUpdateDeleteMultiTableRule().match(statement) assert match @@ -182,8 +154,7 @@ def test_update_delete_multi_table(self): SET o.total_orders = 7 ,item= 'pendrive' WHERE o.order_id = 1 - AND order_detail_id = 1""", - lexer=lexer, + AND order_detail_id = 1""" ) match = PMDUpdateDeleteMultiTableRule().match(statement) assert match @@ -192,9 +163,7 @@ def test_nowait_or_wait(self): statement = parser.parse( """ SELECT * FROM match_record_id FOR UPDATE - """, - lexer=lexer, - debug=True, + """ ) match = PMDNowaitWaitRule().match(statement) assert match @@ -202,8 +171,6 @@ def test_nowait_or_wait(self): """ SELECT * FROM match_record_id FOR UPDATE NOWAIT """, - lexer=lexer, - debug=True, ) match = PMDNowaitWaitRule().match(statement) assert not match @@ -211,8 +178,6 @@ def test_nowait_or_wait(self): """ SELECT * FROM match_record_id FOR UPDATE WAIT 1 """, - lexer=lexer, - debug=True, ) match = PMDNowaitWaitRule().match(statement) assert not match @@ -227,7 +192,6 @@ def test_multi_table(self): JOIN orders o ON p.product_id = o.product_id WHERE o.order_date BETWEEN '2022-01-01' AND '2022-12-31' ORDER BY o.order_date DESC""", - lexer=lexer, ) match = PMDMultiTableRule().match(statement) assert match @@ -240,7 +204,6 @@ def test_multi_table(self): WHERE o.order_date BETWEEN '2022-01-01' AND '2022-12-31' ORDER BY o.order_date DESC """, - lexer=lexer, ) match = PMDMultiTableRule().match(statement) assert not match diff --git a/test/optimizer/test_rewrite.py b/test/optimizer/test_rewrite.py index 3280a05..66835fc 100644 --- a/test/optimizer/test_rewrite.py +++ b/test/optimizer/test_rewrite.py @@ -8,15 +8,12 @@ RemoveOrderByInDeleteUpdateRule, RewriteSupplementColumnRule, ) -from src.parser.mysql_parser.parser import parser -from src.parser.mysql_parser.lexer import lexer +from src.parser.mysql_parser import parser class MyTestCase(unittest.TestCase): def test_or(self): - statement = parser.parse( - "SELECT * FROM T1 WHERE C1 < 20000 OR C2 < 30", lexer=lexer - ) + statement = parser.parse("SELECT * FROM T1 WHERE C1 < 20000 OR C2 < 30") RewriteMySQLORRule().match_action(statement) after_sql_rewrite_format = format_sql(statement, 0) assert ( @@ -32,7 +29,7 @@ def test_or(self): ) def test_supplement_column_rewrite(self): - statement = parser.parse("SELECT * FROM sqless_base", lexer=lexer) + statement = parser.parse("SELECT * FROM sqless_base") catalog_json = """ {"columns": [{"schema":"sqless_test","table":"sqless_base", "name":"a","type":"int(2)","nullable":false},{"schema":"sqless_test","table":"sqless_base", @@ -54,7 +51,7 @@ def test_supplement_column_rewrite(self): ) def test_supplement_column_rewrite_rule_match(self): - statement = parser.parse("SELECT * FROM sqless_base", lexer=lexer) + statement = parser.parse("SELECT * FROM sqless_base") catalog_json = """ {"columns": [{"schema":"sqless_test","table":"sqless_base", "name":"a","type":"int(2)","nullable":false},{"schema":"sqless_test","table":"sqless_base", @@ -66,7 +63,7 @@ def test_supplement_column_rewrite_rule_match(self): catalog_object = MetaDataUtils.json_to_catalog(json.loads(catalog_json)) match = RewriteSupplementColumnRule().match(statement, catalog_object) assert match is True - statement = parser.parse("SELECT a FROM sqless_base", lexer=lexer) + statement = parser.parse("SELECT a FROM sqless_base") match = RewriteSupplementColumnRule().match(statement, catalog_object) assert match is False @@ -92,7 +89,7 @@ def test_completion_column(self): } """ catalog_object = MetaDataUtils.json_to_catalog(json.loads(catalog_json)) - statement = parser.parse("SELECT * FROM d1", lexer=lexer) + statement = parser.parse("SELECT * FROM d1") RewriteSupplementColumnRule().match_action(statement, catalog_object) after_sql_rewrite_format = format_sql(statement, 0) assert ( @@ -103,7 +100,7 @@ def test_completion_column(self): FROM d1""" ) - statement = parser.parse("SELECT a.* FROM d1 a", lexer=lexer) + statement = parser.parse("SELECT a.* FROM d1 a") RewriteSupplementColumnRule().match_action(statement, catalog_object) after_sql_rewrite_format = format_sql(statement, 0) assert ( @@ -114,7 +111,7 @@ def test_completion_column(self): FROM d1 a""" ) - statement = parser.parse("SELECT a.* FROM d1 as a", lexer=lexer) + statement = parser.parse("SELECT a.* FROM d1 as a") RewriteSupplementColumnRule().match_action(statement, catalog_object) after_sql_rewrite_format = format_sql(statement, 0) assert ( @@ -125,7 +122,7 @@ def test_completion_column(self): FROM d1 AS a""" ) - statement = parser.parse("SELECT c.* , d2.b FROM a.d1 c,d2", lexer=lexer) + statement = parser.parse("SELECT c.* , d2.b FROM a.d1 c,d2") RewriteSupplementColumnRule().match_action(statement, catalog_object) after_sql_rewrite_format = format_sql(statement, 0) assert ( @@ -145,30 +142,22 @@ def test_or_same_column(self): T1 WHERE C1 IN (20000, 30)""" - statement = parser.parse( - "SELECT * FROM T1 WHERE C1 = 20000 OR C1 = 30", lexer=lexer - ) + statement = parser.parse("SELECT * FROM T1 WHERE C1 = 20000 OR C1 = 30") RewriteMySQLORRule().match_action(statement) result = format_sql(statement, 0) assert result == after_sql_rewrite_format - statement = parser.parse( - "SELECT * FROM T1 WHERE C1 in (20000) OR C1 = 30", lexer=lexer - ) + statement = parser.parse("SELECT * FROM T1 WHERE C1 in (20000) OR C1 = 30") RewriteMySQLORRule().match_action(statement) result = format_sql(statement, 0) assert result == after_sql_rewrite_format - statement = parser.parse( - "SELECT * FROM T1 WHERE C1 in (20000) OR C1 in (30)", lexer=lexer - ) + statement = parser.parse("SELECT * FROM T1 WHERE C1 in (20000) OR C1 in (30)") RewriteMySQLORRule().match_action(statement) result = format_sql(statement, 0) assert result == after_sql_rewrite_format - statement = parser.parse( - "SELECT * FROM T1 WHERE C1 in (20000) OR C2 in (30)", lexer=lexer - ) + statement = parser.parse("SELECT * FROM T1 WHERE C1 in (20000) OR C2 in (30)") RewriteMySQLORRule().match_action(statement) result = format_sql(statement, 0) assert ( @@ -184,9 +173,7 @@ def test_or_same_column(self): ) def test_like(self): - statement = parser.parse( - "select * from sqless_base where d like 'a%'", lexer=lexer - ) + statement = parser.parse("select * from sqless_base where d like 'a%'") catalog_json = """ { "columns": @@ -223,7 +210,6 @@ def test_qm(self): statement = parser.parse( """select * FROM cm_relation WHERE status = ? AND primary_id = ? AND rel_type = ? AND rel_biz_type = ?""", - lexer=lexer, ) catalog_json = """ {"columns": [{"schema":"luli1","table":"cm_relation", @@ -299,22 +285,20 @@ def test_qm(self): ) def test_delete_update_order(self): - statement = parser.parse( - """delete from tbl where col1 = ? order by col""", lexer=lexer - ) + statement = parser.parse("""delete from tbl where col1 = ? order by col""") match = RemoveOrderByInDeleteUpdateRule().match(statement, None) assert match RemoveOrderByInDeleteUpdateRule().match_action(statement, None) after_sql_rewrite_format = format_sql(statement, 0) assert after_sql_rewrite_format == """DELETE FROM tbl WHERE col1 = ?""" statement = parser.parse( - """delete from tbl where col1 = ? order by col limit 1""", lexer=lexer + """delete from tbl where col1 = ? order by col limit 1""" ) match = RemoveOrderByInDeleteUpdateRule().match(statement, None) assert not match statement = parser.parse( - """update tbl set col1 = ? where col2 = ? order by col""", lexer=lexer + """update tbl set col1 = ? where col2 = ? order by col""" ) match = RemoveOrderByInDeleteUpdateRule().match(statement, None) assert match @@ -323,7 +307,6 @@ def test_delete_update_order(self): assert after_sql_rewrite_format == """UPDATE tbl SET col1 = ? WHERE col2 = ?""" statement = parser.parse( """update tbl set col1 = ? where col2 = ? order by col limit 1""", - lexer=lexer, ) match = RemoveOrderByInDeleteUpdateRule().match(statement, None) assert not match @@ -331,7 +314,6 @@ def test_delete_update_order(self): def test_subquery_or(self): statement = parser.parse( "SELECT t1.* FROM t1 WHERE t1.c1 IN (?) AND t1.c2 = ? AND t1.c3 > ? and c4 not in (select t2.c5 from t2 where t2.c5 = ? or t2.c5 = ?)", - lexer=lexer, ) is_match = RewriteMySQLORRule().match(statement) assert not is_match diff --git a/test/parser/test_parser_batch.py b/test/parser/test_parser_batch.py index ee8ad00..18fd00f 100644 --- a/test/parser/test_parser_batch.py +++ b/test/parser/test_parser_batch.py @@ -16,7 +16,7 @@ import unittest from src.consume.file_parse_common import get_encoding -from src.parser.mysql_parser.parser import parser +from src.parser.mysql_parser import parser from src.parser.parser_utils import ParserUtils from src.common.utils import Utils diff --git a/test/parser/test_parser_ddl.py b/test/parser/test_parser_ddl.py index 28e356f..1c1f3ed 100644 --- a/test/parser/test_parser_ddl.py +++ b/test/parser/test_parser_ddl.py @@ -14,8 +14,7 @@ import unittest -from src.parser.mysql_parser.parser import parser -from src.parser.mysql_parser.lexer import lexer +from src.parser.mysql_parser import parser class MyTestCase(unittest.TestCase): @@ -41,7 +40,6 @@ def test_create_table(self): KEY `pure_dbname` (`pure_dbname`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 """, - lexer=lexer, ) assert result['index_list'][0][0].value == '1.primary' assert result['index_list'][0][1] == 'PRIMARY' @@ -176,7 +174,6 @@ def test_create_table2(self): KEY `I_machine_id_state_type_env_created` (`ticket_machine_id`,`merchant_id`,`order_state`,`occupy_type`,`env`,`gmt_created`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=3529344021213246782 COMMENT='订单表:订单相关信息存储' """, - lexer=lexer, ) assert len(result['index_list']) == 29 assert result['index_list'][0][0].value == '1.primary' diff --git a/test/parser/test_parser_dml.py b/test/parser/test_parser_dml.py index 5cfa36e..5461954 100644 --- a/test/parser/test_parser_dml.py +++ b/test/parser/test_parser_dml.py @@ -17,10 +17,8 @@ from src.parser.tree.with_stmt import WithHasQuery from src.common.utils import Utils -from src.parser.mysql_parser.parser import parser as mysql_parser -from src.parser.mysql_parser.lexer import lexer as mysql_lexer -from src.parser.oceanbase_parser.parser import parser as oceanbase_parser -from src.parser.oceanbase_parser.lexer import lexer as oceanbase_lexer +from src.parser.mysql_parser import parser as mysql_parser +from src.parser.oceanbase_parser import parser as oceanbase_parser from src.parser.tree.expression import ComparisonExpression from src.parser.tree.statement import Statement import warnings @@ -87,7 +85,7 @@ def test_lock_in_share_mode(self): INSERT IGNORE INTO ilimitcenter05.tp_48246_ogt_fc_lc_day (`id`, `tnt_inst_id`, `principal_id`, `principal_type`, `cumulate_code` , `stat_time`, `amount`, `day_count`, `reverse_amount`, `reverse_count` , `max_value`, `min_value`, `cumulate_properties`, `p1`, `p2` , `p3`, `p4`, `p5`, `p6`, `p7` , `p8`, `p9`, `p10`, `p11`, `p12` , `p13`, `p14`, `p15`, `properties_md5`, `gmt_create` , `gmt_modified`, `currency`, `version`) SELECT `id`, `tnt_inst_id`, `principal_id`, `principal_type`, `cumulate_code` , `stat_time`, `amount`, `day_count`, `reverse_amount`, `reverse_count` , `max_value`, `min_value`, `cumulate_properties`, `p1`, `p2` , `p3`, `p4`, `p5`, `p6`, `p7` , `p8`, `p9`, `p10`, `p11`, `p12` , `p13`, `p14`, `p15`, `properties_md5`, `gmt_create` , `gmt_modified`, `currency`, `version` FROM ilimitcenter05.fc_lc_day FORCE INDEX (`PRIMARY`) WHERE `id` > ? AND (`id` < ? OR `id` = ?) LOCK IN SHARE MODE """ sql = Utils.remove_sql_text_affects_parser(sql) - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) def test_insert_now(self): @@ -120,7 +118,7 @@ def test_mysql_logical_opt(self): ] for sql in test_sqls: sql = Utils.remove_sql_text_affects_parser(sql) - result = mysql_parser.parse(sql, lexer=mysql_lexer) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) def test_mysql_regexp_opt(self): @@ -130,7 +128,7 @@ def test_mysql_regexp_opt(self): ] for sql in test_sqls: sql = Utils.remove_sql_text_affects_parser(sql) - result = mysql_parser.parse(sql, lexer=mysql_lexer) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) def test_resvered_word_can_used_as_token_in_mysql_ob(self): @@ -165,9 +163,9 @@ def test_resvered_word_can_used_as_token_in_mysql_ob(self): ] for sql in test_sqls: sql = Utils.remove_sql_text_affects_parser(sql) - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_resvered_word_can_used_as_token_only_in_ob(self): @@ -176,7 +174,7 @@ def test_resvered_word_can_used_as_token_only_in_ob(self): """SELECT groups FROM t""", ] for sql in test_sqls: - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_vector_expression(self): @@ -189,9 +187,9 @@ def test_vector_expression(self): ] for sql in test_sqls: sql = Utils.remove_sql_text_affects_parser(sql) - result = mysql_parser.parse(sql, lexer=mysql_lexer) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_concat_function(self): @@ -202,9 +200,9 @@ def test_concat_function(self): ] for sql in test_sqls: sql = Utils.remove_sql_text_affects_parser(sql) - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_cast_function(self): @@ -218,9 +216,9 @@ def test_cast_function(self): "SELECT CAST(9 AS BINARY(15))", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_inner_join(self): @@ -228,9 +226,9 @@ def test_inner_join(self): " SELECT * FROM ((SELECT * FROM b INNER JOIN a ON a.task_id = b.task_id) x INNER JOIN y ON x.id = y.id)" ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_set_operation(self): @@ -243,9 +241,9 @@ def test_set_operation(self): " ((SELECT * FROM a ) UNION (SELECT * FROM b)) UNION (SELECT * FROM c)", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_with_operation(self): @@ -259,9 +257,9 @@ def test_with_operation(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer) + result = mysql_parser.parse(sql) assert isinstance(result, WithHasQuery) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, WithHasQuery) def test_windows_func(self): @@ -406,9 +404,9 @@ def test_windows_func(self): """, ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_same_name_token(self): @@ -450,9 +448,9 @@ def test_same_name_token(self): 'SELECT row_number FROM my_table', ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_like_escape(self): @@ -461,9 +459,9 @@ def test_like_escape(self): 'select * from t where id like "0049663881" escape "`"', ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_interval(self): @@ -478,9 +476,9 @@ def test_interval(self): "select * from t where a>date_sub(now(),?)", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_when_case(self): @@ -489,9 +487,9 @@ def test_when_case(self): "SELECT CASE WHEN gender = '男' THEN '先生' WHEN gender = '女' THEN '女士' ELSE '未知' END AS title, name FROM user", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_group_concat(p): @@ -501,9 +499,9 @@ def test_group_concat(p): "SELECT category, group_concat(DISTINCT brand ORDER BY brand ASC SEPARATOR ', ') AS brands FROM product GROUP BY category", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_join(p): @@ -514,9 +512,9 @@ def test_join(p): "SELECT o.order_id, c.name FROM orders o FULL OUTER JOIN customers c ON o.customer_id = c.customer_id;", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_subquery_compare(p): @@ -527,9 +525,9 @@ def test_subquery_compare(p): "SELECT o.order_id, o.amount FROM orders o WHERE o.amount > SOME ( SELECT AVG(amount) FROM orders)", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_if(p): @@ -538,9 +536,9 @@ def test_if(p): "SELECT name, IF(gender='male', '先生', '女士') AS title FROM users", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_sconst(p): @@ -552,9 +550,9 @@ def test_sconst(p): r"SELECT * FROM users where a='''aaa\a\a\\' and b='\'b'", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_alias_name(p): @@ -563,9 +561,9 @@ def test_alias_name(p): "SELECT category AS product_category,COUNT(*) AS 'product_count' FROM products", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_match_against(p): @@ -577,9 +575,9 @@ def test_match_against(p): "SELECT * FROM products WHERE MATCH(product_name, product_description) AGAINST('camera' IN NATURAL LANGUAGE MODE WITH QUERY EXPANSION)", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_assignment(p): @@ -589,9 +587,9 @@ def test_assignment(p): "SELECT category_name FROM categories WHERE category_id = (select @rownum.test := 0)", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_binary(p): @@ -600,9 +598,9 @@ def test_binary(p): "SELECT * FROM users WHERE _BINARY 'admin' = 'test'", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_for_update(p): @@ -615,9 +613,9 @@ def test_for_update(p): "SELECT * FROM orders WHERE order_status = 'new' FOR UPDATE SKIP LOCKED", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_for_update_ob(p): @@ -625,7 +623,9 @@ def test_for_update_ob(p): "SELECT * FROM orders WHERE order_status = 'new' FOR UPDATE NO_WAIT", ] for sql in test_sqls: - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_convert(p): @@ -636,9 +636,11 @@ def test_convert(p): "SELECT CONVERT(product_price, DECIMAL(10,2)) AS price_string FROM products", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_aggreate_func_with_window(p): @@ -650,9 +652,11 @@ def test_aggreate_func_with_window(p): "SELECT GROUP_CONCAT(DISTINCT product_name ORDER BY product_name ASC SEPARATOR ', ') OVER(PARTITION BY user_id) AS product_list FROM orders", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_group_by_with_order(p): @@ -664,9 +668,11 @@ def test_group_by_with_order(p): "SELECT * FROM orders GROUP BY order_year DESC, order_month ASC", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_length(p): @@ -674,9 +680,11 @@ def test_length(p): "SELECT length(title) FROM t", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_date_lit(p): @@ -686,9 +694,11 @@ def test_date_lit(p): "SELECT TIMESTAMP '2021-05-05'", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_noeqnull(p): @@ -698,9 +708,11 @@ def test_noeqnull(p): "SELECT 1|1 <=> 1+1*1>2", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_cross_nutural_join(p): @@ -711,9 +723,11 @@ def test_cross_nutural_join(p): "SELECT * FROM products NATURAL JOIN orders CROSS JOIN customers on orders.name=customers.name", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_extract(p): @@ -724,9 +738,11 @@ def test_extract(p): "SELECT EXTRACT(DAY FROM order_date) AS order_day, EXTRACT(HOUR FROM order_date) AS order_hour FROM orders", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_dual(p): @@ -734,9 +750,11 @@ def test_dual(p): "SELECT * FROM DUAL", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_partition(p): @@ -744,9 +762,11 @@ def test_partition(p): "SELECT * FROM t PARTITION(p85,p1)", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_collate(p): @@ -759,9 +779,11 @@ def test_collate(p): 'SELECT DISTINCT k COLLATE latin1_german2_ci FROM t1', ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_number(p): @@ -780,9 +802,11 @@ def test_number(p): "SELECT X''", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_index_hint(p): @@ -795,9 +819,11 @@ def test_index_hint(p): "SELECT * FROM t1 USE INDEX FOR JOIN (i1) FORCE INDEX FOR JOIN (PRIMARY)", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_string_after_string(p): @@ -810,9 +836,11 @@ def test_string_after_string(p): "SELECT * FROM t1 GROUP BY 'a' 'desc'", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_trim_funcion(p): @@ -824,9 +852,11 @@ def test_trim_funcion(p): "SELECT TRIM('x' FROM 'xxxbarxxx')", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_coalesce_function(p): @@ -836,9 +866,11 @@ def test_coalesce_function(p): "SELECT COALESCE(NULL,'a','b')", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_logical_operator(p): @@ -851,9 +883,11 @@ def test_logical_operator(p): "SELECT * FROM products WHERE NOT price BETWEEN 100 AND 200 && brand IN ('Apple', 'Samsung')", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_with_rollup(p): @@ -862,9 +896,11 @@ def test_with_rollup(p): "SELECT * FROM t1 GROUP BY name, size WITH ROLLUP", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_ceil_func(p): @@ -874,9 +910,11 @@ def test_ceil_func(p): "SELECT * FROM t WHERE a=CEIL(0b101)", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_comment(p): @@ -886,9 +924,11 @@ def test_comment(p): "SELECT * /**/FROM t WHERE a=CEIL(0b101)", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse( + sql, + ) assert isinstance(result, Statement) def test_identitifer_name(p): @@ -898,9 +938,9 @@ def test_identitifer_name(p): "SELECT * FROM t WHERE _FA=0", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_position_func(p): @@ -908,9 +948,9 @@ def test_position_func(p): "SELECT * FROM t WHERE a=POSITION('tacode' in 'tacodea')", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) diff --git a/test/parser/test_parser_dql.py b/test/parser/test_parser_dql.py index 8aa0464..4aabb04 100644 --- a/test/parser/test_parser_dql.py +++ b/test/parser/test_parser_dql.py @@ -15,12 +15,9 @@ import unittest from src.common.utils import Utils -from src.parser.mysql_parser.parser import parser as mysql_parser -from src.parser.oceanbase_parser.parser import parser as oceanbase_parser -from src.parser.oceanbase_parser.lexer import lexer as oceanbase_lexer -from src.parser.mysql_parser.lexer import lexer as mysql_lexer -from src.parser.odps_parser.parser import parser as odps_parser -from src.parser.odps_parser.lexer import lexer as odps_lexer +from src.parser.mysql_parser import parser as mysql_parser +from src.parser.oceanbase_parser import parser as oceanbase_parser +from src.parser.odps_parser import parser as odps_parser from src.parser.tree.expression import LikePredicate, ExistsPredicate from src.parser.tree.relation import Join from src.parser.tree.set_operation import Union @@ -33,15 +30,12 @@ def test_simple_sql(self): "select name,age,count(*),avg(age) from blog join a on a.id = blog.id " "where a.b = 1 and blog.c = 2 group by name,age " "having count(*)>2 and avg(age)<20 order by a asc,b desc limit 1 OFFSET 3", - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) assert isinstance(result.query_body.from_, Join) def test_no_filter(self): - result = oceanbase_parser.parse( - "select distinct name from a.blog", lexer=oceanbase_lexer - ) + result = oceanbase_parser.parse("select distinct name from a.blog") query_body = result.query_body assert ( query_body is not None @@ -50,15 +44,11 @@ def test_no_filter(self): ) def test_question_mark(self): - result = oceanbase_parser.parse( - "select n from b where a = ?", lexer=oceanbase_lexer - ) + result = oceanbase_parser.parse("select n from b where a = ?") assert isinstance(result, Statement) def test_like(self): - result = oceanbase_parser.parse( - "SELECT name from blog where a like 'a' ", lexer=oceanbase_lexer - ) + result = oceanbase_parser.parse("SELECT name from blog where a like 'a' ") query_body = result.query_body assert isinstance(query_body.where, LikePredicate) @@ -73,7 +63,6 @@ def test_exists(self): d = ? ) """, - lexer=oceanbase_lexer, ) query_body = result.query_body assert isinstance(query_body.where, ExistsPredicate) @@ -138,7 +127,6 @@ def test_simple_sql2(self): tars_sqldiag_all.tenant_name, tars_sqldiag_all.sql_id, tars_sqldiag_all.diag_type """, - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) @@ -160,7 +148,7 @@ def test_inner_join(self): from obevent where event_id in (?,?) group by event_id) as obe2 on obe.id= obe2.id""" - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_subquery(self): @@ -170,7 +158,6 @@ def test_subquery(self): WHERE ID IN (SELECT ID FROM CUSTOMERS WHERE SALARY > 4500)""", - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) @@ -178,31 +165,26 @@ def test_distinct(self): result = oceanbase_parser.parse( """select max(id) as id, COUNT(distinct uuid) as cnt from obevent""", - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) def test_union(self): result = oceanbase_parser.parse( "SELECT country FROM Websites UNION SELECT country FROM apps ORDER BY country", - lexer=oceanbase_lexer, ) assert isinstance(result.query_body, Union) result = mysql_parser.parse( "SELECT country FROM Websites UNION SELECT country FROM apps ORDER BY country", - lexer=mysql_lexer, ) assert isinstance(result.query_body, Union) def test_union_all(self): result = oceanbase_parser.parse( "SELECT country FROM Websites UNION ALL SELECT country FROM apps ORDER BY country", - lexer=oceanbase_lexer, ) assert isinstance(result.query_body, Union) result = mysql_parser.parse( "SELECT country FROM Websites UNION ALL SELECT country FROM apps ORDER BY country", - lexer=mysql_lexer, ) assert isinstance(result.query_body, Union) @@ -211,7 +193,6 @@ def test_sql_1(self): """ SELECT role.ID, role.NM, role.CODE,role.ORG_ID,role.domain_id,role.ADMINS,role.SCD_ADMINS,role.PRN_ID,role.PATH,role.TYPE_CODE, role.DSC,role.ST,role.EXPR_TM,role.CRT_ID,role.CRT_NM,role.property, role.MOD_ID, role.MOD_NM,role.GMT_CREATE,role.GMT_MODIFIED,role.TNT_INST_ID,role.MNG_MODE,role.APPLY_MODE, role.risk_memo FROM OS_ROLE role WHERE role.TNT_INST_ID='ALIPW3CN' AND (role.TYPE_CODE = 'ROLE' or role.TYPE_CODE is null ) AND role.st !='DELETE' AND (role.apply_mode in ( 'PUBLIC' , 'PUBLIC_COMMON' ) or (role.type_code = 'ROLE' AND 'PUBLIC' in ( 'PUBLIC' , 'PUBLIC_COMMON' ) AND role.apply_mode IS NULL)) and role.isolation_key = 'TENANT_ALIPW3CN' order by role.id desc limit 0, 10 """, - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) @@ -220,7 +201,6 @@ def test_sql_2(self): """ SELECT count(DISTINCT ID) as total FROM OS_ROLE WHERE TNT_INST_ID = 'ALIPW3CN' AND (NM like CONCAT('%', 'CMR-LEADS', '%') or CODE like CONCAT('%','CMR-LEADS','%')) AND (TYPE_CODE = 'ROLE' or TYPE_CODE is null ) AND st !='DELETE' AND (apply_mode in ( 'PUBLIC' , 'PUBLIC_COMMON' ) or (type_code = 'ROLE' AND 'PUBLIC' in ( 'PUBLIC' , 'PUBLIC_COMMON' ) AND apply_mode IS NULL)) and isolation_key = 'TENANT_ALIPW3CN' """, - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) @@ -244,7 +224,6 @@ def test_sql_3(self): LIMIT 100 """, - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) @@ -253,14 +232,12 @@ def test_sql_4(self): """ SELECT oprn.* , b FROM OS_OPRN oprn WHERE oprn.TNT_INST_ID = 'ALIPW3CN' AND oprn.OPT_CODE like CONCAT('%', 'GT_MESSAGE_RECORD_QUERY', '%') and oprn.isolation_key = 'TENANT_ALIPW3CN' order by oprn.id desc limit 0, 5 """, - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) def test_sql_5(self): result = oceanbase_parser.parse( """select * from sqless_base where a = 'sqless_1' or b = 'sqless_2'""", - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) @@ -278,7 +255,6 @@ def test_sql_6(self): order by client_package.id desc limit 0,10 """, - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) @@ -286,30 +262,21 @@ def test_sql_7(self): result = oceanbase_parser.parse( """ SELECT server_release_repo.server_release_repo_id, server_release_repo.instance_id, server_release_repo.repos_name, server_release_repo.branch_url, server_release_repo.revision_enter, server_release_repo.deleted, server_release_repo.weight, server_release_repo.integrate, server_release_repo.create_tag_flag, server_release_repo.merge_record_id, case server_release_repo.merge_record_id when 0 then 0 when -1 then 1 when -2 then 15 else merge_record.merge_result END as merge_result, server_release_repo.completed, server_release_repo.create_time, server_release_repo.update_time FROM server_release_repo left join merge_record on server_release_repo.merge_record_id = merge_record.id WHERE 1 = 1 and integrate = 0 and completed = 1 and deleted = 0 and merge_record_id != -1 """, - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) def test_union_and_union_all(self): - result = mysql_parser.parse( - "select a from b union select a from b", lexer=mysql_lexer - ) + result = mysql_parser.parse("select a from b union select a from b") assert isinstance(result.query_body, Union) assert not result.query_body.all - result = oceanbase_parser.parse( - "select a from b union select a from b", lexer=oceanbase_lexer - ) + result = oceanbase_parser.parse("select a from b union select a from b") assert isinstance(result.query_body, Union) assert not result.query_body.all - result = oceanbase_parser.parse( - "select a from b union all select a from b", lexer=oceanbase_lexer - ) + result = oceanbase_parser.parse("select a from b union all select a from b") assert isinstance(result.query_body, Union) assert result.query_body.all - result = mysql_parser.parse( - "select a from b union all select a from b", lexer=mysql_lexer - ) + result = mysql_parser.parse("select a from b union all select a from b") assert isinstance(result.query_body, Union) assert result.query_body.all @@ -318,7 +285,6 @@ def test_limit_question_mark(self): """ SELECT * FROM `antinvoice93`.einv_base_info WHERE einv_source = ? ORDER BY gmt_create DESC LIMIT ? """, - lexer=oceanbase_lexer, ) assert result.query_body.limit == '?' @@ -326,7 +292,6 @@ def test_limit_question_mark(self): """ SELECT * FROM `antinvoice93`.einv_base_info WHERE einv_source = ? ORDER BY gmt_create DESC LIMIT 1,? """, - lexer=oceanbase_lexer, ) assert result.query_body.limit == '?' @@ -335,7 +300,6 @@ def test_subquery_limit(self): """ SELECT COUNT(*) FROM ( SELECT * FROM customs_script_match_history LIMIT ? ) a """, - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) @@ -344,7 +308,6 @@ def test_current_timestamp(self): """ SELECT device_id, msg_id, short_msg_key, third_msg_id, mission_id , mission_coe, app_id, payload, template_code, business , ruleset_id, strategy, principal_id, tag, priority , expire_time, gmt_create, status, uriextinfo, sub_templates , immediate_product_version, biz_id, immediate_language_type FROM pushcore_msg WHERE device_id = ? AND principal_id = ? AND status = ? AND expire_time > current_timestamp() """, - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) @@ -353,7 +316,6 @@ def test_select_for_update(self): """ SELECT id, gmt_create, gmt_modified, match_id, match_record_id , user_id, complete_status, notice_push_status, result_push_status, reward_status , join_cost, reward, odps_reward, step_number, gmt_complete , gmt_send_reward, match_type, join_stat_bill_id, complete_stat_bill_id, ext_info FROM sports_user_match_record WHERE match_record_id IN (?) FOR UPDATE """, - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) assert result.query_body.for_update is True @@ -362,7 +324,6 @@ def test_select_for_update(self): """ SELECT id, gmt_create, gmt_modified, match_id, match_record_id , user_id, complete_status, notice_push_status, result_push_status, reward_status , join_cost, reward, odps_reward, step_number, gmt_complete , gmt_send_reward, match_type, join_stat_bill_id, complete_stat_bill_id, ext_info FROM sports_user_match_record WHERE match_record_id IN (?) FOR UPDATE NOWAIT """, - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) assert result.query_body.for_update is True @@ -371,7 +332,6 @@ def test_select_for_update(self): """ SELECT id, gmt_create, gmt_modified, match_id, match_record_id , user_id, complete_status, notice_push_status, result_push_status, reward_status , join_cost, reward, odps_reward, step_number, gmt_complete , gmt_send_reward, match_type, join_stat_bill_id, complete_stat_bill_id, ext_info FROM sports_user_match_record WHERE match_record_id IN (?) FOR UPDATE WAIT 6 """, - lexer=oceanbase_lexer, ) assert isinstance(result, Statement) assert result.query_body.for_update is True @@ -387,9 +347,7 @@ def test_interval(self): AND gmt_create > date_sub(now(), INTERVAL ? DAY) GROUP BY biz_id """ - result = oceanbase_parser.parse( - Utils.remove_sql_text_affects_parser(sql), debug=True, lexer=oceanbase_lexer - ) + result = oceanbase_parser.parse(Utils.remove_sql_text_affects_parser(sql)) assert isinstance(result, Statement) def test_force_index(self): @@ -407,14 +365,14 @@ def test_force_index(self): ORDER BY inst_apply_order_id LIMIT ?, ? ) T2 WHERE t1.order_id = t2.order_id """ sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_interval2(self): sql = """ SELECT h.site, h.ip, h.sm_name, h.pre_group, h.nodegroup , host_name, mount, used_pct, size, used , free, m.node FROM ( SELECT host_name, mount, MAX(used_pct) AS used_pct, MAX(size) AS size, MAX(used) AS used , MIN(free) AS free, MAX(check_time) AS check_time FROM host_disk_used h FORCE INDEX (idx_ct_up_m) WHERE check_time > now() - INTERVAL ? HOUR AND mount IN (?) AND host_name NOT LIKE ? AND host_name NOT LIKE ? AND host_name NOT LIKE ? AND host_name NOT LIKE ? AND host_name NOT LIKE ? AND host_name NOT LIKE ? AND host_name NOT LIKE ? AND host_name NOT LIKE ? AND host_name NOT LIKE ? GROUP BY host_name, mount ORDER BY MAX(used) ) i, mt_armory_host h, ( SELECT ip, GROUP_CONCAT(node) AS node FROM mt_mysql_meta WHERE ip IS NOT NULL AND gmt_alive > now() - INTERVAL ? HOUR GROUP BY ip ) m WHERE i.host_name = h.hostname AND h.pre_group = ? AND m.ip = h.ip ORDER BY used""" sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_regexp(self): @@ -422,7 +380,7 @@ def test_regexp(self): SELECT * FROM file_moving_serial WHERE serial_no REGEXP ? """ sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_distinct_2(self): @@ -430,31 +388,31 @@ def test_distinct_2(self): /* trace_id=0b7cad2e168016361004041132631,rpc_id=0.5c88b07f.9.1 */ SELECT /*+ index(midas_record_value idx_tenant_time) */ DISTINCT(trace_id) FROM midas_record_value where tenant='fascore' and is_expired=0 order by gmt_modified asc limit 500 """ sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_quoted(self): sql = "SELECT Original_artist FROM table_15383430_1 WHERE Theme = 'year'" sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) sql = '''SELECT Original_artist FROM table_15383430_1 WHERE Theme = "year"''' sql = Utils.remove_sql_text_affects_parser(sql) - result = mysql_parser.parse(sql, lexer=mysql_lexer) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) def test_chinese_character(self): sql = "SELECT `净值` FROM FundTable WHERE `销售状态` = \"正常申购\"" sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) sql = "SELECT 净值 FROM FundTable WHERE 销售状态 = \"正常申购\"" sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) sql = '''SELECT 赎回状态a FROM FundTable WHERE 重b仓 like \"北部湾港%\"''' sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_current_date(self): @@ -466,7 +424,7 @@ def test_current_date(self): where t1 > (CURRENT_DATE() - INTERVAL 30 day)+'0' """ - result = oceanbase_parser.parse(sql, debug=True, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_double_type(self): @@ -474,7 +432,7 @@ def test_double_type(self): SELECT Winner FROM table_11621915_1 WHERE Purse > 964017.2297960471 AND Date_ds = "may 28" """ sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_union_has_order_limit(self): @@ -482,14 +440,14 @@ def test_union_has_order_limit(self): ( select 球员id from 球员夺冠次数 order by 冠军次数 asc limit 3 ) union ( select 球员id from 球员夺冠次数 order by 亚军次数 desc limit 5 ) """ sql = Utils.remove_sql_text_affects_parser(sql) - result = mysql_parser.parse(sql, lexer=mysql_lexer) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) def test_order_by_has_parentheses(self): sql = """ SELECT channel_code , contact_number FROM customer_contact_channels WHERE active_to_date - active_from_date = (SELECT active_to_date - active_from_date FROM customer_contact_channels ORDER BY (active_to_date - active_from_date) DESC LIMIT 1) """ sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_boolean_expression(self): @@ -497,7 +455,7 @@ def test_boolean_expression(self): SELECT T1.list_followers, T2.user_subscriber = 1 FROM lists AS T1 INNER JOIN lists_users AS T2 ON T1.user_id = T2.user_id AND T2.list_id = T2.list_id WHERE T2.user_id = 4208563 ORDER BY T1.list_followers DESC LIMIT 1 """ sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_date_add(self): @@ -505,7 +463,7 @@ def test_date_add(self): select date_format(date_format(date_add(biz_date, interval -1 day), '%y%m%d'), '%y%m%d') from t """ sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, debug=True, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_as(self): @@ -513,7 +471,7 @@ def test_as(self): select t2.biz_date as biz_date, f1.calculate_field / t2.calculate_field1 as d_remain_rate from ( select t1.biz_date as biz_date , count(DISTINCT if(t1.biz_date_is_visit = '1', t1.user_id, null)) as calculate_field1 from ( select * from pets_user_miaowa_galileo_visit_user_di ) t1 where t1.appname in ('AppPetWXSS', 'HelloPet') and t1.biz_date between date_format(date_sub(date_format(date_sub(curdate(), interval 1 day), '%Y%m%d'), interval 1 day), '%Y%m%d') and date_format(date_sub(curdate(), interval 1 day), '%Y%m%d') group by t1.biz_date ) t2 left join ( select date_format(date_format(date_add(t1.biz_date, interval -1 day), '%Y%m%d'), '%Y%m%d') as biz_date , count(DISTINCT if(datediff(t1.biz_date, t1.last_visit_date) = 1 and t1.biz_date_is_visit = '1', t1.user_id, null)) as calculate_field from ( select * from pets_user_miaowa_galileo_visit_user_di ) t1 where t1.appname in ('AppPetWXSS', 'HelloPet') and t1.biz_date between date_format(date_sub(date_format(date_sub(curdate(), interval 1 day), '%Y%m%d'), interval 1 day), '%Y%m%d') and date_format(date_sub(curdate(), interval 1 day), '%Y%m%d') group by date_format(date_format(date_add(t1.biz_date, interval -1 day), '%Y%m%d'), '%Y%m%d') ) f1 on t2.biz_date = f1.biz_date order by biz_date asc limit 0, 1000 """ sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, debug=True, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_single_quote_escape(self): @@ -521,14 +479,14 @@ def test_single_quote_escape(self): SELECT director_id FROM movies WHERE movie_title = 'It''s Winter' """ sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) sql = """ SELECT director_id FROM movies WHERE movie_title = "It''s Winter" """ sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_operator(self): @@ -536,7 +494,7 @@ def test_operator(self): select count(1) from train_order_info where occupy_type in ( ? ) and order_serial_no = ? and connect_type = ? and merchant_id = '' and order_state in ( ? ) and ticket_machine_id in ( ? ) and lock_state = ? and phone_verify_status = ? and window_no = ? and passenger_info like concat('%',concat( ?,'%')) and departure_station like concat('%',concat( ?,'%')) and arrival_station like concat('%',concat( ?,'%')) and merchant_id = ? and createtime >= ? and createtime <= ? and gmt_booked >= ? and gmt_booked <= ? and gmt_departure >= ? and gmt_departure <= ? and train_code = ? and pay_serial_no = ? and env = ? and gmt_distribute >= ? and gmt_distribute <= ? and inquire_type in ( ? ) and merchant_business_type = ? and ability_require & ? = ? """ sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_engine_non_reserved(self): @@ -559,12 +517,12 @@ def test_engine_non_reserved(self): order by gmt_modified desc """ sql = Utils.remove_sql_text_affects_parser(sql) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) def test_odps_function(self): sql = """select * from tbl where pt=max_pt('myproject.tbl')""" - result = odps_parser.parse(sql, lexer=odps_lexer, debug=True) + result = odps_parser.parse(sql) assert isinstance(result, Statement) def test_ob_tochar_function(self): @@ -597,9 +555,9 @@ def test_ob_tochar_function(self): c.last_ddl_time""", ] for sql in test_sqls: - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) - result = odps_parser.parse(sql, lexer=odps_lexer, debug=True) + result = odps_parser.parse(sql) assert isinstance(result, Statement) def test_regsub_str_function(self): @@ -608,11 +566,11 @@ def test_regsub_str_function(self): "SELECT REGEXP_SUBSTR('hello world', '(.)', 1, 0, 'i')", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) - result = odps_parser.parse(sql, lexer=odps_lexer, debug=True) + result = odps_parser.parse(sql) assert isinstance(result, Statement) def test_pipes_function(self): @@ -622,11 +580,11 @@ def test_pipes_function(self): "SELECT * FROM t WHERE a in (1) || 1+1 || 1%2 || a like '%1'", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) - result = odps_parser.parse(sql, lexer=odps_lexer, debug=True) + result = odps_parser.parse(sql) assert isinstance(result, Statement) def test_ob_function(self): @@ -646,9 +604,9 @@ def test_ob_function(self): "SELECT DECODE(1, 1, 'One', 2, 'Two', 'Other') FROM DUAL", ] for sql in test_sqls: - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) - result = odps_parser.parse(sql, lexer=odps_lexer, debug=True) + result = odps_parser.parse(sql) assert isinstance(result, Statement) def test_select_fetch_condition(self): @@ -663,11 +621,11 @@ def test_select_fetch_condition(self): "SELECT * FROM DUAL FETCH NEXT ? ROW ONLY", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) - result = odps_parser.parse(sql, lexer=odps_lexer, debug=True) + result = odps_parser.parse(sql) assert isinstance(result, Statement) def test_select_stmt_opts(self): @@ -686,11 +644,11 @@ def test_select_stmt_opts(self): "SELECT STRAIGHT_JOIN * FROM DUAL FETCH FIRST 5 ROWS ONLY", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) - result = odps_parser.parse(sql, lexer=odps_lexer, debug=True) + result = odps_parser.parse(sql) assert isinstance(result, Statement) def test_identifier(self): @@ -701,11 +659,11 @@ def test_identifier(self): "SELECT * FROM DUAL WHERE 0b12=1", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) - result = odps_parser.parse(sql, lexer=odps_lexer, debug=True) + result = odps_parser.parse(sql) assert isinstance(result, Statement) def test_date_func(self): @@ -720,11 +678,11 @@ def test_date_func(self): "SELECT DATE_SUB('2022-01-01', 5)", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) - result = odps_parser.parse(sql, lexer=odps_lexer, debug=True) + result = odps_parser.parse(sql) assert isinstance(result, Statement) def test_delete_statement(self): @@ -735,11 +693,11 @@ def test_delete_statement(self): "DELETE t1 FROM t1 LEFT JOIN t2 ON t1.id=t2.id PARTITION(p1,p2) WHERE t2.id IS NULL", ] for sql in test_sqls: - result = mysql_parser.parse(sql, lexer=mysql_lexer, debug=True) + result = mysql_parser.parse(sql) assert isinstance(result, Statement) - result = oceanbase_parser.parse(sql, lexer=oceanbase_lexer, debug=True) + result = oceanbase_parser.parse(sql) assert isinstance(result, Statement) - result = odps_parser.parse(sql, lexer=odps_lexer, debug=True) + result = odps_parser.parse(sql) assert isinstance(result, Statement) diff --git a/test/parser/test_parser_utils.py b/test/parser/test_parser_utils.py index 90d65af..34a8c9c 100644 --- a/test/parser/test_parser_utils.py +++ b/test/parser/test_parser_utils.py @@ -15,8 +15,7 @@ import unittest from src.optimizer.formatter import format_sql -from src.parser.mysql_parser.parser import parser -from src.parser.mysql_parser.lexer import lexer +from src.parser.mysql_parser import parser from src.parser.parser_utils import ParserUtils @@ -27,7 +26,7 @@ def test_get_filter_column(self): "where a.b = 1 and b.c = 2 group by name,age " "having count(*)>2 and avg(age)<20 order by a asc,b desc limit 1,10" ) - visitor = ParserUtils.format_statement(parser.parse(sql, lexer=lexer)) + visitor = ParserUtils.format_statement(parser.parse(sql)) table_list = visitor.table_list projection_column_list = visitor.projection_column_list order_list = visitor.order_list @@ -118,7 +117,7 @@ def test_get_filter_column2(self): tars_sqldiag_all.tenant_name, tars_sqldiag_all.sql_id, tars_sqldiag_all.diag_type """ - visitor = ParserUtils.format_statement(parser.parse(sql, lexer=lexer)) + visitor = ParserUtils.format_statement(parser.parse(sql)) table_list = visitor.table_list projection_column_list = visitor.projection_column_list order_list = visitor.order_list @@ -189,7 +188,7 @@ def test_parameterized_query(self): "where a.b = 1 and b.c = 2 and a.d in ('2','3','6') group by name,age " "having count(*)>2 and avg(age)<20 order by a asc,b desc limit 3,10" ) - statement_node = ParserUtils.parameterized_query(parser.parse(sql, lexer=lexer)) + statement_node = ParserUtils.parameterized_query(parser.parse(sql)) format_sql(statement_node, 0) def test_parameterized_query2(self): @@ -205,19 +204,19 @@ def test_parameterized_query2(self): when -2 then 15 else merge_record.merge_result END as merge_result, server_release_repo.completed, server_release_repo.create_time, server_release_repo.update_time FROM server_release_repo left join merge_record on server_release_repo.merge_record_id = merge_record.id WHERE 1 = 1 and integrate = 0 and completed = 1 and deleted = 0 and merge_record_id != -1 """ - statement_node = ParserUtils.parameterized_query(parser.parse(sql, lexer=lexer)) + statement_node = ParserUtils.parameterized_query(parser.parse(sql)) format_sql(statement_node, 0) def test_parameterized_query3(self): sql = """select id,gmt_create,gmt_modified,proj_code,matter_code,'ATUSER' act_type,content,operator,operator_no,status,biz_code,biz_id,content_detail from lc_opr_biz_activity where id in ( select max(id) id from lc_opr_biz_activity t1 join ( select act_type ,biz_activity_id,task_id from lc_opr_schedule where user_id = '291909' and status = '00' and act_type = 'ATUSER' and matter_code = 'M210713P0689I00007' and task_id is not null ) t2 on t1.id = t2.biz_activity_id group by t2.task_id ) union select id,gmt_create,gmt_modified,proj_code,matter_code,act_type,content,operator,operator_no,status,biz_code,biz_id,content_detail from lc_opr_biz_activity where id in( select max(id) id from lc_opr_biz_activity where matter_code = 'M210713P0689I00007' and biz_code = 'TASK' and biz_id not in ( select distinct task_id from lc_opr_schedule where user_id = '291909' and status = '00' and act_type = 'ATUSER' and matter_code = 'M210713P0689I00007' and task_id is not null ) group by biz_id )""" - statement_node = ParserUtils.parameterized_query(parser.parse(sql, lexer=lexer)) + statement_node = ParserUtils.parameterized_query(parser.parse(sql)) format_sql(statement_node, 0) def test_subquery_expression(self): sql = """ SELECT COUNT(*) FROM ( SELECT * FROM customs_script_match_history LIMIT ? ) a """ - ParserUtils.format_statement(parser.parse(sql, lexer=lexer)) + ParserUtils.format_statement(parser.parse(sql)) def test_sql_1(self): sql = """ @@ -252,11 +251,11 @@ def test_sql_1(self): AND t2.ds = ? AND t1.idc IN (?) AND t1.nc_ip NOT IN ( SELECT DISTINCT ip FROM yusuan_unires_docker_nc_host WHERE pool LIKE ? ) UNION SELECT t1.idc, t2.ds AS ds, SUM(t2.yhat) AS disk FROM `sync_mt_mysql_meta` t1, space_used_forecast_per_inst t2 WHERE t2.node_name = t1.node AND t2.gmt_create = ? AND t1.idc IS NOT NULL AND t1.cluster_name NOT IN (?) AND t2.ds = ? AND t1.idc IN (?) AND t1.nc_ip NOT IN ( SELECT DISTINCT ip FROM yusuan_unires_docker_nc_host WHERE pool LIKE ? ) UNION SELECT t1.idc, t2.ds AS ds, SUM(t2.yhat) AS disk FROM `sync_mt_mysql_meta` t1, space_used_forecast_per_inst t2 WHERE t2.node_name = t1.node AND t2.gmt_create = ? AND t1.idc IS NOT NULL AND t1.cluster_name NOT IN (?) AND t2.ds = ? AND t1.idc IN (?) AND t1.nc_ip NOT IN ( SELECT DISTINCT ip FROM yusuan_unires_docker_nc_host WHERE pool LIKE ? ) UNION SELECT t1.idc, t2.ds AS ds, SUM(t2.yhat) AS disk FROM `sync_mt_mysql_meta` t1, space_used_forecast_per_inst t2 WHERE t2.node_name = t1.node AND t2.gmt_create = ? AND t1.idc IS NOT NULL AND t1.cluster_name NOT IN (?) AND t2.ds = ? AND t1.idc IN (?) AND t1.nc_ip NOT IN ( SELECT DISTINCT ip FROM yusuan_unires_docker_nc_host WHERE pool LIKE ? ) UNION SELECT t1.idc, t2.ds AS ds, SUM(t2.yhat) AS disk FROM `sync_mt_mysql_meta` t1, space_used_forecast_per_inst t2 WHERE t2.node_name = t1.node AND t2.gmt_create = ? AND t1.idc IS NOT NULL AND t1.cluster_name NOT IN (?) AND t2.ds = ? AND t1.idc IN (?) AND t1.nc_ip NOT IN ( SELECT DISTINCT ip FROM yusuan_unires_docker_nc_host WHERE pool LIKE ? ) UNION SELECT t1.idc, t2.ds AS ds, SUM(t2.yhat) AS disk FROM `sync_mt_mysql_meta` t1, space_used_forecast_per_inst t2 WHERE t2.node_name = t1.node AND t2.gmt_create = ? AND t1.idc IS NOT NULL AND t1.cluster_name NOT IN (?) AND t2.ds = ? AND t1.idc IN (?) AND t1.nc_ip NOT IN ( SELECT DISTINCT ip FROM yusuan_unires_docker_nc_host WHERE pool LIKE ? ) UNION SELECT t1.idc, t2.ds AS ds, SUM(t2.yhat) AS disk FROM `sync_mt_mysql_meta` t1, space_used_forecast_per_inst t2 WHERE t2.node_name = t1.node AND t2.gmt_create = ? AND t1.idc IS NOT NULL AND t1.cluster_name NOT IN (?) AND t2.ds = ? AND t1.idc IN (?) AND t1.nc_ip NOT IN ( SELECT DISTINCT ip FROM yusuan_unires_docker_nc_host WHERE pool LIKE ? ) UNION SELECT t1.idc, t2.ds AS ds, SUM(t2.yhat) AS disk FROM `sync_mt_mysql_meta` t1, space_used_forecast_per_inst t2 WHERE t2.node_name = t1.node AND t2.gmt_create = ? AND t1.idc IS NOT NULL AND t1.cluster_name NOT IN (?) AND t2.ds = ? AND t1.idc IN (?) AND t1.nc_ip NOT IN ( SELECT DISTINCT ip FROM yusuan_unires_docker_nc_host WHERE pool LIKE ? ) UNION SELECT t1.idc, t2.ds AS ds, SUM(t2.yhat) AS disk FROM `sync_mt_mysql_meta` t1, space_used_forecast_per_inst t2 WHERE t2.node_name = t1.node AND t2.gmt_create = ? AND t1.idc IS NOT NULL AND t1.cluster_name NOT IN (?) AND t2.ds = ? AND t1.idc IN (?) AND t1.nc_ip NOT IN ( SELECT DISTINCT ip FROM yusuan_unires_docker_nc_host WHERE pool LIKE ? ) UNION SELECT t1.idc, t2.ds AS ds, SUM(t2.yhat) AS disk FROM `sync_mt_mysql_meta` t1, space_used_forecast_per_inst t2 WHERE t2.node_name = t1.node AND t2.gmt_create = ? AND t1.idc IS NOT NULL AND t1.cluster_name NOT IN (?) AND t2.ds = ? AND t1.idc IN (?) AND t1.nc_ip NOT IN ( SELECT DISTINCT ip FROM yusuan_unires_docker_nc_host WHERE pool LIKE ? ) UNION SELECT t1.idc, t2.ds AS ds, SUM(t2.yhat) AS disk FROM `sync_mt_mysql_meta` t1, space_used_forecast_per_inst t2 WHERE t2.node_name = t1.node AND t2.gmt_create = ? AND t1.idc IS NOT NULL AND t1.cluster_name NOT IN (?) AND t2.ds = ? AND t1.idc IN (?) AND t1.nc_ip NOT IN ( SELECT DISTINCT ip FROM yusuan_unires_docker_nc_host WHERE pool LIKE ? ) UNION SELECT t1.idc, t2.ds AS ds, SUM(t2.yhat) AS disk FROM `sync_mt_mysql_meta` t1, space_used_forecast_per_inst t2 WHERE t2.node_name = t1.node AND t2.gmt_create = ? AND t1.idc IS NOT NULL AND t1.cluster_name NOT IN (?) AND t2.ds = ? AND t1.idc IN (?) AND t1.nc_ip NOT IN ( SELECT DISTINCT ip FROM yusuan_unires_docker_nc_host WHERE pool LIKE ? ) """ - ParserUtils.format_statement(parser.parse(sql, lexer=lexer)) + ParserUtils.format_statement(parser.parse(sql)) def test_recursion_error(self): sql = """SELECT id, `table_name`, version, primary_id, template , template_md5, security_level, `nullable`, status, `param_group` , description, operator, global_id, govern_type, utc_create , utc_modified FROM param_template WHERE (table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ? OR table_name = ? AND version = ?) AND status IN (?) ORDER BY table_name ASC, utc_create DESC LIMIT ?, ?""" - statement = parser.parse(sql, lexer=lexer) + statement = parser.parse(sql) ParserUtils.format_statement(statement) def test_in_subquery(self): @@ -264,7 +263,7 @@ def test_in_subquery(self): 'select sum(cost) from costs where eventtype = \'treatment\' and eventid in ' '(select treatmentid from treatment where treatmentname = \'bleeding scan\')' ) - visitor = ParserUtils.format_statement(parser.parse(sql, lexer=lexer)) + visitor = ParserUtils.format_statement(parser.parse(sql)) table_list = visitor.table_list assert table_list == [ { @@ -286,7 +285,7 @@ def test_update_parameterize(self): sql = """ UPDATE `t1` SET `c`='11' WHERE (`id`='1111111') """ - statement_node = ParserUtils.parameterized_query(parser.parse(sql, lexer=lexer)) + statement_node = ParserUtils.parameterized_query(parser.parse(sql)) statement = format_sql(statement_node, 0) assert statement == """UPDATE t1 SET c = ? WHERE id = ?""" @@ -294,7 +293,7 @@ def test_between(self): sql = """ select max(`successRate`) AS `successRate` from `table_850d` where `period` between '2022-07-11 00:00:00' and '2022-07-11 23:59:59' and `successRate` < 0.35; """ - visitor = ParserUtils.format_statement(parser.parse(sql, lexer=lexer)) + visitor = ParserUtils.format_statement(parser.parse(sql)) table_list = visitor.table_list assert table_list == [ {