From 625172adb1a4bc6b988b5413ca31282169976b40 Mon Sep 17 00:00:00 2001 From: dibyendumajumdar Date: Sun, 2 Jun 2024 15:47:24 +0100 Subject: [PATCH 1/5] issue #169 update to v1.0.0 --- CMakeLists.txt | 2 +- mir/.clang-format | 1 - mir/LICENSE | 2 +- mir/c2mir/aarch64/caarch64-ABI-code.c | 17 +- mir/c2mir/aarch64/caarch64-code.c | 5 +- mir/c2mir/aarch64/caarch64.h | 2 +- mir/c2mir/aarch64/mirc_aarch64_float.h | 2 +- mir/c2mir/aarch64/mirc_aarch64_limits.h | 2 +- mir/c2mir/aarch64/mirc_aarch64_linux.h | 2 +- mir/c2mir/aarch64/mirc_aarch64_stdarg.h | 2 +- mir/c2mir/aarch64/mirc_aarch64_stddef.h | 2 +- mir/c2mir/aarch64/mirc_aarch64_stdint.h | 2 +- mir/c2mir/c2mir.c | 2256 +++-- mir/c2mir/c2mir.h | 2 +- mir/c2mir/mirc.h | 2 +- mir/c2mir/mirc_iso646.h | 2 +- mir/c2mir/mirc_stdalign.h | 2 +- mir/c2mir/mirc_stdbool.h | 2 +- mir/c2mir/mirc_stdnoreturn.h | 2 +- mir/c2mir/ppc64/cppc64-ABI-code.c | 28 +- mir/c2mir/ppc64/cppc64-code.c | 5 +- mir/c2mir/ppc64/cppc64.h | 2 +- mir/c2mir/ppc64/mirc_ppc64_float.h | 28 +- mir/c2mir/ppc64/mirc_ppc64_limits.h | 2 +- mir/c2mir/ppc64/mirc_ppc64_linux.h | 18 +- mir/c2mir/ppc64/mirc_ppc64_stdarg.h | 2 +- mir/c2mir/ppc64/mirc_ppc64_stddef.h | 2 +- mir/c2mir/ppc64/mirc_ppc64_stdint.h | 2 +- mir/c2mir/riscv64/criscv64-ABI-code.c | 27 +- mir/c2mir/riscv64/criscv64-code.c | 5 +- mir/c2mir/riscv64/criscv64.h | 2 +- mir/c2mir/riscv64/mirc_riscv64_float.h | 2 +- mir/c2mir/riscv64/mirc_riscv64_limits.h | 2 +- mir/c2mir/riscv64/mirc_riscv64_linux.h | 2 +- mir/c2mir/riscv64/mirc_riscv64_stdarg.h | 2 +- mir/c2mir/riscv64/mirc_riscv64_stddef.h | 2 +- mir/c2mir/riscv64/mirc_riscv64_stdint.h | 2 +- mir/c2mir/s390x/cs390x-ABI-code.c | 17 +- mir/c2mir/s390x/cs390x-code.c | 5 +- mir/c2mir/s390x/cs390x.h | 2 +- mir/c2mir/s390x/mirc_s390x_float.h | 2 +- mir/c2mir/s390x/mirc_s390x_limits.h | 2 +- mir/c2mir/s390x/mirc_s390x_linux.h | 2 +- mir/c2mir/s390x/mirc_s390x_stdarg.h | 2 +- mir/c2mir/s390x/mirc_s390x_stddef.h | 2 +- mir/c2mir/s390x/mirc_s390x_stdint.h | 2 +- mir/c2mir/x86_64/cx86_64-ABI-code.c | 45 +- mir/c2mir/x86_64/cx86_64-code.c | 2 +- mir/c2mir/x86_64/cx86_64.h | 2 +- mir/c2mir/x86_64/mirc_x86_64_float.h | 2 +- mir/c2mir/x86_64/mirc_x86_64_limits.h | 2 +- mir/c2mir/x86_64/mirc_x86_64_linux.h | 2 +- mir/c2mir/x86_64/mirc_x86_64_stdarg.h | 2 +- mir/c2mir/x86_64/mirc_x86_64_stddef.h | 2 +- mir/c2mir/x86_64/mirc_x86_64_stdint.h | 2 +- mir/c2mir/x86_64/mirc_x86_64_win.h | 2 +- mir/mir-aarch64.c | 220 +- mir/mir-aarch64.h | 61 + mir/mir-bitmap.h | 38 +- mir/mir-dlist.h | 2 +- mir/mir-gen-aarch64.c | 780 +- mir/mir-gen-ppc64.c | 823 +- mir/mir-gen-riscv64.c | 839 +- mir/mir-gen-s390x.c | 677 +- mir/mir-gen-stub.c | 3 +- mir/mir-gen-x86_64.c | 2098 +++-- mir/mir-gen.c | 10691 +++++++++++++++------- mir/mir-gen.h | 14 +- mir/mir-hash.h | 18 +- mir/mir-htab.h | 2 +- mir/mir-interp.c | 477 +- mir/mir-ppc64.c | 288 +- mir/mir-ppc64.h | 114 + mir/mir-reduce.h | 15 +- mir/mir-riscv64.c | 385 +- mir/mir-riscv64.h | 78 + mir/mir-s390x.c | 193 +- mir/mir-s390x.h | 57 + mir/mir-varr.h | 8 +- mir/mir-x86_64.c | 391 +- mir/mir-x86_64.h | 59 + mir/mir.c | 1927 ++-- mir/mir.h | 198 +- mir/real-time.h | 2 +- src/ravi_mirjit.c | 4 +- 85 files changed, 15820 insertions(+), 7185 deletions(-) create mode 100644 mir/mir-aarch64.h create mode 100644 mir/mir-ppc64.h create mode 100644 mir/mir-riscv64.h create mode 100644 mir/mir-s390x.h create mode 100644 mir/mir-x86_64.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 259112d3..e20de4b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ option(NO_JIT "Controls whether JIT should be disabled, default is OFF" OFF) option(STATIC_BUILD "Build static version of Ravi, default is OFF" OFF) option(COMPUTED_GOTO "Controls whether the interpreter switch will use computed gotos on gcc/clang, default is ON" ON) option(LTESTS "Controls whether ltests are enabled in Debug mode; note requires Debug build" ON) -option(ASAN "Controls whether address sanitizer should be enabled" OFF) +option(ASAN "Controls whether address sanitizer should be enabled" ON) option(RAVICOMP "Controls whether to link in RaviComp" ON) # By we enable MIR JIT diff --git a/mir/.clang-format b/mir/.clang-format index 23f5787f..03227500 100644 --- a/mir/.clang-format +++ b/mir/.clang-format @@ -8,7 +8,6 @@ PointerAlignment: Right BreakBeforeBinaryOperators: All ConstructorInitializerIndentWidth: 2 ContinuationIndentWidth: 2 -ContinuationIndentWidth: 2 PenaltyBreakBeforeFirstCallParameter: 10000 SortIncludes: false BreakStringLiterals: true diff --git a/mir/LICENSE b/mir/LICENSE index c8e33beb..0c20bb1b 100644 --- a/mir/LICENSE +++ b/mir/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018-2020 Vladimir Makarov +Copyright (c) 2018-2024 Vladimir Makarov Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/mir/c2mir/aarch64/caarch64-ABI-code.c b/mir/c2mir/aarch64/caarch64-ABI-code.c index 687c7292..3eb98bec 100644 --- a/mir/c2mir/aarch64/caarch64-ABI-code.c +++ b/mir/c2mir/aarch64/caarch64-ABI-code.c @@ -1,11 +1,12 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . aarch64 call ABI target specific code. */ typedef int target_arg_info_t; -static void target_init_arg_vars (c2m_ctx_t c2m_ctx, target_arg_info_t *arg_info) {} +static void target_init_arg_vars (c2m_ctx_t c2m_ctx MIR_UNUSED, + target_arg_info_t *arg_info MIR_UNUSED) {} static int target_return_by_addr_p (c2m_ctx_t c2m_ctx, struct type *ret_type) { return ((ret_type->mode == TM_STRUCT || ret_type->mode == TM_UNION) @@ -13,10 +14,10 @@ static int target_return_by_addr_p (c2m_ctx_t c2m_ctx, struct type *ret_type) { } static int reg_aggregate_size (c2m_ctx_t c2m_ctx, struct type *type) { - int size; + size_t size; if (type->mode != TM_STRUCT && type->mode != TM_UNION) return -1; - return (size = type_size (c2m_ctx, type)) <= 2 * 8 ? size : -1; + return (size = type_size (c2m_ctx, type)) <= 2 * 8 ? (int) size : -1; } static void target_add_res_proto (c2m_ctx_t c2m_ctx, struct type *ret_type, @@ -77,7 +78,8 @@ static void target_add_ret_ops (c2m_ctx_t c2m_ctx, struct type *ret_type, op_t r gen_multiple_load_store (c2m_ctx, ret_type, VARR_ADDR (MIR_op_t, ret_ops), res.mir_op, TRUE); } -static MIR_type_t target_get_blk_type (c2m_ctx_t c2m_ctx, struct type *arg_type) { +static MIR_type_t target_get_blk_type (c2m_ctx_t c2m_ctx MIR_UNUSED, + struct type *arg_type MIR_UNUSED) { return MIR_T_BLK; /* one BLK is enough */ } @@ -91,7 +93,8 @@ static void target_add_call_arg_op (c2m_ctx_t c2m_ctx, struct type *arg_type, simple_add_call_arg_op (c2m_ctx, arg_type, arg_info, arg); } -static int target_gen_gather_arg (c2m_ctx_t c2m_ctx, const char *name, struct type *arg_type, - decl_t param_decl, target_arg_info_t *arg_info) { +static int target_gen_gather_arg (c2m_ctx_t c2m_ctx MIR_UNUSED, const char *name MIR_UNUSED, + struct type *arg_type MIR_UNUSED, decl_t param_decl MIR_UNUSED, + target_arg_info_t *arg_info MIR_UNUSED) { return FALSE; } diff --git a/mir/c2mir/aarch64/caarch64-code.c b/mir/c2mir/aarch64/caarch64-code.c index a273b86f..6d9ed04e 100644 --- a/mir/c2mir/aarch64/caarch64-code.c +++ b/mir/c2mir/aarch64/caarch64-code.c @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ #include "../mirc.h" @@ -19,7 +19,8 @@ static string_include_t standard_includes[] #define ADJUST_VAR_ALIGNMENT(c2m_ctx, align, type) \ aarch64_adjust_var_alignment (c2m_ctx, align, type) -static int aarch64_adjust_var_alignment (c2m_ctx_t c2m_ctx, int align, struct type *type) { +static int aarch64_adjust_var_alignment (c2m_ctx_t c2m_ctx MIR_UNUSED, int align, + struct type *type MIR_UNUSED) { return align; } diff --git a/mir/c2mir/aarch64/caarch64.h b/mir/c2mir/aarch64/caarch64.h index 945a2ecd..4ce189df 100644 --- a/mir/c2mir/aarch64/caarch64.h +++ b/mir/c2mir/aarch64/caarch64.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ #include diff --git a/mir/c2mir/aarch64/mirc_aarch64_float.h b/mir/c2mir/aarch64/mirc_aarch64_float.h index 36ca51f7..2dc772e2 100644 --- a/mir/c2mir/aarch64/mirc_aarch64_float.h +++ b/mir/c2mir/aarch64/mirc_aarch64_float.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 5.2.4.2.2 */ diff --git a/mir/c2mir/aarch64/mirc_aarch64_limits.h b/mir/c2mir/aarch64/mirc_aarch64_limits.h index 68a71ed9..f0457866 100644 --- a/mir/c2mir/aarch64/mirc_aarch64_limits.h +++ b/mir/c2mir/aarch64/mirc_aarch64_limits.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See 5.2.4.2 */ diff --git a/mir/c2mir/aarch64/mirc_aarch64_linux.h b/mir/c2mir/aarch64/mirc_aarch64_linux.h index e2295a31..7a3dd26d 100644 --- a/mir/c2mir/aarch64/mirc_aarch64_linux.h +++ b/mir/c2mir/aarch64/mirc_aarch64_linux.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ static char aarch64_mirc[] diff --git a/mir/c2mir/aarch64/mirc_aarch64_stdarg.h b/mir/c2mir/aarch64/mirc_aarch64_stdarg.h index fab5e508..37e9cd14 100644 --- a/mir/c2mir/aarch64/mirc_aarch64_stdarg.h +++ b/mir/c2mir/aarch64/mirc_aarch64_stdarg.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ static char stdarg_str[] diff --git a/mir/c2mir/aarch64/mirc_aarch64_stddef.h b/mir/c2mir/aarch64/mirc_aarch64_stddef.h index 05d63c78..76ff4f0b 100644 --- a/mir/c2mir/aarch64/mirc_aarch64_stddef.h +++ b/mir/c2mir/aarch64/mirc_aarch64_stddef.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 7.19 */ diff --git a/mir/c2mir/aarch64/mirc_aarch64_stdint.h b/mir/c2mir/aarch64/mirc_aarch64_stdint.h index 4bda3749..c7afad93 100644 --- a/mir/c2mir/aarch64/mirc_aarch64_stdint.h +++ b/mir/c2mir/aarch64/mirc_aarch64_stdint.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 7.20 */ diff --git a/mir/c2mir/c2mir.c b/mir/c2mir/c2mir.c index 6a3b8ca0..9e04e545 100644 --- a/mir/c2mir/c2mir.c +++ b/mir/c2mir/c2mir.c @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ /* C to MIR compiler. It is a four pass compiler: @@ -77,7 +77,7 @@ typedef struct stream { fpos_t fpos; /* file pos to resume file stream */ const char *start, *curr; /* non NULL only for string stream */ int ifs_length_at_stream_start; /* length of ifs at the stream start */ -} * stream_t; +} *stream_t; DEF_VARR (stream_t); @@ -294,6 +294,7 @@ enum type_mode { struct type { node_t pos_node; /* set up and used only for checking type correctness */ struct type *arr_type; /* NULL or array type before its adjustment */ + MIR_alias_t antialias; /* it can be non-zero only for pointers */ struct type_qual type_qual; enum type_mode mode; char func_type_before_adjustment_p; @@ -371,14 +372,18 @@ static int char_is_signed_p (void) { return MIR_CHAR_MAX == MIR_SCHAR_MAX; } enum str_flag { FLAG_EXT = 1, FLAG_C89, FLAG_EXT89 }; -static int str_eq (tab_str_t str1, tab_str_t str2, void *arg) { +static int str_eq (tab_str_t str1, tab_str_t str2, void *arg MIR_UNUSED) { return str1.str.len == str2.str.len && memcmp (str1.str.s, str2.str.s, str1.str.len) == 0; } -static htab_hash_t str_hash (tab_str_t str, void *arg) { - return mir_hash (str.str.s, str.str.len, 0x42); +static htab_hash_t str_hash (tab_str_t str, void *arg MIR_UNUSED) { + return (htab_hash_t) mir_hash (str.str.s, str.str.len, 0x42); +} +static int str_key_eq (tab_str_t str1, tab_str_t str2, void *arg MIR_UNUSED) { + return str1.key == str2.key; +} +static htab_hash_t str_key_hash (tab_str_t str, void *arg MIR_UNUSED) { + return (htab_hash_t) mir_hash64 (str.key, 0x24); } -static int str_key_eq (tab_str_t str1, tab_str_t str2, void *arg) { return str1.key == str2.key; } -static htab_hash_t str_key_hash (tab_str_t str, void *arg) { return mir_hash64 (str.key, 0x24); } static str_t uniq_cstr (c2m_ctx_t c2m_ctx, const char *str); @@ -463,9 +468,10 @@ void c2mir_finish (MIR_context_t ctx) { '}'. The input is parse tokens and the output is the following AST nodes (the AST root is transl_unit): -expr : N_I | N_L | N_LL | N_U | N_UL | N_ULL | N_F | N_D | N_LD - | N_CH | N_CH16 | N_CH32 | N_STR | N_STR16 | N_STR32 | N_ID - | N_ADD (expr) | N_SUB (expr) | N_ADD (expr, expr) | N_SUB (expr, expr) +const : N_I | N_L | N_LL | N_U | N_UL | N_ULL | N_F | N_D | N_LD + | N_CH | N_CH16 | N_CH32 | N_STR | N_STR16 | N_STR32 +expr : const | N_ID | N_LABEL_ADDR (N_ID) | N_ADD (expr) + | N_SUB (expr) | N_ADD (expr, expr) | N_SUB (expr, expr) | N_MUL (expr, expr) | N_DIV (expr, expr) | N_MOD (expr, expr) | N_LSH (expr, expr) | N_RSH (expr, expr) | N_NOT (expr) | N_BITWISE_NOT (expr) @@ -488,12 +494,18 @@ label: N_CASE(expr) | N_CASE(expr,expr) | N_DEFAULT | N_LABEL(N_ID) stmt: compound_stmt | N_IF(N_LIST:(label)*, expr, stmt, stmt?) | N_SWITCH(N_LIST:(label)*, expr, stmt) | (N_WHILE|N_DO) (N_LIST:(label)*, expr, stmt) | N_FOR(N_LIST:(label)*,(N_LIST: declaration+ | expr)?, expr?, expr?, stmt) - | N_GOTO(N_LIST:(label)*, N_ID) | (N_CONTINUE|N_BREAK) (N_LIST:(label)*) + | N_GOTO(N_LIST:(label)*, N_ID) | N_INDIRECT_GOTO(N_LIST:(label)*, expr) + | (N_CONTINUE|N_BREAK) (N_LIST:(label)*) | N_RETURN(N_LIST:(label)*, expr?) | N_EXPR(N_LIST:(label)*, expr) compound_stmt: N_BLOCK(N_LIST:(label)*, N_LIST:(declaration | stmt)*) -declaration: N_SPEC_DECL(N_SHARE(declaration_specs), declarator?, initializer?) | st_assert +asm: N_ASM(N_STR | N_STR16 | N_STR32) +attr_arg: const | N_ID +attr: N_ATTR(N_ID, NLIST:(attr_arg)*) +attrs: N_LIST:(attrs)* +declaration: N_SPEC_DECL(N_SHARE(declaration_specs), declarator?, attrs?, asm?, initializer?) + | st_assert st_assert: N_ST_ASSERT(const_expr, N_STR | N_STR16 | N_STR32) -declaration_specs: N_LIST:(align_spec|sc_spec|type_qual|func_spec|type_spec)* +declaration_specs: N_LIST:(align_spec|sc_spec|type_qual|func_spec|type_spec|attr)* align_spec: N_ALIGNAS(type_name|const_expr) sc_spec: N_TYPEDEF|N_EXTERN|N_STATIC|N_AUTO|N_REGISTER|N_THREAD_LOCAL type_qual: N_CONST|N_RESTRICT|N_VOLATILE|N_ATOMIC @@ -502,7 +514,7 @@ type_spec: N_VOID|N_CHAR|N_SHORT|N_INT|N_LONG|N_FLOAT|N_DOUBLE|N_SIGNED|N_UNSIGN | (N_STRUCT|N_UNION) (N_ID?, struct_declaration_list?) | N_ENUM(N_ID?, N_LIST?: N_ENUM_COST(N_ID, const_expr?)*) | typedef_name struct_declaration_list: N_LIST: struct_declaration* -struct_declaration: st_assert | N_MEMBER(N_SHARE(spec_qual_list), declarator?, const_expr?) +struct_declaration: st_assert | N_MEMBER(N_SHARE(spec_qual_list), declarator?, attrs?, const_expr?) spec_qual_list: N_LIST:(type_qual|type_spec)* declarator: the same as direct declarator direct_declarator: N_DECL(N_ID, @@ -511,7 +523,7 @@ direct_declarator: N_DECL(N_ID, (assign_expr|N_STAR)?))*) pointer: N_LIST: N_POINTER(type_qual_list)* type_qual_list : N_LIST: type_qual* -parameter_type_list: N_LIST:(N_SPEC_DECL(declaration_specs, declarator, ignore) +parameter_type_list: N_LIST:(N_SPEC_DECL(declaration_specs, declarator, attrs?, ignore, ignore) | N_TYPE(declaration_specs, abstract_declarator))+ [N_DOTS] id_list: N_LIST: N_ID* initializer: assign_expr | initialize_list @@ -570,13 +582,14 @@ typedef enum { REP8 (NODE_EL, MUL_ASSIGN, DIV, DIV_ASSIGN, MOD, MOD_ASSIGN, IND, FIELD, ADDR), REP8 (NODE_EL, DEREF, DEREF_FIELD, COND, INC, DEC, POST_INC, POST_DEC, ALIGNOF), REP8 (NODE_EL, SIZEOF, EXPR_SIZEOF, CAST, COMPOUND_LITERAL, CALL, GENERIC, GENERIC_ASSOC, IF), - REP8 (NODE_EL, SWITCH, WHILE, DO, FOR, GOTO, CONTINUE, BREAK, RETURN), - REP8 (NODE_EL, EXPR, BLOCK, CASE, DEFAULT, LABEL, LIST, SPEC_DECL, SHARE), - REP8 (NODE_EL, TYPEDEF, EXTERN, STATIC, AUTO, REGISTER, THREAD_LOCAL, DECL, VOID), - REP8 (NODE_EL, CHAR, SHORT, INT, LONG, FLOAT, DOUBLE, SIGNED, UNSIGNED), - REP8 (NODE_EL, BOOL, STRUCT, UNION, ENUM, ENUM_CONST, MEMBER, CONST, RESTRICT), - REP8 (NODE_EL, VOLATILE, ATOMIC, INLINE, NO_RETURN, ALIGNAS, FUNC, STAR, POINTER), - REP8 (NODE_EL, DOTS, ARR, INIT, FIELD_ID, TYPE, ST_ASSERT, FUNC_DEF, MODULE), + REP8 (NODE_EL, SWITCH, WHILE, DO, FOR, GOTO, INDIRECT_GOTO, CONTINUE, BREAK), + REP8 (NODE_EL, RETURN, EXPR, BLOCK, CASE, DEFAULT, LABEL, LABEL_ADDR, LIST), + REP8 (NODE_EL, SPEC_DECL, SHARE, TYPEDEF, EXTERN, STATIC, AUTO, REGISTER, THREAD_LOCAL), + REP8 (NODE_EL, DECL, VOID, CHAR, SHORT, INT, LONG, FLOAT, DOUBLE), + REP8 (NODE_EL, SIGNED, UNSIGNED, BOOL, STRUCT, UNION, ENUM, ENUM_CONST, MEMBER), + REP8 (NODE_EL, CONST, RESTRICT, VOLATILE, ATOMIC, INLINE, NO_RETURN, ALIGNAS, FUNC), + REP8 (NODE_EL, STAR, POINTER, DOTS, ARR, INIT, FIELD_ID, TYPE, ST_ASSERT), + REP4 (NODE_EL, FUNC_DEF, MODULE, ASM, ATTR), } node_code_t; #undef REP_SEP @@ -705,6 +718,14 @@ static node_t new_pos_node4 (c2m_ctx_t c2m_ctx, node_code_t nc, pos_t p, node_t node_t op3, node_t op4) { return add_pos (c2m_ctx, new_node4 (c2m_ctx, nc, op1, op2, op3, op4), p); } +static node_t new_node5 (c2m_ctx_t c2m_ctx, node_code_t nc, node_t op1, node_t op2, node_t op3, + node_t op4, node_t op5) { + return op_append (c2m_ctx, new_node4 (c2m_ctx, nc, op1, op2, op3, op4), op5); +} +static node_t new_pos_node5 (c2m_ctx_t c2m_ctx, node_code_t nc, pos_t p, node_t op1, node_t op2, + node_t op3, node_t op4, node_t op5) { + return add_pos (c2m_ctx, new_node5 (c2m_ctx, nc, op1, op2, op3, op4, op5), p); +} static node_t new_ch_node (c2m_ctx_t c2m_ctx, int ch, pos_t p) { node_t n = new_pos_node (c2m_ctx, N_CH, p); n->u.ch = ch; @@ -942,7 +963,7 @@ static int string_stream_p (stream_t s) { return s->getc_func != NULL; } static void change_stream_pos (c2m_ctx_t c2m_ctx, pos_t pos) { cs->pos = pos; } static void remove_trigraphs (c2m_ctx_t c2m_ctx) { - int len = VARR_LENGTH (char, cs->ln); + int len = (int) VARR_LENGTH (char, cs->ln); char *addr = VARR_ADDR (char, cs->ln); int i, start, to, ch; @@ -1051,32 +1072,32 @@ static void remove_string_stream (c2m_ctx_t c2m_ctx) { /* We use UTF-32 for 32-bit wchars and UTF-16 for 16-bit wchar (LE/BE depending on endianess of the target), UTF-8 for anything else. */ -static void push_str_char (c2m_ctx_t c2m_ctx, VARR (char) * temp, uint64_t ch, int type) { +static void push_str_char (VARR (char) * temp, uint64_t ch, int type) { int i, len = 0; switch (type) { case ' ': if (ch <= 0xFF) { - VARR_PUSH (char, temp, ch); + VARR_PUSH (char, temp, (char) ch); return; } /* Fall through */ case '8': if (ch <= 0x7F) { - VARR_PUSH (char, temp, ch); + VARR_PUSH (char, temp, (char) ch); } else if (ch <= 0x7FF) { - VARR_PUSH (char, temp, 0xC0 | (ch >> 6)); - VARR_PUSH (char, temp, 0x80 | (ch & 0x3F)); + VARR_PUSH (char, temp, (char) (0xC0 | (ch >> 6))); + VARR_PUSH (char, temp, (char) (0x80 | (ch & 0x3F))); } else if (ch <= 0xFFFF) { - VARR_PUSH (char, temp, 0xE0 | (ch >> 12)); - VARR_PUSH (char, temp, 0x80 | ((ch >> 6) & 0x3F)); - VARR_PUSH (char, temp, 0x80 | (ch & 0x3F)); + VARR_PUSH (char, temp, (char) (0xE0 | (ch >> 12))); + VARR_PUSH (char, temp, (char) (0x80 | ((ch >> 6) & 0x3F))); + VARR_PUSH (char, temp, (char) (0x80 | (ch & 0x3F))); } else { assert (ch <= MAX_UTF8); - VARR_PUSH (char, temp, 0xF0 | (ch >> 18)); - VARR_PUSH (char, temp, 0x80 | ((ch >> 12) & 0x3F)); - VARR_PUSH (char, temp, 0x80 | ((ch >> 6) & 0x3F)); - VARR_PUSH (char, temp, 0x80 | (ch & 0x3F)); + VARR_PUSH (char, temp, (char) (0xF0 | (ch >> 18))); + VARR_PUSH (char, temp, (char) (0x80 | ((ch >> 12) & 0x3F))); + VARR_PUSH (char, temp, (char) (0x80 | ((ch >> 6) & 0x3F))); + VARR_PUSH (char, temp, (char) (0x80 | (ch & 0x3F))); } return; case 'L': @@ -1101,10 +1122,10 @@ static int pre_skip_if_part_p (c2m_ctx_t c2m_ctx); static void set_string_val (c2m_ctx_t c2m_ctx, token_t t, VARR (char) * temp, int type) { int i, str_len; int64_t curr_c, last_c = -1; - int64_t max_char = (type == 'u' ? UINT16_MAX - : type == 'U' ? UINT32_MAX - : type == 'L' ? MIR_WCHAR_MAX - : MIR_UCHAR_MAX); + uint64_t max_char = (type == 'u' ? UINT16_MAX + : type == 'U' ? UINT32_MAX + : type == 'L' ? MIR_WCHAR_MAX + : MIR_UCHAR_MAX); int start = type == ' ' ? 0 : type == '8' ? 2 : 1; int string_p = t->repr[start] == '"'; const char *str; @@ -1112,7 +1133,7 @@ static void set_string_val (c2m_ctx_t c2m_ctx, token_t t, VARR (char) * temp, in assert (t->code == T_STR || t->code == T_CH); str = t->repr; VARR_TRUNC (char, temp, 0); - str_len = strlen (str); + str_len = (int) strlen (str); assert (str_len >= start + 2 && (str[start] == '"' || str[start] == '\'') && str[start] == str[str_len - 1]); for (i = start + 1; i < str_len - 1; i++) { @@ -1120,7 +1141,7 @@ static void set_string_val (c2m_ctx_t c2m_ctx, token_t t, VARR (char) * temp, in error (c2m_ctx, t->pos, "multibyte character"); last_c = curr_c = (unsigned char) str[i]; if (curr_c != '\\') { - push_str_char (c2m_ctx, temp, curr_c, type); + push_str_char (temp, curr_c, type); continue; } last_c = curr_c = str[++i]; @@ -1153,12 +1174,12 @@ static void set_string_val (c2m_ctx_t c2m_ctx, token_t t, VARR (char) * temp, in uint64_t v = curr_c - '0'; curr_c = str[++i]; - if (!isdigit (curr_c) || curr_c == '8' || curr_c == '9') { + if (!isdigit ((int) curr_c) || curr_c == '8' || curr_c == '9') { i--; } else { v = v * 8 + curr_c - '0'; curr_c = str[++i]; - if (!isdigit (curr_c) || curr_c == '8' || curr_c == '9') + if (!isdigit ((int) curr_c) || curr_c == '8' || curr_c == '9') i--; else v = v * 8 + curr_c - '0'; @@ -1173,13 +1194,13 @@ static void set_string_val (c2m_ctx_t c2m_ctx, token_t t, VARR (char) * temp, in for (i++;; i++) { curr_c = str[i]; - if (!isxdigit (curr_c)) break; + if (!isxdigit ((int) curr_c)) break; first_p = FALSE; if (v <= UINT32_MAX) { v *= 16; - v += (isdigit (curr_c) ? curr_c - '0' - : islower (curr_c) ? curr_c - 'a' + 10 - : curr_c - 'A' + 10); + v += (isdigit ((int) curr_c) ? curr_c - '0' + : islower ((int) curr_c) ? curr_c - 'a' + 10 + : curr_c - 'A' + 10); } } if (first_p) { @@ -1197,16 +1218,16 @@ static void set_string_val (c2m_ctx_t c2m_ctx, token_t t, VARR (char) * temp, in } case 'u': case 'U': { - int n, start_c = curr_c, digits_num = curr_c == 'u' ? 4 : 8; - int64_t v = 0; + int n, start_c = (int) curr_c, digits_num = curr_c == 'u' ? 4 : 8; + uint64_t v = 0; for (i++, n = 0; n < digits_num; i++, n++) { curr_c = str[i]; - if (!isxdigit (curr_c)) break; + if (!isxdigit ((int) curr_c)) break; v *= 16; - v += (isdigit (curr_c) ? curr_c - '0' - : islower (curr_c) ? curr_c - 'a' + 10 - : curr_c - 'A' + 10); + v += (isdigit ((int) curr_c) ? curr_c - '0' + : islower ((int) curr_c) ? curr_c - 'a' + 10 + : curr_c - 'A' + 10); } last_c = curr_c = v; if (n < digits_num) { @@ -1235,17 +1256,17 @@ static void set_string_val (c2m_ctx_t c2m_ctx, token_t t, VARR (char) * temp, in curr_c = -1; } } - if (!string_p || curr_c >= 0) push_str_char (c2m_ctx, temp, curr_c, type); + if (!string_p || curr_c >= 0) push_str_char (temp, curr_c, type); } - push_str_char (c2m_ctx, temp, '\0', type); + push_str_char (temp, '\0', type); if (string_p) t->node->u.s = uniq_str (c2m_ctx, VARR_ADDR (char, temp), VARR_LENGTH (char, temp)); else if (last_c < 0) { if (!pre_skip_if_part_p (c2m_ctx)) error (c2m_ctx, t->pos, "empty char constant"); } else if (type == 'U' || type == 'u' || type == 'L') { - t->node->u.ul = last_c; + t->node->u.ul = (mir_ulong) last_c; } else { - t->node->u.ch = last_c; + t->node->u.ch = (mir_char) last_c; } } @@ -1271,7 +1292,7 @@ static token_t get_next_pptoken_1 (c2m_ctx_t c2m_ctx, int header_p) { for (comment_char = -1, nl_p = FALSE;; curr_c = cs_get (c2m_ctx)) { switch (curr_c) { case '\t': - cs->pos.ln_pos = round_size (cs->pos.ln_pos, TAB_STOP); + cs->pos.ln_pos = (int) round_size ((mir_size_t) cs->pos.ln_pos, TAB_STOP); /* fall through */ case ' ': case '\f': @@ -1329,7 +1350,7 @@ static token_t get_next_pptoken_1 (c2m_ctx_t c2m_ctx, int header_p) { nl_p ? '\n' : ' ', N_IGNORE); } if (header_p && (curr_c == '<' || curr_c == '\"')) { - int i, stop; + int stop; pos = cs->pos; VARR_TRUNC (char, temp_string, 0); @@ -1350,7 +1371,7 @@ static token_t get_next_pptoken_1 (c2m_ctx_t c2m_ctx, int header_p) { pos)); } else { VARR_PUSH (char, symbol_text, curr_c); - for (i = 0; i < VARR_LENGTH (char, symbol_text); i++) + for (size_t i = 0; i < VARR_LENGTH (char, symbol_text); i++) cs_unget (c2m_ctx, VARR_GET (char, symbol_text, i)); curr_c = (stop == '>' ? '<' : '\"'); } @@ -1523,15 +1544,13 @@ static token_t get_next_pptoken_1 (c2m_ctx_t c2m_ctx, int header_p) { case '}': return new_token (c2m_ctx, cs->pos, "}", curr_c, N_IGNORE); case ']': return new_token (c2m_ctx, cs->pos, "]", curr_c, N_IGNORE); case EOF: { - pos_t pos = cs->pos; - - assert (eof_s != cs); + pos = cs->pos; if (eof_s != NULL) free_stream (eof_s); - if (cs->f != stdin && cs->f != NULL) { + if (eof_s != cs && cs->f != stdin && cs->f != NULL) { fclose (cs->f); cs->f = NULL; } - eof_s = VARR_POP (stream_t, streams); + eof_s = VARR_LENGTH (stream_t, streams) == 0 ? NULL : VARR_POP (stream_t, streams); if (VARR_LENGTH (stream_t, streams) == 0) { return new_token (c2m_ctx, pos, "", T_EOU, N_IGNORE); } @@ -1582,7 +1601,7 @@ static token_t get_next_pptoken_1 (c2m_ctx_t c2m_ctx, int header_p) { } cs_unget (c2m_ctx, curr_c); curr_c = '.'; - /* Fall through: */ + /* falls through */ case '0': case '1': case '2': @@ -1618,7 +1637,7 @@ static token_t get_next_pptoken_1 (c2m_ctx_t c2m_ctx, int header_p) { case '\'': case '\"': wide_type = ' '; - literal : { + literal: { token_t t; int stop = curr_c; @@ -1748,7 +1767,7 @@ static const char *stringify (const char *str, VARR (char) * to) { } static void destringify (const char *repr, VARR (char) * to) { - int i, repr_len = strlen (repr); + int i, repr_len = (int) strlen (repr); VARR_TRUNC (char, to, 0); if (repr_len == 0) return; @@ -1762,14 +1781,12 @@ static void destringify (const char *repr, VARR (char) * to) { /* TS - vector, T defines position for empty vector */ static token_t token_stringify (c2m_ctx_t c2m_ctx, token_t t, VARR (token_t) * ts) { - int i; - if (VARR_LENGTH (token_t, ts) != 0) t = VARR_GET (token_t, ts, 0); t = new_node_token (c2m_ctx, t->pos, "", T_STR, new_str_node (c2m_ctx, N_STR, empty_str, t->pos)); VARR_TRUNC (char, temp_string, 0); for (const char *s = t->repr; *s != 0; s++) VARR_PUSH (char, temp_string, *s); VARR_PUSH (char, temp_string, '"'); - for (i = 0; i < VARR_LENGTH (token_t, ts); i++) + for (size_t i = 0; i < VARR_LENGTH (token_t, ts); i++) if (VARR_GET (token_t, ts, i)->code == ' ' || VARR_GET (token_t, ts, i)->code == '\n') { VARR_PUSH (char, temp_string, ' '); } else { @@ -1801,20 +1818,20 @@ static node_t get_int_node_from_repr (c2m_ctx_t c2m_ctx, const char *repr, char return new_ull_node (c2m_ctx, ull, pos); } if (long_p) { - if (!uns_p && ull <= MIR_LONG_MAX) return new_l_node (c2m_ctx, ull, pos); - if (ull <= MIR_ULONG_MAX) return new_ul_node (c2m_ctx, ull, pos); + if (!uns_p && ull <= MIR_LONG_MAX) return new_l_node (c2m_ctx, (long) ull, pos); + if (ull <= MIR_ULONG_MAX) return new_ul_node (c2m_ctx, (unsigned long) ull, pos); if (!uns_p && (base == 10 || ull <= MIR_LLONG_MAX)) return new_ll_node (c2m_ctx, ull, pos); return new_ull_node (c2m_ctx, ull, pos); } if (uns_p) { - if (ull <= MIR_UINT_MAX) return new_u_node (c2m_ctx, ull, pos); - if (ull <= MIR_ULONG_MAX) return new_ul_node (c2m_ctx, ull, pos); + if (ull <= MIR_UINT_MAX) return new_u_node (c2m_ctx, (unsigned long) ull, pos); + if (ull <= MIR_ULONG_MAX) return new_ul_node (c2m_ctx, (unsigned long) ull, pos); return new_ull_node (c2m_ctx, ull, pos); } - if (ull <= MIR_INT_MAX) return new_i_node (c2m_ctx, ull, pos); - if (base != 10 && ull <= MIR_UINT_MAX) return new_u_node (c2m_ctx, ull, pos); - if (ull <= MIR_LONG_MAX) return new_l_node (c2m_ctx, ull, pos); - if (ull <= MIR_ULONG_MAX) return new_ul_node (c2m_ctx, ull, pos); + if (ull <= MIR_INT_MAX) return new_i_node (c2m_ctx, (long) ull, pos); + if (base != 10 && ull <= MIR_UINT_MAX) return new_u_node (c2m_ctx, (unsigned long) ull, pos); + if (ull <= MIR_LONG_MAX) return new_l_node (c2m_ctx, (long) ull, pos); + if (ull <= MIR_ULONG_MAX) return new_ul_node (c2m_ctx, (unsigned long) ull, pos); if (base == 10 || ull <= MIR_LLONG_MAX) return new_ll_node (c2m_ctx, ull, pos); return new_ull_node (c2m_ctx, ull, pos); } @@ -1828,7 +1845,7 @@ static token_t pptoken2token (c2m_ctx_t c2m_ctx, token_t t, int id2kw_p) { tab_str_t str = str_add (c2m_ctx, t->repr, strlen (t->repr) + 1, T_STR, 0, FALSE); if (str.key != T_STR) { - t->code = str.key; + t->code = (int) str.key; t->node_code = N_IGNORE; t->node = NULL; } @@ -1840,7 +1857,7 @@ static token_t pptoken2token (c2m_ctx_t c2m_ctx, token_t t, int id2kw_p) { int uns_p = FALSE, long_p = FALSE, llong_p = FALSE; const char *repr = t->repr, *start = t->repr; char *stop; - int last = strlen (repr) - 1; + int last = (int) strlen (repr) - 1; assert (last >= 0); if (repr[0] == '0' && (repr[1] == 'x' || repr[1] == 'X')) { @@ -1933,7 +1950,7 @@ typedef struct macro { /* macro definition: */ VARR (token_t) * params; /* (T_ID)* [N_DOTS], NULL means no params */ VARR (token_t) * replacement; /* token*, NULL means a standard macro */ int ignore_p; -} * macro_t; +} *macro_t; DEF_VARR (macro_t); DEF_HTAB (macro_t); @@ -1941,7 +1958,7 @@ DEF_HTAB (macro_t); typedef struct ifstate { int skip_p, true_p, else_p; /* ??? flags that we are in a else part and in a false part */ pos_t if_pos; /* pos for #if and last #else, #elif */ -} * ifstate_t; +} *ifstate_t; DEF_VARR (ifstate_t); @@ -1956,7 +1973,7 @@ typedef struct macro_call { VARR (token_arr_t) * args; int repl_pos; /* position in macro replacement */ VARR (token_t) * repl_buffer; /* LIST:(token nodes)* */ -} * macro_call_t; +} *macro_call_t; DEF_VARR (macro_call_t); @@ -2019,12 +2036,12 @@ static void add_to_temp_string (c2m_ctx_t c2m_ctx, const char *str) { VARR_PUSH (char, temp_string, '\0'); } -static int macro_eq (macro_t macro1, macro_t macro2, void *arg) { +static int macro_eq (macro_t macro1, macro_t macro2, void *arg MIR_UNUSED) { return macro1->id->repr == macro2->id->repr; } -static htab_hash_t macro_hash (macro_t macro, void *arg) { - return mir_hash (macro->id->repr, strlen (macro->id->repr), 0x42); +static htab_hash_t macro_hash (macro_t macro, void *arg MIR_UNUSED) { + return (htab_hash_t) mir_hash (macro->id->repr, strlen (macro->id->repr), 0x42); } static macro_t new_macro (c2m_ctx_t c2m_ctx, token_t id, VARR (token_t) * params, @@ -2045,10 +2062,14 @@ static void init_macros (c2m_ctx_t c2m_ctx) { new_std_macro (c2m_ctx, "__TIME__"); new_std_macro (c2m_ctx, "__FILE__"); new_std_macro (c2m_ctx, "__LINE__"); - VARR_CREATE (token_t, params, 1); - VARR_PUSH (token_t, params, new_id_token (c2m_ctx, no_pos, "$")); - if (!c2m_options->pedantic_p) + if (!c2m_options->pedantic_p) { + VARR_CREATE (token_t, params, 1); + VARR_PUSH (token_t, params, new_id_token (c2m_ctx, no_pos, "$")); new_macro (c2m_ctx, new_id_token (c2m_ctx, no_pos, "__has_include"), params, NULL); + VARR_CREATE (token_t, params, 1); + VARR_PUSH (token_t, params, new_id_token (c2m_ctx, no_pos, "$")); + new_macro (c2m_ctx, new_id_token (c2m_ctx, no_pos, "__has_builtin"), params, NULL); + } } static macro_t new_macro (c2m_ctx_t c2m_ctx, token_t id, VARR (token_t) * params, @@ -2123,7 +2144,7 @@ static void pop_ifstate (c2m_ctx_t c2m_ctx) { static void pre_init (c2m_ctx_t c2m_ctx) { pre_ctx_t pre_ctx; time_t t, time_loc; - struct tm *tm, tm_loc; + struct tm *tm, tm_loc MIR_UNUSED; c2m_ctx->pre_ctx = pre_ctx = c2mir_calloc (c2m_ctx, sizeof (struct pre_ctx)); no_out_p = skip_if_part_p = FALSE; @@ -2177,7 +2198,7 @@ static void add_include_stream (c2m_ctx_t c2m_ctx, const char *fname, const char pre_ctx_t pre_ctx = c2m_ctx->pre_ctx; FILE *f; - for (int i = 0; i < VARR_LENGTH (char_ptr_t, once_include_files); i++) + for (size_t i = 0; i < VARR_LENGTH (char_ptr_t, once_include_files); i++) if (strcmp (fname, VARR_GET (char_ptr_t, once_include_files, i)) == 0) return; assert (fname != NULL); if (content == NULL && (f = fopen (fname, "rb")) == NULL) { @@ -2189,13 +2210,13 @@ static void add_include_stream (c2m_ctx_t c2m_ctx, const char *fname, const char add_stream (c2m_ctx, f, fname, NULL); else add_string_stream (c2m_ctx, fname, content); - cs->ifs_length_at_stream_start = VARR_LENGTH (ifstate_t, ifs); + cs->ifs_length_at_stream_start = (int) VARR_LENGTH (ifstate_t, ifs); } static void skip_nl (c2m_ctx_t c2m_ctx, token_t t, VARR (token_t) * buffer) { /* skip until new line */ if (t == NULL) t = get_next_pptoken (c2m_ctx); - for (; t->code != '\n'; t = get_next_pptoken (c2m_ctx)) // ??> + for (; t->code != '\n' && t->code != T_EOU; t = get_next_pptoken (c2m_ctx)) // ??> if (buffer != NULL) VARR_PUSH (token_t, buffer, t); unget_next_pptoken (c2m_ctx, t); } @@ -2203,15 +2224,14 @@ static void skip_nl (c2m_ctx_t c2m_ctx, token_t t, static const char *varg = "__VA_ARGS__"; static int find_param (VARR (token_t) * params, const char *name) { - size_t len; + size_t len = VARR_LENGTH (token_t, params); token_t param; - len = VARR_LENGTH (token_t, params); if (strcmp (name, varg) == 0 && len != 0 && VARR_LAST (token_t, params)->code == T_DOTS) - return len - 1; - for (int i = 0; i < len; i++) { + return (int) len - 1; + for (size_t i = 0; i < len; i++) { param = VARR_GET (token_t, params, i); - if (strcmp (param->repr, name) == 0) return i; + if (strcmp (param->repr, name) == 0) return (int) i; } return -1; } @@ -2221,7 +2241,7 @@ static int params_eq_p (VARR (token_t) * params1, VARR (token_t) * params2) { if (params1 == NULL || params2 == NULL) return params1 == params2; if (VARR_LENGTH (token_t, params1) != VARR_LENGTH (token_t, params2)) return FALSE; - for (int i = 0; i < VARR_LENGTH (token_t, params1); i++) { + for (size_t i = 0; i < VARR_LENGTH (token_t, params1); i++) { param1 = VARR_GET (token_t, params1, i); param2 = VARR_GET (token_t, params2, i); if (strcmp (param1->repr, param2->repr) != 0) return FALSE; @@ -2233,7 +2253,7 @@ static int replacement_eq_p (VARR (token_t) * r1, VARR (token_t) * r2) { token_t el1, el2; if (VARR_LENGTH (token_t, r1) != VARR_LENGTH (token_t, r2)) return FALSE; - for (int i = 0; i < VARR_LENGTH (token_t, r1); i++) { + for (size_t i = 0; i < VARR_LENGTH (token_t, r1); i++) { el1 = VARR_GET (token_t, r1, i); el2 = VARR_GET (token_t, r2, i); @@ -2481,7 +2501,8 @@ static pos_t check_line_directive_args (c2m_ctx_t c2m_ctx, VARR (token_t) * buff if (i >= len || buffer_arr[i]->code != T_NUMBER) return no_pos; if (!digits_p (buffer_arr[i]->repr)) return no_pos; errno = 0; - lno = l = strtoll (buffer_arr[i]->repr, NULL, 10); + l = strtoll (buffer_arr[i]->repr, NULL, 10); + lno = (int) l; if (errno || l > ((1ul << 31) - 1)) error (c2m_ctx, buffer_arr[i]->pos, "#line with too big value: %s", buffer_arr[i]->repr); i++; @@ -2573,7 +2594,7 @@ static void find_args (c2m_ctx_t c2m_ctx, macro_call_t mc) { /* we have just rea #ifdef C2MIR_PREPRO_DEBUG fprintf (stderr, "# finding args of macro %s call:\n# arg 0:", m->id->repr); #endif - for (;;) { + for (int newln_p = FALSE;; newln_p = t->code == '\n') { t = get_next_pptoken (c2m_ctx); #ifdef C2MIR_PREPRO_DEBUG fprintf (stderr, " <%s>%s", get_token_str (t), t->processed_p ? "*" : ""); @@ -2586,7 +2607,7 @@ static void find_args (c2m_ctx_t c2m_ctx, macro_call_t mc) { /* we have just rea pop_macro_call (c2m_ctx); } if (t->code == T_EOFILE || t->code == T_EOU || t->code == T_EOR || t->code == T_BOA - || t->code == T_EOA) + || t->code == T_EOA || (newln_p && t->code == '#')) break; if (level == 0 && t->code == ')') break; if (level == 0 && !va_p && t->code == ',') { @@ -2609,7 +2630,13 @@ static void find_args (c2m_ctx_t c2m_ctx, macro_call_t mc) { /* we have just rea #ifdef C2MIR_PREPRO_DEBUG fprintf (stderr, "\n"); #endif - if (t->code != ')') error (c2m_ctx, t->pos, "unfinished call of macro %s", m->id->repr); + if (t->code != ')') { + error (c2m_ctx, t->pos, "unfinished call of macro %s", m->id->repr); +#ifdef C2MIR_PREPRO_DEBUG + fprintf (stderr, "# push back <%s>%s\n", get_token_str (t), t->processed_p ? "*" : ""); +#endif + unget_next_pptoken (c2m_ctx, t); + } VARR_PUSH (token_arr_t, args, arg); if (params_len == 0 && VARR_LENGTH (token_arr_t, args) == 1) { token_arr_t arr = VARR_GET (token_arr_t, args, 0); @@ -2623,7 +2650,7 @@ static void find_args (c2m_ctx_t c2m_ctx, macro_call_t mc) { /* we have just rea } } if (VARR_LENGTH (token_arr_t, args) > params_len) { - token_arr_t arg = VARR_GET (token_arr_t, args, params_len); + arg = VARR_GET (token_arr_t, args, params_len); if (VARR_LENGTH (token_t, arg) != 0) t = VARR_GET (token_t, arg, 0); while (VARR_LENGTH (token_arr_t, args) > params_len) { temp_arr = VARR_POP (token_arr_t, args); @@ -2667,7 +2694,7 @@ static void add_token (VARR (token_t) * to, token_t t) { static void add_arg_tokens (VARR (token_t) * to, VARR (token_t) * from) { int start; - for (start = VARR_LENGTH (token_t, from) - 1; start >= 0; start--) + for (start = (int) VARR_LENGTH (token_t, from) - 1; start >= 0; start--) if (VARR_GET (token_t, from, start)->code == T_BOA) break; assert (start >= 0); for (size_t i = start + 1; i < VARR_LENGTH (token_t, from); i++) @@ -2681,7 +2708,7 @@ static void add_tokens (VARR (token_t) * to, VARR (token_t) * from) { } static void del_tokens (VARR (token_t) * tokens, int from, int len) { - int diff, tokens_len = VARR_LENGTH (token_t, tokens); + int diff, tokens_len = (int) VARR_LENGTH (token_t, tokens); token_t *addr = VARR_ADDR (token_t, tokens); if (len < 0) len = tokens_len - from; @@ -2692,7 +2719,7 @@ static void del_tokens (VARR (token_t) * tokens, int from, int len) { } static VARR (token_t) * do_concat (c2m_ctx_t c2m_ctx, VARR (token_t) * tokens) { - int i, j, k, empty_j_p, empty_k_p, len = VARR_LENGTH (token_t, tokens); + int i, j, k, empty_j_p, empty_k_p, len = (int) VARR_LENGTH (token_t, tokens); token_t t; for (i = len - 1; i >= 0; i--) @@ -2733,7 +2760,7 @@ static VARR (token_t) * do_concat (c2m_ctx_t c2m_ctx, VARR (token_t) * tokens) { VARR_SET (token_t, tokens, k, t); } i = k; - len = VARR_LENGTH (token_t, tokens); + len = (int) VARR_LENGTH (token_t, tokens); } for (i = len - 1; i >= 0; i--) VARR_GET (token_t, tokens, i)->processed_p = TRUE; return tokens; @@ -2748,7 +2775,7 @@ static void process_replacement (c2m_ctx_t c2m_ctx, macro_call_t mc) { m = mc->macro; sharp_pos = -1; m_repl = VARR_ADDR (token_t, m->replacement); - m_repl_len = VARR_LENGTH (token_t, m->replacement); + m_repl_len = (int) VARR_LENGTH (token_t, m->replacement); for (;;) { if (mc->repl_pos >= m_repl_len) { t = get_next_pptoken (c2m_ctx); @@ -2811,7 +2838,7 @@ static void process_replacement (c2m_ctx_t c2m_ctx, macro_call_t mc) { } } } else if (t->code == '#') { - sharp_pos = VARR_LENGTH (token_t, mc->repl_buffer); + sharp_pos = (int) VARR_LENGTH (token_t, mc->repl_buffer); } else if (t->code != ' ') { sharp_pos = -1; } @@ -2899,18 +2926,18 @@ struct val { static void move_tokens (VARR (token_t) * to, VARR (token_t) * from) { VARR_TRUNC (token_t, to, 0); - for (int i = 0; i < VARR_LENGTH (token_t, from); i++) + for (size_t i = 0; i < VARR_LENGTH (token_t, from); i++) VARR_PUSH (token_t, to, VARR_GET (token_t, from, i)); VARR_TRUNC (token_t, from, 0); } -static void reverse_move_tokens (c2m_ctx_t c2m_ctx, VARR (token_t) * to, VARR (token_t) * from) { +static void reverse_move_tokens (VARR (token_t) * to, VARR (token_t) * from) { VARR_TRUNC (token_t, to, 0); while (VARR_LENGTH (token_t, from) != 0) VARR_PUSH (token_t, to, VARR_POP (token_t, from)); } static void transform_to_header (c2m_ctx_t c2m_ctx, VARR (token_t) * buffer) { - int i, j, k; + size_t i, j, k; token_t t; pos_t pos; @@ -2936,7 +2963,7 @@ static void transform_to_header (c2m_ctx_t c2m_ctx, VARR (token_t) * buffer) { VARR_PUSH (char, symbol_text, '>'); VARR_PUSH (char, symbol_text, '\0'); VARR_PUSH (char, temp_string, '\0'); - del_tokens (buffer, i, j - i); + del_tokens (buffer, (int) i, (int) (j - i)); t = new_node_token (c2m_ctx, pos, VARR_ADDR (char, symbol_text), T_HEADER, new_str_node (c2m_ctx, N_STR, uniq_cstr (c2m_ctx, VARR_ADDR (char, temp_string)), pos)); @@ -2949,7 +2976,7 @@ static struct val eval_expr (c2m_ctx_t c2m_ctx, VARR (token_t) * buffer, token_t static const char *get_header_name (c2m_ctx_t c2m_ctx, VARR (token_t) * buffer, pos_t err_pos, const char **content) { - int i; + size_t i; *content = NULL; transform_to_header (c2m_ctx, buffer); @@ -3014,7 +3041,7 @@ static void process_directive (c2m_ctx_t c2m_ctx) { error (c2m_ctx, t1->pos, "garbage at the end of #%s", t1->repr); skip_nl (c2m_ctx, NULL, NULL); } - if (VARR_LENGTH (ifstate_t, ifs) <= cs->ifs_length_at_stream_start) + if ((int) VARR_LENGTH (ifstate_t, ifs) <= cs->ifs_length_at_stream_start) error (c2m_ctx, t1->pos, "unmatched #%s", t1->repr); else if (strcmp (t1->repr, "endif") == 0) { pop_ifstate (c2m_ctx); @@ -3087,7 +3114,7 @@ static void process_directive (c2m_ctx_t c2m_ctx) { goto ret; } } - if (VARR_LENGTH (stream_t, streams) >= max_nested_includes + 1) { + if ((int) VARR_LENGTH (stream_t, streams) >= max_nested_includes + 1) { error (c2m_ctx, t->pos, "more %d include levels", VARR_LENGTH (stream_t, streams) - 1); goto ret; } @@ -3281,7 +3308,7 @@ static node_t parse_pre_expr (c2m_ctx_t c2m_ctx, VARR (token_t) * expr) { static void replace_defined (c2m_ctx_t c2m_ctx, VARR (token_t) * expr_buffer) { pre_ctx_t pre_ctx = c2m_ctx->pre_ctx; - int i, j, k, len; + size_t i, j, k, len; token_t t, id; const char *res; struct macro macro_struct; @@ -3300,7 +3327,7 @@ static void replace_defined (c2m_ctx_t c2m_ctx, VARR (token_t) * expr_buffer) { res = HTAB_DO (macro_t, macro_tab, ¯o_struct, HTAB_FIND, tab_macro) ? "1" : "0"; VARR_SET (token_t, expr_buffer, i, new_token (c2m_ctx, t->pos, res, T_NUMBER, N_IGNORE)); // ??? - del_tokens (expr_buffer, i + 1, j - i); + del_tokens (expr_buffer, (int) i + 1, (int) (j - i)); continue; } if (j >= len || VARR_GET (token_t, expr_buffer, j)->code != '(') continue; @@ -3315,7 +3342,7 @@ static void replace_defined (c2m_ctx_t c2m_ctx, VARR (token_t) * expr_buffer) { res = HTAB_DO (macro_t, macro_tab, ¯o_struct, HTAB_FIND, tab_macro) ? "1" : "0"; VARR_SET (token_t, expr_buffer, i, new_token (c2m_ctx, t->pos, res, T_NUMBER, N_IGNORE)); // ??? - del_tokens (expr_buffer, i + 1, j - i); + del_tokens (expr_buffer, (int) i + 1, (int) (j - i)); } } } @@ -3324,7 +3351,7 @@ static struct val eval (c2m_ctx_t c2m_ctx, node_t tree); static struct val eval_expr (c2m_ctx_t c2m_ctx, VARR (token_t) * expr_buffer, token_t if_token) { pre_ctx_t pre_ctx = c2m_ctx->pre_ctx; - int i, j; + size_t i, j; token_t t, ppt; VARR (token_t) * temp_buffer; node_t tree; @@ -3340,7 +3367,7 @@ static struct val eval_expr (c2m_ctx_t c2m_ctx, VARR (token_t) * expr_buffer, to processing (c2m_ctx, TRUE); replace_defined (c2m_ctx, output_buffer); no_out_p = FALSE; - reverse_move_tokens (c2m_ctx, expr_buffer, output_buffer); + reverse_move_tokens (expr_buffer, output_buffer); VARR_CREATE (token_t, temp_buffer, VARR_LENGTH (token_t, expr_buffer)); for (i = j = 0; i < VARR_LENGTH (token_t, expr_buffer); i++) { int change_p = TRUE; @@ -3548,6 +3575,16 @@ static macro_call_t try_param_macro_call (c2m_ctx_t c2m_ctx, macro_t m, token_t return mc; } +#define ADD_OVERFLOW "__builtin_add_overflow" +#define SUB_OVERFLOW "__builtin_sub_overflow" +#define MUL_OVERFLOW "__builtin_mul_overflow" +#define EXPECT "__builtin_expect" +#define JCALL "__builtin_jcall" +#define JRET "__builtin_jret" +#define PROP_SET "__builtin_prop_set" +#define PROP_EQ "__builtin_prop_eq" +#define PROP_NE "__builtin_prop_ne" + static void processing (c2m_ctx_t c2m_ctx, int ignore_directive_p) { pre_ctx_t pre_ctx = c2m_ctx->pre_ctx; token_t t; @@ -3571,11 +3608,12 @@ static void processing (c2m_ctx_t c2m_ctx, int ignore_directive_p) { out_token (c2m_ctx, t); continue; } else if (t->code == T_EOFILE || t->code == T_EOU) { - if (VARR_LENGTH (ifstate_t, ifs) > eof_s->ifs_length_at_stream_start) { + if ((int) VARR_LENGTH (ifstate_t, ifs) + > (eof_s == NULL ? 0 : eof_s->ifs_length_at_stream_start)) { error (c2m_ctx, VARR_LAST (ifstate_t, ifs)->if_pos, "unfinished #if"); } if (t->code == T_EOU) return; - while (VARR_LENGTH (ifstate_t, ifs) > eof_s->ifs_length_at_stream_start) + while ((int) VARR_LENGTH (ifstate_t, ifs) > eof_s->ifs_length_at_stream_start) pop_ifstate (c2m_ctx); skip_if_part_p = VARR_LENGTH (ifstate_t, ifs) == 0 ? 0 : VARR_LAST (ifstate_t, ifs)->skip_p; newln_p = TRUE; @@ -3660,6 +3698,40 @@ static void processing (c2m_ctx_t c2m_ctx, int ignore_directive_p) { unget_next_pptoken (c2m_ctx, new_node_token (c2m_ctx, t->pos, res ? "1" : "0", T_NUMBER, new_i_node (c2m_ctx, res, t->pos))); } + } else if (strcmp (t->repr, "__has_builtin") == 0) { + int res; + size_t i, len; + VARR (token_t) * arg; + + res = 0; + if ((mc = try_param_macro_call (c2m_ctx, m, t)) != NULL) { + unget_next_pptoken (c2m_ctx, new_token (c2m_ctx, t->pos, "", T_EOR, N_IGNORE)); + if (VARR_LENGTH (token_arr_t, mc->args) != 1) { + error (c2m_ctx, t->pos, "wrong number of args for __has_builtin"); + } else { + arg = VARR_LAST (token_arr_t, mc->args); + len = VARR_LENGTH (token_t, arg); + i = 0; + if (i < len && VARR_GET (token_t, arg, i)->code == ' ') i++; + if (i >= len || (t = VARR_GET (token_t, arg, i))->code != T_ID) { + error (c2m_ctx, t->pos, "__has_builtin requires identifier"); + } else { + i++; + if (i < len && VARR_GET (token_t, arg, i)->code == ' ') i++; + if (i != len) + error (c2m_ctx, t->pos, "garbage after identifier in __has_builtin"); + else + res = (strcmp (t->repr, ADD_OVERFLOW) == 0 || strcmp (t->repr, SUB_OVERFLOW) == 0 + || strcmp (t->repr, MUL_OVERFLOW) == 0 || strcmp (t->repr, EXPECT) == 0 + || strcmp (t->repr, JCALL) == 0 || strcmp (t->repr, JRET) == 0 + || strcmp (t->repr, PROP_SET) == 0 || strcmp (t->repr, PROP_EQ) == 0 + || strcmp (t->repr, PROP_NE) == 0); + } + } + m->ignore_p = TRUE; + unget_next_pptoken (c2m_ctx, new_node_token (c2m_ctx, t->pos, res ? "1" : "0", T_NUMBER, + new_i_node (c2m_ctx, res, t->pos))); + } } else { assert (FALSE); } @@ -3787,7 +3859,7 @@ static void common_pre_out (c2m_ctx_t c2m_ctx, token_t t) { (c2m_options->prepro_only_p ? pre_text_out : pre_out) (c2m_ctx, t); } -static void pre (c2m_ctx_t c2m_ctx, const char *start_source_name) { +static void pre (c2m_ctx_t c2m_ctx) { pre_ctx_t pre_ctx = c2m_ctx->pre_ctx; pre_last_token = NULL; @@ -3887,12 +3959,12 @@ static void syntax_error (c2m_ctx_t c2m_ctx, const char *expected_name) { n_errors++; } -static int tpname_eq (tpname_t tpname1, tpname_t tpname2, void *arg) { +static int tpname_eq (tpname_t tpname1, tpname_t tpname2, void *arg MIR_UNUSED) { return tpname1.id->u.s.s == tpname2.id->u.s.s && tpname1.scope == tpname2.scope; } -static htab_hash_t tpname_hash (tpname_t tpname, void *arg) { - return (mir_hash_finish ( +static htab_hash_t tpname_hash (tpname_t tpname, void *arg MIR_UNUSED) { + return (htab_hash_t) (mir_hash_finish ( mir_hash_step (mir_hash_step (mir_hash_init (0x42), (uint64_t) tpname.id->u.s.s), (uint64_t) tpname.scope))); } @@ -3978,7 +4050,7 @@ static void tpname_finish (c2m_ctx_t c2m_ctx) { typedef node_t (*nonterm_func_t) (c2m_ctx_t c2m_ctx, int); typedef node_t (*nonterm_arg_func_t) (c2m_ctx_t c2m_ctx, int, node_t); -#define D(f) static node_t f (c2m_ctx_t c2m_ctx, int no_err_p) +#define D(f) static node_t f (c2m_ctx_t c2m_ctx, int no_err_p MIR_UNUSED) #define DA(f) static node_t f (c2m_ctx_t c2m_ctx, int no_err_p, node_t arg) #define C(c) (curr_token->code == c) @@ -4043,6 +4115,9 @@ D (primary_expr) { if (MN (T_ID, r) || MN (T_NUMBER, r) || MN (T_CH, r) || MN (T_STR, r)) { return r; + } else if (MP (T_ANDAND, pos)) { + PTN (T_ID); + return new_pos_node1 (c2m_ctx, N_LABEL_ADDR, pos, r); } else if (M ('(')) { if (C ('{')) { P (compound_stmt); @@ -4260,32 +4335,38 @@ D (st_assert); D (asm_spec) { parse_ctx_t parse_ctx = c2m_ctx->parse_ctx; - node_t r; + node_t r, id; PTN (T_ID); - if (strcmp (r->u.s.s, "__asm") != 0) PTFAIL (T_ID); + if (strcmp (r->u.s.s, "__asm") != 0 && strcmp (r->u.s.s, "asm") != 0) PTFAIL (T_ID); + id = r; PT ('('); - while (!C (')')) { - PT (T_STR); - } + PTN (T_STR); PT (')'); - return NULL; + return new_pos_node1 (c2m_ctx, N_ASM, POS (id), r); } -static node_t try_attr_spec (c2m_ctx_t c2m_ctx, pos_t pos) { +static node_t try_attr_spec (c2m_ctx_t c2m_ctx, pos_t pos, node_t *asm_part) { node_t r; if (c2m_options->pedantic_p) return NULL; + if (asm_part != NULL) { + *asm_part = NULL; + if ((r = TRY (asm_spec)) != err_node) { + if (c2m_options->pedantic_p) + error (c2m_ctx, pos, "asm is not implemented"); + else { + /*warning (c2m_ctx, pos, "asm is not implemented -- ignoring it")*/ + } + *asm_part = r; + } + } if ((r = TRY (attr_spec)) != err_node) { if (c2m_options->pedantic_p) error (c2m_ctx, pos, "GCC attributes are not implemented"); - else - /*warning (c2m_ctx, pos, "GCC attributes are not implemented -- ignoring them")*/; - } else if ((r = TRY (asm_spec)) != err_node) { - if (c2m_options->pedantic_p) - error (c2m_ctx, pos, "asm is not implemented"); - else - /*warning (c2m_ctx, pos, "asm is not implemented -- ignoring it")*/; + else { + /*warning (c2m_ctx, pos, "GCC attributes are not implemented -- ignoring them")*/ + } } return r; } @@ -4293,7 +4374,7 @@ static node_t try_attr_spec (c2m_ctx_t c2m_ctx, pos_t pos) { D (declaration) { parse_ctx_t parse_ctx = c2m_ctx->parse_ctx; int typedef_p; - node_t op, list, decl, spec, r; + node_t op, list, decl, spec, r, attrs, asm_part = NULL; pos_t pos, last_pos; if (C (T_STATIC_ASSERT)) { @@ -4303,14 +4384,15 @@ D (declaration) { if (curr_scope == top_scope && c2m_options->pedantic_p) warning (c2m_ctx, pos, "extra ; outside of a function"); } else { - try_attr_spec (c2m_ctx, curr_token->pos); + try_attr_spec (c2m_ctx, curr_token->pos, NULL); PA (declaration_specs, curr_scope == top_scope ? (node_t) 1 : NULL); spec = r; last_pos = POS (spec); list = new_node (c2m_ctx, N_LIST); if (C (';')) { op_append (c2m_ctx, list, - new_node3 (c2m_ctx, N_SPEC_DECL, spec, new_node (c2m_ctx, N_IGNORE), + new_node5 (c2m_ctx, N_SPEC_DECL, spec, new_node (c2m_ctx, N_IGNORE), + new_node (c2m_ctx, N_IGNORE), new_node (c2m_ctx, N_IGNORE), new_node (c2m_ctx, N_IGNORE))); } else { /* init-declarator-list */ for (op = NL_HEAD (spec->u.ops); op != NULL; op = NL_NEXT (op)) @@ -4323,15 +4405,17 @@ D (declaration) { assert (decl->code == N_DECL); op = NL_HEAD (decl->u.ops); tpname_add (c2m_ctx, op, curr_scope, typedef_p); - try_attr_spec (c2m_ctx, last_pos); + attrs = try_attr_spec (c2m_ctx, last_pos, &asm_part); + if (attrs == err_node) attrs = new_node (c2m_ctx, N_IGNORE); + if (asm_part == NULL) asm_part = new_node (c2m_ctx, N_IGNORE); if (M ('=')) { P (initializer); } else { r = new_node (c2m_ctx, N_IGNORE); } op_append (c2m_ctx, list, - new_pos_node3 (c2m_ctx, N_SPEC_DECL, POS (decl), - new_node1 (c2m_ctx, N_SHARE, spec), decl, r)); + new_pos_node5 (c2m_ctx, N_SPEC_DECL, POS (decl), + new_node1 (c2m_ctx, N_SHARE, spec), decl, attrs, asm_part, r)); if (!M (',')) break; } } @@ -4343,43 +4427,54 @@ D (declaration) { D (attr) { parse_ctx_t parse_ctx = c2m_ctx->parse_ctx; + node_t r, res, list; if (C (')') || C (',')) /* empty */ return NULL; - if (FIRST_KW <= curr_token->code && curr_token->code <= LAST_KW) + if (FIRST_KW <= (token_code_t) curr_token->code && (token_code_t) curr_token->code <= LAST_KW) { + token_t kw = curr_token; PT (curr_token->code); - else - PT (T_ID); + r = new_str_node (c2m_ctx, N_ID, uniq_cstr (c2m_ctx, kw->repr), kw->pos); + } else { + PTN (T_ID); + } + list = new_node (c2m_ctx, N_LIST); + res = new_node2 (c2m_ctx, N_ATTR, r, list); if (C ('(')) { PT ('('); while (!C (')')) { if (C (T_NUMBER) || C (T_CH) || C (T_STR)) - PT (curr_token->code); + PTN (curr_token->code); else - PT (T_ID); + PTN (T_ID); + op_append (c2m_ctx, list, r); if (!C (')')) PT (','); } PT (')'); } - return NULL; + return res; } D (attr_spec) { parse_ctx_t parse_ctx = c2m_ctx->parse_ctx; - node_t r; + node_t list, r; PTN (T_ID); - if (strcmp (r->u.s.s, "__attribute__") != 0) PTFAIL (T_ID); + /* libc can define empty __attribute__ for non-GNU C compiler -- define also __mirc_attribute__ */ + if (strcmp (r->u.s.s, "__attribute__") != 0 && strcmp (r->u.s.s, "__mirc_attribute__") != 0) + PTFAIL (T_ID); PT ('('); PT ('('); + list = new_node (c2m_ctx, N_LIST); for (;;) { P (attr); + op_append (c2m_ctx, list, r); if (C (')')) break; PT (','); } PT (')'); PT (')'); - return NULL; + return list; } DA (declaration_specs) { @@ -4401,8 +4496,8 @@ DA (declaration_specs) { prev_type_spec = r; } else if ((r = TRY_A (type_spec, prev_type_spec)) != err_node) { prev_type_spec = r; - } else if ((r = try_attr_spec (c2m_ctx, spec_pos)) != err_node && r != NULL) { - continue; + } else if ((r = try_attr_spec (c2m_ctx, spec_pos, FALSE)) != err_node && r != NULL) { + continue; /* ignore attrs for declaration specs (type attrs) */ } else break; op_append (c2m_ctx, list, r); @@ -4431,6 +4526,11 @@ D (sc_spec) { } else if (MP (T_REGISTER, pos)) { r = new_pos_node (c2m_ctx, N_REGISTER, pos); } else if (MP (T_THREAD_LOCAL, pos)) { + if (c2m_options->pedantic_p) + error (c2m_ctx, pos, "Thread local is not implemented"); + else + warning (c2m_ctx, pos, + "Thread local is not implemented -- program might not work as assumed"); r = new_pos_node (c2m_ctx, N_THREAD_LOCAL, pos); } else { if (record_level == 0) syntax_error (c2m_ctx, "a storage specifier"); @@ -4553,7 +4653,7 @@ D (struct_declaration_list) { D (struct_declaration) { parse_ctx_t parse_ctx = c2m_ctx->parse_ctx; - node_t list, spec, op, r; + node_t list, spec, op, r, attrs; if (C (T_STATIC_ASSERT)) { P (st_assert); @@ -4562,23 +4662,28 @@ D (struct_declaration) { spec = r; list = new_node (c2m_ctx, N_LIST); if (M (';')) { - op = new_pos_node3 (c2m_ctx, N_MEMBER, POS (spec), new_node1 (c2m_ctx, N_SHARE, spec), - new_node (c2m_ctx, N_IGNORE), new_node (c2m_ctx, N_IGNORE)); + op = new_pos_node4 (c2m_ctx, N_MEMBER, POS (spec), new_node1 (c2m_ctx, N_SHARE, spec), + new_node (c2m_ctx, N_IGNORE), new_node (c2m_ctx, N_IGNORE), + new_node (c2m_ctx, N_IGNORE)); op_append (c2m_ctx, list, op); } else { /* struct-declarator-list */ for (;;) { /* struct-declarator */ if (!C (':')) { P (declarator); + attrs = try_attr_spec (c2m_ctx, curr_token->pos, NULL); op = r; } else { + attrs = err_node; op = new_node (c2m_ctx, N_IGNORE); } + if (attrs == err_node) attrs = new_node (c2m_ctx, N_IGNORE); if (M (':')) { P (const_expr); } else { r = new_node (c2m_ctx, N_IGNORE); } - op = new_pos_node3 (c2m_ctx, N_MEMBER, POS (op), new_node1 (c2m_ctx, N_SHARE, spec), op, r); + op = new_pos_node4 (c2m_ctx, N_MEMBER, POS (op), new_node1 (c2m_ctx, N_SHARE, spec), op, + attrs, r); op_append (c2m_ctx, list, op); if (!M (',')) break; } @@ -4791,7 +4896,7 @@ D (param_type_abstract_declarator) { D (param_type_list) { parse_ctx_t parse_ctx = c2m_ctx->parse_ctx; - node_t list, op1, op2, r = err_node; + node_t list, attrs, op1, op2, r = err_node; int comma_p; pos_t pos; @@ -4811,7 +4916,10 @@ D (param_type_list) { r = new_node2 (c2m_ctx, N_TYPE, op1, op2); } else { P (declarator); - r = new_pos_node3 (c2m_ctx, N_SPEC_DECL, POS (op2), op1, r, new_node (c2m_ctx, N_IGNORE)); + attrs = try_attr_spec (c2m_ctx, curr_token->pos, NULL); + if (attrs == err_node) attrs = new_node (c2m_ctx, N_IGNORE); + r = new_pos_node5 (c2m_ctx, N_SPEC_DECL, POS (op2), op1, r, attrs, + new_node (c2m_ctx, N_IGNORE), new_node (c2m_ctx, N_IGNORE)); } op_append (c2m_ctx, list, r); comma_p = FALSE; @@ -4956,8 +5064,8 @@ D (initializer) { P (assign_expr); } else { P (initializer_list); - if (M (',')) - ; + if (M (',')) { + } PT ('}'); } return r; @@ -5147,9 +5255,15 @@ D (stmt) { op_append (c2m_ctx, n, r); r = n; } else if (MP (T_GOTO, pos)) { /* jump-statement */ - PTN (T_ID); + int indirect_p = FALSE; + if (!M ('*')) { + PTN (T_ID); + } else { + indirect_p = TRUE; + P (expr); + } PT (';'); - r = new_pos_node2 (c2m_ctx, N_GOTO, pos, l, r); + r = new_pos_node2 (c2m_ctx, indirect_p ? N_INDIRECT_GOTO : N_GOTO, pos, l, r); } else if (MP (T_CONTINUE, pos)) { /* continue-statement */ PT (';'); r = new_pos_node1 (c2m_ctx, N_CONTINUE, pos, l); @@ -5280,7 +5394,7 @@ D (transl_unit) { return new_node1 (c2m_ctx, N_MODULE, list); } -static void fatal_error (c2m_ctx_t c2m_ctx, C_error_code_t code, const char *message) { +static void fatal_error (c2m_ctx_t c2m_ctx, C_error_code_t code MIR_UNUSED, const char *message) { if (c2m_options->message_file != NULL) fprintf (c2m_options->message_file, "%s\n", message); longjmp (c2m_ctx->env, 1); } @@ -5355,7 +5469,7 @@ static void parse_init (c2m_ctx_t c2m_ctx) { static void add_standard_includes (c2m_ctx_t c2m_ctx) { const char *str, *name; - for (int i = 0; i < sizeof (standard_includes) / sizeof (string_include_t); i++) { + for (size_t i = 0; i < sizeof (standard_includes) / sizeof (string_include_t); i++) { if ((name = standard_includes[i].name) != NULL) continue; str = standard_includes[i].content; add_string_stream (c2m_ctx, "", str); @@ -5404,7 +5518,7 @@ static void parse_finish (c2m_ctx_t c2m_ctx) { 1. expr nodes have attribute "struct expr", N_ID not expr context has NULL attribute. 2. N_SWITCH has attribute "struct switch_attr" 3. N_SPEC_DECL (only with ID), N_MEMBER, N_FUNC_DEF have attribute "struct decl" - 4. N_GOTO hash attribute node_t (target stmt) + 4. N_GOTO has attribute node_t (target stmt) 5. N_STRUCT, N_UNION have attribute "struct node_scope" if they have a decl list 6. N_MODULE, N_BLOCK, N_FOR, N_FUNC have attribute "struct node_scope" 7. declaration_specs or spec_qual_list N_LISTs have attribute "struct decl_spec", @@ -5425,10 +5539,10 @@ DEF_HTAB (case_t); struct check_ctx { node_t curr_scope; - VARR (node_t) * gotos; + VARR (node_t) * label_uses; node_t func_block_scope; unsigned curr_func_scope_num; - int in_params_p; + unsigned char in_params_p, jump_ret_p; node_t curr_unnamed_anon_struct_union_member; node_t curr_switch; VARR (decl_t) * func_decls_for_allocation; @@ -5441,10 +5555,11 @@ struct check_ctx { }; #define curr_scope check_ctx->curr_scope -#define gotos check_ctx->gotos +#define label_uses check_ctx->label_uses #define func_block_scope check_ctx->func_block_scope #define curr_func_scope_num check_ctx->curr_func_scope_num #define in_params_p check_ctx->in_params_p +#define jump_ret_p check_ctx->jump_ret_p #define curr_unnamed_anon_struct_union_member check_ctx->curr_unnamed_anon_struct_union_member #define curr_switch check_ctx->curr_switch #define func_decls_for_allocation check_ctx->func_decls_for_allocation @@ -5457,20 +5572,20 @@ struct check_ctx { #define curr_call_arg_area_offset check_ctx->curr_call_arg_area_offset #define context_stack check_ctx->context_stack -static int supported_alignment_p (mir_llong align) { return TRUE; } // ??? +static int supported_alignment_p (mir_llong align MIR_UNUSED) { return TRUE; } // ??? -static int symbol_eq (symbol_t s1, symbol_t s2, void *arg) { +static int symbol_eq (symbol_t s1, symbol_t s2, void *arg MIR_UNUSED) { return s1.mode == s2.mode && s1.id->u.s.s == s2.id->u.s.s && s1.scope == s2.scope; } -static htab_hash_t symbol_hash (symbol_t s, void *arg) { - return (mir_hash_finish ( +static htab_hash_t symbol_hash (symbol_t s, void *arg MIR_UNUSED) { + return (htab_hash_t) (mir_hash_finish ( mir_hash_step (mir_hash_step (mir_hash_step (mir_hash_init (0x42), (uint64_t) s.mode), (uint64_t) s.id->u.s.s), (uint64_t) s.scope))); } -static void symbol_clear (symbol_t sym, void *arg) { VARR_DESTROY (node_t, sym.defs); } +static void symbol_clear (symbol_t sym, void *arg MIR_UNUSED) { VARR_DESTROY (node_t, sym.defs); } static void symbol_init (c2m_ctx_t c2m_ctx) { HTAB_CREATE_WITH_FREE_FUNC (symbol_t, symbol_tab, 5000, symbol_hash, symbol_eq, symbol_clear, @@ -5554,8 +5669,10 @@ static struct type_qual type_qual_union (const struct type_qual *tq1, const stru static void init_type (struct type *type) { clear_type_qual (&type->type_qual); + type->mode = TM_UNDEF; type->pos_node = NULL; type->arr_type = NULL; + type->antialias = 0; type->align = -1; type->raw_size = MIR_SIZE_MAX; type->func_type_before_adjustment_p = FALSE; @@ -5697,15 +5814,18 @@ static struct type arithmetic_conversion (const struct type *type1, const struct struct expr { unsigned int const_p : 1, const_addr_p : 1, builtin_call_p : 1; - node_t lvalue_node; - node_t def_node; /* defined for id or const address (ref) */ + union { + node_t lvalue_node; /* for id, str, field, deref field, ind, deref, compound literal */ + node_t label_addr_target; /* for label address */ + } u; + node_t def_node; /* defined for id or const address (ref) or label address node */ struct type *type; /* type of the result */ struct type *type2; /* used for assign expr type */ union { /* defined for const or const addr (i_val is offset) */ mir_llong i_val; mir_ullong u_val; mir_ldouble d_val; - } u; + } c; }; struct decl_spec { @@ -5738,19 +5858,22 @@ struct node_scope { struct decl { /* true if address is taken, reg can be used or is used: */ - unsigned addr_p : 1, reg_p : 1, used_p : 1; + unsigned addr_p : 1, reg_p : 1, asm_p : 1, used_p : 1; int bit_offset, width; /* for bitfields, -1 bit_offset for non bitfields. */ mir_size_t offset; /* var offset in frame or bss */ node_t scope; /* declaration scope */ /* The next 2 members are used only for param decls. The 1st member is number of the start MIR - func arg. The 2nd one is number or MIR func args used to pass param value, it is positive only - for aggregates passed by value. */ + func arg. The 2nd one is number or MIR func args used to pass param value, it is positive + only for aggregates passed by value. */ uint32_t param_args_start, param_args_num; struct decl_spec decl_spec; /* Unnamed member if this scope is anon struct/union for the member, NULL otherwise: */ node_t containing_unnamed_anon_struct_union_member; - MIR_item_t item; /* MIR_item for some declarations */ + union { + const char *asm_str; /* register name for global reg used and defined only if asm_p */ + MIR_item_t item; /* MIR_item for some declarations */ + } u; c2m_ctx_t c2m_ctx; }; @@ -5787,7 +5910,7 @@ static int type_eq_p (struct type *type1, struct type *type2) { && at1->size->code != N_IGNORE && at2->size->code != N_IGNORE && (cexpr1 = at1->size->attr)->const_p && (cexpr2 = at2->size->attr)->const_p && integer_type_p (cexpr2->type) && integer_type_p (cexpr2->type) - && cexpr1->u.i_val == cexpr2->u.i_val); + && cexpr1->c.i_val == cexpr2->c.i_val); } case TM_FUNC: { struct func_type *ft1 = type1->u.func_type, *ft2 = type2->u.func_type; @@ -5832,7 +5955,7 @@ static int compatible_types_p (struct type *type1, struct type *type2, int ignor if (at1->size->code == N_IGNORE || at2->size->code == N_IGNORE) return TRUE; if ((cexpr1 = at1->size->attr)->const_p && (cexpr2 = at2->size->attr)->const_p && integer_type_p (cexpr2->type) && integer_type_p (cexpr2->type)) - return cexpr1->u.i_val == cexpr2->u.i_val; + return cexpr1->c.i_val == cexpr2->c.i_val; return TRUE; } else if (type1->mode == TM_FUNC) { struct func_type *ft1 = type1->u.func_type, *ft2 = type2->u.func_type; @@ -5858,7 +5981,7 @@ static struct type composite_type (c2m_ctx_t c2m_ctx, struct type *tp1, struct t t.u.arr_type = arr_type = reg_malloc (c2m_ctx, sizeof (struct arr_type)); *arr_type = *tp1->u.arr_type; - if (arr_type->size == N_IGNORE) arr_type->size = tp2->u.arr_type->size; + if (arr_type->size->code == N_IGNORE) arr_type->size = tp2->u.arr_type->size; *arr_type->el_type = composite_type (c2m_ctx, tp1->u.arr_type->el_type, tp2->u.arr_type->el_type); } else if (tp1->mode == TM_FUNC) { /* ??? */ @@ -5955,11 +6078,11 @@ static void aux_set_type_align (c2m_ctx_t c2m_ctx, struct type *type) { member = NL_NEXT (member)) if (member->code == N_MEMBER) { decl_t decl = member->attr; - node_t width = NL_EL (member->u.ops, 2); + node_t width = NL_EL (member->u.ops, 3); struct expr *expr; if (type->mode == TM_UNION && width->code != N_IGNORE && (expr = width->attr)->const_p - && expr->u.u_val == 0) + && expr->c.u_val == 0) continue; member_align = type_align (decl->decl_spec.type); if (align < member_align) align = member_align; @@ -6002,9 +6125,9 @@ static void update_field_layout (int *bf_p, mir_size_t *overall_size, mir_size_t assert (field_type_size > 0 && field_type_align > 0); start_offset = curr_offset = (*overall_size + field_type_align - 1) / field_type_align * field_type_align; - if (start_offset < field_type_align && bits >= 0) *bound_bit = 0; + if ((long) start_offset < field_type_align && bits >= 0) *bound_bit = 0; for (;; start_offset = curr_offset) { - if (curr_offset < field_type_align) { + if ((long) curr_offset < field_type_align) { if (bits >= 0) *bound_bit += bits; break; } @@ -6012,11 +6135,13 @@ static void update_field_layout (int *bf_p, mir_size_t *overall_size, mir_size_t if (!*bf_p) { /* previous is a regular field: */ if (curr_offset < prev_field_offset + prev_field_type_size) { if (bits >= 0) { - *bound_bit = (prev_field_offset + prev_field_type_size - curr_offset) * MIR_CHAR_BIT; - if (*bound_bit + bits <= field_type_size * MIR_CHAR_BIT) continue; + *bound_bit + = (int) (prev_field_offset + prev_field_type_size - curr_offset) * MIR_CHAR_BIT; + if (*bound_bit + bits <= (long) field_type_size * MIR_CHAR_BIT) continue; *bound_bit = bits; if (prev_field_offset + prev_field_type_size > start_offset) - *bound_bit += (prev_field_offset + prev_field_type_size - start_offset) * MIR_CHAR_BIT; + *bound_bit + += (int) (prev_field_offset + prev_field_type_size - start_offset) * MIR_CHAR_BIT; } break; } @@ -6028,8 +6153,8 @@ static void update_field_layout (int *bf_p, mir_size_t *overall_size, mir_size_t if (start_offset * MIR_CHAR_BIT >= prev_field_offset * MIR_CHAR_BIT + *bound_bit) { *bound_bit = bits; } else { - *bound_bit - = prev_field_offset * MIR_CHAR_BIT + *bound_bit + bits - start_offset * MIR_CHAR_BIT; + *bound_bit = (int) (prev_field_offset * MIR_CHAR_BIT + *bound_bit + bits + - start_offset * MIR_CHAR_BIT); } break; } @@ -6074,7 +6199,7 @@ static void set_type_layout (c2m_ctx_t c2m_ctx, struct type *type) { struct arr_type *arr_type = type->u.arr_type; struct expr *cexpr = arr_type->size->attr; mir_size_t nel - = (arr_type->size->code == N_IGNORE || cexpr == NULL || !cexpr->const_p ? 1 : cexpr->u.i_val); + = (arr_type->size->code == N_IGNORE || cexpr == NULL || !cexpr->const_p ? 1 : cexpr->c.i_val); set_type_layout (c2m_ctx, arr_type->el_type); overall_size = type_size (c2m_ctx, arr_type->el_type) * nel; @@ -6094,7 +6219,7 @@ static void set_type_layout (c2m_ctx_t c2m_ctx, struct type *type) { decl_t decl = el->attr; int member_align; mir_size_t member_size; - node_t width = NL_EL (el->u.ops, 2); + node_t width = NL_EL (el->u.ops, 3); struct expr *expr; int anon_process_p = (!type->unnamed_anon_struct_union_member_type_p && decl->decl_spec.type->unnamed_anon_struct_union_member_type_p @@ -6104,7 +6229,8 @@ static void set_type_layout (c2m_ctx_t c2m_ctx, struct type *type) { set_type_layout (c2m_ctx, decl->decl_spec.type); if ((member_size = type_size (c2m_ctx, decl->decl_spec.type)) == 0) continue; member_align = type_align (decl->decl_spec.type); - bits = width->code == N_IGNORE || !(expr = width->attr)->const_p ? -1 : expr->u.u_val; + bits + = width->code == N_IGNORE || !(expr = width->attr)->const_p ? -1 : (int) expr->c.u_val; update_field_layout (&bf_p, &overall_size, &offset, &bound_bit, prev_size, member_size, member_align, bits); prev_size = member_size; @@ -6173,8 +6299,8 @@ static int incomplete_type_p (c2m_ctx_t c2m_ctx, struct type *type) { } static int null_const_p (struct expr *expr, struct type *type) { - return ((integer_type_p (type) && expr->const_p && expr->u.u_val == 0) - || (void_ptr_p (type) && expr->const_p && expr->u.u_val == 0 + return ((integer_type_p (type) && expr->const_p && expr->c.u_val == 0) + || (void_ptr_p (type) && expr->const_p && expr->c.u_val == 0 && type_qual_eq_p (&type->type_qual, &zero_type_qual))); } @@ -6182,10 +6308,8 @@ static void cast_value (struct expr *to_e, struct expr *from_e, struct type *to) assert (to_e->const_p && from_e->const_p); struct type *from = from_e->type; -#define CONV(TP, cast, mto, mfrom) \ - case TP: \ - to_e->u.mto = (cast) from_e->u.mfrom; \ - break; +#define CONV(TP, cast, mto, mfrom) \ + case TP: to_e->c.mto = (cast) from_e->c.mfrom; break; #define BASIC_FROM_CONV(mfrom) \ switch (to->u.basic_type) { \ CONV (TP_BOOL, mir_bool, u_val, mfrom) CONV (TP_UCHAR, mir_uchar, u_val, mfrom); \ @@ -6198,9 +6322,9 @@ static void cast_value (struct expr *to_e, struct expr *from_e, struct type *to) CONV (TP_LDOUBLE, mir_ldouble, d_val, mfrom); \ case TP_CHAR: \ if (char_is_signed_p ()) \ - to_e->u.i_val = (mir_char) from_e->u.mfrom; \ + to_e->c.i_val = (mir_char) from_e->c.mfrom; \ else \ - to_e->u.u_val = (mir_char) from_e->u.mfrom; \ + to_e->c.u_val = (mir_char) from_e->c.mfrom; \ break; \ default: assert (FALSE); \ } @@ -6212,21 +6336,21 @@ static void cast_value (struct expr *to_e, struct expr *from_e, struct type *to) case TP_USHORT: \ case TP_UINT: \ case TP_ULONG: \ - case TP_ULLONG: to_e->u.mto = (cast) from_e->u.u_val; break; \ + case TP_ULLONG: to_e->c.mto = (cast) from_e->c.u_val; break; \ case TP_CHAR: \ if (!char_is_signed_p ()) { \ - to_e->u.mto = (cast) from_e->u.u_val; \ + to_e->c.mto = (cast) from_e->c.u_val; \ break; \ } \ - /* Fall through: */ \ + /* falls through */ \ case TP_SCHAR: \ case TP_SHORT: \ case TP_INT: \ case TP_LONG: \ - case TP_LLONG: to_e->u.mto = (cast) from_e->u.i_val; break; \ + case TP_LLONG: to_e->c.mto = (cast) from_e->c.i_val; break; \ case TP_FLOAT: \ case TP_DOUBLE: \ - case TP_LDOUBLE: to_e->u.mto = (cast) from_e->u.d_val; break; \ + case TP_LDOUBLE: to_e->c.mto = (cast) from_e->c.d_val; break; \ default: assert (FALSE); \ } @@ -6242,7 +6366,7 @@ static void cast_value (struct expr *to_e, struct expr *from_e, struct type *to) from = &temp2; } if (to->mode == from->mode && (from->mode == TM_PTR || from->mode == TM_ENUM)) { - to_e->u = from_e->u; + to_e->c = from_e->c; } else if (from->mode == TM_PTR) { BASIC_FROM_CONV (u_val); } else if (to->mode == TM_PTR) { @@ -6260,7 +6384,7 @@ static void cast_value (struct expr *to_e, struct expr *from_e, struct type *to) BASIC_FROM_CONV (u_val); break; } - /* Fall through: */ + /* falls through */ case TP_SCHAR: case TP_SHORT: case TP_INT: @@ -6401,7 +6525,11 @@ static void def_symbol (c2m_ctx_t c2m_ctx, enum symbol_mode mode, node_t id, nod return; } tab_decl_spec = ((decl_t) sym.def_node->attr)->decl_spec; - if (linkage == N_IGNORE) { + if ((def_node->code == N_ENUM_CONST || sym.def_node->code == N_ENUM_CONST) + && def_node->code != sym.def_node->code) { + error (c2m_ctx, POS (id), "%s redeclared as a different kind of symbol", id->u.s.s); + return; + } else if (linkage == N_IGNORE) { if (!decl_spec.typedef_p || !tab_decl_spec.typedef_p || !type_eq_p (decl_spec.type, tab_decl_spec.type)) #if defined(__APPLE__) @@ -6437,17 +6565,17 @@ static node_t skip_struct_scopes (node_t scope) { } static void check (c2m_ctx_t c2m_ctx, node_t node, node_t context); -static struct decl_spec check_decl_spec (c2m_ctx_t c2m_ctx, node_t r, node_t decl) { +static struct decl_spec check_decl_spec (c2m_ctx_t c2m_ctx, node_t r, node_t decl_node) { check_ctx_t check_ctx = c2m_ctx->check_ctx; int n_sc = 0, sign = 0, size = 0, func_p = FALSE; struct decl_spec *res; struct type *type; if (r->attr != NULL) return *(struct decl_spec *) r->attr; - if (decl->code == N_FUNC_DEF) { + if (decl_node->code == N_FUNC_DEF) { func_p = TRUE; - } else if (decl->code == N_SPEC_DECL) { - node_t declarator = NL_EL (decl->u.ops, 1); + } else if (decl_node->code == N_SPEC_DECL) { + node_t declarator = NL_EL (decl_node->u.ops, 1); node_t list = NL_EL (declarator->u.ops, 1); func_p = list != NULL && NL_HEAD (list->u.ops) != NULL && NL_HEAD (list->u.ops)->code == N_FUNC; @@ -6625,39 +6753,39 @@ static struct decl_spec check_decl_spec (c2m_ctx_t c2m_ctx, node_t r, node_t dec case N_STRUCT: case N_UNION: { int new_scope_p; - node_t res, id = NL_HEAD (n->u.ops); + node_t res_tag_type, id = NL_HEAD (n->u.ops); node_t decl_list = NL_NEXT (id); node_t saved_unnamed_anon_struct_union_member = curr_unnamed_anon_struct_union_member; set_type_pos_node (type, n); - res = process_tag (c2m_ctx, n, id, decl_list); + res_tag_type = process_tag (c2m_ctx, n, id, decl_list); check_type_duplication (c2m_ctx, type, n, n->code == N_STRUCT ? "struct" : "union", size, sign); type->mode = n->code == N_STRUCT ? TM_STRUCT : TM_UNION; - type->u.tag_type = res; - new_scope_p = (id->code != N_IGNORE || decl->code != N_MEMBER - || NL_EL (decl->u.ops, 1)->code != N_IGNORE); + type->u.tag_type = res_tag_type; + new_scope_p = (id->code != N_IGNORE || decl_node->code != N_MEMBER + || NL_EL (decl_node->u.ops, 1)->code != N_IGNORE); type->unnamed_anon_struct_union_member_type_p = !new_scope_p; - curr_unnamed_anon_struct_union_member = new_scope_p ? NULL : decl; + curr_unnamed_anon_struct_union_member = new_scope_p ? NULL : decl_node; if (decl_list->code != N_IGNORE) { - if (new_scope_p) create_node_scope (c2m_ctx, res); + if (new_scope_p) create_node_scope (c2m_ctx, res_tag_type); check (c2m_ctx, decl_list, n); if (new_scope_p) finish_scope (c2m_ctx); - if (res != n) make_type_complete (c2m_ctx, type); /* recalculate size */ + if (res_tag_type != n) make_type_complete (c2m_ctx, type); /* recalculate size */ } curr_unnamed_anon_struct_union_member = saved_unnamed_anon_struct_union_member; break; } case N_ENUM: { - node_t res, id = NL_HEAD (n->u.ops); + node_t res_tag_type, id = NL_HEAD (n->u.ops); node_t enum_list = NL_NEXT (id); node_t enum_const_scope = skip_struct_scopes (curr_scope); set_type_pos_node (type, n); - res = process_tag (c2m_ctx, n, id, enum_list); + res_tag_type = process_tag (c2m_ctx, n, id, enum_list); check_type_duplication (c2m_ctx, type, n, "enum", size, sign); type->mode = TM_ENUM; - type->u.tag_type = res; + type->u.tag_type = res_tag_type; if (enum_list->code == N_IGNORE) { if (incomplete_type_p (c2m_ctx, type)) error (c2m_ctx, POS (n), "enum storage size is unknown"); @@ -6670,7 +6798,7 @@ static struct decl_spec check_decl_spec (c2m_ctx_t c2m_ctx, node_t r, node_t dec n->attr = enum_type = reg_malloc (c2m_ctx, sizeof (struct enum_type)); enum_type->enum_basic_type = TP_INT; // ??? for (node_t en = NL_HEAD (enum_list->u.ops); en != NULL; en = NL_NEXT (en)) { // ??? id - node_t id, const_expr; + node_t const_expr; symbol_t sym; struct enum_value *enum_value; @@ -6695,8 +6823,8 @@ static struct decl_spec check_decl_spec (c2m_ctx_t c2m_ctx, node_t r, node_t dec error (c2m_ctx, POS (const_expr), "enum const expression is not of an integer type"); continue; } - curr_val = cexpr->u.i_val; - neg_p = signed_integer_type_p (cexpr->type) && cexpr->u.i_val < 0; + curr_val = cexpr->c.i_val; + neg_p = signed_integer_type_p (cexpr->type) && cexpr->c.i_val < 0; } en->attr = enum_value = reg_malloc (c2m_ctx, sizeof (struct enum_value)); if (!neg_p) { @@ -6711,7 +6839,7 @@ static struct decl_spec check_decl_spec (c2m_ctx_t c2m_ctx, node_t r, node_t dec if (min_val < 0 && max_val >= MIR_LLONG_MAX) error (c2m_ctx, POS (const_expr), "enum const expression is not represented by an int"); - } else if (curr_val >= 0 && max_val < curr_val) { + } else if (curr_val >= 0 && max_val < (mir_ullong) curr_val) { max_val = curr_val; } enum_value->u.i_val = curr_val; @@ -6731,12 +6859,12 @@ static struct decl_spec check_decl_spec (c2m_ctx_t c2m_ctx, node_t r, node_t dec node_t el; int align = -1; - if (decl->code == N_FUNC_DEF) { + if (decl_node->code == N_FUNC_DEF) { error (c2m_ctx, POS (n), "_Alignas for function"); - } else if (decl->code == N_MEMBER && (el = NL_EL (decl->u.ops, 3)) != NULL + } else if (decl_node->code == N_MEMBER && (el = NL_EL (decl_node->u.ops, 3)) != NULL && el->code != N_IGNORE) { error (c2m_ctx, POS (n), "_Alignas for a bit-field"); - } else if (decl->code == N_SPEC_DECL && in_params_p) { + } else if (decl_node->code == N_SPEC_DECL && in_params_p) { error (c2m_ctx, POS (n), "_Alignas for a function parameter"); } else { node_t op = NL_HEAD (n->u.ops); @@ -6754,12 +6882,12 @@ static struct decl_spec check_decl_spec (c2m_ctx_t c2m_ctx, node_t r, node_t dec } else if (!integer_type_p (cexpr->type)) { error (c2m_ctx, POS (op), "constant value in _Alignas is not of an integer type"); } else if (!signed_integer_type_p (cexpr->type) - || !supported_alignment_p (cexpr->u.i_val)) { + || !supported_alignment_p (cexpr->c.i_val)) { error (c2m_ctx, POS (op), "constant value in _Alignas specifies unsupported alignment"); - } else if (invalid_alignment (cexpr->u.i_val)) { + } else if (invalid_alignment (cexpr->c.i_val)) { error (c2m_ctx, POS (op), "unsupported alignmnent"); } else { - align = cexpr->u.i_val; + align = (int) cexpr->c.i_val; } } if (align != 0 && res->align < align) { @@ -7049,9 +7177,9 @@ static void check_type (c2m_ctx_t c2m_ctx, struct type *type, int level, int fun error (c2m_ctx, POS (size_node), "non-integer array size type"); } else if (!cexpr->const_p) { error (c2m_ctx, POS (size_node), "variable size arrays are not supported"); - } else if (signed_integer_type_p (cexpr->type) && cexpr->u.i_val < 0) { + } else if (signed_integer_type_p (cexpr->type) && cexpr->c.i_val < 0) { error (c2m_ctx, POS (size_node), "array size should be not negative"); - } else if (cexpr->u.i_val == 0) { + } else if (cexpr->c.i_val == 0) { (c2m_options->pedantic_p ? error : warning) (c2m_ctx, POS (size_node), "zero array size"); } } @@ -7222,7 +7350,7 @@ static int update_init_object_path (c2m_ctx_t c2m_ctx, size_t mark, struct type size_node = init_object.container_type->u.arr_type->size; sexpr = size_node->attr; size_val = (size_node->code != N_IGNORE && sexpr->const_p && integer_type_p (sexpr->type) - ? sexpr->u.i_val + ? sexpr->c.i_val : -1); init_object.u.curr_index++; if (size_val < 0 || init_object.u.curr_index < size_val) break; @@ -7342,7 +7470,7 @@ static int check_const_addr_p (c2m_ctx_t c2m_ctx, node_t r, node_t *base, mir_ll if (e->const_p && integer_type_p (e->type)) { *base = NULL; - *offset = (mir_size_t) e->u.u_val; + *offset = (mir_size_t) e->c.u_val; *deref = 0; return TRUE; } @@ -7354,6 +7482,11 @@ static int check_const_addr_p (c2m_ctx_t c2m_ctx, node_t r, node_t *base, mir_ll *offset = 0; *deref = 0; return curr_scope == top_scope; + case N_LABEL_ADDR: + *base = r; + *offset = 0; + *deref = 0; + return TRUE; case N_ID: if (e->def_node == NULL) return FALSE; @@ -7362,8 +7495,8 @@ static int check_const_addr_p (c2m_ctx_t c2m_ctx, node_t r, node_t *base, mir_ll && ((decl_t) e->def_node->attr)->decl_spec.type->mode == TM_FUNC)) { *base = e->def_node; *deref = 0; - } else if (e->lvalue_node == NULL - || ((decl = e->lvalue_node->attr)->scope != top_scope + } else if (e->u.lvalue_node == NULL + || ((decl = e->u.lvalue_node->attr)->scope != top_scope && decl->decl_spec.linkage != N_IGNORE)) { return FALSE; } else { @@ -7375,16 +7508,17 @@ static int check_const_addr_p (c2m_ctx_t c2m_ctx, node_t r, node_t *base, mir_ll case N_DEREF: case N_ADDR: { node_t op = NL_HEAD (r->u.ops); - struct expr *e = op->attr; + struct expr *op_e = op->attr; if (!check_const_addr_p (c2m_ctx, op, base, offset, deref)) return FALSE; if (r->code == N_ADDR - && (e->type->mode == TM_ARR || (e->type->mode == TM_PTR && e->type->arr_type != NULL))) { + && (op_e->type->mode == TM_ARR + || (op_e->type->mode == TM_PTR && op_e->type->arr_type != NULL))) { if (*deref > 0) (*deref)--; } else if (op->code != N_ID - || (e->def_node->code != N_FUNC_DEF - && (e->def_node->code != N_SPEC_DECL - || ((decl_t) e->def_node->attr)->decl_spec.type->mode != TM_FUNC))) { + || (op_e->def_node->code != N_FUNC_DEF + && (op_e->def_node->code != N_SPEC_DECL + || ((decl_t) op_e->def_node->attr)->decl_spec.type->mode != TM_FUNC))) { r->code == N_DEREF ? (*deref)++ : (*deref)--; } return TRUE; @@ -7395,8 +7529,8 @@ static int check_const_addr_p (c2m_ctx_t c2m_ctx, node_t r, node_t *base, mir_ll if (*deref != (r->code == N_FIELD ? 1 : 0)) return FALSE; *deref = 1; e = r->attr; - if (e->lvalue_node != NULL) { - decl = e->lvalue_node->attr; + if (e->u.lvalue_node != NULL) { + decl = e->u.lvalue_node->attr; *offset += decl->offset; } return TRUE; @@ -7407,7 +7541,7 @@ static int check_const_addr_p (c2m_ctx_t c2m_ctx, node_t r, node_t *base, mir_ll type = ((struct expr *) r->attr)->type; size = type_size (c2m_ctx, type->arr_type != NULL ? type->arr_type : type); *deref = 1; - *offset += e->u.i_val * size; + *offset += e->c.i_val * size; return TRUE; case N_ADD: case N_SUB: @@ -7416,7 +7550,9 @@ static int check_const_addr_p (c2m_ctx_t c2m_ctx, node_t r, node_t *base, mir_ll if (r->code == N_ADD && (e = op1->attr)->const_p) SWAP (op1, op2, temp); if (!check_const_addr_p (c2m_ctx, op1, base, offset, deref)) return FALSE; if (*deref != 0 && ((struct expr *) op1->attr)->type->arr_type == NULL) return FALSE; - if (!(e = op2->attr)->const_p) return FALSE; + if (!(e = op2->attr)->const_p + && (!e->const_addr_p || e->def_node == NULL || e->def_node->code != N_LABEL_ADDR)) + return FALSE; type = ((struct expr *) r->attr)->type; assert (type->mode == TM_BASIC || type->mode == TM_PTR); size = (type->mode == TM_BASIC || type->u.ptr_type->mode == TM_FUNC @@ -7424,9 +7560,9 @@ static int check_const_addr_p (c2m_ctx_t c2m_ctx, node_t r, node_t *base, mir_ll : type_size (c2m_ctx, type->u.ptr_type->arr_type != NULL ? type->u.ptr_type->arr_type : type->u.ptr_type)); if (r->code == N_ADD) - *offset += e->u.i_val * size; + *offset += e->c.i_val * size; else - *offset -= e->u.i_val * size; + *offset -= e->c.i_val * size; return TRUE; case N_CAST: decl_spec = NL_HEAD (r->u.ops)->attr; @@ -7446,7 +7582,7 @@ static void setup_const_addr_p (c2m_ctx_t c2m_ctx, node_t r) { e = r->attr; e->const_addr_p = TRUE; e->def_node = base; - e->u.i_val = offset; + e->c.i_val = offset; } static void process_init_field_designator (c2m_ctx_t c2m_ctx, node_t designator_member, @@ -7508,7 +7644,7 @@ static node_t get_compound_literal (node_t n, int *addr_p) { return NULL; } -static mir_llong get_arr_type_size (c2m_ctx_t c2m_ctx, struct type *arr_type) { +static mir_llong get_arr_type_size (struct type *arr_type) { node_t size_node; struct expr *sexpr; @@ -7516,12 +7652,13 @@ static mir_llong get_arr_type_size (c2m_ctx_t c2m_ctx, struct type *arr_type) { size_node = arr_type->u.arr_type->size; sexpr = size_node->attr; return (size_node->code != N_IGNORE && sexpr->const_p && integer_type_p (sexpr->type) - ? sexpr->u.i_val + ? sexpr->c.i_val : -1); } -static void check_initializer (c2m_ctx_t c2m_ctx, decl_t member_decl, struct type **type_ptr, - node_t initializer, int const_only_p, int top_p) { +static void check_initializer (c2m_ctx_t c2m_ctx, decl_t member_decl MIR_UNUSED, + struct type **type_ptr, node_t initializer, int const_only_p, + int top_p) { struct type *type = *type_ptr; struct expr *cexpr; node_t literal, des_list, curr_des, init, str, value, size_node, temp; @@ -7569,15 +7706,18 @@ static void check_initializer (c2m_ctx_t c2m_ctx, decl_t member_decl, struct typ len = str->u.s.len; if (incomplete_type_p (c2m_ctx, type)) { assert (len < MIR_INT_MAX); - type->u.arr_type->size = new_i_node (c2m_ctx, len, POS (type->u.arr_type->size)); + type->u.arr_type->size = new_i_node (c2m_ctx, (long) len, POS (type->u.arr_type->size)); check (c2m_ctx, type->u.arr_type->size, NULL); make_type_complete (c2m_ctx, type); - } else if (len > ((struct expr *) type->u.arr_type->size->attr)->u.i_val + 1) { + } else if (len > (size_t) ((struct expr *) type->u.arr_type->size->attr)->c.i_val + 1) { error (c2m_ctx, POS (initializer), "string is too long for array initializer"); } return; } - if (init == NULL) return; + if (init == NULL) { + if (scalar_type_p (type)) error (c2m_ctx, POS (initializer), "empty scalar initializer"); + return; + } assert (init->code == N_INIT); des_list = NL_HEAD (init->u.ops); assert (des_list->code == N_LIST); @@ -7602,7 +7742,7 @@ static void check_initializer (c2m_ctx_t c2m_ctx, decl_t member_decl, struct typ init_object.container_type = type; init_object.field_designator_p = FALSE; if (type->mode == TM_ARR) { - size_val = get_arr_type_size (c2m_ctx, type); + size_val = get_arr_type_size (type); init_object.u.curr_index = -1; } else { init_object.u.curr_member = NULL; @@ -7664,14 +7804,14 @@ static void check_initializer (c2m_ctx_t c2m_ctx, decl_t member_decl, struct typ } else if (!integer_type_p (cexpr->type)) { error (c2m_ctx, POS (curr_des), "array index in initializer not of integer type"); } else if (incomplete_type_p (c2m_ctx, curr_type) && signed_integer_type_p (cexpr->type) - && cexpr->u.i_val < 0) { + && cexpr->c.i_val < 0) { error (c2m_ctx, POS (curr_des), "negative array index in initializer for array without size"); - } else if ((arr_size_val = get_arr_type_size (c2m_ctx, curr_type)) >= 0 - && arr_size_val <= cexpr->u.u_val) { + } else if ((arr_size_val = get_arr_type_size (curr_type)) >= 0 + && (mir_ullong) arr_size_val <= cexpr->c.u_val) { error (c2m_ctx, POS (curr_des), "array index in initializer exceeds array bounds"); } else { - init_object.u.curr_index = cexpr->u.i_val - 1; /* previous el */ + init_object.u.curr_index = cexpr->c.i_val - 1; /* previous el */ init_object.field_designator_p = FALSE; init_object.container_type = curr_type; VARR_PUSH (init_object_t, init_object_path, init_object); @@ -7696,8 +7836,8 @@ static void check_initializer (c2m_ctx_t c2m_ctx, decl_t member_decl, struct typ type->u.arr_type = arr_type; size_node = type->u.arr_type->size; type->u.arr_type->size - = (max_index < MIR_INT_MAX ? new_i_node (c2m_ctx, max_index + 1, POS (size_node)) - : max_index < MIR_LONG_MAX ? new_l_node (c2m_ctx, max_index + 1, POS (size_node)) + = (max_index < MIR_INT_MAX ? new_i_node (c2m_ctx, (long) max_index + 1, POS (size_node)) + : max_index < MIR_LONG_MAX ? new_l_node (c2m_ctx, (long) max_index + 1, POS (size_node)) : new_ll_node (c2m_ctx, max_index + 1, POS (size_node))); check (c2m_ctx, type->u.arr_type->size, NULL); make_type_complete (c2m_ctx, type); @@ -7718,19 +7858,18 @@ static void init_decl (c2m_ctx_t c2m_ctx, decl_t decl) { check_ctx_t check_ctx = c2m_ctx->check_ctx; decl->addr_p = FALSE; - decl->reg_p = decl->used_p = FALSE; + decl->reg_p = decl->asm_p = decl->used_p = FALSE; decl->offset = 0; decl->bit_offset = -1; decl->param_args_start = decl->param_args_num = 0; decl->scope = curr_scope; decl->containing_unnamed_anon_struct_union_member = curr_unnamed_anon_struct_union_member; - decl->item = NULL; + decl->u.item = NULL; decl->c2m_ctx = c2m_ctx; } static void create_decl (c2m_ctx_t c2m_ctx, node_t scope, node_t decl_node, - struct decl_spec decl_spec, node_t width, node_t initializer, - int param_p) { + struct decl_spec decl_spec, node_t initializer, int param_p) { check_ctx_t check_ctx = c2m_ctx->check_ctx; int func_def_p = decl_node->code == N_FUNC_DEF, func_p = FALSE; node_t id = NULL; /* to remove an uninitialized warning */ @@ -7858,7 +7997,7 @@ static struct expr *create_expr (c2m_ctx_t c2m_ctx, node_t r) { e->type = create_type (c2m_ctx, NULL); e->type2 = NULL; e->type->pos_node = r; - e->lvalue_node = NULL; + e->u.lvalue_node = NULL; e->const_p = e->const_addr_p = e->builtin_call_p = FALSE; return e; } @@ -7878,7 +8017,7 @@ static void get_int_node (c2m_ctx_t c2m_ctx, node_t *op, struct expr **e, struct if (i == 1) { *op = n_i1_node; } else { - *op = new_i_node (c2m_ctx, i, no_pos); + *op = new_i_node (c2m_ctx, (long) i, no_pos); check (c2m_ctx, *op, NULL); } *e = (*op)->attr; @@ -7886,12 +8025,11 @@ static void get_int_node (c2m_ctx_t c2m_ctx, node_t *op, struct expr **e, struct init_type (*t); (*e)->type->mode = TM_BASIC; (*e)->type->u.basic_type = TP_INT; - (*e)->u.i_val = i; // ??? + (*e)->c.i_val = i; // ??? } -static struct expr *check_assign_op (c2m_ctx_t c2m_ctx, node_t r, node_t op1, node_t op2, - struct expr *e1, struct expr *e2, struct type *t1, - struct type *t2) { +static struct expr *check_assign_op (c2m_ctx_t c2m_ctx, node_t r, struct expr *e1, struct expr *e2, + struct type *t1, struct type *t2) { struct expr *e = NULL; struct expr *te; struct type t, *tt; @@ -7916,13 +8054,13 @@ static struct expr *check_assign_op (c2m_ctx_t c2m_ctx, node_t r, node_t op1, no convert_value (e2, &t); e->const_p = TRUE; if (signed_integer_type_p (&t)) - e->u.i_val = (r->code == N_AND ? e1->u.i_val & e2->u.i_val - : r->code == N_OR ? e1->u.i_val | e2->u.i_val - : e1->u.i_val ^ e2->u.i_val); + e->c.i_val = (r->code == N_AND ? e1->c.i_val & e2->c.i_val + : r->code == N_OR ? e1->c.i_val | e2->c.i_val + : e1->c.i_val ^ e2->c.i_val); else - e->u.u_val = (r->code == N_AND ? e1->u.u_val & e2->u.u_val - : r->code == N_OR ? e1->u.u_val | e2->u.u_val - : e1->u.u_val ^ e2->u.u_val); + e->c.u_val = (r->code == N_AND ? e1->c.u_val & e2->c.u_val + : r->code == N_OR ? e1->c.u_val | e2->c.u_val + : e1->c.u_val ^ e2->c.u_val); } } break; @@ -7946,13 +8084,13 @@ static struct expr *check_assign_op (c2m_ctx_t c2m_ctx, node_t r, node_t op1, no e->const_p = TRUE; if (signed_integer_type_p (&t)) { if (signed_integer_type_p (&rt)) - e->u.i_val = r->code == N_LSH ? e1->u.i_val << e2->u.i_val : e1->u.i_val >> e2->u.i_val; + e->c.i_val = r->code == N_LSH ? e1->c.i_val << e2->c.i_val : e1->c.i_val >> e2->c.i_val; else - e->u.i_val = r->code == N_LSH ? e1->u.i_val << e2->u.u_val : e1->u.i_val >> e2->u.u_val; + e->c.i_val = r->code == N_LSH ? e1->c.i_val << e2->c.u_val : e1->c.i_val >> e2->c.u_val; } else if (signed_integer_type_p (&rt)) { - e->u.u_val = r->code == N_LSH ? e1->u.u_val << e2->u.i_val : e1->u.u_val >> e2->u.i_val; + e->c.u_val = r->code == N_LSH ? e1->c.u_val << e2->c.i_val : e1->c.u_val >> e2->c.i_val; } else { - e->u.u_val = r->code == N_LSH ? e1->u.u_val << e2->u.u_val : e1->u.u_val >> e2->u.u_val; + e->c.u_val = r->code == N_LSH ? e1->c.u_val << e2->c.u_val : e1->c.u_val >> e2->c.u_val; } } } @@ -7980,11 +8118,11 @@ static struct expr *check_assign_op (c2m_ctx_t c2m_ctx, node_t r, node_t op1, no convert_value (e1, &t); convert_value (e2, &t); if (floating_type_p (&t)) - e->u.d_val = (add_p ? e1->u.d_val + e2->u.d_val : e1->u.d_val - e2->u.d_val); + e->c.d_val = (add_p ? e1->c.d_val + e2->c.d_val : e1->c.d_val - e2->c.d_val); else if (signed_integer_type_p (&t)) - e->u.i_val = (add_p ? e1->u.i_val + e2->u.i_val : e1->u.i_val - e2->u.i_val); + e->c.i_val = (add_p ? e1->c.i_val + e2->c.i_val : e1->c.i_val - e2->c.i_val); else - e->u.u_val = (add_p ? e1->u.u_val + e2->u.u_val : e1->u.u_val - e2->u.u_val); + e->c.u_val = (add_p ? e1->c.u_val + e2->c.u_val : e1->c.u_val - e2->c.u_val); } } else if (add_p) { if (t2->mode == TM_PTR) { @@ -8000,8 +8138,8 @@ static struct expr *check_assign_op (c2m_ctx_t c2m_ctx, node_t r, node_t op1, no if (e1->const_p && e2->const_p) { size = type_size (c2m_ctx, t1->u.ptr_type); e->const_p = TRUE; - e->u.u_val = (signed_integer_type_p (t2) ? e1->u.u_val + e2->u.i_val * size - : e1->u.u_val + e2->u.u_val * size); + e->c.u_val = (signed_integer_type_p (t2) ? e1->c.u_val + e2->c.i_val * size + : e1->c.u_val + e2->c.u_val * size); } } } else if (t1->mode == TM_PTR && integer_type_p (t2)) { @@ -8012,8 +8150,8 @@ static struct expr *check_assign_op (c2m_ctx_t c2m_ctx, node_t r, node_t op1, no if (e1->const_p && e2->const_p) { size = type_size (c2m_ctx, t1->u.ptr_type); e->const_p = TRUE; - e->u.u_val = (signed_integer_type_p (t2) ? e1->u.u_val - e2->u.i_val * size - : e1->u.u_val - e2->u.u_val * size); + e->c.u_val = (signed_integer_type_p (t2) ? e1->c.u_val - e2->c.i_val * size + : e1->c.u_val - e2->c.u_val * size); } } } else if (t1->mode == TM_PTR && t2->mode == TM_PTR && compatible_types_p (t1, t2, TRUE)) { @@ -8029,9 +8167,9 @@ static struct expr *check_assign_op (c2m_ctx_t c2m_ctx, node_t r, node_t op1, no if (e1->const_p && e2->const_p) { size = type_size (c2m_ctx, t1->u.ptr_type); e->const_p = TRUE; - e->u.i_val - = (e1->u.u_val > e2->u.u_val ? (mir_ptrdiff_t) ((e1->u.u_val - e2->u.u_val) / size) - : -(mir_ptrdiff_t) ((e2->u.u_val - e1->u.u_val) / size)); + e->c.i_val + = (e1->c.u_val > e2->c.u_val ? (mir_ptrdiff_t) ((e1->c.u_val - e2->c.u_val) / size) + : -(mir_ptrdiff_t) ((e2->c.u_val - e1->c.u_val) / size)); } } } else { @@ -8061,29 +8199,29 @@ static struct expr *check_assign_op (c2m_ctx_t c2m_ctx, node_t r, node_t op1, no convert_value (e2, &t); if (r->code == N_MUL) { if (floating_type_p (&t)) - e->u.d_val = e1->u.d_val * e2->u.d_val; + e->c.d_val = e1->c.d_val * e2->c.d_val; else if (signed_integer_type_p (&t)) - e->u.i_val = e1->u.i_val * e2->u.i_val; + e->c.i_val = e1->c.i_val * e2->c.i_val; else - e->u.u_val = e1->u.u_val * e2->u.u_val; - } else if ((floating_type_p (&t) && e1->u.d_val == 0.0 && e2->u.d_val == 0.0) - || (signed_integer_type_p (&t) && e2->u.i_val == 0) - || (integer_type_p (&t) && !signed_integer_type_p (&t) && e2->u.u_val == 0)) { + e->c.u_val = e1->c.u_val * e2->c.u_val; + } else if ((floating_type_p (&t) && e1->c.d_val == 0.0 && e2->c.d_val == 0.0) + || (signed_integer_type_p (&t) && e2->c.i_val == 0) + || (integer_type_p (&t) && !signed_integer_type_p (&t) && e2->c.u_val == 0)) { if (floating_type_p (&t)) { - e->u.d_val = nanl (""); /* Use NaN */ + e->c.d_val = nanl (""); /* Use NaN */ } else { if (signed_integer_type_p (&t)) - e->u.i_val = 0; + e->c.i_val = 0; else - e->u.u_val = 0; + e->c.u_val = 0; error (c2m_ctx, POS (r), "Division by zero"); } } else if (r->code != N_MOD && floating_type_p (&t)) { - e->u.d_val = e1->u.d_val / e2->u.d_val; + e->c.d_val = e1->c.d_val / e2->c.d_val; } else if (signed_integer_type_p (&t)) { // ??? zero - e->u.i_val = r->code == N_DIV ? e1->u.i_val / e2->u.i_val : e1->u.i_val % e2->u.i_val; + e->c.i_val = r->code == N_DIV ? e1->c.i_val / e2->c.i_val : e1->c.i_val % e2->c.i_val; } else { - e->u.u_val = r->code == N_DIV ? e1->u.u_val / e2->u.u_val : e1->u.u_val % e2->u.u_val; + e->c.u_val = r->code == N_DIV ? e1->c.u_val / e2->c.u_val : e1->c.u_val % e2->c.u_val; } } } @@ -8093,7 +8231,7 @@ static struct expr *check_assign_op (c2m_ctx_t c2m_ctx, node_t r, node_t op1, no return e; } -static unsigned case_hash (case_t el, void *arg) { +static unsigned case_hash (case_t el, void *arg MIR_UNUSED) { node_t case_expr = NL_HEAD (el->case_node->u.ops); struct expr *expr; @@ -8101,11 +8239,11 @@ static unsigned case_hash (case_t el, void *arg) { expr = case_expr->attr; assert (expr->const_p); if (signed_integer_type_p (expr->type)) - return mir_hash (&expr->u.i_val, sizeof (expr->u.i_val), 0x42); - return mir_hash (&expr->u.u_val, sizeof (expr->u.u_val), 0x42); + return (unsigned) mir_hash (&expr->c.i_val, sizeof (expr->c.i_val), 0x42); + return (unsigned) mir_hash (&expr->c.u_val, sizeof (expr->c.u_val), 0x42); } -static int case_eq (case_t el1, case_t el2, void *arg) { +static int case_eq (case_t el1, case_t el2, void *arg MIR_UNUSED) { node_t case_expr1 = NL_HEAD (el1->case_node->u.ops); node_t case_expr2 = NL_HEAD (el2->case_node->u.ops); struct expr *expr1, *expr2; @@ -8115,8 +8253,8 @@ static int case_eq (case_t el1, case_t el2, void *arg) { expr2 = case_expr2->attr; assert (expr1->const_p && expr2->const_p); assert (signed_integer_type_p (expr1->type) == signed_integer_type_p (expr2->type)); - if (signed_integer_type_p (expr1->type)) return expr1->u.i_val == expr2->u.i_val; - return expr1->u.u_val == expr2->u.u_val; + if (signed_integer_type_p (expr1->type)) return expr1->c.i_val == expr2->c.i_val; + return expr1->c.u_val == expr2->c.u_val; } static void update_call_arg_area_offset (c2m_ctx_t c2m_ctx, struct type *type, int update_scope_p) { @@ -8136,17 +8274,17 @@ static void classify_node (node_t n, int *expr_attr_p, int *stmt_p) { switch (n->code) { REP8 (NODE_CASE, I, L, LL, U, UL, ULL, F, D) REP7 (NODE_CASE, LD, CH, CH16, CH32, STR, STR16, STR32) - REP6 (NODE_CASE, ID, COMMA, ANDAND, OROR, EQ, STMTEXPR) + REP7 (NODE_CASE, ID, LABEL_ADDR, COMMA, ANDAND, OROR, EQ, STMTEXPR) REP8 (NODE_CASE, NE, LT, LE, GT, GE, ASSIGN, BITWISE_NOT, NOT) REP8 (NODE_CASE, AND, AND_ASSIGN, OR, OR_ASSIGN, XOR, XOR_ASSIGN, LSH, LSH_ASSIGN) REP8 (NODE_CASE, RSH, RSH_ASSIGN, ADD, ADD_ASSIGN, SUB, SUB_ASSIGN, MUL, MUL_ASSIGN) REP8 (NODE_CASE, DIV, DIV_ASSIGN, MOD, MOD_ASSIGN, IND, FIELD, ADDR, DEREF) REP8 (NODE_CASE, DEREF_FIELD, COND, INC, DEC, POST_INC, POST_DEC, ALIGNOF, SIZEOF) - REP6 (NODE_CASE, EXPR_SIZEOF, CAST, COMPOUND_LITERAL, CALL, GENERIC, GENERIC_ASSOC) *expr_attr_p - = TRUE; + REP6 (NODE_CASE, EXPR_SIZEOF, CAST, COMPOUND_LITERAL, CALL, GENERIC, GENERIC_ASSOC) + *expr_attr_p = TRUE; break; - REP8 (NODE_CASE, IF, SWITCH, WHILE, DO, FOR, GOTO, CONTINUE, BREAK) - REP4 (NODE_CASE, RETURN, EXPR, BLOCK, SPEC_DECL) /* SPEC DECL may have an initializer */ + REP8 (NODE_CASE, IF, SWITCH, WHILE, DO, FOR, GOTO, INDIRECT_GOTO, CONTINUE) + REP5 (NODE_CASE, BREAK, RETURN, EXPR, BLOCK, SPEC_DECL) /* SPEC DECL may have an initializer */ *stmt_p = TRUE; break; REP8 (NODE_CASE, IGNORE, CASE, DEFAULT, LABEL, LIST, SHARE, TYPEDEF, EXTERN) @@ -8180,7 +8318,8 @@ static void add__func__def (c2m_ctx_t c2m_ctx, node_t func_block, str_t func_nam new_pos_node (c2m_ctx, N_IGNORE, pos))); declarator = new_pos_node2 (c2m_ctx, N_DECL, pos, new_str_node (c2m_ctx, N_ID, str.str, pos), list); - decl = new_pos_node3 (c2m_ctx, N_SPEC_DECL, pos, decl_specs, declarator, + decl = new_pos_node5 (c2m_ctx, N_SPEC_DECL, pos, decl_specs, declarator, + new_node (c2m_ctx, N_IGNORE), new_node (c2m_ctx, N_IGNORE), new_str_node (c2m_ctx, N_STR, func_name, pos)); NL_PREPEND (NL_EL (func_block->u.ops, 1)->u.ops, decl); } @@ -8212,7 +8351,7 @@ static void process_func_decls_for_allocation (c2m_ctx_t c2m_ctx) { decl = VARR_GET (decl_t, func_decls_for_allocation, i); type = decl->decl_spec.type; ns = decl->scope->attr; - if (scalar_type_p (type) && !decl->addr_p) { + if (scalar_type_p (type)) { decl->reg_p = TRUE; continue; } @@ -8255,9 +8394,46 @@ static void process_func_decls_for_allocation (c2m_ctx_t c2m_ctx) { } } -#define BUILTIN_VA_START (const char *[]){"__builtin_va_start", NULL} -#define BUILTIN_VA_ARG (const char *[]){"__builtin_va_arg", NULL} -#define ALLOCA (const char *[]){"alloca", "__builtin_alloca", NULL} +static const char *check_attrs (c2m_ctx_t c2m_ctx, node_t r, decl_t decl, node_t attrs, + int check_p) { + node_t n, list, id, alias_id; + if (attrs->code == N_IGNORE) return NULL; + assert (attrs->code == N_LIST); + alias_id = NULL; + for (n = NL_HEAD (attrs->u.ops); n != NULL; n = NL_NEXT (n)) { + assert (n->code == N_ATTR); + id = NL_HEAD (n->u.ops); + assert (id->code == N_ID); + if (strcmp (id->u.s.s, "antialias") != 0) continue; + list = NL_NEXT (id); + assert (list->code == N_LIST); + id = NL_HEAD (list->u.ops); + if (id == NULL) continue; + if (!check_p) { + if (id->code == N_ID) return id->u.s.s; + } else if (NL_NEXT (id) != NULL) { + error (c2m_ctx, POS (r), "antialias attribute has more one arg"); + } else if (id->code != N_ID) { + error (c2m_ctx, POS (r), "antialias attribute arg should be an identifier"); + } else if (alias_id != NULL && strcmp (id->u.s.s, alias_id->u.s.s) != 0) { + error (c2m_ctx, POS (r), "antialias attributes have different ids %s and %s", id->u.s.s, + alias_id->u.s.s); + } + alias_id = id; + } + if (alias_id == NULL) return NULL; + if (decl->decl_spec.type->mode != TM_PTR) { + error (c2m_ctx, POS (r), "antialias attribute should be given for a pointer type"); + } + return alias_id->u.s.s; +} + +#define BUILTIN_VA_START \ + (const char *[]) { "__builtin_va_start", NULL } +#define BUILTIN_VA_ARG \ + (const char *[]) { "__builtin_va_arg", NULL } +#define ALLOCA \ + (const char *[]) { "alloca", "__builtin_alloca", NULL } static int str_eq_p (const char *str, const char *v[]) { for (int i = 0; v[i] != NULL; i++) @@ -8286,52 +8462,52 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { case N_L: e = create_basic_type_expr (c2m_ctx, r, r->code == N_I ? TP_INT : TP_LONG); e->const_p = TRUE; - e->u.i_val = r->u.l; + e->c.i_val = r->u.l; break; case N_LL: e = create_basic_type_expr (c2m_ctx, r, TP_LLONG); e->const_p = TRUE; - e->u.i_val = r->u.ll; + e->c.i_val = r->u.ll; break; case N_U: case N_UL: e = create_basic_type_expr (c2m_ctx, r, r->code == N_U ? TP_UINT : TP_ULONG); e->const_p = TRUE; - e->u.u_val = r->u.ul; + e->c.u_val = r->u.ul; break; case N_ULL: e = create_basic_type_expr (c2m_ctx, r, TP_ULLONG); e->const_p = TRUE; - e->u.u_val = r->u.ull; + e->c.u_val = r->u.ull; break; case N_F: e = create_basic_type_expr (c2m_ctx, r, TP_FLOAT); e->const_p = TRUE; - e->u.d_val = r->u.f; + e->c.d_val = r->u.f; break; case N_D: e = create_basic_type_expr (c2m_ctx, r, TP_DOUBLE); e->const_p = TRUE; - e->u.d_val = r->u.d; + e->c.d_val = r->u.d; break; case N_LD: e = create_basic_type_expr (c2m_ctx, r, TP_LDOUBLE); e->const_p = TRUE; - e->u.d_val = r->u.ld; + e->c.d_val = r->u.ld; break; case N_CH: e = create_basic_type_expr (c2m_ctx, r, TP_CHAR); e->const_p = TRUE; if (char_is_signed_p ()) - e->u.i_val = r->u.ch; + e->c.i_val = r->u.ch; else - e->u.u_val = r->u.ch; + e->c.u_val = r->u.ch; break; case N_CH16: case N_CH32: e = create_basic_type_expr (c2m_ctx, r, get_uint_basic_type (r->code == N_CH16 ? 2 : 4)); e->const_p = TRUE; - e->u.u_val = r->u.ul; + e->c.u_val = r->u.ul; break; case N_STR: case N_STR16: @@ -8340,7 +8516,7 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { int size = r->code == N_STR ? 1 : r->code == N_STR16 ? 2 : 4; e = create_expr (c2m_ctx, r); - e->lvalue_node = r; + e->u.lvalue_node = r; e->type->mode = TM_ARR; e->type->pos_node = r; e->type->u.arr_type = arr_type = reg_malloc (c2m_ctx, sizeof (struct arr_type)); @@ -8350,7 +8526,7 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { arr_type->el_type->pos_node = r; arr_type->el_type->mode = TM_BASIC; arr_type->el_type->u.basic_type = size == 1 ? TP_CHAR : get_uint_basic_type (size); - arr_type->size = new_i_node (c2m_ctx, r->u.s.len, POS (r)); + arr_type->size = new_i_node (c2m_ctx, (long) r->u.s.len, POS (r)); check (c2m_ctx, arr_type->size, NULL); break; } @@ -8372,19 +8548,21 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { error (c2m_ctx, POS (r), "typedef name %s as an operand", r->u.s.s); decl->used_p = TRUE; *e->type = *decl->decl_spec.type; - if (e->type->mode != TM_FUNC) e->lvalue_node = op1; + if (e->type->mode != TM_FUNC) e->u.lvalue_node = op1; } else if (op1->code == N_FUNC_DEF) { decl = op1->attr; decl->used_p = TRUE; assert (decl->decl_spec.type->mode == TM_FUNC); *e->type = *decl->decl_spec.type; - } else { - assert (op1->code == N_ENUM_CONST && aux_node && aux_node->code == N_ENUM); + } else if (op1->code == N_ENUM_CONST) { + assert (aux_node && aux_node->code == N_ENUM); e->type->mode = TM_ENUM; e->type->pos_node = r; e->type->u.tag_type = aux_node; e->const_p = TRUE; - e->u.i_val = ((struct enum_value *) op1->attr)->u.i_val; + e->c.i_val = ((struct enum_value *) op1->attr)->u.i_val; + } else { /* it is a member reference inside struct/union */ + assert (op1->code == N_MEMBER); } break; } @@ -8405,26 +8583,26 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { int v; if (floating_type_p (t1)) - v = e1->u.d_val != 0.0; + v = e1->c.d_val != 0.0; else if (signed_integer_type_p (t1)) - v = e1->u.i_val != 0; + v = e1->c.i_val != 0; else - v = e1->u.u_val != 0; + v = e1->c.u_val != 0; if (v && r->code == N_OROR) { e->const_p = TRUE; - e->u.i_val = v; + e->c.i_val = v; } else if (!v && r->code == N_ANDAND) { e->const_p = TRUE; - e->u.i_val = v; + e->c.i_val = v; } else if (e2->const_p) { e->const_p = TRUE; if (floating_type_p (t2)) - v = e2->u.d_val != 0.0; + v = e2->c.d_val != 0.0; else if (signed_integer_type_p (t2)) - v = e2->u.i_val != 0; + v = e2->c.i_val != 0; else - v = e2->u.u_val != 0; - e->u.i_val = v; + v = e2->c.u_val != 0; + e->c.i_val = v; } } break; @@ -8451,12 +8629,12 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { error (c2m_ctx, POS (r), "pointer to atomic type as a comparison operand"); } else if (e1->const_p && e2->const_p) { e->const_p = TRUE; - e->u.i_val = (r->code == N_EQ ? e1->u.u_val == e2->u.u_val - : r->code == N_NE ? e1->u.u_val != e2->u.u_val - : r->code == N_LT ? e1->u.u_val < e2->u.u_val - : r->code == N_LE ? e1->u.u_val <= e2->u.u_val - : r->code == N_GT ? e1->u.u_val > e2->u.u_val - : e1->u.u_val >= e2->u.u_val); + e->c.i_val = (r->code == N_EQ ? e1->c.u_val == e2->c.u_val + : r->code == N_NE ? e1->c.u_val != e2->c.u_val + : r->code == N_LT ? e1->c.u_val < e2->c.u_val + : r->code == N_LE ? e1->c.u_val <= e2->c.u_val + : r->code == N_GT ? e1->c.u_val > e2->c.u_val + : e1->c.u_val >= e2->c.u_val); } } else if (arithmetic_type_p (t1) && arithmetic_type_p (t2)) { if (e1->const_p && e2->const_p) { @@ -8465,27 +8643,30 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { convert_value (e2, &t); e->const_p = TRUE; if (floating_type_p (&t)) - e->u.i_val = (r->code == N_EQ ? e1->u.d_val == e2->u.d_val - : r->code == N_NE ? e1->u.d_val != e2->u.d_val - : r->code == N_LT ? e1->u.d_val < e2->u.d_val - : r->code == N_LE ? e1->u.d_val <= e2->u.d_val - : r->code == N_GT ? e1->u.d_val > e2->u.d_val - : e1->u.d_val >= e2->u.d_val); + e->c.i_val = (r->code == N_EQ ? e1->c.d_val == e2->c.d_val + : r->code == N_NE ? e1->c.d_val != e2->c.d_val + : r->code == N_LT ? e1->c.d_val < e2->c.d_val + : r->code == N_LE ? e1->c.d_val <= e2->c.d_val + : r->code == N_GT ? e1->c.d_val > e2->c.d_val + : e1->c.d_val >= e2->c.d_val); else if (signed_integer_type_p (&t)) - e->u.i_val = (r->code == N_EQ ? e1->u.i_val == e2->u.i_val - : r->code == N_NE ? e1->u.i_val != e2->u.i_val - : r->code == N_LT ? e1->u.i_val < e2->u.i_val - : r->code == N_LE ? e1->u.i_val <= e2->u.i_val - : r->code == N_GT ? e1->u.i_val > e2->u.i_val - : e1->u.i_val >= e2->u.i_val); + e->c.i_val = (r->code == N_EQ ? e1->c.i_val == e2->c.i_val + : r->code == N_NE ? e1->c.i_val != e2->c.i_val + : r->code == N_LT ? e1->c.i_val < e2->c.i_val + : r->code == N_LE ? e1->c.i_val <= e2->c.i_val + : r->code == N_GT ? e1->c.i_val > e2->c.i_val + : e1->c.i_val >= e2->c.i_val); else - e->u.i_val = (r->code == N_EQ ? e1->u.u_val == e2->u.u_val - : r->code == N_NE ? e1->u.u_val != e2->u.u_val - : r->code == N_LT ? e1->u.u_val < e2->u.u_val - : r->code == N_LE ? e1->u.u_val <= e2->u.u_val - : r->code == N_GT ? e1->u.u_val > e2->u.u_val - : e1->u.u_val >= e2->u.u_val); - } + e->c.i_val = (r->code == N_EQ ? e1->c.u_val == e2->c.u_val + : r->code == N_NE ? e1->c.u_val != e2->c.u_val + : r->code == N_LT ? e1->c.u_val < e2->c.u_val + : r->code == N_LE ? e1->c.u_val <= e2->c.u_val + : r->code == N_GT ? e1->c.u_val > e2->c.u_val + : e1->c.u_val >= e2->c.u_val); + } + } else if ((t1->mode == TM_PTR && integer_type_p (t2)) + || (t2->mode == TM_PTR && integer_type_p (t1))) { + warning (c2m_ctx, POS (r), "comparison of integer with a pointer"); } else { error (c2m_ctx, POS (r), "invalid types of comparison operands"); } @@ -8507,18 +8688,18 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { convert_value (e1, &t); e->const_p = TRUE; if (signed_integer_type_p (&t)) - e->u.i_val = ~e1->u.i_val; + e->c.i_val = ~e1->c.i_val; else - e->u.u_val = ~e1->u.u_val; + e->c.u_val = ~e1->c.u_val; } } else if (e1->const_p) { e->const_p = TRUE; if (floating_type_p (t1)) - e->u.i_val = e1->u.d_val == 0.0; + e->c.i_val = e1->c.d_val == 0.0; else if (signed_integer_type_p (t1)) - e->u.i_val = e1->u.i_val == 0; + e->c.i_val = e1->c.i_val == 0; else - e->u.i_val = e1->u.u_val == 0; + e->c.i_val = e1->c.u_val == 0; } break; case N_INC: @@ -8532,7 +8713,7 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { t1 = e1->type = adjust_type (c2m_ctx, e1->type); get_int_node (c2m_ctx, &op2, &e2, &t2, t1->mode != TM_PTR ? 1 : type_size (c2m_ctx, t1->u.ptr_type)); - e = check_assign_op (c2m_ctx, r, op1, op2, e1, e2, t1, t2); + e = check_assign_op (c2m_ctx, r, e1, e2, t1, t2); t2 = ((struct expr *) r->attr)->type; *e1 = saved_expr; t1 = e1->type; @@ -8554,22 +8735,22 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { if (e1->const_p) e->const_p = TRUE; if (floating_type_p (t1)) { e->type->u.basic_type = t1->u.basic_type; - if (e->const_p) e->u.d_val = (r->code == N_ADD ? +e1->u.d_val : -e1->u.d_val); + if (e->const_p) e->c.d_val = (r->code == N_ADD ? +e1->c.d_val : -e1->c.d_val); } else { t = integer_promotion (t1); e->type->u.basic_type = t.u.basic_type; if (e1->const_p) { convert_value (e1, &t); if (signed_integer_type_p (&t)) - e->u.i_val = (r->code == N_ADD ? +e1->u.i_val : -e1->u.i_val); + e->c.i_val = (r->code == N_ADD ? +e1->c.i_val : -e1->c.i_val); else - e->u.u_val = (r->code == N_ADD ? +e1->u.u_val : -e1->u.u_val); + e->c.u_val = (r->code == N_ADD ? +e1->c.u_val : -e1->c.u_val); } } } break; } - /* Fall through: */ + /* falls through */ case N_AND: case N_OR: case N_XOR: @@ -8579,7 +8760,7 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { case N_DIV: case N_MOD: process_bin_ops (c2m_ctx, r, &op1, &op2, &e1, &e2, &t1, &t2, r); - e = check_assign_op (c2m_ctx, r, op1, op2, e1, e2, t1, t2); + e = check_assign_op (c2m_ctx, r, e1, e2, t1, t2); break; case N_AND_ASSIGN: case N_OR_ASSIGN: @@ -8597,7 +8778,7 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { saved_expr = *e1; t1 = e1->type = adjust_type (c2m_ctx, e1->type); t2 = e2->type = adjust_type (c2m_ctx, e2->type); - e = check_assign_op (c2m_ctx, r, op1, op2, e1, e2, t1, t2); + e = check_assign_op (c2m_ctx, r, e1, e2, t1, t2); assign_expr_type = create_type (c2m_ctx, NULL); *assign_expr_type = *e->type; t2 = ((struct expr *) r->attr)->type; @@ -8612,7 +8793,7 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { assign_expr_type = NULL; assign: e = create_expr (c2m_ctx, r); - if (!e1->lvalue_node) { + if (!e1->u.lvalue_node) { error (c2m_ctx, POS (r), "lvalue required as left operand of assignment"); } check_assignment_types (c2m_ctx, t1, t2, e2, r); @@ -8632,7 +8813,7 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { NL_APPEND (r->u.ops, op1); } e = create_expr (c2m_ctx, r); - e->lvalue_node = r; + e->u.lvalue_node = r; e->type->mode = TM_BASIC; e->type->u.basic_type = TP_INT; if (t1->mode != TM_PTR && t1->mode != TM_ARR) { @@ -8653,6 +8834,12 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { error (c2m_ctx, POS (r), "array subscript is not an integer"); } break; + case N_LABEL_ADDR: + e = create_expr (c2m_ctx, r); + e->type->mode = TM_PTR; + e->type->u.ptr_type = &VOID_TYPE; + VARR_PUSH (node_t, label_uses, r); + break; case N_ADDR: process_unop (c2m_ctx, r, &op1, &e1, &t1, r); assert (t1->mode != TM_ARR); @@ -8669,7 +8856,7 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { && e1->type->func_type_before_adjustment_p) { *e->type = *e1->type; break; - } else if (!e1->lvalue_node) { + } else if (!e1->u.lvalue_node) { e->type->mode = TM_BASIC; e->type->u.basic_type = TP_INT; error (c2m_ctx, POS (r), "lvalue required as unary & operand"); @@ -8678,17 +8865,17 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { if (op1->code == N_IND) { t2 = create_type (c2m_ctx, t1); } else if (op1->code == N_ID) { - node_t decl_node = e1->lvalue_node; + node_t decl_node = e1->u.lvalue_node; decl_t decl = decl_node->attr; decl->addr_p = TRUE; if (decl->decl_spec.register_p) error (c2m_ctx, POS (r), "address of register variable %s requested", op1->u.s.s); t2 = create_type (c2m_ctx, decl->decl_spec.type); - } else if (e1->lvalue_node->code == N_MEMBER) { - node_t declarator = NL_EL (e1->lvalue_node->u.ops, 1); - node_t width = NL_NEXT (declarator); - decl_t decl = e1->lvalue_node->attr; + } else if (e1->u.lvalue_node->code == N_MEMBER) { + node_t declarator = NL_EL (e1->u.lvalue_node->u.ops, 1); + node_t width = NL_NEXT (NL_NEXT (declarator)); + decl_t decl = e1->u.lvalue_node->attr; assert (declarator->code == N_DECL); if (width->code != N_IGNORE) { @@ -8698,13 +8885,13 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { t2 = create_type (c2m_ctx, decl->decl_spec.type); if (op1->code == N_DEREF_FIELD && (e2 = NL_HEAD (op1->u.ops)->attr)->const_p) { e->const_p = TRUE; - e->u.u_val = e2->u.u_val + decl->offset; + e->c.u_val = e2->c.u_val + decl->offset; } - } else if (e1->lvalue_node->code == N_COMPOUND_LITERAL) { + } else if (e1->u.lvalue_node->code == N_COMPOUND_LITERAL) { t2 = t1; } else { - assert (e1->lvalue_node->code == N_STR || e1->lvalue_node->code == N_STR16 - || e1->lvalue_node->code == N_STR32); + assert (e1->u.lvalue_node->code == N_STR || e1->u.lvalue_node->code == N_STR16 + || e1->u.lvalue_node->code == N_STR32); t2 = t1; } if (t2->mode == TM_ARR) { @@ -8723,7 +8910,7 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { error (c2m_ctx, POS (r), "invalid type argument of unary *"); } else { *e->type = *t1->u.ptr_type; - e->lvalue_node = r; + e->u.lvalue_node = r; } break; case N_FIELD: @@ -8752,10 +8939,10 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { assert (sym.def_node->code == N_MEMBER); decl = sym.def_node->attr; *e->type = *decl->decl_spec.type; - e->lvalue_node = sym.def_node; - if ((width = NL_EL (sym.def_node->u.ops, 2))->code != N_IGNORE && e->type->mode == TM_BASIC + e->u.lvalue_node = sym.def_node; + if ((width = NL_EL (sym.def_node->u.ops, 3))->code != N_IGNORE && e->type->mode == TM_BASIC && (width_expr = width->attr)->const_p - && width_expr->u.i_val < sizeof (mir_int) * MIR_CHAR_BIT) + && width_expr->c.i_val < (mir_llong) sizeof (mir_int) * MIR_CHAR_BIT) e->type->u.basic_type = TP_INT; } break; @@ -8781,11 +8968,11 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { } if (e1->const_p) { if (floating_type_p (t1)) - v = e1->u.d_val != 0.0; + v = e1->c.d_val != 0.0; else if (signed_integer_type_p (t1)) - v = e1->u.i_val != 0; + v = e1->c.i_val != 0; else - v = e1->u.u_val != 0; + v = e1->c.u_val != 0; } if (arithmetic_type_p (t2) && arithmetic_type_p (t3)) { t = arithmetic_conversion (t2, t3); @@ -8794,11 +8981,11 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { if (v && e2->const_p) { e->const_p = TRUE; convert_value (e2, &t); - e->u = e2->u; + e->c = e2->c; } else if (!v && e3->const_p) { e->const_p = TRUE; convert_value (e3, &t); - e->u = e3->u; + e->c = e3->c; } } break; @@ -8849,10 +9036,10 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { if (e1->const_p) { if (v && e2->const_p) { e->const_p = TRUE; - e->u = e2->u; + e->c = e2->c; } else if (!v && e3->const_p) { e->const_p = TRUE; - e->u = e3->u; + e->c = e3->c; } } break; @@ -8879,8 +9066,8 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { r->code == N_ALIGNOF ? "_Alignof" : "sizeof"); } else { e->const_p = TRUE; - e->u.i_val = (r->code == N_SIZEOF ? type_size (c2m_ctx, decl_spec->type) - : type_align (decl_spec->type)); + e->c.i_val = (r->code == N_SIZEOF ? type_size (c2m_ctx, decl_spec->type) + : (mir_size_t) type_align (decl_spec->type)); } break; } @@ -8893,9 +9080,9 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { error (c2m_ctx, POS (r), "sizeof of incomplete type requested"); } else if (t1->mode == TM_FUNC) { error (c2m_ctx, POS (r), "sizeof of function type requested"); - } else if (e1->lvalue_node && e1->lvalue_node->code == N_MEMBER) { - node_t declarator = NL_EL (e1->lvalue_node->u.ops, 1); - node_t width = NL_NEXT (declarator); + } else if (e1->u.lvalue_node && e1->u.lvalue_node->code == N_MEMBER) { + node_t declarator = NL_EL (e1->u.lvalue_node->u.ops, 1); + node_t width = NL_NEXT (NL_NEXT (declarator)); assert (declarator->code == N_DECL); if (width->code != N_IGNORE) { @@ -8904,7 +9091,7 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { } } e->const_p = TRUE; - e->u.i_val = type_size (c2m_ctx, t1); + e->c.i_val = type_size (c2m_ctx, t1); break; case N_CAST: { struct decl_spec *decl_spec; @@ -8949,12 +9136,12 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { error (c2m_ctx, POS (r), "compound literal of incomplete type"); break; } - for (n_spec_index = VARR_LENGTH (node_t, context_stack) - 1; + for (n_spec_index = (int) VARR_LENGTH (node_t, context_stack) - 1; n_spec_index >= 0 && (n = VARR_GET (node_t, context_stack, n_spec_index)) != NULL && n->code != N_SPEC_DECL; n_spec_index--) ; - if (n_spec_index < VARR_LENGTH (node_t, context_stack) - 1 + if (n_spec_index < (int) VARR_LENGTH (node_t, context_stack) - 1 && (n_spec_index < 0 || !get_compound_literal (VARR_GET (node_t, context_stack, n_spec_index + 1), &addr_p) || addr_p)) @@ -8964,7 +9151,7 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { FALSE); decl->decl_spec.type = t1; e = create_expr (c2m_ctx, r); - e->lvalue_node = r; + e->u.lvalue_node = r; *e->type = *t1; if (curr_scope != top_scope) VARR_PUSH (decl_t, func_decls_for_allocation, decl); break; @@ -8972,27 +9159,42 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { case N_CALL: { struct func_type *func_type = NULL; /* to remove an uninitialized warning */ struct type *ret_type; - node_t list, spec_list, decl, param_list, start_param, param, arg_list, arg; + node_t list, spec_list, decl, param_list, start_param, param, arg_list, first_arg, arg; node_t saved_scope = curr_scope; struct decl_spec *decl_spec; mir_size_t saved_call_arg_area_offset_before_args; struct type res_type; - int builtin_call_p, alloca_p, va_arg_p = FALSE, va_start_p = FALSE; + int builtin_call_p, alloca_p = FALSE, va_arg_p = FALSE, va_start_p = FALSE; + int add_overflow_p = FALSE, sub_overflow_p = FALSE, mul_overflow_p = FALSE, expect_p = FALSE; + int jcall_p = FALSE, jret_p = FALSE, prop_set_p = FALSE, prop_eq_p = FALSE, prop_ne_p = FALSE; op1 = NL_HEAD (r->u.ops); - alloca_p = op1->code == N_ID && str_eq_p (op1->u.s.s, ALLOCA); + if (op1->code == N_ID) { + alloca_p = str_eq_p (op1->u.s.s, ALLOCA); + add_overflow_p = strcmp (op1->u.s.s, ADD_OVERFLOW) == 0; + sub_overflow_p = strcmp (op1->u.s.s, SUB_OVERFLOW) == 0; + mul_overflow_p = strcmp (op1->u.s.s, MUL_OVERFLOW) == 0; + expect_p = strcmp (op1->u.s.s, EXPECT) == 0; + jcall_p = strcmp (op1->u.s.s, JCALL) == 0; + jret_p = strcmp (op1->u.s.s, JRET) == 0; + prop_set_p = strcmp (op1->u.s.s, PROP_SET) == 0; + prop_eq_p = strcmp (op1->u.s.s, PROP_EQ) == 0; + prop_ne_p = strcmp (op1->u.s.s, PROP_NE) == 0; + } if (op1->code == N_ID && find_def (c2m_ctx, S_REGULAR, op1, curr_scope, NULL) == NULL) { va_arg_p = str_eq_p (op1->u.s.s, BUILTIN_VA_ARG); va_start_p = str_eq_p (op1->u.s.s, BUILTIN_VA_START); if (!va_arg_p && !va_start_p && !alloca_p) { - /* N_SPEC_DECL (N_SHARE (N_LIST (N_INT)), N_DECL (N_ID, N_FUNC (N_LIST)), N_IGNORE) */ + /* N_SPEC_DECL (N_SHARE (N_LIST (N_INT)), N_DECL (N_ID, N_FUNC (N_LIST)), N_IGNORE, + N_IGNORE, N_IGNORE) */ spec_list = new_node (c2m_ctx, N_LIST); op_append (c2m_ctx, spec_list, new_node (c2m_ctx, N_INT)); list = new_node (c2m_ctx, N_LIST); op_append (c2m_ctx, list, new_node1 (c2m_ctx, N_FUNC, new_node (c2m_ctx, N_LIST))); decl - = new_pos_node3 (c2m_ctx, N_SPEC_DECL, POS (op1), new_node1 (c2m_ctx, N_SHARE, spec_list), + = new_pos_node5 (c2m_ctx, N_SPEC_DECL, POS (op1), new_node1 (c2m_ctx, N_SHARE, spec_list), new_node2 (c2m_ctx, N_DECL, copy_node (c2m_ctx, op1), list), + new_node (c2m_ctx, N_IGNORE), new_node (c2m_ctx, N_IGNORE), new_node (c2m_ctx, N_IGNORE)); curr_scope = top_scope; check (c2m_ctx, decl, NULL); @@ -9003,8 +9205,10 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { op_prepend (c2m_ctx, list, decl); } } - builtin_call_p = alloca_p || va_arg_p || va_start_p; - if (!builtin_call_p) VARR_PUSH (node_t, call_nodes, r); + builtin_call_p = alloca_p || va_arg_p || va_start_p || add_overflow_p || sub_overflow_p + || mul_overflow_p || expect_p || jcall_p || jret_p || prop_set_p || prop_eq_p + || prop_ne_p; + if (!builtin_call_p || jcall_p) VARR_PUSH (node_t, call_nodes, r); arg_list = NL_NEXT (op1); if (builtin_call_p) { if (func_block_scope != NULL && (va_arg_p || va_start_p)) @@ -9017,14 +9221,23 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { res_type.u.ptr_type = &VOID_TYPE; } else { res_type.mode = TM_BASIC; - res_type.u.basic_type = va_arg_p ? TP_INT : TP_VOID; + res_type.u.basic_type + = (va_arg_p || add_overflow_p || sub_overflow_p || mul_overflow_p ? TP_INT + : expect_p || prop_eq_p || prop_ne_p ? TP_LONG + : TP_VOID); } ret_type = &res_type; - if (va_start_p && NL_LENGTH (arg_list->u.ops) != 1) { - error (c2m_ctx, POS (op1), "wrong number of arguments in %s call", op1->u.s.s); - } else if (alloca_p && NL_LENGTH (arg_list->u.ops) != 1) { - error (c2m_ctx, POS (op1), "wrong number of arguments in %s call", op1->u.s.s); - } else if (va_arg_p && NL_LENGTH (arg_list->u.ops) != 2) { + if (builtin_call_p + && ((va_start_p && NL_LENGTH (arg_list->u.ops) != 1) + || (alloca_p && NL_LENGTH (arg_list->u.ops) != 1) + || (add_overflow_p && NL_LENGTH (arg_list->u.ops) != 3) + || (sub_overflow_p && NL_LENGTH (arg_list->u.ops) != 3) + || (mul_overflow_p && NL_LENGTH (arg_list->u.ops) != 3) + || (expect_p && NL_LENGTH (arg_list->u.ops) != 2) + || (jret_p && NL_LENGTH (arg_list->u.ops) != 1) + || (va_arg_p && NL_LENGTH (arg_list->u.ops) != 2) + || (prop_set_p && NL_LENGTH (arg_list->u.ops) != 2) + || ((prop_eq_p || prop_ne_p) && NL_LENGTH (arg_list->u.ops) != 2))) { error (c2m_ctx, POS (op1), "wrong number of arguments in %s call", op1->u.s.s); } else { /* first argument type ??? */ @@ -9036,6 +9249,64 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { error (c2m_ctx, POS (arg), "wrong type of 2nd argument of %s call", BUILTIN_VA_ARG); else ret_type = t2->u.ptr_type; + } else if (add_overflow_p || sub_overflow_p || mul_overflow_p) { + arg = NL_EL (arg_list->u.ops, 2); + e2 = arg->attr; + t2 = e2->type; + if (t2->mode != TM_PTR || !standard_integer_type_p (t2->u.ptr_type) + || t2->u.ptr_type->u.basic_type < TP_INT) /* only [u]int, [u]long, [u]longlong */ + error (c2m_ctx, POS (arg), "wrong type of 3rd argument of %s call", + add_overflow_p ? ADD_OVERFLOW + : sub_overflow_p ? SUB_OVERFLOW + : MUL_OVERFLOW); + for (int i = 0; i < 2; i++) { + arg = NL_EL (arg_list->u.ops, i); + e2 = arg->attr; + if (!integer_type_p (e2->type)) + error (c2m_ctx, POS (arg), "non-integer type of %d argument of %s call", i, + add_overflow_p ? ADD_OVERFLOW + : sub_overflow_p ? SUB_OVERFLOW + : MUL_OVERFLOW); + } + } else if (expect_p) { + for (int i = 0; i < 2; i++) { + arg = NL_EL (arg_list->u.ops, i); + e2 = arg->attr; + if (!integer_type_p (e2->type)) + error (c2m_ctx, POS (arg), "non-integer type of %d argument of %s call", i, EXPECT); + } + } else if (jret_p) { + arg = NL_HEAD (arg_list->u.ops); + e2 = arg->attr; + t2 = e2->type; + if (t2->mode != TM_PTR) + error (c2m_ctx, POS (arg), "non-pointer type of 1st argument of %s call", JRET); + } else if (jcall_p) { + arg = NL_HEAD (arg_list->u.ops); + e2 = arg->attr; + t2 = e2->type; + if (t2->mode != TM_PTR || (t2 = t2->u.ptr_type)->mode != TM_FUNC) { + error (c2m_ctx, POS (r), "calling non-function in %s", JCALL); + break; + } + func_type = t2->u.func_type; + ret_type = func_type->ret_type; + if (!void_type_p (ret_type)) { + error (c2m_ctx, POS (arg), "calling non-void function in %s", JCALL); + break; + } + } else if (prop_set_p || prop_eq_p || prop_ne_p) { + arg = NL_HEAD (arg_list->u.ops); + e2 = arg->attr; + if (!e2->u.lvalue_node) + error (c2m_ctx, POS (r), "1st arg of %s should be lvalue", op1->u.s.s); + arg = NL_NEXT (arg); + e2 = arg->attr; + t2 = e2->type; + if (!e2->const_p || !integer_type_p (t2)) + error (c2m_ctx, POS (arg), + "property value in 2nd arg of %s call should be an integer constant", + op1->u.s.s); } } } else { @@ -9058,18 +9329,19 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { } if (ret_type->mode == TM_STRUCT || ret_type->mode == TM_UNION) { set_type_layout (c2m_ctx, ret_type); - if (!builtin_call_p) update_call_arg_area_offset (c2m_ctx, ret_type, TRUE); + if (!builtin_call_p && curr_scope != top_scope) + update_call_arg_area_offset (c2m_ctx, ret_type, TRUE); } - if (builtin_call_p) break; + if (builtin_call_p && !jcall_p) break; + first_arg = jcall_p ? NL_EL (arg_list->u.ops, 1) : NL_HEAD (arg_list->u.ops); param_list = func_type->param_list; param = start_param = NL_HEAD (param_list->u.ops); if (void_param_p (start_param)) { /* f(void) */ - if ((arg = NL_HEAD (arg_list->u.ops)) != NULL) - error (c2m_ctx, POS (arg), "too many arguments"); + if (first_arg != NULL) error (c2m_ctx, POS (first_arg), "too many arguments"); break; } saved_call_arg_area_offset_before_args = curr_call_arg_area_offset; - for (arg = NL_HEAD (arg_list->u.ops); arg != NULL; arg = NL_NEXT (arg)) { + for (arg = first_arg; arg != NULL; arg = NL_NEXT (arg)) { check (c2m_ctx, arg, r); e2 = arg->attr; if (start_param == NULL || start_param->code == N_ID) continue; /* no params or ident list */ @@ -9147,17 +9419,49 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { case N_SPEC_DECL: { node_t specs = NL_HEAD (r->u.ops); node_t declarator = NL_NEXT (specs); - node_t initializer = NL_NEXT (declarator); + node_t attrs = NL_NEXT (declarator); + node_t asm_part = NL_NEXT (attrs); + node_t initializer = NL_NEXT (asm_part); node_t unshared_specs = specs->code != N_SHARE ? specs : NL_HEAD (specs->u.ops); struct decl_spec decl_spec = check_decl_spec (c2m_ctx, unshared_specs, r); - + int i; + const char *asm_str = NULL; + + if (asm_part->code == N_ASM && decl_spec.register_p) { + /* func can have asm which ignore here */ + if (initializer->code != N_IGNORE) { + error (c2m_ctx, POS (r), "asm register decl with initializer"); + } else if (curr_scope != top_scope) { + error (c2m_ctx, POS (r), "asm register decl should be at the top level"); + } else { + asm_str = NL_HEAD (asm_part->u.ops)->u.s.s; + for (i = 0; asm_str[i] != '\0' && _MIR_name_char_p (c2m_ctx->ctx, asm_str[i], i == 0); i++) + ; + if (asm_str[i] != '\0') { + error (c2m_ctx, POS (r), "asm register name %s contains wrong char '%c'", asm_str, + asm_str[i]); + asm_str = NULL; + } + } + } if (declarator->code != N_IGNORE) { - create_decl (c2m_ctx, curr_scope, r, decl_spec, NULL, initializer, + create_decl (c2m_ctx, curr_scope, r, decl_spec, initializer, context != NULL && context->code == N_FUNC); decl_t decl = r->attr; - if ((initializer == NULL || initializer->code == N_IGNORE) && !decl->decl_spec.typedef_p - && !decl->decl_spec.extern_p - && (decl->decl_spec.type->mode == TM_STRUCT || decl->decl_spec.type->mode == TM_UNION)) { + const char *antialias = check_attrs (c2m_ctx, r, decl, attrs, TRUE); + if (decl->decl_spec.type->mode == TM_PTR && antialias != NULL) + decl->decl_spec.type->antialias = MIR_alias (c2m_ctx->ctx, antialias); + if (asm_str != NULL) { + if (!scalar_type_p (decl->decl_spec.type)) { + error (c2m_ctx, POS (r), "asm register decl should have a scalar type"); + } else { + decl->reg_p = decl->asm_p = TRUE; + decl->u.asm_str = asm_str; + } + } else if ((initializer == NULL || initializer->code == N_IGNORE) + && !decl->decl_spec.typedef_p && !decl->decl_spec.extern_p + && (decl->decl_spec.type->mode == TM_STRUCT + || decl->decl_spec.type->mode == TM_UNION)) { VARR_PUSH (node_t, possible_incomplete_decls, r); } } else if (decl_spec.type->mode == TM_STRUCT || decl_spec.type->mode == TM_UNION) { @@ -9182,9 +9486,9 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { error (c2m_ctx, POS (r), "expression in static assertion is not an integer"); } else { if (signed_integer_type_p (t1)) - ok_p = e1->u.i_val != 0; + ok_p = e1->c.i_val != 0; else - ok_p = e1->u.u_val != 0; + ok_p = e1->c.u_val != 0; if (!ok_p) { assert (NL_NEXT (op1) != NULL && (NL_NEXT (op1)->code == N_STR || NL_NEXT (op1)->code == N_STR16 @@ -9199,15 +9503,15 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { struct type *type; node_t specs = NL_HEAD (r->u.ops); node_t declarator = NL_NEXT (specs); - node_t const_expr = NL_NEXT (declarator); + node_t attrs = NL_NEXT (declarator); + node_t const_expr = NL_NEXT (attrs); node_t unshared_specs = specs->code != N_SHARE ? specs : NL_HEAD (specs->u.ops); struct decl_spec decl_spec = check_decl_spec (c2m_ctx, unshared_specs, r); - create_decl (c2m_ctx, curr_scope, r, decl_spec, const_expr, NULL, FALSE); + create_decl (c2m_ctx, curr_scope, r, decl_spec, NULL, FALSE); type = ((decl_t) r->attr)->decl_spec.type; if (const_expr->code != N_IGNORE) { struct expr *cexpr; - check (c2m_ctx, const_expr, r); cexpr = const_expr->attr; if (cexpr != NULL) { @@ -9221,14 +9525,15 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { } else if (!integer_type_p (cexpr->type) && (cexpr->type->mode != TM_BASIC || cexpr->type->u.basic_type != TP_BOOL)) { error (c2m_ctx, POS (const_expr), "bit field width is not of an integer type"); - } else if (signed_integer_type_p (cexpr->type) && cexpr->u.i_val < 0) { + } else if (signed_integer_type_p (cexpr->type) && cexpr->c.i_val < 0) { error (c2m_ctx, POS (const_expr), "bit field width is negative"); - } else if (cexpr->u.i_val == 0 && declarator->code == N_DECL) { + } else if (cexpr->c.i_val == 0 && declarator->code == N_DECL) { error (c2m_ctx, POS (const_expr), "zero bit field width for %s", NL_HEAD (declarator->u.ops)->u.s.s); - } else if ((!signed_integer_type_p (cexpr->type) && cexpr->u.u_val > int_bit_size (type)) + } else if ((!signed_integer_type_p (cexpr->type) + && cexpr->c.u_val > (mir_ullong) int_bit_size (type)) || (signed_integer_type_p (cexpr->type) - && cexpr->u.i_val > int_bit_size (type))) { + && cexpr->c.i_val > int_bit_size (type))) { error (c2m_ctx, POS (const_expr), "bit field width exceeds its type"); } } @@ -9240,7 +9545,9 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { error (c2m_ctx, POS (r), "no declarator in struct or union declaration"); } else { node_t id = NL_HEAD (declarator->u.ops); - + const char *antialias = check_attrs (c2m_ctx, r, r->attr, attrs, TRUE); + if (type->mode == TM_PTR && antialias != NULL) + type->antialias = MIR_alias (c2m_ctx->ctx, antialias); if (type->mode == TM_FUNC) { error (c2m_ctx, POS (id), "field %s is declared as a function", id->u.s.s); } else if (incomplete_type_p (c2m_ctx, type)) { @@ -9270,7 +9577,10 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { struct node_scope *ns; if (str_eq_p (id->u.s.s, ALLOCA) || str_eq_p (id->u.s.s, BUILTIN_VA_START) - || str_eq_p (id->u.s.s, BUILTIN_VA_ARG)) { + || str_eq_p (id->u.s.s, BUILTIN_VA_ARG) || strcmp (id->u.s.s, ADD_OVERFLOW) == 0 + || strcmp (id->u.s.s, SUB_OVERFLOW) == 0 || strcmp (id->u.s.s, MUL_OVERFLOW) == 0 + || strcmp (id->u.s.s, EXPECT) == 0 || strcmp (id->u.s.s, JCALL) == 0 + || strcmp (id->u.s.s, JRET) == 0) { error (c2m_ctx, POS (id), "%s is a builtin function", id->u.s.s); break; } @@ -9278,10 +9588,11 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { create_node_scope (c2m_ctx, block); func_block_scope = curr_scope; curr_func_def = r; + jump_ret_p = FALSE; curr_switch = curr_loop = curr_loop_switch = NULL; curr_call_arg_area_offset = 0; VARR_TRUNC (decl_t, func_decls_for_allocation, 0); - create_decl (c2m_ctx, top_scope, r, decl_spec, NULL, NULL, FALSE); + create_decl (c2m_ctx, top_scope, r, decl_spec, NULL, FALSE); curr_scope = func_block_scope; check (c2m_ctx, declarations, r); /* Process parameter identifier list: */ @@ -9300,19 +9611,20 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { error (c2m_ctx, POS (p), "parameter %s has no type", p->u.s.s); } else { warning (c2m_ctx, POS (p), "type of parameter %s defaults to int", p->u.s.s); - decl_node = new_pos_node3 (c2m_ctx, N_SPEC_DECL, POS (p), + decl_node = new_pos_node5 (c2m_ctx, N_SPEC_DECL, POS (p), new_node1 (c2m_ctx, N_SHARE, new_node1 (c2m_ctx, N_LIST, new_pos_node (c2m_ctx, N_INT, POS (p)))), new_pos_node2 (c2m_ctx, N_DECL, POS (p), new_str_node (c2m_ctx, N_ID, p->u.s, POS (p)), new_node (c2m_ctx, N_LIST)), + new_node (c2m_ctx, N_IGNORE), new_node (c2m_ctx, N_IGNORE), new_node (c2m_ctx, N_IGNORE)); NL_APPEND (param_list->u.ops, decl_node); check (c2m_ctx, decl_node, r); } } else { - struct decl_spec decl_spec, *decl_spec_ptr; + struct decl_spec *decl_spec_ptr; decl_node = sym.def_node; assert (decl_node->code == N_SPEC_DECL); @@ -9321,7 +9633,7 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { param_declarator = NL_EL (decl_node->u.ops, 1); assert (param_declarator->code == N_DECL); param_id = NL_HEAD (param_declarator->u.ops); - if (NL_NEXT (param_declarator)->code != N_IGNORE) { + if (NL_NEXT (NL_NEXT (param_declarator))->code != N_IGNORE) { error (c2m_ctx, POS (p), "initialized parameter %s", param_id->u.s.s); } decl_spec_ptr = &((decl_t) decl_node->attr)->decl_spec; @@ -9348,19 +9660,20 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { } add__func__def (c2m_ctx, block, id->u.s); check (c2m_ctx, block, r); - /* Process all gotos: */ - for (size_t i = 0; i < VARR_LENGTH (node_t, gotos); i++) { - symbol_t sym; - node_t n = VARR_GET (node_t, gotos, i); - node_t id = NL_NEXT (NL_HEAD (n->u.ops)); + /* Process all label uses: */ + for (size_t i = 0; i < VARR_LENGTH (node_t, label_uses); i++) { + node_t n = VARR_GET (node_t, label_uses, i); + id = n->code == N_LABEL_ADDR ? NL_HEAD (n->u.ops) : NL_NEXT (NL_HEAD (n->u.ops)); if (!symbol_find (c2m_ctx, S_LABEL, id, func_block_scope, &sym)) { error (c2m_ctx, POS (id), "undefined label %s", id->u.s.s); + } else if (n->code == N_LABEL_ADDR) { + ((struct expr *) n->attr)->u.label_addr_target = sym.def_node; } else { n->attr = sym.def_node; } } - VARR_TRUNC (node_t, gotos, 0); + VARR_TRUNC (node_t, label_uses, 0); assert (curr_scope == top_scope); /* set up in the block */ func_block_scope = top_scope; process_func_decls_for_allocation (c2m_ctx); @@ -9450,11 +9763,11 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { node_t labels = NL_HEAD (r->u.ops); node_t expr = NL_NEXT (labels); node_t stmt = NL_NEXT (expr); - struct type t, *type; + struct type *type; struct switch_attr *switch_attr; case_t el; node_t case_expr, case_expr2, another_case_expr, another_case_expr2; - struct expr *e, *e2, *another_e, *another_e2; + struct expr *case_e, *case_e2, *another_case_e, *another_case_e2; int signed_p, skip_range_p; check_labels (c2m_ctx, labels, r); @@ -9488,12 +9801,12 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { switch_attr->min_val_case = switch_attr->max_val_case = c; continue; } - e = NL_HEAD (c->case_node->u.ops)->attr; - e2 = NL_HEAD (switch_attr->min_val_case->case_node->u.ops)->attr; - if (signed_p ? e->u.i_val < e2->u.i_val : e->u.u_val < e2->u.u_val) + case_e = NL_HEAD (c->case_node->u.ops)->attr; + case_e2 = NL_HEAD (switch_attr->min_val_case->case_node->u.ops)->attr; + if (signed_p ? case_e->c.i_val < case_e2->c.i_val : case_e->c.u_val < case_e2->c.u_val) switch_attr->min_val_case = c; - e2 = NL_HEAD (switch_attr->max_val_case->case_node->u.ops)->attr; - if (signed_p ? e->u.i_val > e2->u.i_val : e->u.u_val > e2->u.u_val) + case_e2 = NL_HEAD (switch_attr->max_val_case->case_node->u.ops)->attr; + if (signed_p ? case_e->c.i_val > case_e2->c.i_val : case_e->c.u_val > case_e2->c.u_val) switch_attr->max_val_case = c; } HTAB_CLEAR (case_t, case_tab); @@ -9504,8 +9817,8 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { continue; switch_attr->ranges_p = TRUE; case_expr = NL_HEAD (c->case_node->u.ops); - e = case_expr->attr; - e2 = case_expr2->attr; + case_e = case_expr->attr; + case_e2 = case_expr2->attr; skip_range_p = FALSE; for (case_t c2 = DLIST_HEAD (case_t, switch_attr->case_labels); c2 != NULL; c2 = DLIST_NEXT (case_t, c2)) { @@ -9517,25 +9830,29 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { another_case_expr = NL_HEAD (c2->case_node->u.ops); another_case_expr2 = NL_EL (c2->case_node->u.ops, 1); if (skip_range_p && another_case_expr2 != NULL) continue; - another_e = another_case_expr->attr; - assert (another_e->const_p && integer_type_p (another_e->type)); + another_case_e = another_case_expr->attr; + assert (another_case_e->const_p && integer_type_p (another_case_e->type)); if (another_case_expr2 == NULL) { - if ((signed_p && e->u.i_val <= another_e->u.i_val && another_e->u.i_val <= e2->u.i_val) - || (!signed_p && e->u.u_val <= another_e->u.u_val - && another_e->u.u_val <= e2->u.u_val)) { + if ((signed_p && case_e->c.i_val <= another_case_e->c.i_val + && another_case_e->c.i_val <= case_e2->c.i_val) + || (!signed_p && case_e->c.u_val <= another_case_e->c.u_val + && another_case_e->c.u_val <= case_e2->c.u_val)) { error (c2m_ctx, POS (c->case_node), "duplicate value in a range case"); break; } } else { - another_e2 = another_case_expr2->attr; - assert (another_e2->const_p && integer_type_p (another_e2->type)); + another_case_e2 = another_case_expr2->attr; + assert (another_case_e2->const_p && integer_type_p (another_case_e2->type)); if ((signed_p - && ((e->u.i_val <= another_e->u.i_val && another_e->u.i_val <= e2->u.i_val) - || (e->u.i_val <= another_e2->u.i_val && another_e2->u.i_val <= e2->u.i_val))) + && ((case_e->c.i_val <= another_case_e->c.i_val + && another_case_e->c.i_val <= case_e2->c.i_val) + || (case_e->c.i_val <= another_case_e2->c.i_val + && another_case_e2->c.i_val <= case_e2->c.i_val))) || (!signed_p - && ((e->u.u_val <= another_e->u.u_val && another_e->u.u_val <= e2->u.u_val) - || (e->u.u_val <= another_e2->u.u_val - && another_e2->u.u_val <= e2->u.u_val)))) { + && ((case_e->c.u_val <= another_case_e->c.u_val + && another_case_e->c.u_val <= case_e2->c.u_val) + || (case_e->c.u_val <= another_case_e2->c.u_val + && another_case_e2->c.u_val <= case_e2->c.u_val)))) { error (c2m_ctx, POS (c->case_node), "duplicate value in a range case"); break; } @@ -9613,7 +9930,17 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { node_t labels = NL_HEAD (r->u.ops); check_labels (c2m_ctx, labels, r); - VARR_PUSH (node_t, gotos, r); + VARR_PUSH (node_t, label_uses, r); + break; + } + case N_INDIRECT_GOTO: { + node_t labels = NL_HEAD (r->u.ops); + node_t expr = NL_NEXT (labels); + + check_labels (c2m_ctx, labels, r); + check (c2m_ctx, expr, r); + e1 = expr->attr; + if (e1->type->mode != TM_PTR) error (c2m_ctx, POS (r), "computed goto must be pointer type"); break; } case N_CONTINUE: @@ -9668,10 +9995,14 @@ static void check (c2m_ctx_t c2m_ctx, node_t r, node_t context) { && context->code != N_EXPR_SIZEOF) e->type = adjust_type (c2m_ctx, e->type); set_type_layout (c2m_ctx, e->type); - if (!e->const_p && check_const_addr_p (c2m_ctx, r, &base, &offset, &deref) && deref == 0 - && base == NULL) { - e->const_p = TRUE; - e->u.i_val = offset; + if (!e->const_p && check_const_addr_p (c2m_ctx, r, &base, &offset, &deref) && deref == 0) { + if (base == NULL) { + e->const_p = TRUE; + e->c.i_val = offset; + } else if (base->code == N_LABEL_ADDR) { + e->const_addr_p = TRUE; + e->c.i_val = offset; + } } if (e->const_p) convert_value (e, e->type); } else if (stmt_p) { @@ -9707,7 +10038,7 @@ static void context_init (c2m_ctx_t c2m_ctx) { VARR_CREATE (node_t, context_stack, 64); check (c2m_ctx, n_i1_node, NULL); func_block_scope = curr_scope = NULL; - VARR_CREATE (node_t, gotos, 0); + VARR_CREATE (node_t, label_uses, 0); symbol_init (c2m_ctx); in_params_p = FALSE; curr_unnamed_anon_struct_union_member = NULL; @@ -9721,7 +10052,7 @@ static void context_finish (c2m_ctx_t c2m_ctx) { if (c2m_ctx == NULL || (check_ctx = c2m_ctx->check_ctx) == NULL) return; if (context_stack != NULL) VARR_DESTROY (node_t, context_stack); - if (gotos != NULL) VARR_DESTROY (node_t, gotos); + if (label_uses != NULL) VARR_DESTROY (node_t, label_uses); symbol_finish (c2m_ctx); if (case_tab != NULL) HTAB_DESTROY (case_t, case_tab); if (func_decls_for_allocation != NULL) VARR_DESTROY (decl_t, func_decls_for_allocation); @@ -9743,7 +10074,7 @@ static const char *RET_ADDR_NAME = "Ret_Addr"; /* MIR var naming: {I|U|i|u|f|d}_ -- temporary of I64, U64, I32, U32, F, D type {I|U|i|u|f|d}_ -- variable with of original of the corresponding - type in scope with + type in scope with */ #if MIR_PTR64 @@ -9774,7 +10105,7 @@ struct init_el { c2m_ctx_t c2m_ctx; /* for sorting */ mir_size_t num, offset; decl_t member_decl; /* NULL for non-member initialization */ - struct type *el_type; + struct type *el_type, *container_type; node_t init; }; @@ -9788,6 +10119,7 @@ DEF_HTAB (MIR_item_t); struct gen_ctx { op_t zero_op, one_op, minus_one_op; MIR_item_t curr_func; + DLIST (MIR_insn_t) slow_code_part; HTAB (reg_var_t) * reg_var_tab; int reg_free_mark; MIR_label_t continue_label, break_label; @@ -9804,12 +10136,14 @@ struct gen_ctx { VARR (case_t) * switch_cases; int curr_mir_proto_num; HTAB (MIR_item_t) * proto_tab; + VARR (node_t) * node_stack; }; #define zero_op gen_ctx->zero_op #define one_op gen_ctx->one_op #define minus_one_op gen_ctx->minus_one_op #define curr_func gen_ctx->curr_func +#define slow_code_part gen_ctx->slow_code_part #define reg_var_tab gen_ctx->reg_var_tab #define reg_free_mark gen_ctx->reg_free_mark #define continue_label gen_ctx->continue_label @@ -9827,6 +10161,7 @@ struct gen_ctx { #define switch_cases gen_ctx->switch_cases #define curr_mir_proto_num gen_ctx->curr_mir_proto_num #define proto_tab gen_ctx->proto_tab +#define node_stack gen_ctx->node_stack static op_t new_op (decl_t decl, MIR_op_t mir_op) { op_t res; @@ -9836,10 +10171,10 @@ static op_t new_op (decl_t decl, MIR_op_t mir_op) { return res; } -static htab_hash_t reg_var_hash (reg_var_t r, void *arg) { - return mir_hash (r.name, strlen (r.name), 0x42); +static htab_hash_t reg_var_hash (reg_var_t r, void *arg MIR_UNUSED) { + return (htab_hash_t) mir_hash (r.name, strlen (r.name), 0x42); } -static int reg_var_eq (reg_var_t r1, reg_var_t r2, void *arg) { +static int reg_var_eq (reg_var_t r1, reg_var_t r2, void *arg MIR_UNUSED) { return strcmp (r1.name, r2.name) == 0; } @@ -9863,7 +10198,8 @@ static void finish_reg_vars (c2m_ctx_t c2m_ctx) { if (reg_var_tab != NULL) HTAB_DESTROY (reg_var_t, reg_var_tab); } -static reg_var_t get_reg_var (c2m_ctx_t c2m_ctx, MIR_type_t t, const char *reg_name) { +static reg_var_t get_reg_var (c2m_ctx_t c2m_ctx, MIR_type_t t, const char *reg_name, + const char *asm_str) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; MIR_context_t ctx = c2m_ctx->ctx; reg_var_t reg_var, el; @@ -9873,8 +10209,12 @@ static reg_var_t get_reg_var (c2m_ctx_t c2m_ctx, MIR_type_t t, const char *reg_n reg_var.name = reg_name; if (HTAB_DO (reg_var_t, reg_var_tab, reg_var, HTAB_FIND, el)) return el; t = t == MIR_T_I32 || t == MIR_T_U32 || t == MIR_T_U64 ? MIR_T_I64 : t; - reg = (t != MIR_T_UNDEF ? MIR_new_func_reg (ctx, curr_func->u.func, t, reg_name) - : MIR_reg (ctx, reg_name, curr_func->u.func)); + if (asm_str == NULL) { + reg = (t != MIR_T_UNDEF ? MIR_new_func_reg (ctx, curr_func->u.func, t, reg_name) + : MIR_reg (ctx, reg_name, curr_func->u.func)); + } else { + reg = MIR_new_global_func_reg (ctx, curr_func->u.func, t, reg_name, asm_str); + } str = reg_malloc (c2m_ctx, (strlen (reg_name) + 1) * sizeof (char)); strcpy (str, reg_name); reg_var.name = str; @@ -9926,7 +10266,7 @@ static op_t get_new_temp (c2m_ctx_t c2m_ctx, MIR_type_t t) { : t == MIR_T_D ? "d_%u" : "D_%u", reg_free_mark++); - reg = get_reg_var (c2m_ctx, t, reg_name).reg; + reg = get_reg_var (c2m_ctx, t, reg_name, NULL).reg; return new_op (NULL, MIR_new_reg_op (ctx, reg)); } @@ -9975,8 +10315,128 @@ static MIR_type_t get_op_type (c2m_ctx_t c2m_ctx, op_t op) { } } +static void push_val (VARR (char) * repr, mir_long val) { + mir_long bound; + + for (bound = 10; val >= bound;) bound *= 10; + while (bound != 1) { + bound /= 10; + VARR_PUSH (char, repr, '0' + val / bound); + val %= bound; + } +} + +static void get_type_alias_name (c2m_ctx_t c2m_ctx, struct type *type, VARR (char) * name) { + gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; + enum basic_type basic_type; + size_t i; + + switch (type->mode) { + case TM_ENUM: basic_type = get_enum_basic_type (type); goto basic; + case TM_BASIC: + basic_type = type->u.basic_type; + basic: + switch (basic_type) { + case TP_VOID: VARR_PUSH (char, name, 'v'); break; + case TP_BOOL: VARR_PUSH (char, name, 'b'); break; + case TP_CHAR: + case TP_SCHAR: + case TP_UCHAR: VARR_PUSH (char, name, 'c'); break; + case TP_SHORT: + case TP_USHORT: VARR_PUSH (char, name, 's'); break; + case TP_INT: + case TP_UINT: VARR_PUSH (char, name, 'i'); break; + case TP_LONG: + case TP_ULONG: VARR_PUSH (char, name, 'l'); break; + case TP_LLONG: + case TP_ULLONG: VARR_PUSH (char, name, 'L'); break; + case TP_FLOAT: VARR_PUSH (char, name, 'f'); break; + case TP_DOUBLE: VARR_PUSH (char, name, 'd'); break; + case TP_LDOUBLE: VARR_PUSH (char, name, 'D'); break; + default: assert (FALSE); + } + break; + case TM_PTR: + VARR_PUSH (char, name, 'p'); + get_type_alias_name (c2m_ctx, type->u.ptr_type, name); + break; + case TM_STRUCT: + case TM_UNION: + VARR_PUSH (char, name, type->mode == TM_STRUCT ? 'S' : 'U'); + for (i = 0; i < VARR_LENGTH (node_t, node_stack); i++) + if (VARR_GET (node_t, node_stack, i) == type->u.tag_type) break; + if (i < VARR_LENGTH (node_t, node_stack)) { + VARR_PUSH (char, name, 'r'); + push_val (name, (mir_long) i); + } else { + VARR_PUSH (node_t, node_stack, type->u.tag_type); + for (node_t member = NL_HEAD (NL_EL (type->u.tag_type->u.ops, 1)->u.ops); member != NULL; + member = NL_NEXT (member)) + if (member->code == N_MEMBER) { + decl_t decl = member->attr; + node_t width = NL_EL (member->u.ops, 3); + struct expr *expr; + + get_type_alias_name (c2m_ctx, decl->decl_spec.type, name); + if (width->code != N_IGNORE && (expr = width->attr)->const_p) { + VARR_PUSH (char, name, 'w'); + push_val (name, (mir_long) expr->c.u_val); + for (mir_ullong v = expr->c.u_val;;) { + VARR_PUSH (char, name, v % 10 + '0'); + v /= 10; + if (v == 0) break; + } + } + } + } + VARR_PUSH (char, name, 'e'); + break; + case TM_ARR: + VARR_PUSH (char, name, 'A'); + get_type_alias_name (c2m_ctx, type->u.arr_type->el_type, name); + break; + case TM_FUNC: + VARR_PUSH (char, name, 'F'); + get_type_alias_name (c2m_ctx, type->u.func_type->ret_type, name); + for (node_t p = NL_HEAD (type->u.func_type->param_list->u.ops); p != NULL; p = NL_NEXT (p)) { + struct decl_spec *ds = get_param_decl_spec (p); + get_type_alias_name (c2m_ctx, ds->type, name); + } + VARR_PUSH (char, name, type->u.func_type->dots_p ? 'E' : 'e'); + break; + default: assert (FALSE); + } +} + +static MIR_alias_t get_type_alias (c2m_ctx_t c2m_ctx, struct type *type) { + MIR_context_t ctx = c2m_ctx->ctx; + gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; + + switch (type->mode) { + case TM_BASIC: + if (type->u.basic_type != TP_CHAR && type->u.basic_type != TP_SCHAR + && type->u.basic_type != TP_UCHAR) + break; + /* fall through */ + case TM_UNDEF: + case TM_STRUCT: + case TM_ARR: + case TM_FUNC: return 0; + default: break; + } + VARR_TRUNC (node_t, node_stack, 0); + VARR_TRUNC (char, temp_string, 0); + get_type_alias_name (c2m_ctx, type, temp_string); + VARR_PUSH (char, temp_string, '\0'); + return MIR_alias (ctx, VARR_ADDR (char, temp_string)); +} + static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_t false_label, - int val_p, op_t *desirable_dest); + int val_p, op_t *desirable_dest, int *expect_res); + +static op_t val_gen (c2m_ctx_t c2m_ctx, node_t r) { + return gen (c2m_ctx, r, NULL, NULL, TRUE, NULL, NULL); +} static int push_const_val (c2m_ctx_t c2m_ctx, node_t r, op_t *res) { MIR_context_t ctx = c2m_ctx->ctx; @@ -9987,13 +10447,13 @@ static int push_const_val (c2m_ctx_t c2m_ctx, node_t r, op_t *res) { if (floating_type_p (e->type)) { /* MIR support only IEEE float and double */ mir_type = get_mir_type (c2m_ctx, e->type); - *res = new_op (NULL, (mir_type == MIR_T_F ? MIR_new_float_op (ctx, e->u.d_val) - : mir_type == MIR_T_D ? MIR_new_double_op (ctx, e->u.d_val) - : MIR_new_ldouble_op (ctx, e->u.d_val))); + *res = new_op (NULL, (mir_type == MIR_T_F ? MIR_new_float_op (ctx, (float) e->c.d_val) + : mir_type == MIR_T_D ? MIR_new_double_op (ctx, e->c.d_val) + : MIR_new_ldouble_op (ctx, e->c.d_val))); } else { assert (integer_type_p (e->type) || e->type->mode == TM_PTR); - *res = new_op (NULL, (signed_integer_type_p (e->type) ? MIR_new_int_op (ctx, e->u.i_val) - : MIR_new_uint_op (ctx, e->u.u_val))); + *res = new_op (NULL, (signed_integer_type_p (e->type) ? MIR_new_int_op (ctx, e->c.i_val) + : MIR_new_uint_op (ctx, e->c.u_val))); } return TRUE; } @@ -10313,7 +10773,7 @@ static op_t force_val (c2m_ctx_t c2m_ctx, op_t op, int arr_p) { if (op.decl == NULL || op.decl->bit_offset < 0) return op; assert (op.mir_op.mode == MIR_OP_MEM); temp_op = get_new_temp (c2m_ctx, MIR_T_I64); - emit2 (c2m_ctx, MIR_MOV, temp_op.mir_op, op.mir_op); + emit2 (c2m_ctx, MIR_MOV, temp_op.mir_op, op.mir_op); /* ??? */ #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ sh = 64 - op.decl->bit_offset - op.decl->width; #else @@ -10323,7 +10783,7 @@ static op_t force_val (c2m_ctx_t c2m_ctx, op_t op, int arr_p) { emit3 (c2m_ctx, signed_integer_type_p (op.decl->decl_spec.type) && (op.decl->decl_spec.type->mode != TM_ENUM - || op.decl->width >= sizeof (mir_int) * MIR_CHAR_BIT) + || op.decl->width >= (int) sizeof (mir_int) * MIR_CHAR_BIT) ? MIR_RSH : MIR_URSH, temp_op.mir_op, temp_op.mir_op, MIR_new_int_op (ctx, 64 - op.decl->width)); @@ -10334,7 +10794,7 @@ static void gen_unary_op (c2m_ctx_t c2m_ctx, node_t r, op_t *op, op_t *res) { MIR_type_t t; assert (!((struct expr *) r->attr)->const_p); - *op = gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, TRUE, NULL); + *op = val_gen (c2m_ctx, NL_HEAD (r->u.ops)); t = get_mir_type (c2m_ctx, ((struct expr *) r->attr)->type); *op = promote (c2m_ctx, *op, t, FALSE); *res = get_new_temp (c2m_ctx, t); @@ -10347,8 +10807,8 @@ static void gen_assign_bin_op (c2m_ctx_t c2m_ctx, node_t r, struct type *assign_ assert (!((struct expr *) r->attr)->const_p); t = get_mir_type (c2m_ctx, assign_expr_type); - *op1 = gen (c2m_ctx, e, NULL, NULL, FALSE, NULL); - *op2 = gen (c2m_ctx, NL_NEXT (e), NULL, NULL, TRUE, NULL); + *op1 = gen (c2m_ctx, e, NULL, NULL, FALSE, NULL, NULL); + *op2 = val_gen (c2m_ctx, NL_NEXT (e)); *op2 = promote (c2m_ctx, *op2, t, FALSE); *var = *op1; *op1 = force_val (c2m_ctx, *op1, ((struct expr *) e->attr)->type->arr_type != NULL); @@ -10360,8 +10820,8 @@ static void gen_bin_op (c2m_ctx_t c2m_ctx, node_t r, op_t *op1, op_t *op2, op_t MIR_type_t t = get_mir_type (c2m_ctx, e->type); assert (!e->const_p); - *op1 = gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, TRUE, NULL); - *op2 = gen (c2m_ctx, NL_EL (r->u.ops, 1), NULL, NULL, TRUE, NULL); + *op1 = val_gen (c2m_ctx, NL_HEAD (r->u.ops)); + *op2 = val_gen (c2m_ctx, NL_EL (r->u.ops, 1)); *op1 = promote (c2m_ctx, *op1, t, FALSE); *op2 = promote (c2m_ctx, *op2, t, FALSE); *res = get_new_temp (c2m_ctx, t); @@ -10372,8 +10832,8 @@ static void gen_cmp_op (c2m_ctx_t c2m_ctx, node_t r, struct type *type, op_t *op MIR_type_t t = get_mir_type (c2m_ctx, type), res_t = get_int_mir_type (sizeof (mir_int)); assert (!((struct expr *) r->attr)->const_p); - *op1 = gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, TRUE, NULL); - *op2 = gen (c2m_ctx, NL_EL (r->u.ops, 1), NULL, NULL, TRUE, NULL); + *op1 = val_gen (c2m_ctx, NL_HEAD (r->u.ops)); + *op2 = val_gen (c2m_ctx, NL_EL (r->u.ops, 1)); *op1 = promote (c2m_ctx, *op1, t, FALSE); *op2 = promote (c2m_ctx, *op2, t, FALSE); *res = get_new_temp (c2m_ctx, res_t); @@ -10496,8 +10956,7 @@ static MIR_insn_code_t get_compare_branch_code (MIR_insn_code_t code) { case MIR_##n##S: return MIR_B##n##S; \ case MIR_F##n: return MIR_FB##n; \ case MIR_D##n: return MIR_DB##n; \ - case MIR_LD##n: \ - return MIR_LDB##n; + case MIR_LD##n: return MIR_LDB##n; #define BCMP(n) \ B (n) \ case MIR_U##n: return MIR_UB##n; \ @@ -10542,9 +11001,10 @@ static MIR_label_t get_label (c2m_ctx_t c2m_ctx, node_t target) { return labels->attr = MIR_new_label (c2m_ctx->ctx); } -static void top_gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_t false_label) { +static void top_gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_t false_label, + int *expect_res) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; - top_gen_last_op = gen (c2m_ctx, r, true_label, false_label, FALSE, NULL); + top_gen_last_op = gen (c2m_ctx, r, true_label, false_label, FALSE, NULL, expect_res); } static op_t modify_for_block_move (c2m_ctx_t c2m_ctx, op_t mem, op_t index) { @@ -10568,6 +11028,7 @@ static op_t modify_for_block_move (c2m_ctx_t c2m_ctx, op_t mem, op_t index) { mem.mir_op.u.mem.index = index.mir_op.u.reg; mem.mir_op.u.mem.scale = 1; } + mem.mir_op.u.mem.alias = mem.mir_op.u.mem.nonalias = 0; return mem; } @@ -10646,14 +11107,16 @@ static const char *get_param_name (c2m_ctx_t c2m_ctx, struct type *param_type, c return get_reg_var_name (c2m_ctx, promote_mir_int_type (type), name, 0); } -static void MIR_UNUSED simple_init_arg_vars (c2m_ctx_t c2m_ctx, void *arg_info) {} +static void MIR_UNUSED simple_init_arg_vars (c2m_ctx_t c2m_ctx MIR_UNUSED, + void *arg_info MIR_UNUSED) {} -static int simple_return_by_addr_p (c2m_ctx_t c2m_ctx, struct type *ret_type) { +static int simple_return_by_addr_p (c2m_ctx_t c2m_ctx MIR_UNUSED, struct type *ret_type) { return ret_type->mode == TM_STRUCT || ret_type->mode == TM_UNION; } static void MIR_UNUSED simple_add_res_proto (c2m_ctx_t c2m_ctx, struct type *ret_type, - void *arg_info, VARR (MIR_type_t) * res_types, + void *arg_info MIR_UNUSED, + VARR (MIR_type_t) * res_types, VARR (MIR_var_t) * arg_vars) { MIR_var_t var; @@ -10669,7 +11132,8 @@ static void MIR_UNUSED simple_add_res_proto (c2m_ctx_t c2m_ctx, struct type *ret } static int MIR_UNUSED simple_add_call_res_op (c2m_ctx_t c2m_ctx, struct type *ret_type, - void *arg_info, size_t call_arg_area_offset) { + void *arg_info MIR_UNUSED, + size_t call_arg_area_offset) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; MIR_context_t ctx = c2m_ctx->ctx; MIR_type_t type; @@ -10692,9 +11156,10 @@ static int MIR_UNUSED simple_add_call_res_op (c2m_ctx_t c2m_ctx, struct type *re return 0; } -static op_t MIR_UNUSED simple_gen_post_call_res_code (c2m_ctx_t c2m_ctx, struct type *ret_type, - op_t res, MIR_insn_t call, - size_t call_ops_start) { +static op_t MIR_UNUSED simple_gen_post_call_res_code (c2m_ctx_t c2m_ctx MIR_UNUSED, + struct type *ret_type MIR_UNUSED, op_t res, + MIR_insn_t call MIR_UNUSED, + size_t call_ops_start MIR_UNUSED) { return res; } @@ -10714,12 +11179,13 @@ static void MIR_UNUSED simple_add_ret_ops (c2m_ctx_t c2m_ctx, struct type *ret_t } } -static MIR_type_t MIR_UNUSED simple_target_get_blk_type (c2m_ctx_t c2m_ctx, struct type *arg_type) { +static MIR_type_t MIR_UNUSED simple_target_get_blk_type (c2m_ctx_t c2m_ctx MIR_UNUSED, + struct type *arg_type MIR_UNUSED) { return MIR_T_BLK; } static void MIR_UNUSED simple_add_arg_proto (c2m_ctx_t c2m_ctx, const char *name, - struct type *arg_type, void *arg_info, + struct type *arg_type, void *arg_info MIR_UNUSED, VARR (MIR_var_t) * arg_vars) { MIR_var_t var; MIR_type_t type; @@ -10734,7 +11200,7 @@ static void MIR_UNUSED simple_add_arg_proto (c2m_ctx_t c2m_ctx, const char *name } static void MIR_UNUSED simple_add_call_arg_op (c2m_ctx_t c2m_ctx, struct type *arg_type, - void *arg_info, op_t arg) { + void *arg_info MIR_UNUSED, op_t arg) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; MIR_type_t type; @@ -10752,9 +11218,11 @@ static void MIR_UNUSED simple_add_call_arg_op (c2m_ctx_t c2m_ctx, struct type *a } } -static int MIR_UNUSED simple_gen_gather_arg (c2m_ctx_t c2m_ctx, const char *name, - struct type *arg_type, decl_t param_decl, - void *arg_info) { +static int MIR_UNUSED simple_gen_gather_arg (c2m_ctx_t c2m_ctx MIR_UNUSED, + const char *name MIR_UNUSED, + struct type *arg_type MIR_UNUSED, + decl_t param_decl MIR_UNUSED, + void *arg_info MIR_UNUSED) { return FALSE; } @@ -10778,7 +11246,7 @@ static inline void MIR_UNUSED gen_multiple_load_store (c2m_ctx_t c2m_ctx, struct MIR_context_t ctx = c2m_ctx->ctx; MIR_op_t op, var_op; MIR_insn_t insn; - int i, sh, size = type_size (c2m_ctx, type); + int i, sh, size = (int) type_size (c2m_ctx, type); if (size == 0) return; if (type_align (type) == 8) { @@ -10975,7 +11443,7 @@ static mir_size_t get_object_path_offset (c2m_ctx_t c2m_ctx) { /* The function has the same structure as check_initializer. Keep it this way. */ static void collect_init_els (c2m_ctx_t c2m_ctx, decl_t member_decl, struct type **type_ptr, - node_t initializer, int const_only_p, int top_p) { + node_t initializer, int const_only_p, int top_p MIR_UNUSED) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; struct type *type = *type_ptr; struct expr *cexpr; @@ -11006,6 +11474,9 @@ static void collect_init_els (c2m_ctx_t c2m_ctx, decl_t member_decl, struct type init_el.offset = get_object_path_offset (c2m_ctx); init_el.member_decl = member_decl; init_el.el_type = type; + init_el.container_type = VARR_LENGTH (init_object_t, init_object_path) == 0 + ? NULL + : VARR_LAST (init_object_t, init_object_path).container_type; init_el.init = initializer; VARR_PUSH (init_el_t, init_els, init_el); return; @@ -11024,6 +11495,9 @@ static void collect_init_els (c2m_ctx_t c2m_ctx, decl_t member_decl, struct type init_el.offset = get_object_path_offset (c2m_ctx); init_el.member_decl = NULL; init_el.el_type = type; + init_el.container_type = VARR_LENGTH (init_object_t, init_object_path) == 0 + ? NULL + : VARR_LAST (init_object_t, init_object_path).container_type; init_el.init = str; VARR_PUSH (init_el_t, init_els, init_el); return; @@ -11043,7 +11517,7 @@ static void collect_init_els (c2m_ctx_t c2m_ctx, decl_t member_decl, struct type init_object.container_type = type; init_object.field_designator_p = FALSE; if (type->mode == TM_ARR) { - size_val = get_arr_type_size (c2m_ctx, type); + size_val = get_arr_type_size (type); /* we already figured out the array size during check: */ assert (size_val >= 0); init_object.u.curr_index = -1; @@ -11097,9 +11571,9 @@ static void collect_init_els (c2m_ctx_t c2m_ctx, decl_t member_decl, struct type /* index should be in array initializer and const expr of right type and value: */ assert (curr_type->mode == TM_ARR && cexpr->const_p && integer_type_p (cexpr->type) && !incomplete_type_p (c2m_ctx, curr_type) - && (arr_size_val = get_arr_type_size (c2m_ctx, curr_type)) >= 0 - && arr_size_val > cexpr->u.u_val); - init_object.u.curr_index = cexpr->u.i_val - 1; + && (arr_size_val = get_arr_type_size (curr_type)) >= 0 + && (mir_ullong) arr_size_val > cexpr->c.u_val); + init_object.u.curr_index = cexpr->c.i_val - 1; init_object.field_designator_p = FALSE; init_object.container_type = curr_type; VARR_PUSH (init_object_t, init_object_path, init_object); @@ -11249,7 +11723,7 @@ static void emit_scalar_assign (c2m_ctx_t c2m_ctx, op_t var, op_t *val, MIR_type op_t temp_op1, temp_op2, temp_op3, temp_op4; size_t MIR_UNUSED size = type_size (c2m_ctx, var.decl->decl_spec.type) * MIR_CHAR_BIT; - assert (var.mir_op.mode == MIR_OP_MEM); + assert (var.mir_op.mode == MIR_OP_MEM); /*???*/ mask = 0xffffffffffffffff >> (64 - width); #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ mask2 = ~(mask << var.decl->bit_offset); @@ -11369,12 +11843,12 @@ static void gen_initializer (c2m_ctx_t c2m_ctx, size_t init_start, op_t var, if (var.mir_op.mode == MIR_OP_REG) { /* scalar initialization: */ assert (local_p && offset == 0 && VARR_LENGTH (init_el_t, init_els) - init_start == 1); init_el = VARR_GET (init_el_t, init_els, init_start); - val = gen (c2m_ctx, init_el.init, NULL, NULL, TRUE, NULL); + val = val_gen (c2m_ctx, init_el.init); t = get_op_type (c2m_ctx, var); val = cast (c2m_ctx, val, get_mir_type (c2m_ctx, init_el.el_type), FALSE); emit_scalar_assign (c2m_ctx, var, &val, t, FALSE); } else if (local_p) { /* local variable initialization: */ - assert (var.mir_op.mode == MIR_OP_MEM && var.mir_op.u.mem.index == 0); + assert (var.mir_op.mode == MIR_OP_MEM && var.mir_op.u.mem.index == 0); /*???*/ offset = var.mir_op.u.mem.disp; base = var.mir_op.u.mem.base; for (size_t i = init_start; i < VARR_LENGTH (init_el_t, init_els); i++) { @@ -11386,8 +11860,8 @@ static void gen_initializer (c2m_ctx_t c2m_ctx, size_t init_start, op_t var, } if (t == MIR_T_UNDEF) val = new_op (NULL, MIR_new_mem_op (ctx, t, offset + rel_offset, base, 0, 1)); - val - = gen (c2m_ctx, init_el.init, NULL, NULL, t != MIR_T_UNDEF, t != MIR_T_UNDEF ? NULL : &val); + val = gen (c2m_ctx, init_el.init, NULL, NULL, t != MIR_T_UNDEF, + t != MIR_T_UNDEF ? NULL : &val, NULL); if (!scalar_type_p (init_el.el_type)) { mir_size_t s = init_el.init->code == N_STR ? init_el.init->u.s.len : init_el.init->code == N_STR16 ? init_el.init->u.s.len / 2 @@ -11396,11 +11870,17 @@ static void gen_initializer (c2m_ctx_t c2m_ctx, size_t init_start, op_t var, gen_memcpy (c2m_ctx, offset + rel_offset, base, val, s); rel_offset = init_el.offset + s; } else { + MIR_op_t mem + = MIR_new_alias_mem_op (ctx, t, offset + init_el.offset, base, 0, 1, + get_type_alias (c2m_ctx, + init_el.container_type != NULL + && init_el.container_type->mode == TM_UNION + ? init_el.container_type + : init_el.el_type), + 0); val = cast (c2m_ctx, val, get_mir_type (c2m_ctx, init_el.el_type), FALSE); - emit_scalar_assign (c2m_ctx, - new_op (init_el.member_decl, - MIR_new_mem_op (ctx, t, offset + init_el.offset, base, 0, 1)), - &val, t, i == init_start || rel_offset == init_el.offset); + emit_scalar_assign (c2m_ctx, new_op (init_el.member_decl, mem), &val, t, + i == init_start || rel_offset == init_el.offset); rel_offset = init_el.offset + _MIR_type_size (ctx, t); } } @@ -11419,7 +11899,7 @@ static void gen_initializer (c2m_ctx_t c2m_ctx, size_t init_start, op_t var, convert_value (e, init_el.el_type); e->type = init_el.el_type; /* to get the right value in the subsequent gen call */ } - val = gen (c2m_ctx, init_el.init, NULL, NULL, TRUE, NULL); + val = val_gen (c2m_ctx, init_el.init); assert (val.mir_op.mode == MIR_OP_INT || val.mir_op.mode == MIR_OP_UINT || val.mir_op.mode == MIR_OP_FLOAT || val.mir_op.mode == MIR_OP_DOUBLE || val.mir_op.mode == MIR_OP_LDOUBLE || val.mir_op.mode == MIR_OP_STR @@ -11427,7 +11907,7 @@ static void gen_initializer (c2m_ctx_t c2m_ctx, size_t init_start, op_t var, } if (rel_offset < init_el.offset) { /* fill the gap: */ data = MIR_new_bss (ctx, global_name, init_el.offset - rel_offset); - if (global_name != NULL) var.decl->item = data; + if (global_name != NULL) var.decl->u.item = data; global_name = NULL; } t = get_mir_type (c2m_ctx, init_el.el_type); @@ -11435,17 +11915,22 @@ static void gen_initializer (c2m_ctx_t c2m_ctx, size_t init_start, op_t var, node_t def; if ((def = e->def_node) == NULL) { /* constant address */ - mir_size_t s = e->u.i_val; - + mir_size_t s = e->c.i_val; data = MIR_new_data (ctx, global_name, MIR_T_P, 1, &s); data_size = _MIR_type_size (ctx, MIR_T_P); + } else if (def->code == N_LABEL_ADDR) { + data = MIR_new_lref_data (ctx, global_name, + get_label (c2m_ctx, + ((struct expr *) def->attr)->u.label_addr_target), + NULL, e->c.i_val); + data_size = _MIR_type_size (ctx, t); } else { if (def->code != N_STR && def->code != N_STR16 && def->code != N_STR32) { - data = ((decl_t) def->attr)->item; + data = ((decl_t) def->attr)->u.item; } else { data = get_string_data (c2m_ctx, def); } - data = MIR_new_ref_data (ctx, global_name, data, e->u.i_val); + data = MIR_new_ref_data (ctx, global_name, data, e->c.i_val); data_size = _MIR_type_size (ctx, t); } } else if (val.mir_op.mode == MIR_OP_REF) { @@ -11469,32 +11954,32 @@ static void gen_initializer (c2m_ctx_t c2m_ctx, size_t init_start, op_t var, start_offset = 0; el_size = data_size = _MIR_type_size (ctx, t); if (init_el.member_decl != NULL && init_el.member_decl->bit_offset >= 0) { - uint64_t u = 0; + uint64_t uval = 0; assert (val.mir_op.mode == MIR_OP_INT || val.mir_op.mode == MIR_OP_UINT); assert (init_el.member_decl->bit_offset % 8 == 0); /* first in the group of bitfields */ start_offset = init_el.member_decl->bit_offset / 8; - add_bit_field (c2m_ctx, &u, val.mir_op.u.u, init_el.member_decl); + add_bit_field (c2m_ctx, &uval, val.mir_op.u.u, init_el.member_decl); for (; i + 1 < VARR_LENGTH (init_el_t, init_els); i++, init_el = next_init_el) { next_init_el = VARR_GET (init_el_t, init_els, i + 1); if (next_init_el.offset != init_el.offset) break; if (next_init_el.member_decl->bit_offset == init_el.member_decl->bit_offset) continue; - val = gen (c2m_ctx, next_init_el.init, NULL, NULL, TRUE, NULL); + val = val_gen (c2m_ctx, next_init_el.init); assert (val.mir_op.mode == MIR_OP_INT || val.mir_op.mode == MIR_OP_UINT); - add_bit_field (c2m_ctx, &u, val.mir_op.u.u, next_init_el.member_decl); + add_bit_field (c2m_ctx, &uval, val.mir_op.u.u, next_init_el.member_decl); } - val.mir_op.u.u = u; + val.mir_op.u.u = uval; if (i + 1 < VARR_LENGTH (init_el_t, init_els) && next_init_el.offset - init_el.offset < data_size) data_size = next_init_el.offset - init_el.offset; } switch (t) { - case MIR_T_I8: u.i8 = val.mir_op.u.i; break; - case MIR_T_U8: u.u8 = val.mir_op.u.u; break; - case MIR_T_I16: u.i16 = val.mir_op.u.i; break; - case MIR_T_U16: u.u16 = val.mir_op.u.u; break; - case MIR_T_I32: u.i32 = val.mir_op.u.i; break; - case MIR_T_U32: u.u32 = val.mir_op.u.u; break; + case MIR_T_I8: u.i8 = (int8_t) val.mir_op.u.i; break; + case MIR_T_U8: u.u8 = (uint8_t) val.mir_op.u.u; break; + case MIR_T_I16: u.i16 = (int16_t) val.mir_op.u.i; break; + case MIR_T_U16: u.u16 = (uint16_t) val.mir_op.u.u; break; + case MIR_T_I32: u.i32 = (int32_t) val.mir_op.u.i; break; + case MIR_T_U32: u.u32 = (uint32_t) val.mir_op.u.u; break; case MIR_T_I64: u.i64 = val.mir_op.u.i; break; case MIR_T_U64: u.u64 = val.mir_op.u.u; break; case MIR_T_F: u.f = val.mir_op.u.f; break; @@ -11526,13 +12011,13 @@ static void gen_initializer (c2m_ctx_t c2m_ctx, size_t init_start, op_t var, data = MIR_new_ref_data (ctx, global_name, data, 0); data_size = _MIR_type_size (ctx, t); } - if (global_name != NULL) var.decl->item = data; + if (global_name != NULL) var.decl->u.item = data; global_name = NULL; rel_offset = init_el.offset + data_size; } if (rel_offset < size || size == 0) { /* fill the tail: */ data = MIR_new_bss (ctx, global_name, size - rel_offset); - if (global_name != NULL) var.decl->item = data; + if (global_name != NULL) var.decl->u.item = data; } } } @@ -11585,8 +12070,8 @@ static int signed_case_compare (const void *v1, const void *v2) { struct expr *e1 = NL_HEAD (c1->case_node->u.ops)->attr; struct expr *e2 = NL_HEAD (c2->case_node->u.ops)->attr; - assert (e1->u.i_val != e2->u.i_val); - return e1->u.i_val < e2->u.i_val ? -1 : 1; + assert (e1->c.i_val != e2->c.i_val); + return e1->c.i_val < e2->c.i_val ? -1 : 1; } static int unsigned_case_compare (const void *v1, const void *v2) { @@ -11594,16 +12079,31 @@ static int unsigned_case_compare (const void *v1, const void *v2) { struct expr *e1 = NL_HEAD (c1->case_node->u.ops)->attr; struct expr *e2 = NL_HEAD (c2->case_node->u.ops)->attr; - assert (e1->u.u_val != e2->u.u_val); - return e1->u.u_val < e2->u.u_val ? -1 : 1; + assert (e1->c.u_val != e2->c.u_val); + return e1->c.u_val < e2->c.u_val ? -1 : 1; +} + +static void make_cond_val (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, + MIR_label_t false_label, op_t *res) { + MIR_context_t ctx = c2m_ctx->ctx; + gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; + struct type *type = ((struct expr *) r->attr)->type; + MIR_label_t end_label = MIR_new_label (ctx); + *res = get_new_temp (c2m_ctx, get_mir_type (c2m_ctx, type)); + emit_label_insn_opt (c2m_ctx, true_label); + emit2 (c2m_ctx, MIR_MOV, res->mir_op, one_op.mir_op); + emit1 (c2m_ctx, MIR_JMP, MIR_new_label_op (ctx, end_label)); + emit_label_insn_opt (c2m_ctx, false_label); + emit2 (c2m_ctx, MIR_MOV, res->mir_op, zero_op.mir_op); + emit_label_insn_opt (c2m_ctx, end_label); } static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_t false_label, - int val_p, op_t *desirable_dest) { + int val_p, op_t *desirable_dest, int *expect_res) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; check_ctx_t check_ctx = c2m_ctx->check_ctx; /* check and gen share curr_scope */ MIR_context_t ctx = c2m_ctx->ctx; - op_t res, op1, op2, var, val; + op_t res, op1, op2, op3, var, val; MIR_type_t t = MIR_T_UNDEF; /* to remove an uninitialized warning */ MIR_insn_code_t insn_code; MIR_type_t mir_type; @@ -11616,15 +12116,16 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ int expr_attr_p, stmt_p; classify_node (r, &expr_attr_p, &stmt_p); - assert ((true_label == NULL && false_label == NULL) + assert ((true_label == NULL && false_label == NULL && expect_res == NULL) || (true_label != NULL && false_label != NULL)); assert (!val_p || desirable_dest == NULL); + if (expect_res != NULL) *expect_res = 0; /* no expected result */ if (r->code != N_ANDAND && r->code != N_OROR && expr_attr_p && push_const_val (c2m_ctx, r, &res)) goto finish; switch (r->code) { case N_LIST: for (node_t n = NL_HEAD (r->u.ops); n != NULL; n = NL_NEXT (n)) - gen (c2m_ctx, n, true_label, false_label, val_p, NULL); + gen (c2m_ctx, n, true_label, false_label, val_p, NULL, expect_res); break; case N_IGNORE: break; /* do nothing */ case N_I: @@ -11647,7 +12148,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ ld = r->u.ld; float_val: mir_type = get_mir_type (c2m_ctx, ((struct expr *) r->attr)->type); - res = new_op (NULL, (mir_type == MIR_T_F ? MIR_new_float_op (ctx, ld) + res = new_op (NULL, (mir_type == MIR_T_F ? MIR_new_float_op (ctx, (float) ld) : mir_type == MIR_T_D ? MIR_new_double_op (ctx, ld) : MIR_new_ldouble_op (ctx, ld))); break; @@ -11663,9 +12164,10 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ // and str in initializer break; case N_COMMA: - gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, FALSE, NULL); + gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, FALSE, NULL, NULL); res = gen (c2m_ctx, NL_EL (r->u.ops, 1), true_label, false_label, - true_label == NULL && !void_type_p (((struct expr *) r->attr)->type), NULL); + true_label == NULL && !void_type_p (((struct expr *) r->attr)->type), NULL, + expect_res); if (true_label != NULL) { true_label = false_label = NULL; val_p = FALSE; @@ -11683,21 +12185,10 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ } assert (t_label != NULL && f_label != NULL); gen (c2m_ctx, NL_HEAD (r->u.ops), r->code == N_ANDAND ? temp_label : t_label, - r->code == N_ANDAND ? f_label : temp_label, FALSE, NULL); + r->code == N_ANDAND ? f_label : temp_label, FALSE, NULL, NULL); emit_label_insn_opt (c2m_ctx, temp_label); - gen (c2m_ctx, NL_EL (r->u.ops, 1), t_label, f_label, FALSE, NULL); - if (make_val_p) { - MIR_label_t end_label = MIR_new_label (ctx); - - type = ((struct expr *) r->attr)->type; - res = get_new_temp (c2m_ctx, get_mir_type (c2m_ctx, type)); - emit_label_insn_opt (c2m_ctx, t_label); - emit2 (c2m_ctx, MIR_MOV, res.mir_op, one_op.mir_op); - emit1 (c2m_ctx, MIR_JMP, MIR_new_label_op (ctx, end_label)); - emit_label_insn_opt (c2m_ctx, f_label); - emit2 (c2m_ctx, MIR_MOV, res.mir_op, zero_op.mir_op); - emit_label_insn_opt (c2m_ctx, end_label); - } + gen (c2m_ctx, NL_EL (r->u.ops, 1), t_label, f_label, FALSE, NULL, NULL); + if (make_val_p) make_cond_val (c2m_ctx, r, t_label, f_label, &res); true_label = false_label = NULL; } else if (true_label != NULL) { int true_p; @@ -11720,14 +12211,14 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ break; case N_NOT: if (true_label != NULL) { - gen (c2m_ctx, NL_HEAD (r->u.ops), false_label, true_label, FALSE, NULL); + gen (c2m_ctx, NL_HEAD (r->u.ops), false_label, true_label, FALSE, NULL, NULL); true_label = false_label = NULL; } else { MIR_label_t end_label = MIR_new_label (ctx); MIR_label_t t_label = MIR_new_label (ctx), f_label = MIR_new_label (ctx); res = get_new_temp (c2m_ctx, MIR_T_I64); - gen (c2m_ctx, NL_HEAD (r->u.ops), t_label, f_label, FALSE, NULL); + gen (c2m_ctx, NL_HEAD (r->u.ops), t_label, f_label, FALSE, NULL, NULL); emit_label_insn_opt (c2m_ctx, t_label); emit2 (c2m_ctx, MIR_MOV, res.mir_op, zero_op.mir_op); emit1 (c2m_ctx, MIR_JMP, MIR_new_label_op (ctx, end_label)); @@ -11758,7 +12249,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ } break; } - /* Fall through: */ + /* falls through */ case N_AND: case N_OR: case N_XOR: @@ -11778,13 +12269,13 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ case N_GE: { struct type *type1 = ((struct expr *) NL_HEAD (r->u.ops)->attr)->type; struct type *type2 = ((struct expr *) NL_EL (r->u.ops, 1)->attr)->type; - struct type type, ptr_type = get_ptr_int_type (FALSE); + struct type type_s, ptr_type_s = get_ptr_int_type (FALSE); - type = arithmetic_conversion (type1->mode == TM_PTR ? &ptr_type : type1, - type2->mode == TM_PTR ? &ptr_type : type2); - set_type_layout (c2m_ctx, &type); - gen_cmp_op (c2m_ctx, r, &type, &op1, &op2, &res); - insn_code = get_mir_type_insn_code (c2m_ctx, &type, r); + type_s = arithmetic_conversion (type1->mode == TM_PTR ? &ptr_type_s : type1, + type2->mode == TM_PTR ? &ptr_type_s : type2); + set_type_layout (c2m_ctx, &type_s); + gen_cmp_op (c2m_ctx, r, &type_s, &op1, &op2, &res); + insn_code = get_mir_type_insn_code (c2m_ctx, &type_s, r); if (true_label == NULL) { emit3 (c2m_ctx, insn_code, res.mir_op, op1.mir_op, op2.mir_op); } else { @@ -11797,10 +12288,9 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ } case N_POST_INC: case N_POST_DEC: { - struct type *type = ((struct expr *) r->attr)->type2; - + type = ((struct expr *) r->attr)->type2; t = get_mir_type (c2m_ctx, type); - var = gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, FALSE, NULL); + var = gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, FALSE, NULL, NULL); op1 = force_val (c2m_ctx, var, FALSE); if (val_p || true_label != NULL) { res = get_new_temp (c2m_ctx, t); @@ -11818,10 +12308,9 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ } case N_INC: case N_DEC: { - struct type *type = ((struct expr *) r->attr)->type2; - + type = ((struct expr *) r->attr)->type2; t = get_mir_type (c2m_ctx, type); - var = gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, FALSE, NULL); + var = gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, FALSE, NULL, NULL); val = promote (c2m_ctx, force_val (c2m_ctx, var, FALSE), t, TRUE); op2 = promote (c2m_ctx, type->mode != TM_PTR @@ -11851,10 +12340,10 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ goto assign; break; case N_ASSIGN: - var = gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, FALSE, NULL); + var = gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, FALSE, NULL, NULL); t = get_op_type (c2m_ctx, var); op2 = gen (c2m_ctx, NL_EL (r->u.ops, 1), NULL, NULL, t != MIR_T_UNDEF, - t != MIR_T_UNDEF ? NULL : &var); + t != MIR_T_UNDEF ? NULL : &var, NULL); if ((!val_p && true_label == NULL) || t == MIR_T_UNDEF) { res = var; val = op2; @@ -11880,18 +12369,21 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ case N_ID: { e = r->attr; assert (!e->const_p); - if (e->lvalue_node == NULL) { - res = new_op (NULL, MIR_new_ref_op (ctx, ((decl_t) e->def_node->attr)->item)); - } else if ((decl = e->lvalue_node->attr)->scope == top_scope || decl->decl_spec.static_p - || decl->decl_spec.linkage != N_IGNORE) { + if (e->u.lvalue_node == NULL) { + res = new_op (NULL, MIR_new_ref_op (ctx, ((decl_t) e->def_node->attr)->u.item)); + } else if (((decl = e->u.lvalue_node->attr)->scope == top_scope || decl->decl_spec.static_p + || decl->decl_spec.linkage != N_IGNORE) + && !decl->asm_p) { t = get_mir_type (c2m_ctx, e->type); res = get_new_temp (c2m_ctx, MIR_T_I64); - emit2 (c2m_ctx, MIR_MOV, res.mir_op, MIR_new_ref_op (ctx, decl->item)); - res = new_op (decl, MIR_new_mem_op (ctx, t, 0, res.mir_op.u.reg, 0, 1)); + emit2 (c2m_ctx, MIR_MOV, res.mir_op, MIR_new_ref_op (ctx, decl->u.item)); + res = new_op (decl, MIR_new_alias_mem_op (ctx, t, 0, res.mir_op.u.reg, 0, 1, + get_type_alias (c2m_ctx, e->type), 0)); } else if (!decl->reg_p) { t = get_mir_type (c2m_ctx, e->type); - res = new_op (decl, MIR_new_mem_op (ctx, t, decl->offset, - MIR_reg (ctx, FP_NAME, curr_func->u.func), 0, 1)); + res = new_op (decl, MIR_new_alias_mem_op (ctx, t, decl->offset, + MIR_reg (ctx, FP_NAME, curr_func->u.func), 0, 1, + get_type_alias (c2m_ctx, e->type), 0)); } else { const char *name; reg_var_t reg_var; @@ -11901,7 +12393,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ t = promote_mir_int_type (t); name = get_reg_var_name (c2m_ctx, t, r->u.s.s, ((struct node_scope *) decl->scope->attr)->func_scope_num); - reg_var = get_reg_var (c2m_ctx, t, name); + reg_var = get_reg_var (c2m_ctx, t, name, decl->u.asm_str); res = new_op (decl, MIR_new_reg_op (ctx, reg_var.reg)); } break; @@ -11909,13 +12401,13 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ case N_IND: { MIR_type_t ind_t; node_t arr = NL_HEAD (r->u.ops); - struct type *type = ((struct expr *) r->attr)->type; + struct type *el_type = ((struct expr *) r->attr)->type; struct type *arr_type = ((struct expr *) arr->attr)->type; - mir_size_t size = type_size (c2m_ctx, type); + mir_size_t size = type_size (c2m_ctx, el_type); - t = get_mir_type (c2m_ctx, type); - op1 = gen (c2m_ctx, arr, NULL, NULL, TRUE, NULL); - op2 = gen (c2m_ctx, NL_EL (r->u.ops, 1), NULL, NULL, TRUE, NULL); + t = get_mir_type (c2m_ctx, el_type); + op1 = val_gen (c2m_ctx, arr); + op2 = val_gen (c2m_ctx, NL_EL (r->u.ops, 1)); ind_t = get_mir_type (c2m_ctx, ((struct expr *) NL_EL (r->u.ops, 1)->attr)->type); #if MIR_PTR32 op2 = force_reg (c2m_ctx, op2, ind_t); @@ -11926,12 +12418,12 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ op2 = cast (c2m_ctx, op2, ind_t == MIR_T_I32 ? MIR_T_I64 : MIR_T_U64, FALSE); } #endif - if (type->mode == TM_PTR && type->arr_type != NULL) { /* elem is an array */ - size = type_size (c2m_ctx, type->arr_type); + if (el_type->mode == TM_PTR && el_type->arr_type != NULL) { /* elem is an array */ + size = type_size (c2m_ctx, el_type->arr_type); } if (arr_type->mode == TM_PTR && arr_type->arr_type != NULL) { /* indexing an array */ op1 = force_reg_or_mem (c2m_ctx, op1, MIR_T_I64); - assert (op1.mir_op.mode == MIR_OP_REG || op1.mir_op.mode == MIR_OP_MEM); + assert (op1.mir_op.mode == MIR_OP_REG || op1.mir_op.mode == MIR_OP_MEM); /*???*/ } else { op1 = force_reg (c2m_ctx, op1, MIR_T_I64); assert (op1.mir_op.mode == MIR_OP_REG); @@ -11939,12 +12431,13 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ res = op1; res.decl = NULL; if (res.mir_op.mode == MIR_OP_REG) - res.mir_op = MIR_new_mem_op (ctx, t, 0, res.mir_op.u.reg, 0, 1); + res.mir_op = MIR_new_alias_mem_op (ctx, t, 0, res.mir_op.u.reg, 0, 1, + get_type_alias (c2m_ctx, el_type), arr_type->antialias); if (res.mir_op.u.mem.base == 0 && size == 1) { res.mir_op.u.mem.base = op2.mir_op.u.reg; } else if (res.mir_op.u.mem.index == 0 && size <= MIR_MAX_SCALE) { res.mir_op.u.mem.index = op2.mir_op.u.reg; - res.mir_op.u.mem.scale = size; + res.mir_op.u.mem.scale = (MIR_scale_t) size; } else { op_t temp_op; @@ -11958,18 +12451,44 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ res.mir_op.u.mem.type = t; break; } + case N_LABEL_ADDR: { + node_t target; + + e = r->attr; + type = e->type; + target = e->u.label_addr_target; + t = get_mir_type (c2m_ctx, type); + res = get_new_temp (c2m_ctx, t); + emit2 (c2m_ctx, MIR_LADDR, res.mir_op, MIR_new_label_op (ctx, get_label (c2m_ctx, target))); + break; + } case N_ADDR: { int add_p = FALSE; - op1 = gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, FALSE, NULL); - if (op1.mir_op.mode == MIR_OP_REG || op1.mir_op.mode == MIR_OP_REF - || op1.mir_op.mode == MIR_OP_STR) { /* array or func */ + op1 = gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, FALSE, NULL, NULL); + type = ((struct expr *) r->attr)->type; + t = get_mir_type (c2m_ctx, type); + if (op1.mir_op.mode == MIR_OP_REG && type->mode == TM_PTR && scalar_type_p (type->u.ptr_type)) { + MIR_insn_code_t code; + res = get_new_temp (c2m_ctx, t); + switch (get_mir_type (c2m_ctx, type->u.ptr_type)) { + case MIR_T_I8: + case MIR_T_U8: code = MIR_ADDR8; break; + case MIR_T_I16: + case MIR_T_U16: code = MIR_ADDR16; break; + case MIR_T_I32: + case MIR_T_U32: code = MIR_ADDR32; break; + default: code = MIR_ADDR; break; + } + emit2 (c2m_ctx, code, res.mir_op, MIR_new_reg_op (ctx, op1.mir_op.u.reg)); + break; + } else if (op1.mir_op.mode == MIR_OP_REG || op1.mir_op.mode == MIR_OP_REF + || op1.mir_op.mode == MIR_OP_STR) { /* array or func */ res = op1; res.decl = NULL; break; } assert (op1.mir_op.mode == MIR_OP_MEM); - t = get_mir_type (c2m_ctx, ((struct expr *) r->attr)->type); res = get_new_temp (c2m_ctx, t); if (op1.mir_op.u.mem.index != 0) { emit3 (c2m_ctx, MIR_MUL, res.mir_op, MIR_new_reg_op (ctx, op1.mir_op.u.mem.index), @@ -11994,37 +12513,51 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ break; } case N_DEREF: - op1 = gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, TRUE, NULL); + op1 = val_gen (c2m_ctx, NL_HEAD (r->u.ops)); op1 = force_reg (c2m_ctx, op1, MIR_T_I64); assert (op1.mir_op.mode == MIR_OP_REG); if ((type = ((struct expr *) r->attr)->type)->mode == TM_PTR && type->u.ptr_type->mode == TM_FUNC && type->func_type_before_adjustment_p) { res = op1; } else { + struct expr *op_e = NL_HEAD (r->u.ops)->attr; t = get_mir_type (c2m_ctx, type); - op1.mir_op = MIR_new_mem_op (ctx, t, 0, op1.mir_op.u.reg, 0, 1); + op1.mir_op = MIR_new_alias_mem_op (ctx, t, 0, op1.mir_op.u.reg, 0, 1, + get_type_alias (c2m_ctx, type), op_e->type->antialias); res = new_op (NULL, op1.mir_op); } break; case N_FIELD: case N_DEREF_FIELD: { node_t def_node; + MIR_alias_t alias; e = r->attr; - def_node = e->lvalue_node; + def_node = e->u.lvalue_node; assert (def_node != NULL && def_node->code == N_MEMBER); decl = def_node->attr; - op1 = gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, r->code == N_DEREF_FIELD, NULL); + op1 = gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, r->code == N_DEREF_FIELD, NULL, NULL); t = get_mir_type (c2m_ctx, decl->decl_spec.type); if (r->code == N_FIELD) { assert (op1.mir_op.mode == MIR_OP_MEM); + alias = (op1.mir_op.u.mem.alias != 0 && MIR_alias_name (ctx, op1.mir_op.u.mem.alias)[0] == 'U' + ? op1.mir_op.u.mem.alias + : get_type_alias (c2m_ctx, e->type)); op1.mir_op - = MIR_new_mem_op (ctx, t, op1.mir_op.u.mem.disp + decl->offset, op1.mir_op.u.mem.base, - op1.mir_op.u.mem.index, op1.mir_op.u.mem.scale); + = MIR_new_alias_mem_op (ctx, t, op1.mir_op.u.mem.disp + decl->offset, op1.mir_op.u.mem.base, + op1.mir_op.u.mem.index, op1.mir_op.u.mem.scale, alias, + decl->decl_spec.type->antialias); } else { + struct expr *left = NL_HEAD (r->u.ops)->attr; + assert (left->type->mode == TM_PTR); op1 = force_reg (c2m_ctx, op1, MIR_T_I64); assert (op1.mir_op.mode == MIR_OP_REG); - op1.mir_op = MIR_new_mem_op (ctx, t, decl->offset, op1.mir_op.u.reg, 0, 1); + op1.mir_op + = MIR_new_alias_mem_op (ctx, t, decl->offset, op1.mir_op.u.reg, 0, 1, + get_type_alias (c2m_ctx, left->type->u.ptr_type->mode == TM_UNION + ? left->type->u.ptr_type + : e->type), + decl->decl_spec.type->antialias); } res = new_op (decl, op1.mir_op); break; @@ -12033,17 +12566,17 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ node_t cond = NL_HEAD (r->u.ops); node_t true_expr = NL_NEXT (cond); node_t false_expr = NL_NEXT (true_expr); - MIR_label_t true_label = MIR_new_label (ctx), false_label = MIR_new_label (ctx); + MIR_label_t cond_true_label = MIR_new_label (ctx), cond_false_label = MIR_new_label (ctx); MIR_label_t end_label = MIR_new_label (ctx); - struct type *type = ((struct expr *) r->attr)->type; + struct type *cond_res_type = ((struct expr *) r->attr)->type; op_t addr; - int void_p = void_type_p (type); - mir_size_t size = type_size (c2m_ctx, ((struct expr *) r->attr)->type); + int void_p = void_type_p (cond_res_type), cond_expect_res; + mir_size_t size = type_size (c2m_ctx, cond_res_type); - if (!void_p) t = get_mir_type (c2m_ctx, type); - gen (c2m_ctx, cond, true_label, false_label, FALSE, NULL); - emit_label_insn_opt (c2m_ctx, true_label); - op1 = gen (c2m_ctx, true_expr, NULL, NULL, !void_p && t != MIR_T_UNDEF, NULL); + if (!void_p) t = get_mir_type (c2m_ctx, cond_res_type); + gen (c2m_ctx, cond, cond_true_label, cond_false_label, FALSE, NULL, &cond_expect_res); + emit_label_insn_opt (c2m_ctx, cond_true_label); + op1 = gen (c2m_ctx, true_expr, NULL, NULL, !void_p && t != MIR_T_UNDEF, NULL, NULL); if (!void_p) { if (t != MIR_T_UNDEF) { res = get_new_temp (c2m_ctx, t); @@ -12059,8 +12592,8 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ } } emit1 (c2m_ctx, MIR_JMP, MIR_new_label_op (ctx, end_label)); - emit_label_insn_opt (c2m_ctx, false_label); - op1 = gen (c2m_ctx, false_expr, NULL, NULL, !void_p && t != MIR_T_UNDEF, NULL); + emit_label_insn_opt (c2m_ctx, cond_false_label); + op1 = gen (c2m_ctx, false_expr, NULL, NULL, !void_p && t != MIR_T_UNDEF, NULL, NULL); if (!void_p) { if (t != MIR_T_UNDEF) { op1 = cast (c2m_ctx, op1, t, FALSE); @@ -12082,7 +12615,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ case N_CAST: assert (!((struct expr *) r->attr)->const_p); type = ((struct expr *) r->attr)->type; - op1 = gen (c2m_ctx, NL_EL (r->u.ops, 1), NULL, NULL, !void_type_p (type), NULL); + op1 = gen (c2m_ctx, NL_EL (r->u.ops, 1), NULL, NULL, !void_type_p (type), NULL, NULL); if (void_type_p (type)) { res = op1; res.decl = NULL; @@ -12096,13 +12629,13 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ const char *global_name = NULL; char buff[50]; node_t type_name = NL_HEAD (r->u.ops); - decl_t decl = type_name->attr; struct expr *expr = r->attr; MIR_module_t module = DLIST_TAIL (MIR_module_t, *MIR_get_module_list (ctx)); size_t init_start; + decl = type_name->attr; if (decl->scope == top_scope) { - assert (decl->item == NULL); + assert (decl->u.item == NULL); _MIR_get_temp_item_name (ctx, module, buff, sizeof (buff)); global_name = buff; } @@ -12118,47 +12651,144 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ var = new_op (decl, MIR_new_ref_op (ctx, NULL)); } else { t = get_mir_type (c2m_ctx, expr->type); - var = new_op (decl, MIR_new_mem_op (ctx, t, decl->offset, - MIR_reg (ctx, FP_NAME, curr_func->u.func), 0, 1)); + var = new_op (decl, MIR_new_alias_mem_op (ctx, t, decl->offset, + MIR_reg (ctx, FP_NAME, curr_func->u.func), 0, 1, + get_type_alias (c2m_ctx, expr->type), 0)); } + int local_p + = decl->scope != top_scope && !decl->decl_spec.static_p && !decl->decl_spec.thread_local_p; gen_initializer (c2m_ctx, init_start, var, global_name, - raw_type_size (c2m_ctx, decl->decl_spec.type), - decl->scope != top_scope && !decl->decl_spec.static_p - && !decl->decl_spec.thread_local_p); + (local_p ? raw_type_size : type_size) (c2m_ctx, decl->decl_spec.type), + local_p); VARR_TRUNC (init_el_t, init_els, init_start); - if (var.mir_op.mode == MIR_OP_REF) var.mir_op.u.ref = var.decl->item; + if (var.mir_op.mode == MIR_OP_REF) var.mir_op.u.ref = var.decl->u.item; res = var; break; } case N_CALL: { - node_t func = NL_HEAD (r->u.ops), param_list, param, args = NL_EL (r->u.ops, 1); + node_t func = NL_HEAD (r->u.ops), param_list, param, args = NL_EL (r->u.ops, 1), first_arg; struct decl_spec *decl_spec; size_t ops_start; struct expr *call_expr = r->attr, *func_expr; struct type *func_type = NULL; /* to remove an uninitialized warning */ - struct type *type = call_expr->type; MIR_item_t proto_item; - MIR_insn_t call_insn; + MIR_insn_t call_insn, label; mir_size_t saved_call_arg_area_offset_before_args, arg_area_offset; int va_arg_p = call_expr->builtin_call_p && str_eq_p (func->u.s.s, BUILTIN_VA_ARG); int va_start_p = call_expr->builtin_call_p && str_eq_p (func->u.s.s, BUILTIN_VA_START); int alloca_p = call_expr->builtin_call_p && str_eq_p (func->u.s.s, ALLOCA); + int add_overflow_p = call_expr->builtin_call_p && strcmp (func->u.s.s, ADD_OVERFLOW) == 0; + int sub_overflow_p = call_expr->builtin_call_p && strcmp (func->u.s.s, SUB_OVERFLOW) == 0; + int mul_overflow_p = call_expr->builtin_call_p && strcmp (func->u.s.s, MUL_OVERFLOW) == 0; + int expect_p = call_expr->builtin_call_p && strcmp (func->u.s.s, EXPECT) == 0; + int jcall_p = call_expr->builtin_call_p && strcmp (func->u.s.s, JCALL) == 0; + int jret_p = call_expr->builtin_call_p && strcmp (func->u.s.s, JRET) == 0; + int prop_set_p = call_expr->builtin_call_p && strcmp (func->u.s.s, PROP_SET) == 0; + int prop_eq_p = call_expr->builtin_call_p && strcmp (func->u.s.s, PROP_EQ) == 0; + int prop_ne_p = call_expr->builtin_call_p && strcmp (func->u.s.s, PROP_NE) == 0; int builtin_call_p = alloca_p || va_arg_p || va_start_p, inline_p = FALSE; node_t block = NL_EL (curr_func_def->u.ops, 3); struct node_scope *ns = block->attr; target_arg_info_t arg_info; int n, struct_p; + type = call_expr->type; + if (add_overflow_p || sub_overflow_p || mul_overflow_p) { + op1 = val_gen (c2m_ctx, NL_HEAD (args->u.ops)); + op2 = val_gen (c2m_ctx, NL_EL (args->u.ops, 1)); + op3 = val_gen (c2m_ctx, NL_EL (args->u.ops, 2)); + e = NL_EL (args->u.ops, 2)->attr; + assert (e->type->mode == TM_PTR && standard_integer_type_p (e->type->u.ptr_type)); + t = get_mir_type (c2m_ctx, e->type->u.ptr_type); + assert (op3.mir_op.mode == MIR_OP_REG); + MIR_append_insn (ctx, curr_func, + MIR_new_insn (ctx, + t == MIR_T_I32 || t == MIR_T_U32 + ? (add_overflow_p ? MIR_ADDOS + : sub_overflow_p ? MIR_SUBOS + : t == MIR_T_I32 ? MIR_MULOS + : MIR_UMULOS) + : (add_overflow_p ? MIR_ADDO + : sub_overflow_p ? MIR_SUBO + : t == MIR_T_I64 ? MIR_MULO + : MIR_UMULO), + MIR_new_mem_op (ctx, t, 0, op3.mir_op.u.reg, 0, 1), op1.mir_op, + op2.mir_op)); + if (true_label != NULL) { + MIR_op_t lab_op = MIR_new_label_op (ctx, true_label); + emit1 (c2m_ctx, t == MIR_T_I32 || t == MIR_T_I64 ? MIR_BO : MIR_UBO, lab_op); + emit1 (c2m_ctx, MIR_JMP, MIR_new_label_op (ctx, false_label)); + } else { + label = MIR_new_label (ctx); + res = get_new_temp (c2m_ctx, MIR_T_I64); + emit1 (c2m_ctx, t == MIR_T_I32 || t == MIR_T_I64 ? MIR_BO : MIR_UBO, + MIR_new_label_op (ctx, label)); + emit2 (c2m_ctx, MIR_MOV, res.mir_op, MIR_new_int_op (ctx, 0)); + emit_label_insn_opt (c2m_ctx, label); + } + true_label = false_label = NULL; + break; + } + if (expect_p) { + e = NL_EL (args->u.ops, 1)->attr; + if (e->const_p && true_label != NULL && expect_res != NULL) + *expect_res = e->c.u_val == 0 ? -1 : 1; + res = gen (c2m_ctx, NL_HEAD (args->u.ops), true_label, false_label, val_p, desirable_dest, + NULL); + true_label = false_label = NULL; + val_p = FALSE; + break; + } + if (jret_p) { + op1 = val_gen (c2m_ctx, NL_HEAD (args->u.ops)); + emit1 (c2m_ctx, MIR_JRET, op1.mir_op); + true_label = false_label = NULL; + val_p = FALSE; + break; + } + if (prop_set_p) { + op1 = gen (c2m_ctx, NL_HEAD (args->u.ops), NULL, NULL, FALSE, NULL, NULL); + op2 = val_gen (c2m_ctx, NL_EL (args->u.ops, 1)); + emit2 (c2m_ctx, MIR_PRSET, op1.mir_op, op2.mir_op); + true_label = false_label = NULL; + val_p = FALSE; + break; + } + if (prop_eq_p || prop_ne_p) { + MIR_label_t t_label = true_label, f_label = false_label; + int make_val_p = t_label == NULL; + if (make_val_p) { + t_label = MIR_new_label (ctx); + f_label = MIR_new_label (ctx); + } + node_t arg = NL_HEAD (args->u.ops); + op1 = gen (c2m_ctx, arg, NULL, NULL, FALSE, NULL, NULL); + arg = NL_NEXT (arg); + op2 = val_gen (c2m_ctx, arg); + emit3 (c2m_ctx, prop_eq_p ? MIR_PRBEQ : MIR_PRBNE, MIR_new_label_op (ctx, t_label), + op1.mir_op, op2.mir_op); + emit1 (c2m_ctx, MIR_JMP, MIR_new_label_op (ctx, f_label)); + if (make_val_p) make_cond_val (c2m_ctx, r, t_label, f_label, &res); + true_label = false_label = NULL; + val_p = FALSE; + break; + } + first_arg = NL_HEAD (args->u.ops); + if (jcall_p) { + func = NL_HEAD (args->u.ops); + first_arg = NL_EL (args->u.ops, 1); + assert (void_type_p (type)); + } ops_start = VARR_LENGTH (MIR_op_t, call_ops); - if (!builtin_call_p) { + if (!builtin_call_p || jcall_p) { func_expr = func->attr; func_type = func_expr->type; assert (func_type->mode == TM_PTR && func_type->u.ptr_type->mode == TM_FUNC); func_type = func_type->u.ptr_type; proto_item = func_type->u.func_type->proto_item; // ??? VARR_PUSH (MIR_op_t, call_ops, MIR_new_ref_op (ctx, proto_item)); - op1 = gen (c2m_ctx, func, NULL, NULL, TRUE, NULL); - if (op1.mir_op.mode == MIR_OP_REF && func->code == N_ID + op1 = val_gen (c2m_ctx, func); + if (!jcall_p && op1.mir_op.mode == MIR_OP_REF && func->code == N_ID && ((decl_t) func_expr->def_node->attr)->decl_spec.inline_p) inline_p = TRUE; VARR_PUSH (MIR_op_t, call_ops, op1.mir_op); @@ -12168,7 +12798,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ if ((n = target_add_call_res_op (c2m_ctx, type, &arg_info, arg_area_offset)) < 0) { /* pass nothing */ } else if (n == 0) { /* by addr */ - if (!builtin_call_p) update_call_arg_area_offset (c2m_ctx, type, FALSE); + if (!builtin_call_p || jcall_p) update_call_arg_area_offset (c2m_ctx, type, FALSE); res = new_op (NULL, VARR_LAST (MIR_op_t, call_ops)); assert (res.mir_op.mode == MIR_OP_MEM && res.mir_op.u.mem.type == MIR_T_RBLK); res.mir_op = MIR_new_mem_op (ctx, MIR_T_UNDEF, 0, res.mir_op.u.mem.base, 0, 1); @@ -12191,7 +12821,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ saved_call_arg_area_offset_before_args = curr_call_arg_area_offset; if (va_arg_p) { op1 = get_new_temp (c2m_ctx, MIR_T_I64); - op2 = gen (c2m_ctx, NL_HEAD (args->u.ops), NULL, NULL, TRUE, NULL); + op2 = val_gen (c2m_ctx, NL_HEAD (args->u.ops)); if (op2.mir_op.mode == MIR_OP_MEM) { #ifndef _WIN32 if (op2.mir_op.u.mem.type == MIR_T_UNDEF) @@ -12199,14 +12829,20 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ op2 = mem_to_address (c2m_ctx, op2, FALSE); } if (type->mode == TM_STRUCT || type->mode == TM_UNION) { - assert (desirable_dest != NULL && desirable_dest->mir_op.mode == MIR_OP_MEM); - res = mem_to_address (c2m_ctx, *desirable_dest, TRUE); + if (desirable_dest == NULL) { + res = get_new_temp (c2m_ctx, MIR_T_I64); + MIR_append_insn (ctx, curr_func, + MIR_new_insn (ctx, MIR_MOV, res.mir_op, MIR_new_int_op (ctx, 0))); + } else { + assert (desirable_dest->mir_op.mode == MIR_OP_MEM); + res = mem_to_address (c2m_ctx, *desirable_dest, TRUE); + } MIR_append_insn (ctx, curr_func, MIR_new_insn (ctx, MIR_VA_BLOCK_ARG, res.mir_op, op2.mir_op, MIR_new_int_op (ctx, type_size (c2m_ctx, type)), MIR_new_int_op (ctx, target_get_blk_type (c2m_ctx, type) - MIR_T_BLK))); - res = *desirable_dest; + if (desirable_dest != NULL) res = *desirable_dest; } else { MIR_append_insn (ctx, curr_func, MIR_new_insn (ctx, MIR_VA_ARG, op1.mir_op, op2.mir_op, @@ -12214,7 +12850,8 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ op2 = get_new_temp (c2m_ctx, t); MIR_append_insn (ctx, curr_func, MIR_new_insn (ctx, tp_mov (t), op2.mir_op, - MIR_new_mem_op (ctx, t, 0, op1.mir_op.u.reg, 0, 1))); + MIR_new_alias_mem_op (ctx, t, 0, op1.mir_op.u.reg, 0, 1, + get_type_alias (c2m_ctx, type), 0))); if (res.mir_op.mode == MIR_OP_REG) { res = op2; } else { @@ -12223,7 +12860,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ } } } else if (va_start_p) { - op1 = gen (c2m_ctx, NL_HEAD (args->u.ops), NULL, NULL, TRUE, NULL); + op1 = val_gen (c2m_ctx, NL_HEAD (args->u.ops)); if (op1.mir_op.mode == MIR_OP_MEM) { #ifndef _WIN32 if (op1.mir_op.u.mem.type == MIR_T_UNDEF) @@ -12233,16 +12870,16 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ MIR_append_insn (ctx, curr_func, MIR_new_insn (ctx, MIR_VA_START, op1.mir_op)); } else if (alloca_p) { res = get_new_temp (c2m_ctx, t); - op1 = gen (c2m_ctx, NL_HEAD (args->u.ops), NULL, NULL, TRUE, NULL); + op1 = val_gen (c2m_ctx, NL_HEAD (args->u.ops)); MIR_append_insn (ctx, curr_func, MIR_new_insn (ctx, MIR_ALLOCA, res.mir_op, op1.mir_op)); } else { param_list = func_type->u.func_type->param_list; param = NL_HEAD (param_list->u.ops); - for (node_t arg = NL_HEAD (args->u.ops); arg != NULL; arg = NL_NEXT (arg)) { + for (node_t arg = first_arg; arg != NULL; arg = NL_NEXT (arg)) { struct type *arg_type; e = arg->attr; struct_p = e->type->mode == TM_STRUCT || e->type->mode == TM_UNION; - op2 = gen (c2m_ctx, arg, NULL, NULL, !struct_p, NULL); + op2 = gen (c2m_ctx, arg, NULL, NULL, !struct_p, NULL, NULL); assert (param != NULL || NL_HEAD (param_list->u.ops) == NULL || func_type->u.func_type->dots_p); arg_type = e->type; @@ -12262,7 +12899,10 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ target_add_call_arg_op (c2m_ctx, arg_type, &arg_info, op2); if (param != NULL) param = NL_NEXT (param); } - call_insn = MIR_new_insn_arr (ctx, (inline_p ? MIR_INLINE : MIR_CALL), + call_insn = MIR_new_insn_arr (ctx, + (jcall_p ? MIR_JCALL + : inline_p ? MIR_INLINE + : MIR_CALL), VARR_LENGTH (MIR_op_t, call_ops) - ops_start, VARR_ADDR (MIR_op_t, call_ops) + ops_start); MIR_append_insn (ctx, curr_func, call_insn); @@ -12278,7 +12918,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ node_t ga_case = NL_HEAD (list->u.ops); /* first element is now a compatible generic association case */ - op1 = gen (c2m_ctx, NL_EL (ga_case->u.ops, 1), NULL, NULL, TRUE, NULL); + op1 = val_gen (c2m_ctx, NL_EL (ga_case->u.ops, 1)); t = get_mir_type (c2m_ctx, ((struct expr *) r->attr)->type); res = promote (c2m_ctx, op1, t, TRUE); break; @@ -12286,7 +12926,9 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ case N_SPEC_DECL: { // ??? export and defintion with external declaration node_t specs = NL_HEAD (r->u.ops); node_t declarator = NL_NEXT (specs); - node_t initializer = NL_NEXT (declarator); + node_t attrs = NL_NEXT (declarator); + node_t asm_part = NL_NEXT (attrs); + node_t initializer = NL_NEXT (asm_part); node_t id, curr_node; symbol_t sym; decl_t curr_decl; @@ -12294,43 +12936,45 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ const char *name; decl = (decl_t) r->attr; - if (declarator != NULL && declarator->code != N_IGNORE && decl->item == NULL) { + if (declarator != NULL && declarator->code != N_IGNORE && decl->u.item == NULL) { id = NL_HEAD (declarator->u.ops); name = (decl->scope != top_scope && decl->decl_spec.static_p ? get_func_static_var_name (c2m_ctx, id->u.s.s, decl) : id->u.s.s); - if (decl->used_p && decl->scope != top_scope && decl->decl_spec.linkage == N_STATIC) { - decl->item = MIR_new_forward (ctx, name); - move_item_forward (c2m_ctx, decl->item); + if (decl->asm_p) { + } else if (decl->used_p && decl->scope != top_scope && decl->decl_spec.linkage == N_STATIC) { + decl->u.item = MIR_new_forward (ctx, name); + move_item_forward (c2m_ctx, decl->u.item); } else if (decl->used_p && decl->decl_spec.linkage != N_IGNORE) { if (symbol_find (c2m_ctx, S_REGULAR, id, decl->decl_spec.linkage == N_EXTERN ? top_scope : decl->scope, &sym) - && (decl->item = get_ref_item (c2m_ctx, sym.def_node, name)) == NULL) { + && (decl->u.item = get_ref_item (c2m_ctx, sym.def_node, name)) == NULL) { for (i = 0; i < VARR_LENGTH (node_t, sym.defs); i++) - if ((decl->item = get_ref_item (c2m_ctx, VARR_GET (node_t, sym.defs, i), name)) != NULL) + if ((decl->u.item = get_ref_item (c2m_ctx, VARR_GET (node_t, sym.defs, i), name)) + != NULL) break; } - if (decl->item == NULL) decl->item = MIR_new_import (ctx, name); - if (decl->scope != top_scope) move_item_forward (c2m_ctx, decl->item); + if (decl->u.item == NULL) decl->u.item = MIR_new_import (ctx, name); + if (decl->scope != top_scope) move_item_forward (c2m_ctx, decl->u.item); } if (declarator->code == N_DECL && decl->decl_spec.type->mode != TM_FUNC - && !decl->decl_spec.typedef_p && !decl->decl_spec.extern_p) { + && !decl->decl_spec.typedef_p && !decl->decl_spec.extern_p && !decl->asm_p) { if (initializer->code == N_IGNORE) { if (decl->scope != top_scope && decl->decl_spec.static_p) { - decl->item = MIR_new_bss (ctx, name, raw_type_size (c2m_ctx, decl->decl_spec.type)); + decl->u.item = MIR_new_bss (ctx, name, raw_type_size (c2m_ctx, decl->decl_spec.type)); } else if (decl->scope == top_scope && symbol_find (c2m_ctx, S_REGULAR, id, top_scope, &sym) - && ((curr_decl = sym.def_node->attr)->item == NULL - || curr_decl->item->item_type != MIR_bss_item)) { + && ((curr_decl = sym.def_node->attr)->u.item == NULL + || curr_decl->u.item->item_type != MIR_bss_item)) { for (i = 0; i < VARR_LENGTH (node_t, sym.defs); i++) { curr_node = VARR_GET (node_t, sym.defs, i); curr_decl = curr_node->attr; - if ((curr_decl->item != NULL && curr_decl->item->item_type == MIR_bss_item) - || NL_EL (curr_node->u.ops, 2)->code != N_IGNORE) + if ((curr_decl->u.item != NULL && curr_decl->u.item->item_type == MIR_bss_item) + || NL_EL (curr_node->u.ops, 4)->code != N_IGNORE) break; } if (i >= VARR_LENGTH (node_t, sym.defs)) /* No item yet or no decl with intializer: */ - decl->item = MIR_new_bss (ctx, name, raw_type_size (c2m_ctx, decl->decl_spec.type)); + decl->u.item = MIR_new_bss (ctx, name, raw_type_size (c2m_ctx, decl->decl_spec.type)); } } else if (initializer->code != N_IGNORE) { // ??? general code init_start = VARR_LENGTH (init_el_t, init_els); @@ -12352,20 +12996,21 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ || decl->decl_spec.thread_local_p) { var = new_op (decl, MIR_new_ref_op (ctx, NULL)); } else { - var = gen (c2m_ctx, id, NULL, NULL, FALSE, NULL); + var = gen (c2m_ctx, id, NULL, NULL, FALSE, NULL, NULL); assert (var.decl != NULL && (var.mir_op.mode == MIR_OP_REG || (var.mir_op.mode == MIR_OP_MEM && var.mir_op.u.mem.index == 0))); } + int local_p = (decl->scope != top_scope && !decl->decl_spec.static_p + && !decl->decl_spec.thread_local_p); gen_initializer (c2m_ctx, init_start, var, name, - raw_type_size (c2m_ctx, decl->decl_spec.type), - decl->scope != top_scope && !decl->decl_spec.static_p - && !decl->decl_spec.thread_local_p); + (local_p ? raw_type_size : type_size) (c2m_ctx, decl->decl_spec.type), + local_p); VARR_TRUNC (init_el_t, init_els, init_start); } - if (decl->item != NULL && decl->scope == top_scope && !decl->decl_spec.static_p) { + if (decl->u.item != NULL && decl->scope == top_scope && !decl->decl_spec.static_p) { MIR_new_export (ctx, name); - } else if (decl->item != NULL && decl->scope != top_scope && decl->decl_spec.static_p) { + } else if (decl->u.item != NULL && decl->scope != top_scope && decl->decl_spec.static_p) { MIR_item_t item = MIR_new_forward (ctx, name); move_item_forward (c2m_ctx, item); @@ -12382,8 +13027,8 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ node_t decls = NL_NEXT (declarator); node_t stmt = NL_NEXT (decls); struct node_scope *ns = stmt->attr; - decl_t param_decl, decl = r->attr; - struct type *decl_type = decl->decl_spec.type; + decl_t param_decl, func_decl = r->attr; + struct type *decl_type = func_decl->decl_spec.type; node_t first_param, param, param_declarator, param_id; struct type *param_type; MIR_insn_t insn; @@ -12406,7 +13051,8 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ VARR_ADDR (MIR_type_t, proto_info.ret_types), VARR_LENGTH (MIR_var_t, proto_info.arg_vars), VARR_ADDR (MIR_var_t, proto_info.arg_vars))); - decl->item = curr_func; + func_decl->u.item = curr_func; + DLIST_INIT (MIR_insn_t, slow_code_part); if (ns->stack_var_p /* we can have empty struct only with size 0 and still need a frame: */ || ns->size > 0) { fp_reg = MIR_new_func_reg (ctx, curr_func->u.func, MIR_T_I64, FP_NAME); @@ -12415,7 +13061,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ MIR_new_int_op (ctx, ns->size))); } for (size_t i = 0; i < VARR_LENGTH (MIR_var_t, proto_info.arg_vars); i++) - get_reg_var (c2m_ctx, MIR_T_UNDEF, VARR_GET (MIR_var_t, proto_info.arg_vars, i).name); + get_reg_var (c2m_ctx, MIR_T_UNDEF, VARR_GET (MIR_var_t, proto_info.arg_vars, i).name, NULL); target_init_arg_vars (c2m_ctx, &arg_info); if ((first_param = NL_HEAD (decl_type->u.func_type->param_list->u.ops)) != NULL && !void_param_p (first_param)) { @@ -12432,7 +13078,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ if (param_decl->reg_p) continue; if (param_type->mode == TM_STRUCT || param_type->mode == TM_UNION) { /* ??? only block pass */ - param_reg = get_reg_var (c2m_ctx, MIR_POINTER_TYPE, name).reg; + param_reg = get_reg_var (c2m_ctx, MIR_POINTER_TYPE, name, NULL).reg; val = new_op (NULL, MIR_new_mem_op (ctx, MIR_T_UNDEF, 0, param_reg, 0, 1)); var = new_op (param_decl, MIR_new_mem_op (ctx, MIR_T_UNDEF, param_decl->offset, @@ -12441,17 +13087,21 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ } else { param_mir_type = get_mir_type (c2m_ctx, param_type); emit2 (c2m_ctx, tp_mov (param_mir_type), - MIR_new_mem_op (ctx, param_mir_type, param_decl->offset, - MIR_reg (ctx, FP_NAME, curr_func->u.func), 0, 1), - MIR_new_reg_op (ctx, get_reg_var (c2m_ctx, MIR_T_UNDEF, name).reg)); + MIR_new_alias_mem_op (ctx, param_mir_type, param_decl->offset, + MIR_reg (ctx, FP_NAME, curr_func->u.func), 0, 1, + get_type_alias (c2m_ctx, param_type), 0), + MIR_new_reg_op (ctx, get_reg_var (c2m_ctx, MIR_T_UNDEF, name, NULL).reg)); } } } - gen (c2m_ctx, stmt, NULL, NULL, FALSE, NULL); + gen (c2m_ctx, stmt, NULL, NULL, FALSE, NULL, NULL); if ((insn = DLIST_TAIL (MIR_insn_t, curr_func->u.func->insns)) == NULL - || (insn->code != MIR_RET && insn->code != MIR_JMP)) { + || (insn->code != MIR_RET && insn->code != MIR_JRET && insn->code != MIR_JMP)) { if (VARR_LENGTH (MIR_type_t, proto_info.ret_types) == 0) { - emit_insn (c2m_ctx, MIR_new_ret_insn (ctx, 0)); + if (jump_ret_p) + emit1 (c2m_ctx, MIR_JRET, MIR_new_int_op (ctx, 0)); + else + emit_insn (c2m_ctx, MIR_new_ret_insn (ctx, 0)); } else { VARR_TRUNC (MIR_op_t, ret_ops, 0); for (size_t i = 0; i < VARR_LENGTH (MIR_type_t, proto_info.ret_types); i++) { @@ -12470,37 +13120,65 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ VARR_ADDR (MIR_op_t, ret_ops))); } } + while ((insn = DLIST_HEAD (MIR_insn_t, slow_code_part)) != NULL) { + DLIST_REMOVE (MIR_insn_t, slow_code_part, insn); + DLIST_APPEND (MIR_insn_t, curr_func->u.func->insns, insn); + } MIR_finish_func (ctx); - if (decl->decl_spec.linkage == N_EXTERN) + if (func_decl->decl_spec.linkage == N_EXTERN) MIR_new_export (ctx, NL_HEAD (declarator->u.ops)->u.s.s); finish_curr_func_reg_vars (c2m_ctx); break; } case N_STMTEXPR: { - gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, FALSE, NULL); + gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, FALSE, NULL, NULL); res = top_gen_last_op; break; } case N_BLOCK: emit_label (c2m_ctx, r); - gen (c2m_ctx, NL_EL (r->u.ops, 1), NULL, NULL, FALSE, NULL); + gen (c2m_ctx, NL_EL (r->u.ops, 1), NULL, NULL, FALSE, NULL, NULL); break; - case N_MODULE: gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, FALSE, NULL); break; // ??? + case N_MODULE: gen (c2m_ctx, NL_HEAD (r->u.ops), NULL, NULL, FALSE, NULL, NULL); break; // ??? case N_IF: { node_t expr = NL_EL (r->u.ops, 1); node_t if_stmt = NL_NEXT (expr); node_t else_stmt = NL_NEXT (if_stmt); MIR_label_t if_label = MIR_new_label (ctx), else_label = MIR_new_label (ctx); MIR_label_t end_label = MIR_new_label (ctx); + int cond_expect_res; + MIR_insn_t insn, last = NULL; assert (false_label == NULL && true_label == NULL); emit_label (c2m_ctx, r); - top_gen (c2m_ctx, expr, if_label, else_label); - emit_label_insn_opt (c2m_ctx, if_label); - gen (c2m_ctx, if_stmt, NULL, NULL, FALSE, NULL); - emit1 (c2m_ctx, MIR_JMP, MIR_new_label_op (ctx, end_label)); - emit_label_insn_opt (c2m_ctx, else_label); - gen (c2m_ctx, else_stmt, NULL, NULL, FALSE, NULL); + top_gen (c2m_ctx, expr, if_label, else_label, &cond_expect_res); + if (cond_expect_res == 0 || cond_expect_res == 1) { /* fall through on true */ + emit_label_insn_opt (c2m_ctx, if_label); + gen (c2m_ctx, if_stmt, NULL, NULL, FALSE, NULL, NULL); + if (cond_expect_res == 0) { + emit1 (c2m_ctx, MIR_JMP, MIR_new_label_op (ctx, end_label)); + emit_label_insn_opt (c2m_ctx, else_label); + gen (c2m_ctx, else_stmt, NULL, NULL, FALSE, NULL, NULL); + } else { + last = DLIST_TAIL (MIR_insn_t, curr_func->u.func->insns); + MIR_append_insn (ctx, curr_func, else_label); + gen (c2m_ctx, else_stmt, NULL, NULL, FALSE, NULL, NULL); + } + } else { /* fall on false */ + emit_label_insn_opt (c2m_ctx, else_label); + gen (c2m_ctx, else_stmt, NULL, NULL, FALSE, NULL, NULL); + last = DLIST_TAIL (MIR_insn_t, curr_func->u.func->insns); + MIR_append_insn (ctx, curr_func, if_label); + gen (c2m_ctx, if_stmt, NULL, NULL, FALSE, NULL, NULL); + } + if (last != NULL) { /* move to slow part of code */ + while ((insn = DLIST_NEXT (MIR_insn_t, last)) != NULL) { + DLIST_REMOVE (MIR_insn_t, curr_func->u.func->insns, insn); + DLIST_APPEND (MIR_insn_t, slow_code_part, insn); + } + DLIST_APPEND (MIR_insn_t, slow_code_part, + MIR_new_insn (ctx, MIR_JMP, MIR_new_label_op (ctx, end_label))); + } emit_label_insn_opt (c2m_ctx, end_label); break; } @@ -12519,7 +13197,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ assert (false_label == NULL && true_label == NULL); emit_label (c2m_ctx, r); break_label = MIR_new_label (ctx); - case_reg_op = gen (c2m_ctx, expr, NULL, NULL, TRUE, NULL); + case_reg_op = val_gen (c2m_ctx, expr); type = ((struct expr *) expr->attr)->type; signed_p = signed_integer_type_p (type); mir_type = get_mir_type (c2m_ctx, type); @@ -12528,7 +13206,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ if (switch_attr->min_val_case != NULL) { e = NL_HEAD (switch_attr->min_val_case->case_node->u.ops)->attr; e2 = NL_HEAD (switch_attr->max_val_case->case_node->u.ops)->attr; - range = signed_p ? e2->u.i_val - e->u.i_val : e2->u.u_val - e->u.u_val; + range = signed_p ? (mir_ullong) (e2->c.i_val - e->c.i_val) : e2->c.u_val - e->c.u_val; } len = DLIST_LENGTH (case_t, switch_attr->case_labels); if (!switch_attr->ranges_p && len > 4 && range != 0 && range / len < 3) { /* use MIR_SWITCH */ @@ -12542,7 +13220,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ label = get_label (c2m_ctx, c->case_target_node); } emit3 (c2m_ctx, short_p ? MIR_SUBS : MIR_SUB, index.mir_op, case_reg_op.mir_op, - signed_p ? MIR_new_int_op (ctx, e->u.i_val) : MIR_new_uint_op (ctx, e->u.u_val)); + signed_p ? MIR_new_int_op (ctx, e->c.i_val) : MIR_new_uint_op (ctx, e->c.u_val)); emit3 (c2m_ctx, short_p ? MIR_UBGTS : MIR_UBGT, MIR_new_label_op (ctx, label), index.mir_op, MIR_new_uint_op (ctx, range)); if (short_p) emit2 (c2m_ctx, MIR_UEXT32, index.mir_op, index.mir_op); @@ -12557,7 +13235,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ for (size_t i = 0; i < VARR_LENGTH (case_t, switch_cases); i++) { c = VARR_GET (case_t, switch_cases, i); e2 = NL_HEAD (c->case_node->u.ops)->attr; - curr_val = signed_p ? e2->u.i_val - e->u.i_val : e2->u.u_val - e->u.u_val; + curr_val = signed_p ? (mir_ullong) (e2->c.i_val - e->c.i_val) : e2->c.u_val - e->c.u_val; if (i != 0) { for (n = prev_val + 1; n < curr_val; n++) VARR_PUSH (MIR_op_t, switch_ops, MIR_new_label_op (ctx, label)); @@ -12585,21 +13263,21 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ assert (e->const_p && integer_type_p (e->type)); if (case_expr2 == NULL) { emit3 (c2m_ctx, short_p ? MIR_BEQS : MIR_BEQ, MIR_new_label_op (ctx, label), - case_reg_op.mir_op, MIR_new_int_op (ctx, e->u.i_val)); + case_reg_op.mir_op, MIR_new_int_op (ctx, e->c.i_val)); } else { e2 = case_expr2->attr; assert (e2->const_p && integer_type_p (e2->type)); cont_label = MIR_new_label (ctx); if (signed_p) { emit3 (c2m_ctx, short_p ? MIR_BLTS : MIR_BLT, MIR_new_label_op (ctx, cont_label), - case_reg_op.mir_op, MIR_new_int_op (ctx, e->u.i_val)); + case_reg_op.mir_op, MIR_new_int_op (ctx, e->c.i_val)); emit3 (c2m_ctx, short_p ? MIR_BLES : MIR_BLE, MIR_new_label_op (ctx, label), - case_reg_op.mir_op, MIR_new_int_op (ctx, e2->u.i_val)); + case_reg_op.mir_op, MIR_new_int_op (ctx, e2->c.i_val)); } else { emit3 (c2m_ctx, short_p ? MIR_UBLTS : MIR_UBLT, MIR_new_label_op (ctx, cont_label), - case_reg_op.mir_op, MIR_new_int_op (ctx, e->u.i_val)); + case_reg_op.mir_op, MIR_new_int_op (ctx, e->c.i_val)); emit3 (c2m_ctx, short_p ? MIR_UBLES : MIR_UBLE, MIR_new_label_op (ctx, label), - case_reg_op.mir_op, MIR_new_int_op (ctx, e2->u.i_val)); + case_reg_op.mir_op, MIR_new_int_op (ctx, e2->c.i_val)); } emit_label_insn_opt (c2m_ctx, cont_label); } @@ -12607,7 +13285,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ if (c == NULL) /* no default: */ emit1 (c2m_ctx, MIR_JMP, MIR_new_label_op (ctx, break_label)); } - top_gen (c2m_ctx, stmt, NULL, NULL); + top_gen (c2m_ctx, stmt, NULL, NULL, NULL); emit_label_insn_opt (c2m_ctx, break_label); break_label = saved_break_label; break; @@ -12623,9 +13301,9 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ break_label = MIR_new_label (ctx); emit_label (c2m_ctx, r); emit_label_insn_opt (c2m_ctx, start_label); - gen (c2m_ctx, stmt, NULL, NULL, FALSE, NULL); + gen (c2m_ctx, stmt, NULL, NULL, FALSE, NULL, NULL); emit_label_insn_opt (c2m_ctx, continue_label); - top_gen (c2m_ctx, expr, start_label, break_label); + top_gen (c2m_ctx, expr, start_label, break_label, NULL); emit_label_insn_opt (c2m_ctx, break_label); continue_label = saved_continue_label; break_label = saved_break_label; @@ -12642,10 +13320,10 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ break_label = MIR_new_label (ctx); emit_label (c2m_ctx, r); emit_label_insn_opt (c2m_ctx, continue_label); - top_gen (c2m_ctx, expr, stmt_label, break_label); + top_gen (c2m_ctx, expr, stmt_label, break_label, NULL); emit_label_insn_opt (c2m_ctx, stmt_label); - gen (c2m_ctx, stmt, NULL, NULL, FALSE, NULL); - top_gen (c2m_ctx, expr, stmt_label, break_label); + gen (c2m_ctx, stmt, NULL, NULL, FALSE, NULL, NULL); + top_gen (c2m_ctx, expr, stmt_label, break_label, NULL); emit_label_insn_opt (c2m_ctx, break_label); continue_label = saved_continue_label; break_label = saved_break_label; @@ -12663,17 +13341,17 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ continue_label = MIR_new_label (ctx); break_label = MIR_new_label (ctx); emit_label (c2m_ctx, r); - top_gen (c2m_ctx, init, NULL, NULL); + top_gen (c2m_ctx, init, NULL, NULL, NULL); if (cond->code != N_IGNORE) /* non-empty condition: */ - top_gen (c2m_ctx, cond, stmt_label, break_label); + top_gen (c2m_ctx, cond, stmt_label, break_label, NULL); emit_label_insn_opt (c2m_ctx, stmt_label); - gen (c2m_ctx, stmt, NULL, NULL, FALSE, NULL); + gen (c2m_ctx, stmt, NULL, NULL, FALSE, NULL, NULL); emit_label_insn_opt (c2m_ctx, continue_label); - top_gen (c2m_ctx, iter, NULL, NULL); + top_gen (c2m_ctx, iter, NULL, NULL, NULL); if (cond->code == N_IGNORE) { /* empty condition: */ emit1 (c2m_ctx, MIR_JMP, MIR_new_label_op (ctx, stmt_label)); } else { - top_gen (c2m_ctx, cond, stmt_label, break_label); + top_gen (c2m_ctx, cond, stmt_label, break_label, NULL); } emit_label_insn_opt (c2m_ctx, break_label); continue_label = saved_continue_label; @@ -12688,6 +13366,15 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ emit1 (c2m_ctx, MIR_JMP, MIR_new_label_op (ctx, get_label (c2m_ctx, target))); break; } + case N_INDIRECT_GOTO: { + node_t expr = NL_EL (r->u.ops, 1); + + assert (false_label == NULL && true_label == NULL); + emit_label (c2m_ctx, r); + val = val_gen (c2m_ctx, expr); + emit1 (c2m_ctx, MIR_JMPI, val.mir_op); + break; + } case N_CONTINUE: assert (false_label == NULL && true_label == NULL); emit_label (c2m_ctx, r); @@ -12717,7 +13404,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ var = new_op (NULL, MIR_new_mem_op (ctx, MIR_T_I8, 0, ret_addr_reg, 0, 1)); } val = gen (c2m_ctx, NL_EL (r->u.ops, 1), NULL, NULL, !ret_by_addr_p && scalar_p, - !ret_by_addr_p || scalar_p ? NULL : &var); + !ret_by_addr_p || scalar_p ? NULL : &var, NULL); if (!ret_by_addr_p && scalar_p) { t = get_mir_type (c2m_ctx, ret_type); t = promote_mir_int_type (t); @@ -12732,7 +13419,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ case N_EXPR: assert (false_label == NULL && true_label == NULL); emit_label (c2m_ctx, r); - top_gen (c2m_ctx, NL_EL (r->u.ops, 1), NULL, NULL); + top_gen (c2m_ctx, NL_EL (r->u.ops, 1), NULL, NULL, NULL); break; default: abort (); } @@ -12760,7 +13447,7 @@ static op_t gen (c2m_ctx_t c2m_ctx, node_t r, MIR_label_t true_label, MIR_label_ return res; } -static htab_hash_t proto_hash (MIR_item_t pi, void *arg) { +static htab_hash_t proto_hash (MIR_item_t pi, void *arg MIR_UNUSED) { MIR_proto_t p = pi->u.proto; MIR_var_t *args = VARR_ADDR (MIR_var_t, p->args); uint64_t h = mir_hash_init (42); @@ -12773,10 +13460,10 @@ static htab_hash_t proto_hash (MIR_item_t pi, void *arg) { h = mir_hash_step (h, mir_hash (args[i].name, strlen (args[i].name), 24)); if (MIR_all_blk_type_p (args[i].type)) h = mir_hash_step (h, args[i].size); } - return mir_hash_finish (h); + return (htab_hash_t) mir_hash_finish (h); } -static int proto_eq (MIR_item_t pi1, MIR_item_t pi2, void *arg) { +static int proto_eq (MIR_item_t pi1, MIR_item_t pi2, void *arg MIR_UNUSED) { MIR_proto_t p1 = pi1->u.proto, p2 = pi2->u.proto; if (p1->nres != p2->nres || p1->vararg_p != p2->vararg_p @@ -12794,8 +13481,7 @@ static int proto_eq (MIR_item_t pi1, MIR_item_t pi2, void *arg) { return TRUE; } -static MIR_item_t get_mir_proto (c2m_ctx_t c2m_ctx, int vararg_p, VARR (MIR_type_t) * ret_types, - VARR (MIR_var_t) * vars) { +static MIR_item_t get_mir_proto (c2m_ctx_t c2m_ctx, int vararg_p) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; struct MIR_item pi, *el; struct MIR_proto p; @@ -12803,9 +13489,9 @@ static MIR_item_t get_mir_proto (c2m_ctx_t c2m_ctx, int vararg_p, VARR (MIR_type pi.u.proto = &p; p.vararg_p = vararg_p; - p.nres = VARR_LENGTH (MIR_type_t, proto_info.ret_types); + p.nres = (uint32_t) VARR_LENGTH (MIR_type_t, proto_info.ret_types); p.res_types = VARR_ADDR (MIR_type_t, proto_info.ret_types); - p.args = vars; + p.args = proto_info.arg_vars; if (HTAB_DO (MIR_item_t, proto_tab, &pi, HTAB_FIND, el)) return el; sprintf (buff, "proto%d", curr_mir_proto_num++); el = (vararg_p ? MIR_new_vararg_proto_arr @@ -12818,7 +13504,7 @@ static MIR_item_t get_mir_proto (c2m_ctx_t c2m_ctx, int vararg_p, VARR (MIR_type static void gen_mir_protos (c2m_ctx_t c2m_ctx) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; - node_t call, func; + node_t call, func, op1; struct type *type; struct func_type *func_type; @@ -12827,7 +13513,11 @@ static void gen_mir_protos (c2m_ctx_t c2m_ctx) { for (size_t i = 0; i < VARR_LENGTH (node_t, call_nodes); i++) { call = VARR_GET (node_t, call_nodes, i); assert (call->code == N_CALL); - func = NL_HEAD (call->u.ops); + op1 = NL_HEAD (call->u.ops); + if (op1->code == N_ID && strcmp (op1->u.s.s, JCALL) == 0) + func = NL_HEAD (NL_NEXT (op1)->u.ops); + else + func = NL_HEAD (call->u.ops); type = ((struct expr *) func->attr)->type; assert (type->mode == TM_PTR && type->u.ptr_type->mode == TM_FUNC); set_type_layout (c2m_ctx, type); @@ -12835,8 +13525,8 @@ static void gen_mir_protos (c2m_ctx_t c2m_ctx) { assert (func_type->param_list->code == N_LIST); collect_args_and_func_types (c2m_ctx, func_type); func_type->proto_item - = get_mir_proto (c2m_ctx, func_type->dots_p || NL_HEAD (func_type->param_list->u.ops) == NULL, - proto_info.ret_types, proto_info.arg_vars); + = get_mir_proto (c2m_ctx, + func_type->dots_p || NL_HEAD (func_type->param_list->u.ops) == NULL); } HTAB_DESTROY (MIR_item_t, proto_tab); } @@ -12853,6 +13543,7 @@ static void gen_finish (c2m_ctx_t c2m_ctx) { if (switch_ops != NULL) VARR_DESTROY (MIR_op_t, switch_ops); if (switch_cases != NULL) VARR_DESTROY (case_t, switch_cases); if (init_els != NULL) VARR_DESTROY (init_el_t, init_els); + if (node_stack != NULL) VARR_DESTROY (node_t, node_stack); free (c2m_ctx->gen_ctx); } @@ -12873,8 +13564,9 @@ static void gen_mir (c2m_ctx_t c2m_ctx, node_t r) { VARR_CREATE (MIR_op_t, switch_ops, 128); VARR_CREATE (case_t, switch_cases, 64); VARR_CREATE (init_el_t, init_els, 128); + VARR_CREATE (node_t, node_stack, 8); memset_proto = memset_item = memcpy_proto = memcpy_item = NULL; - top_gen (c2m_ctx, r, NULL, NULL); + top_gen (c2m_ctx, r, NULL, NULL, NULL); gen_finish (c2m_ctx); } @@ -12897,13 +13589,14 @@ static const char *get_node_name (node_code_t code) { REP8 (C, DIV, DIV_ASSIGN, MOD, MOD_ASSIGN, IND, FIELD, ADDR, DEREF); REP8 (C, DEREF_FIELD, COND, INC, DEC, POST_INC, POST_DEC, ALIGNOF, SIZEOF); REP8 (C, EXPR_SIZEOF, CAST, COMPOUND_LITERAL, CALL, GENERIC, GENERIC_ASSOC, IF, SWITCH); - REP8 (C, WHILE, DO, FOR, GOTO, CONTINUE, BREAK, RETURN, EXPR); - REP8 (C, BLOCK, CASE, DEFAULT, LABEL, LIST, SPEC_DECL, SHARE, TYPEDEF); - REP8 (C, EXTERN, STATIC, AUTO, REGISTER, THREAD_LOCAL, DECL, VOID, CHAR); - REP8 (C, SHORT, INT, LONG, FLOAT, DOUBLE, SIGNED, UNSIGNED, BOOL); - REP8 (C, STRUCT, UNION, ENUM, ENUM_CONST, MEMBER, CONST, RESTRICT, VOLATILE); - REP8 (C, ATOMIC, INLINE, NO_RETURN, ALIGNAS, FUNC, STAR, POINTER, DOTS); - REP7 (C, ARR, INIT, FIELD_ID, TYPE, ST_ASSERT, FUNC_DEF, MODULE); + REP8 (C, WHILE, DO, FOR, GOTO, INDIRECT_GOTO, CONTINUE, BREAK, RETURN); + REP8 (C, EXPR, BLOCK, CASE, DEFAULT, LABEL, LABEL_ADDR, LIST, SPEC_DECL); + REP8 (C, SHARE, TYPEDEF, EXTERN, STATIC, AUTO, REGISTER, THREAD_LOCAL, DECL); + REP8 (C, VOID, CHAR, SHORT, INT, LONG, FLOAT, DOUBLE, SIGNED); + REP8 (C, UNSIGNED, BOOL, STRUCT, UNION, ENUM, ENUM_CONST, MEMBER, CONST); + REP8 (C, RESTRICT, VOLATILE, ATOMIC, INLINE, NO_RETURN, ALIGNAS, FUNC, STAR); + REP8 (C, POINTER, DOTS, ARR, INIT, FIELD_ID, TYPE, ST_ASSERT, FUNC_DEF); + REP3 (C, MODULE, ASM, ATTR); default: abort (); } #undef C @@ -12911,7 +13604,6 @@ static const char *get_node_name (node_code_t code) { } static void print_char (FILE *f, mir_ulong ch) { - assert (ch >= 0); if (ch >= 0x100) { fprintf (f, ch <= 0xFFFF ? "\\u%04x" : "\\U%08x", (unsigned int) ch); } else { @@ -13045,21 +13737,22 @@ static void print_decl (c2m_ctx_t c2m_ctx, FILE *f, decl_t decl) { fprintf (f, ", offset = %llu", (unsigned long long) decl->offset); if (decl->bit_offset >= 0) fprintf (f, ", bit offset = %d", decl->bit_offset); } + if (decl->asm_p) fprintf (f, ", asm=%s", decl->u.asm_str); } static void print_expr (c2m_ctx_t c2m_ctx, FILE *f, struct expr *e) { if (e == NULL) return; /* e.g. N_ID which is not an expr */ fprintf (f, ": "); - if (e->lvalue_node) fprintf (f, "lvalue, "); + if (e->u.lvalue_node) fprintf (f, "lvalue, "); print_type (c2m_ctx, f, e->type); if (e->const_p) { fprintf (f, ", const = "); if (!integer_type_p (e->type)) { - fprintf (f, " %.*Lg\n", LDBL_MANT_DIG, (long double) e->u.d_val); + fprintf (f, " %.*Lg\n", LDBL_MANT_DIG, (long double) e->c.d_val); } else if (signed_integer_type_p (e->type)) { - fprintf (f, "%lld", (long long) e->u.i_val); + fprintf (f, "%lld", (long long) e->c.i_val); } else { - fprintf (f, "%llu", (unsigned long long) e->u.u_val); + fprintf (f, "%llu", (unsigned long long) e->c.u_val); } } } @@ -13159,6 +13852,7 @@ static void print_node (c2m_ctx_t c2m_ctx, FILE *f, node_t n, int indent, int at case N_CALL: case N_GENERIC: case N_STMTEXPR: + case N_LABEL_ADDR: if (attr_p && n->attr != NULL) print_expr (c2m_ctx, f, n->attr); fprintf (f, "\n"); print_ops (c2m_ctx, f, n, indent, attr_p); @@ -13207,6 +13901,11 @@ static void print_node (c2m_ctx_t c2m_ctx, FILE *f, node_t n, int indent, int at case N_FIELD_ID: case N_TYPE: case N_ST_ASSERT: + case N_ASM: + fprintf (f, "\n"); + print_ops (c2m_ctx, f, n, indent, attr_p); + break; + case N_ATTR: fprintf (f, "\n"); print_ops (c2m_ctx, f, n, indent, attr_p); break; @@ -13231,7 +13930,7 @@ static void print_node (c2m_ctx_t c2m_ctx, FILE *f, node_t n, int indent, int at print_ops (c2m_ctx, f, n, indent, attr_p); break; } - /* fall through: */ + /* fall through */ case N_STRUCT: case N_UNION: case N_MODULE: @@ -13265,6 +13964,7 @@ static void print_node (c2m_ctx_t c2m_ctx, FILE *f, node_t n, int indent, int at if (attr_p && n->attr != NULL) fprintf (f, ": target node %u\n", ((node_t) n->attr)->uid); print_ops (c2m_ctx, f, n, indent, attr_p); break; + case N_INDIRECT_GOTO: print_ops (c2m_ctx, f, n, indent, attr_p); break; case N_ENUM: if (attr_p && n->attr != NULL) { fprintf (f, ": enum_basic_type = "); @@ -13461,7 +14161,7 @@ int c2mir_compile (MIR_context_t ctx, struct c2mir_options *ops, int (*getc_func real_usec_time () - start_time); add_stream (c2m_ctx, NULL, source_name, top_level_getc); if (!c2m_options->no_prepro_p) add_standard_includes (c2m_ctx); - pre (c2m_ctx, source_name); + pre (c2m_ctx); if (c2m_options->verbose_p && c2m_options->message_file != NULL) fprintf (c2m_options->message_file, " C2MIR preprocessor end -- %.0f usec\n", real_usec_time () - start_time); diff --git a/mir/c2mir/c2mir.h b/mir/c2mir/c2mir.h index c90e2be9..fff5b800 100644 --- a/mir/c2mir/c2mir.h +++ b/mir/c2mir/c2mir.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ #ifndef C2MIR_H diff --git a/mir/c2mir/mirc.h b/mir/c2mir/mirc.h index 34eecb1d..4c00a5cc 100644 --- a/mir/c2mir/mirc.h +++ b/mir/c2mir/mirc.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ static const char mirc[] diff --git a/mir/c2mir/mirc_iso646.h b/mir/c2mir/mirc_iso646.h index e42438a5..071674d3 100644 --- a/mir/c2mir/mirc_iso646.h +++ b/mir/c2mir/mirc_iso646.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 7.9 */ diff --git a/mir/c2mir/mirc_stdalign.h b/mir/c2mir/mirc_stdalign.h index 99d48bef..8a61e200 100644 --- a/mir/c2mir/mirc_stdalign.h +++ b/mir/c2mir/mirc_stdalign.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 7.15 */ diff --git a/mir/c2mir/mirc_stdbool.h b/mir/c2mir/mirc_stdbool.h index 2621a05d..dd98b13d 100644 --- a/mir/c2mir/mirc_stdbool.h +++ b/mir/c2mir/mirc_stdbool.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 7.18 */ diff --git a/mir/c2mir/mirc_stdnoreturn.h b/mir/c2mir/mirc_stdnoreturn.h index 7782eed0..2f449ceb 100644 --- a/mir/c2mir/mirc_stdnoreturn.h +++ b/mir/c2mir/mirc_stdnoreturn.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 7.23 */ diff --git a/mir/c2mir/ppc64/cppc64-ABI-code.c b/mir/c2mir/ppc64/cppc64-ABI-code.c index 5833d814..f4f46222 100644 --- a/mir/c2mir/ppc64/cppc64-ABI-code.c +++ b/mir/c2mir/ppc64/cppc64-ABI-code.c @@ -1,11 +1,12 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . ppc64 call ABI target specific code. */ typedef int target_arg_info_t; -static void target_init_arg_vars (c2m_ctx_t c2m_ctx, target_arg_info_t *arg_info) {} +static void target_init_arg_vars (c2m_ctx_t c2m_ctx MIR_UNUSED, + target_arg_info_t *arg_info MIR_UNUSED) {} #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ static MIR_type_t fp_homogeneous_type (c2m_ctx_t c2m_ctx, struct type *param_type, int *num) { @@ -26,7 +27,7 @@ static MIR_type_t fp_homogeneous_type_1 (c2m_ctx_t c2m_ctx, MIR_type_t curr_type if ((t = fp_homogeneous_type_1 (c2m_ctx, curr_type, type->u.arr_type->el_type, &n)) == MIR_T_UNDEF) return MIR_T_UNDEF; - *num = arr_type->size->code == N_IGNORE || !cexpr->const_p ? 1 : cexpr->u.i_val; + *num = arr_type->size->code == N_IGNORE || !cexpr->const_p ? 1 : cexpr->c.i_val; return t; } case TM_STRUCT: @@ -83,8 +84,8 @@ static int target_return_by_addr_p (c2m_ctx_t c2m_ctx, struct type *ret_type) { } static void target_add_res_proto (c2m_ctx_t c2m_ctx, struct type *ret_type, - target_arg_info_t *arg_info, VARR (MIR_type_t) * res_types, - VARR (MIR_var_t) * arg_vars) { + target_arg_info_t *arg_info MIR_UNUSED, + VARR (MIR_type_t) * res_types, VARR (MIR_var_t) * arg_vars) { MIR_var_t var; MIR_type_t type; int i, n, size; @@ -107,7 +108,8 @@ static void target_add_res_proto (c2m_ctx_t c2m_ctx, struct type *ret_type, } static int target_add_call_res_op (c2m_ctx_t c2m_ctx, struct type *ret_type, - target_arg_info_t *arg_info, size_t call_arg_area_offset) { + target_arg_info_t *arg_info MIR_UNUSED, + size_t call_arg_area_offset) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; MIR_context_t ctx = c2m_ctx->ctx; MIR_type_t type; @@ -149,7 +151,7 @@ static int target_add_call_res_op (c2m_ctx_t c2m_ctx, struct type *ret_type, } static op_t target_gen_post_call_res_code (c2m_ctx_t c2m_ctx, struct type *ret_type, op_t res, - MIR_insn_t call, size_t call_ops_start) { + MIR_insn_t call MIR_UNUSED, size_t call_ops_start) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; MIR_context_t ctx = c2m_ctx->ctx; MIR_type_t type; @@ -218,12 +220,14 @@ static void target_add_ret_ops (c2m_ctx_t c2m_ctx, struct type *ret_type, op_t r } } -static MIR_type_t target_get_blk_type (c2m_ctx_t c2m_ctx, struct type *arg_type) { +static MIR_type_t target_get_blk_type (c2m_ctx_t c2m_ctx MIR_UNUSED, + struct type *arg_type MIR_UNUSED) { return MIR_T_BLK; /* one BLK is enough */ } static void target_add_arg_proto (c2m_ctx_t c2m_ctx, const char *name, struct type *arg_type, - target_arg_info_t *arg_info, VARR (MIR_var_t) * arg_vars) { + target_arg_info_t *arg_info MIR_UNUSED, + VARR (MIR_var_t) * arg_vars) { MIR_var_t var; MIR_type_t type; int n; @@ -247,7 +251,7 @@ static void target_add_arg_proto (c2m_ctx_t c2m_ctx, const char *name, struct ty } static void target_add_call_arg_op (c2m_ctx_t c2m_ctx, struct type *arg_type, - target_arg_info_t *arg_info, op_t arg) { + target_arg_info_t *arg_info MIR_UNUSED, op_t arg) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; MIR_context_t ctx = c2m_ctx->ctx; MIR_type_t type; @@ -280,7 +284,7 @@ static void target_add_call_arg_op (c2m_ctx_t c2m_ctx, struct type *arg_type, } static int target_gen_gather_arg (c2m_ctx_t c2m_ctx, const char *name, struct type *arg_type, - decl_t param_decl, target_arg_info_t *arg_info) { + decl_t param_decl, target_arg_info_t *arg_info MIR_UNUSED) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; MIR_context_t ctx = c2m_ctx->ctx; MIR_type_t type; @@ -291,7 +295,7 @@ static int target_gen_gather_arg (c2m_ctx_t c2m_ctx, const char *name, struct ty && n <= 8) { for (i = 0; i < n; i++) { assert (!param_decl->reg_p); - reg_var = get_reg_var (c2m_ctx, type, gen_get_indexed_name (c2m_ctx, name, i)); + reg_var = get_reg_var (c2m_ctx, type, gen_get_indexed_name (c2m_ctx, name, i), NULL); MIR_append_insn (ctx, curr_func, MIR_new_insn (ctx, tp_mov (type), MIR_new_mem_op (ctx, type, diff --git a/mir/c2mir/ppc64/cppc64-code.c b/mir/c2mir/ppc64/cppc64-code.c index 0c8813c4..eb60c408 100644 --- a/mir/c2mir/ppc64/cppc64-code.c +++ b/mir/c2mir/ppc64/cppc64-code.c @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ #include "../mirc.h" @@ -18,7 +18,8 @@ static string_include_t standard_includes[] #define ADJUST_VAR_ALIGNMENT(c2m_ctx, align, type) ppc64_adjust_var_alignment (c2m_ctx, align, type) -static int ppc64_adjust_var_alignment (c2m_ctx_t c2m_ctx, int align, struct type *type) { +static int ppc64_adjust_var_alignment (c2m_ctx_t c2m_ctx MIR_UNUSED, int align MIR_UNUSED, + struct type *type MIR_UNUSED) { return align; } diff --git a/mir/c2mir/ppc64/cppc64.h b/mir/c2mir/ppc64/cppc64.h index 945a2ecd..4ce189df 100644 --- a/mir/c2mir/ppc64/cppc64.h +++ b/mir/c2mir/ppc64/cppc64.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ #include diff --git a/mir/c2mir/ppc64/mirc_ppc64_float.h b/mir/c2mir/ppc64/mirc_ppc64_float.h index bdcab189..1a14c5c5 100644 --- a/mir/c2mir/ppc64/mirc_ppc64_float.h +++ b/mir/c2mir/ppc64/mirc_ppc64_float.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 5.2.4.2.2 */ @@ -11,48 +11,48 @@ static char float_str[] "\n" "#define FLT_MANT_DIG 24\n" "#define DBL_MANT_DIG 53\n" - "#define LDBL_MANT_DIG 113\n" + "#define LDBL_MANT_DIG 106\n" "\n" "#define FLT_DECIMAL_DIG 9\n" "#define DBL_DECIMAL_DIG 17\n" - "#define LDBL_DECIMAL_DIG 36\n" - "#define FLT_DIG FLT_DECIMAL_DIG\n" - "#define DBL_DIG DBL_DECIMAL_DIG\n" - "#define LDBL_DIG LDBL_DECIMAL_DIG\n" + "#define LDBL_DECIMAL_DIG 33\n" + "#define FLT_DIG 6\n" + "#define DBL_DIG 15\n" + "#define LDBL_DIG 31\n" "\n" "#define DECIMAL_DIG LDBL_DECIMAL_DIG\n" "\n" "#define FLT_MIN_EXP (-125)\n" "#define DBL_MIN_EXP (-1021)\n" - "#define LDBL_MIN_EXP (-16381)\n" + "#define LDBL_MIN_EXP (-968)\n" "\n" "#define FLT_MIN_10_EXP (-37)\n" "#define DBL_MIN_10_EXP (-307)\n" - "#define LDBL_MIN_10_EXP (-4931)\n" + "#define LDBL_MIN_10_EXP (-291)\n" "\n" "#define FLT_MAX_EXP 128\n" "#define DBL_MAX_EXP 1024\n" - "#define LDBL_MAX_EXP 16384\n" + "#define LDBL_MAX_EXP 1024\n" "\n" "#define FLT_MAX_10_EXP 38\n" "#define DBL_MAX_10_EXP 308\n" - "#define LDBL_MAX_10_EXP 4932\n" + "#define LDBL_MAX_10_EXP 308\n" "\n" "#define FLT_MAX 0x1.fffffep+127\n" "#define DBL_MAX 0x1.fffffffffffffp+1023\n" - "#define LDBL_MAX 0x1.ffffffffffffffffffffffffffffp+16383l\n" + "#define LDBL_MAX 0x1.fffffffffffff7ffffffffffff8p+1023l\n" "\n" "#define FLT_EPSILON 0x1p-23\n" "#define DBL_EPSILON 0x1p-52\n" - "#define LDBL_EPSILON 0x1p-112l\n" + "#define LDBL_EPSILON 0x0.0000000000001p-1022l\n" "\n" "#define FLT_MIN 0x1p-126\n" "#define DBL_MIN 0x1p-1022\n" - "#define LDBL_MIN 0x1p-16382\n" + "#define LDBL_MIN 0x1p-969\n" "\n" "#define FLT_TRUE_MIN 0x1p-149\n" "#define DBL_TRUE_MIN 0x0.0000000000001p-1022\n" - "#define LDBL_TRUE_MIN 0x0.0000000000000000000000000001p-16382l\n" + "#define LDBL_TRUE_MIN 0x0.0000000000001p-1022l\n" "\n" "#define FLT_EVAL_METHOD 0\n" "#define FLT_ROUNDS 1 /* round to the nearest */\n" diff --git a/mir/c2mir/ppc64/mirc_ppc64_limits.h b/mir/c2mir/ppc64/mirc_ppc64_limits.h index c0e0d9a1..e95c8ca2 100644 --- a/mir/c2mir/ppc64/mirc_ppc64_limits.h +++ b/mir/c2mir/ppc64/mirc_ppc64_limits.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See 5.2.4.2 */ diff --git a/mir/c2mir/ppc64/mirc_ppc64_linux.h b/mir/c2mir/ppc64/mirc_ppc64_linux.h index be5b8313..17f2c03f 100644 --- a/mir/c2mir/ppc64/mirc_ppc64_linux.h +++ b/mir/c2mir/ppc64/mirc_ppc64_linux.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ static char ppc64_mirc[] @@ -94,14 +94,14 @@ static char ppc64_mirc[] "#define __DBL_MIN__ ((double) 2.22507385850720138309023271733240406e-308L)\n" "#define __DBL_EPSILON__ ((double) 2.22044604925031308084726333618164062e-16L)\n" "\n" - "#define __LDBL_MIN_EXP__ (-16381)\n" - "#define __LDBL_MAX_EXP__ 16384\n" - "#define __LDBL_DIG__ 33\n" - "#define __LDBL_DECIMAL_DIG__ 36\n" - "#define __LDBL_MANT_DIG__ 113\n" - "#define __LDBL_MAX__ 1.18973149535723176508575932662800702e+4932L\n" - "#define __LDBL_MIN__ 3.36210314311209350626267781732175260e-4932L\n" - "#define __LDBL_EPSILON__ 1.92592994438723585305597794258492732e-34L\n" + "#define __LDBL_MIN_EXP__ (-968)\n" + "#define __LDBL_MAX_EXP__ 1024\n" + "#define __LDBL_DIG__ 31\n" + "#define __LDBL_DECIMAL_DIG__ 33\n" + "#define __LDBL_MANT_DIG__ 106\n" + "#define __LDBL_MAX__ 1.79769313486231580793728971405301199e+308L\n" + "#define __LDBL_MIN__ 2.00416836000897277799610805135016205e-292L\n" + "#define __LDBL_EPSILON__ 4.94065645841246544176568792868221372e-324L\n" "\n" "typedef unsigned short char16_t;\n" "typedef unsigned int char32_t;\n" diff --git a/mir/c2mir/ppc64/mirc_ppc64_stdarg.h b/mir/c2mir/ppc64/mirc_ppc64_stdarg.h index 3ac20829..1e36f327 100644 --- a/mir/c2mir/ppc64/mirc_ppc64_stdarg.h +++ b/mir/c2mir/ppc64/mirc_ppc64_stdarg.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ static char stdarg_str[] diff --git a/mir/c2mir/ppc64/mirc_ppc64_stddef.h b/mir/c2mir/ppc64/mirc_ppc64_stddef.h index 05d63c78..76ff4f0b 100644 --- a/mir/c2mir/ppc64/mirc_ppc64_stddef.h +++ b/mir/c2mir/ppc64/mirc_ppc64_stddef.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 7.19 */ diff --git a/mir/c2mir/ppc64/mirc_ppc64_stdint.h b/mir/c2mir/ppc64/mirc_ppc64_stdint.h index aa06f822..0f5ee6dc 100644 --- a/mir/c2mir/ppc64/mirc_ppc64_stdint.h +++ b/mir/c2mir/ppc64/mirc_ppc64_stdint.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 7.20 */ diff --git a/mir/c2mir/riscv64/criscv64-ABI-code.c b/mir/c2mir/riscv64/criscv64-ABI-code.c index b381750e..ebfe770a 100644 --- a/mir/c2mir/riscv64/criscv64-ABI-code.c +++ b/mir/c2mir/riscv64/criscv64-ABI-code.c @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . riscv64 call ABI target specific code. */ @@ -7,7 +7,7 @@ typedef struct target_arg_info { int n_iregs, n_fregs; } target_arg_info_t; -static void target_init_arg_vars (c2m_ctx_t c2m_ctx, target_arg_info_t *arg_info) { +static void target_init_arg_vars (c2m_ctx_t c2m_ctx MIR_UNUSED, target_arg_info_t *arg_info) { arg_info->n_iregs = arg_info->n_fregs = 0; } @@ -17,10 +17,10 @@ static int target_return_by_addr_p (c2m_ctx_t c2m_ctx, struct type *ret_type) { } static int reg_aggregate_size (c2m_ctx_t c2m_ctx, struct type *type) { - int size; + size_t size; if (type->mode != TM_STRUCT && type->mode != TM_UNION) return -1; - return (size = type_size (c2m_ctx, type)) <= 2 * 8 ? size : -1; + return (size = type_size (c2m_ctx, type)) <= 2 * 8 ? (int) size : -1; } #define MAX_MEMBERS 2 @@ -53,10 +53,10 @@ static int small_struct_p (c2m_ctx_t c2m_ctx, struct type *type, int struct_only uint64_t nel; if (at->size->code == N_IGNORE || !(cexpr = at->size->attr)->const_p) return FALSE; - nel = cexpr->u.i_val; + nel = cexpr->c.i_val; if (!small_struct_p (c2m_ctx, at->el_type, FALSE, 0, &sub_n, sub_members)) return FALSE; if (sub_n * nel > 2) return FALSE; - for (int i = 0; i < sub_n * nel; i++) { + for (size_t i = 0; i < sub_n * nel; i++) { members[i].type = sub_members[i].type; members[i].offset = start_offset + i * type_size (c2m_ctx, at->el_type); } @@ -69,9 +69,13 @@ static int small_struct_p (c2m_ctx_t c2m_ctx, struct type *type, int struct_only el = NL_NEXT (el)) if (el->code == N_MEMBER) { decl_t decl = el->attr; + mir_size_t member_offset = decl->offset; if (decl->width == 0) continue; - if (!small_struct_p (c2m_ctx, decl->decl_spec.type, FALSE, decl->offset + start_offset, + if (decl->containing_unnamed_anon_struct_union_member != NULL) { + member_offset = 0; + } + if (!small_struct_p (c2m_ctx, decl->decl_spec.type, FALSE, member_offset + start_offset, &sub_n, sub_members)) return FALSE; if (sub_n + *members_n > 2) return FALSE; @@ -179,7 +183,7 @@ static int target_add_call_res_op (c2m_ctx_t c2m_ctx, struct type *ret_type, } static op_t target_gen_post_call_res_code (c2m_ctx_t c2m_ctx, struct type *ret_type, op_t res, - MIR_insn_t call, size_t call_ops_start) { + MIR_insn_t call MIR_UNUSED, size_t call_ops_start) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; MIR_context_t ctx = c2m_ctx->ctx; struct type_offset members[MAX_MEMBERS]; @@ -231,8 +235,7 @@ static void target_add_ret_ops (c2m_ctx_t c2m_ctx, struct type *ret_type, op_t r } else { ret_addr_reg = MIR_reg (ctx, RET_ADDR_NAME, curr_func->u.func); var = new_op (NULL, MIR_new_mem_op (ctx, MIR_T_I8, 0, ret_addr_reg, 0, 1)); - size = type_size (c2m_ctx, ret_type); - block_move (c2m_ctx, var, res, size); + block_move (c2m_ctx, var, res, type_size (c2m_ctx, ret_type)); } return; } @@ -431,7 +434,7 @@ static int target_gen_gather_arg (c2m_ctx_t c2m_ctx, const char *name, struct ty assert (!param_decl->reg_p); type = members[0].type; reg_var = get_reg_var (c2m_ctx, promote_mir_int_type (type), - gen_get_indexed_name (c2m_ctx, name, 0)); + gen_get_indexed_name (c2m_ctx, name, 0), NULL); MIR_append_insn (ctx, curr_func, MIR_new_insn (ctx, tp_mov (type), MIR_new_mem_op (ctx, type, @@ -442,7 +445,7 @@ static int target_gen_gather_arg (c2m_ctx_t c2m_ctx, const char *name, struct ty if (n == 2) { type = members[1].type; reg_var = get_reg_var (c2m_ctx, promote_mir_int_type (type), - gen_get_indexed_name (c2m_ctx, name, 1)); + gen_get_indexed_name (c2m_ctx, name, 1), NULL); MIR_append_insn (ctx, curr_func, MIR_new_insn (ctx, tp_mov (type), MIR_new_mem_op (ctx, type, diff --git a/mir/c2mir/riscv64/criscv64-code.c b/mir/c2mir/riscv64/criscv64-code.c index d5deb7f1..c6828980 100644 --- a/mir/c2mir/riscv64/criscv64-code.c +++ b/mir/c2mir/riscv64/criscv64-code.c @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ #include "../mirc.h" @@ -19,7 +19,8 @@ static string_include_t standard_includes[] #define ADJUST_VAR_ALIGNMENT(c2m_ctx, align, type) \ riscv64_adjust_var_alignment (c2m_ctx, align, type) -static int riscv64_adjust_var_alignment (c2m_ctx_t c2m_ctx, int align, struct type *type) { +static int riscv64_adjust_var_alignment (c2m_ctx_t c2m_ctx MIR_UNUSED, int align, + struct type *type MIR_UNUSED) { return align; } diff --git a/mir/c2mir/riscv64/criscv64.h b/mir/c2mir/riscv64/criscv64.h index 945a2ecd..4ce189df 100644 --- a/mir/c2mir/riscv64/criscv64.h +++ b/mir/c2mir/riscv64/criscv64.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ #include diff --git a/mir/c2mir/riscv64/mirc_riscv64_float.h b/mir/c2mir/riscv64/mirc_riscv64_float.h index 25872578..d0131907 100644 --- a/mir/c2mir/riscv64/mirc_riscv64_float.h +++ b/mir/c2mir/riscv64/mirc_riscv64_float.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 5.2.4.2.2 */ diff --git a/mir/c2mir/riscv64/mirc_riscv64_limits.h b/mir/c2mir/riscv64/mirc_riscv64_limits.h index 68a71ed9..f0457866 100644 --- a/mir/c2mir/riscv64/mirc_riscv64_limits.h +++ b/mir/c2mir/riscv64/mirc_riscv64_limits.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See 5.2.4.2 */ diff --git a/mir/c2mir/riscv64/mirc_riscv64_linux.h b/mir/c2mir/riscv64/mirc_riscv64_linux.h index 8a296a96..48aea70d 100644 --- a/mir/c2mir/riscv64/mirc_riscv64_linux.h +++ b/mir/c2mir/riscv64/mirc_riscv64_linux.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ static char riscv64_mirc[] diff --git a/mir/c2mir/riscv64/mirc_riscv64_stdarg.h b/mir/c2mir/riscv64/mirc_riscv64_stdarg.h index ac8b045a..b0cbcca9 100644 --- a/mir/c2mir/riscv64/mirc_riscv64_stdarg.h +++ b/mir/c2mir/riscv64/mirc_riscv64_stdarg.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ static char stdarg_str[] diff --git a/mir/c2mir/riscv64/mirc_riscv64_stddef.h b/mir/c2mir/riscv64/mirc_riscv64_stddef.h index 05d63c78..76ff4f0b 100644 --- a/mir/c2mir/riscv64/mirc_riscv64_stddef.h +++ b/mir/c2mir/riscv64/mirc_riscv64_stddef.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 7.19 */ diff --git a/mir/c2mir/riscv64/mirc_riscv64_stdint.h b/mir/c2mir/riscv64/mirc_riscv64_stdint.h index aa06f822..0f5ee6dc 100644 --- a/mir/c2mir/riscv64/mirc_riscv64_stdint.h +++ b/mir/c2mir/riscv64/mirc_riscv64_stdint.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 7.20 */ diff --git a/mir/c2mir/s390x/cs390x-ABI-code.c b/mir/c2mir/s390x/cs390x-ABI-code.c index 8d580534..24610e77 100644 --- a/mir/c2mir/s390x/cs390x-ABI-code.c +++ b/mir/c2mir/s390x/cs390x-ABI-code.c @@ -1,11 +1,12 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . s390x call ABI target specific code. */ typedef int target_arg_info_t; -static void target_init_arg_vars (c2m_ctx_t c2m_ctx, target_arg_info_t *arg_info) {} +static void target_init_arg_vars (c2m_ctx_t c2m_ctx MIR_UNUSED, + target_arg_info_t *arg_info MIR_UNUSED) {} static int target_return_by_addr_p (c2m_ctx_t c2m_ctx, struct type *ret_type) { return simple_return_by_addr_p (c2m_ctx, ret_type); @@ -36,12 +37,14 @@ static int reg_aggregate_p (c2m_ctx_t c2m_ctx, struct type *arg_type) { return size == 1 || size == 2 || size == 4 || size == 8; } -static MIR_type_t target_get_blk_type (c2m_ctx_t c2m_ctx, struct type *arg_type) { +static MIR_type_t target_get_blk_type (c2m_ctx_t c2m_ctx MIR_UNUSED, + struct type *arg_type MIR_UNUSED) { return MIR_T_BLK; /* one BLK is enough */ } static void target_add_arg_proto (c2m_ctx_t c2m_ctx, const char *name, struct type *arg_type, - target_arg_info_t *arg_info, VARR (MIR_var_t) * arg_vars) { + target_arg_info_t *arg_info MIR_UNUSED, + VARR (MIR_var_t) * arg_vars) { MIR_var_t var; MIR_type_t type; @@ -58,7 +61,7 @@ static void target_add_arg_proto (c2m_ctx_t c2m_ctx, const char *name, struct ty } static void target_add_call_arg_op (c2m_ctx_t c2m_ctx, struct type *arg_type, - target_arg_info_t *arg_info, op_t arg) { + target_arg_info_t *arg_info MIR_UNUSED, op_t arg) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; op_t temp; @@ -79,7 +82,7 @@ static void target_add_call_arg_op (c2m_ctx_t c2m_ctx, struct type *arg_type, } static int target_gen_gather_arg (c2m_ctx_t c2m_ctx, const char *name, struct type *arg_type, - decl_t param_decl, target_arg_info_t *arg_info) { + decl_t param_decl, target_arg_info_t *arg_info MIR_UNUSED) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; MIR_context_t ctx = c2m_ctx->ctx; MIR_op_t param_op; @@ -89,7 +92,7 @@ static int target_gen_gather_arg (c2m_ctx_t c2m_ctx, const char *name, struct ty || !reg_aggregate_p (c2m_ctx, arg_type)) return FALSE; assert (!param_decl->reg_p); - reg_var = get_reg_var (c2m_ctx, MIR_T_I64, name); + reg_var = get_reg_var (c2m_ctx, MIR_T_I64, name, NULL); param_op = MIR_new_reg_op (ctx, reg_var.reg); gen_multiple_load_store (c2m_ctx, arg_type, ¶m_op, MIR_new_mem_op (ctx, MIR_T_UNDEF, param_decl->offset, diff --git a/mir/c2mir/s390x/cs390x-code.c b/mir/c2mir/s390x/cs390x-code.c index 6544431b..c6cbba45 100644 --- a/mir/c2mir/s390x/cs390x-code.c +++ b/mir/c2mir/s390x/cs390x-code.c @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ #include "../mirc.h" @@ -18,7 +18,8 @@ static string_include_t standard_includes[] #define ADJUST_VAR_ALIGNMENT(c2m_ctx, align, type) s390x_adjust_var_alignment (c2m_ctx, align, type) -static int s390x_adjust_var_alignment (c2m_ctx_t c2m_ctx, int align, struct type *type) { +static int s390x_adjust_var_alignment (c2m_ctx_t c2m_ctx MIR_UNUSED, int align, + struct type *type MIR_UNUSED) { return align; } diff --git a/mir/c2mir/s390x/cs390x.h b/mir/c2mir/s390x/cs390x.h index dfa031e9..8b258dcc 100644 --- a/mir/c2mir/s390x/cs390x.h +++ b/mir/c2mir/s390x/cs390x.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ #include diff --git a/mir/c2mir/s390x/mirc_s390x_float.h b/mir/c2mir/s390x/mirc_s390x_float.h index 36ca51f7..2dc772e2 100644 --- a/mir/c2mir/s390x/mirc_s390x_float.h +++ b/mir/c2mir/s390x/mirc_s390x_float.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 5.2.4.2.2 */ diff --git a/mir/c2mir/s390x/mirc_s390x_limits.h b/mir/c2mir/s390x/mirc_s390x_limits.h index c0e0d9a1..e95c8ca2 100644 --- a/mir/c2mir/s390x/mirc_s390x_limits.h +++ b/mir/c2mir/s390x/mirc_s390x_limits.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See 5.2.4.2 */ diff --git a/mir/c2mir/s390x/mirc_s390x_linux.h b/mir/c2mir/s390x/mirc_s390x_linux.h index c4ca2db0..83dffac0 100644 --- a/mir/c2mir/s390x/mirc_s390x_linux.h +++ b/mir/c2mir/s390x/mirc_s390x_linux.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ static char s390x_mirc[] diff --git a/mir/c2mir/s390x/mirc_s390x_stdarg.h b/mir/c2mir/s390x/mirc_s390x_stdarg.h index e7aca7b4..d4db4d24 100644 --- a/mir/c2mir/s390x/mirc_s390x_stdarg.h +++ b/mir/c2mir/s390x/mirc_s390x_stdarg.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ static char stdarg_str[] diff --git a/mir/c2mir/s390x/mirc_s390x_stddef.h b/mir/c2mir/s390x/mirc_s390x_stddef.h index 05d63c78..76ff4f0b 100644 --- a/mir/c2mir/s390x/mirc_s390x_stddef.h +++ b/mir/c2mir/s390x/mirc_s390x_stddef.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 7.19 */ diff --git a/mir/c2mir/s390x/mirc_s390x_stdint.h b/mir/c2mir/s390x/mirc_s390x_stdint.h index aa06f822..0f5ee6dc 100644 --- a/mir/c2mir/s390x/mirc_s390x_stdint.h +++ b/mir/c2mir/s390x/mirc_s390x_stdint.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* See C11 7.20 */ diff --git a/mir/c2mir/x86_64/cx86_64-ABI-code.c b/mir/c2mir/x86_64/cx86_64-ABI-code.c index 78de581e..a84ff2f7 100644 --- a/mir/c2mir/x86_64/cx86_64-ABI-code.c +++ b/mir/c2mir/x86_64/cx86_64-ABI-code.c @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . x86_64 ABI target specific code. */ @@ -34,25 +34,24 @@ static MIR_type_t get_result_type (MIR_type_t arg_type1, MIR_type_t arg_type2) { } static int classify_arg (c2m_ctx_t c2m_ctx, struct type *type, MIR_type_t types[MAX_QWORDS], - int bit_field_p) { - size_t size = type_size (c2m_ctx, type); - int i, n_el_qwords, n_qwords = (size + 7) / 8; + int bit_field_p MIR_UNUSED) { + size_t size = type_size (c2m_ctx, type), n_qwords = (size + 7) / 8; MIR_type_t mir_type; if (type->mode == TM_STRUCT || type->mode == TM_UNION || type->mode == TM_ARR) { - MIR_type_t subtypes[MAX_QWORDS]; - if (n_qwords > MAX_QWORDS) return 0; /* too big aggregate */ #ifndef _WIN32 - for (i = 0; i < n_qwords; i++) types[i] = (MIR_type_t) NO_CLASS; + MIR_type_t subtypes[MAX_QWORDS]; + int i, n_el_qwords; + for (i = 0; (size_t) i < n_qwords; i++) types[i] = (MIR_type_t) NO_CLASS; switch (type->mode) { case TM_ARR: { /* Arrays are handled as small records. */ n_el_qwords = classify_arg (c2m_ctx, type->u.arr_type->el_type, subtypes, FALSE); if (n_el_qwords == 0) return 0; /* make full types: */ - for (i = 0; i < n_qwords; i++) + for (i = 0; (size_t) i < n_qwords; i++) types[i] = get_result_type (types[i], subtypes[i % n_el_qwords]); break; } @@ -62,8 +61,16 @@ static int classify_arg (c2m_ctx_t c2m_ctx, struct type *type, MIR_type_t types[ el = NL_NEXT (el)) if (el->code == N_MEMBER) { decl_t decl = el->attr; - int start_qword = decl->offset / 8; - int end_qword = (decl->offset + type_size (c2m_ctx, decl->decl_spec.type) - 1) / 8; + mir_size_t offset = decl->offset; + node_t container; + if ((container = decl->containing_unnamed_anon_struct_union_member) != NULL) { + decl_t decl2 = container->attr; + assert (decl2->decl_spec.type->mode == TM_STRUCT + || decl2->decl_spec.type->mode == TM_UNION); + offset -= decl2->offset; + } + int start_qword = offset / 8; + int end_qword = (offset + type_size (c2m_ctx, decl->decl_spec.type) - 1) / 8; int span_qwords = end_qword - start_qword + 1; if (decl->bit_offset >= 0) { @@ -72,7 +79,7 @@ static int classify_arg (c2m_ctx_t c2m_ctx, struct type *type, MIR_type_t types[ n_el_qwords = classify_arg (c2m_ctx, decl->decl_spec.type, subtypes, decl->bit_offset >= 0); if (n_el_qwords == 0) return 0; - for (i = 0; i < n_el_qwords && (i + start_qword) < n_qwords; i++) { + for (i = 0; i < n_el_qwords && (size_t) (i + start_qword) < n_qwords; i++) { types[i + start_qword] = get_result_type (subtypes[i], types[i + start_qword]); if (span_qwords > n_el_qwords) types[i + start_qword + 1] @@ -86,7 +93,7 @@ static int classify_arg (c2m_ctx_t c2m_ctx, struct type *type, MIR_type_t types[ if (n_qwords > 2) return 0; /* as we don't have vector values (see SSEUP_CLASS) */ - for (i = 0; i < n_qwords; i++) { + for (i = 0; (size_t) i < n_qwords; i++) { if (types[i] == MIR_T_UNDEF) return 0; /* pass in memory if a word class is memory. */ if ((enum add_arg_class) types[i] == X87UP_CLASS && (i == 0 || types[i - 1] != MIR_T_LD)) return 0; @@ -114,7 +121,7 @@ typedef struct target_arg_info { int n_iregs, n_fregs; } target_arg_info_t; -static void target_init_arg_vars (c2m_ctx_t c2m_ctx, target_arg_info_t *arg_info) { +static void target_init_arg_vars (c2m_ctx_t c2m_ctx MIR_UNUSED, target_arg_info_t *arg_info) { arg_info->n_iregs = arg_info->n_fregs = 0; } @@ -235,7 +242,7 @@ static int target_add_call_res_op (c2m_ctx_t c2m_ctx, struct type *ret_type, } static op_t target_gen_post_call_res_code (c2m_ctx_t c2m_ctx, struct type *ret_type, op_t res, - MIR_insn_t call, size_t call_ops_start) { + MIR_insn_t call MIR_UNUSED, size_t call_ops_start) { gen_ctx_t gen_ctx = c2m_ctx->gen_ctx; MIR_context_t ctx = c2m_ctx->ctx; MIR_type_t type; @@ -268,7 +275,7 @@ static void target_add_ret_ops (c2m_ctx_t c2m_ctx, struct type *ret_type, op_t r MIR_insn_t insn; MIR_reg_t ret_addr_reg; op_t temp, var; - int i, size, n_qwords; + int i, n_qwords; if (void_type_p (ret_type)) return; n_qwords = process_ret_type (c2m_ctx, ret_type, qword_types); @@ -288,8 +295,7 @@ static void target_add_ret_ops (c2m_ctx_t c2m_ctx, struct type *ret_type, op_t r } else { ret_addr_reg = MIR_reg (ctx, RET_ADDR_NAME, curr_func->u.func); var = new_op (NULL, MIR_new_mem_op (ctx, MIR_T_I8, 0, ret_addr_reg, 0, 1)); - size = type_size (c2m_ctx, ret_type); - block_move (c2m_ctx, var, res, size); + block_move (c2m_ctx, var, res, type_size (c2m_ctx, ret_type)); } } @@ -401,7 +407,8 @@ static void target_add_call_arg_op (c2m_ctx_t c2m_ctx, struct type *arg_type, } } -static int target_gen_gather_arg (c2m_ctx_t c2m_ctx, const char *name, struct type *arg_type, - decl_t param_decl, target_arg_info_t *arg_info) { +static int target_gen_gather_arg (c2m_ctx_t c2m_ctx MIR_UNUSED, const char *name MIR_UNUSED, + struct type *arg_type MIR_UNUSED, decl_t param_decl MIR_UNUSED, + target_arg_info_t *arg_info MIR_UNUSED) { return FALSE; } diff --git a/mir/c2mir/x86_64/cx86_64-code.c b/mir/c2mir/x86_64/cx86_64-code.c index 682fef05..f57f6917 100644 --- a/mir/c2mir/x86_64/cx86_64-code.c +++ b/mir/c2mir/x86_64/cx86_64-code.c @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ #include "../mirc.h" diff --git a/mir/c2mir/x86_64/cx86_64.h b/mir/c2mir/x86_64/cx86_64.h index 83a912ef..13293a5a 100644 --- a/mir/c2mir/x86_64/cx86_64.h +++ b/mir/c2mir/x86_64/cx86_64.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ #include diff --git a/mir/c2mir/x86_64/mirc_x86_64_float.h b/mir/c2mir/x86_64/mirc_x86_64_float.h index 90d4675c..34bbf188 100644 --- a/mir/c2mir/x86_64/mirc_x86_64_float.h +++ b/mir/c2mir/x86_64/mirc_x86_64_float.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2019-2023 Vladimir Makarov . + Copyright (C) 2019-2024 Vladimir Makarov . */ /* See C11 5.2.4.2.2 */ diff --git a/mir/c2mir/x86_64/mirc_x86_64_limits.h b/mir/c2mir/x86_64/mirc_x86_64_limits.h index 19efff4f..843513ce 100644 --- a/mir/c2mir/x86_64/mirc_x86_64_limits.h +++ b/mir/c2mir/x86_64/mirc_x86_64_limits.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2019-2023 Vladimir Makarov . + Copyright (C) 2019-2024 Vladimir Makarov . */ /* See 5.2.4.2 */ diff --git a/mir/c2mir/x86_64/mirc_x86_64_linux.h b/mir/c2mir/x86_64/mirc_x86_64_linux.h index 85eeb0d3..29533688 100644 --- a/mir/c2mir/x86_64/mirc_x86_64_linux.h +++ b/mir/c2mir/x86_64/mirc_x86_64_linux.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2019-2023 Vladimir Makarov . + Copyright (C) 2019-2024 Vladimir Makarov . */ static char x86_64_mirc[] diff --git a/mir/c2mir/x86_64/mirc_x86_64_stdarg.h b/mir/c2mir/x86_64/mirc_x86_64_stdarg.h index c15feb37..3ef106d1 100644 --- a/mir/c2mir/x86_64/mirc_x86_64_stdarg.h +++ b/mir/c2mir/x86_64/mirc_x86_64_stdarg.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2019-2023 Vladimir Makarov . + Copyright (C) 2019-2024 Vladimir Makarov . */ /* See C11 7.16 and https://www.uclibc.org/docs/psABI-x86_64.pdf */ diff --git a/mir/c2mir/x86_64/mirc_x86_64_stddef.h b/mir/c2mir/x86_64/mirc_x86_64_stddef.h index 0b84adf6..77e6d109 100644 --- a/mir/c2mir/x86_64/mirc_x86_64_stddef.h +++ b/mir/c2mir/x86_64/mirc_x86_64_stddef.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2019-2023 Vladimir Makarov . + Copyright (C) 2019-2024 Vladimir Makarov . */ /* See C11 7.19 */ diff --git a/mir/c2mir/x86_64/mirc_x86_64_stdint.h b/mir/c2mir/x86_64/mirc_x86_64_stdint.h index 9ccbf61a..4d1c6918 100644 --- a/mir/c2mir/x86_64/mirc_x86_64_stdint.h +++ b/mir/c2mir/x86_64/mirc_x86_64_stdint.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2019-2023 Vladimir Makarov . + Copyright (C) 2019-2024 Vladimir Makarov . */ /* See C11 7.20 */ diff --git a/mir/c2mir/x86_64/mirc_x86_64_win.h b/mir/c2mir/x86_64/mirc_x86_64_win.h index 6b0c998c..073271e7 100644 --- a/mir/c2mir/x86_64/mirc_x86_64_win.h +++ b/mir/c2mir/x86_64/mirc_x86_64_win.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2019-2023 Vladimir Makarov . + Copyright (C) 2019-2024 Vladimir Makarov . */ static char x86_64_mirc[] diff --git a/mir/mir-aarch64.c b/mir/mir-aarch64.c index a9372598..4ece324d 100644 --- a/mir/mir-aarch64.c +++ b/mir/mir-aarch64.c @@ -1,7 +1,9 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ +#include "mir-aarch64.h" + /* x31 - sp; x30 - link reg; x29 - fp; x0-x7, v0-v7 - arg/result regs; x19-x29, v8-v15 - callee-saved (only bottom 64-bits are saved for v8-v15); x9-x15, v0-v7, v16-v31 - temp regs @@ -98,7 +100,7 @@ void *va_arg_builtin (void *p, uint64_t t) { return a; } -void va_block_arg_builtin (void *res, void *p, size_t s, uint64_t ncase) { +void va_block_arg_builtin (void *res, void *p, size_t s, uint64_t ncase MIR_UNUSED) { struct aarch64_va_list *va = p; #if defined(__APPLE__) void *a = (void *) va->arg_area; @@ -108,7 +110,7 @@ void va_block_arg_builtin (void *res, void *p, size_t s, uint64_t ncase) { a = *(void **) a; va->arg_area++; } - memcpy (res, a, s); + if (res != NULL) memcpy (res, a, s); #else void *a; long size = (s + 7) / 8 * 8; @@ -117,7 +119,7 @@ void va_block_arg_builtin (void *res, void *p, size_t s, uint64_t ncase) { a = va->__stack; va->__stack = (char *) va->__stack + size; va->__gr_offs += size; - memcpy (res, a, s); + if (res != NULL) memcpy (res, a, s); return; } if (size > 2 * 8) size = 8; @@ -129,11 +131,11 @@ void va_block_arg_builtin (void *res, void *p, size_t s, uint64_t ncase) { va->__stack = (char *) va->__stack + size; } if (s > 2 * 8) a = *(void **) a; /* address */ - memcpy (res, a, s); + if (res != NULL) memcpy (res, a, s); #endif } -void va_start_interp_builtin (MIR_context_t ctx, void *p, void *a) { +void va_start_interp_builtin (MIR_context_t ctx MIR_UNUSED, void *p, void *a) { struct aarch64_va_list *va = p; va_list *vap = a; @@ -141,7 +143,7 @@ void va_start_interp_builtin (MIR_context_t ctx, void *p, void *a) { *va = *(struct aarch64_va_list *) vap; } -void va_end_interp_builtin (MIR_context_t ctx, void *p) {} +void va_end_interp_builtin (MIR_context_t ctx MIR_UNUSED, void *p MIR_UNUSED) {} static int setup_imm64_insns (uint32_t *to, int reg, uint64_t imm64) { /* xd=imm64 */ @@ -198,10 +200,9 @@ static void gen_call_addr (VARR (uint8_t) * insn_varr, void *base_addr, int temp push_insns (insn_varr, &insn, sizeof (insn)); } -#define NOP 0xd503201f - void *_MIR_get_thunk (MIR_context_t ctx) { - int pat[5] = {NOP, NOP, NOP, NOP, NOP}; /* maximal size thunk -- see _MIR_redirect_thunk */ + /* maximal size thunk -- see _MIR_redirect_thunk */ + int pat[4] = {TARGET_NOP, TARGET_NOP, TARGET_NOP, TARGET_NOP}; return _MIR_publish_code (ctx, (uint8_t *) pat, sizeof (pat)); } @@ -210,21 +211,32 @@ void _MIR_redirect_thunk (MIR_context_t ctx, void *thunk, void *to) { static const uint32_t branch_pat1 = 0xd61f0120; /* br x9 */ static const uint32_t branch_pat2 = 0x14000000; /* b x */ int64_t offset = (uint32_t *) to - (uint32_t *) thunk; - uint32_t code[5]; + uint32_t code[4]; mir_assert (((uint64_t) thunk & 0x3) == 0 && ((uint64_t) to & 0x3) == 0); /* alignment */ if (-(int64_t) MAX_BR_OFFSET <= offset && offset < (int64_t) MAX_BR_OFFSET) { code[0] = branch_pat2 | ((uint32_t) offset & BR_OFFSET_MASK); _MIR_change_code (ctx, thunk, (uint8_t *) &code[0], sizeof (code[0])); } else { - int n = setup_imm64_insns (code, 9, (uint64_t) to); - - mir_assert (n == 4); - code[4] = branch_pat1; + code[0] = 0x58000049; /* ldr x9,8 (pc-relative) */ + code[1] = branch_pat1; + *(void **) &code[2] = to; _MIR_change_code (ctx, thunk, (uint8_t *) code, sizeof (code)); } } +void *_MIR_get_thunk_addr (MIR_context_t ctx MIR_UNUSED, void *thunk) { + void *addr; + int short_p = (*(uint32_t *) thunk >> BR_OFFSET_BITS) == 0x5; + if (short_p) { + int32_t offset = *(uint32_t *) thunk & BR_OFFSET_MASK; + addr = (uint8_t *) thunk + ((offset << (32 - BR_OFFSET_BITS)) >> (30 - BR_OFFSET_BITS)); + } else { + addr = *(void **) ((char *) thunk + 8); + } + return addr; +} + static void gen_blk_mov (VARR (uint8_t) * insn_varr, uint32_t offset, uint32_t addr_offset, uint32_t qwords, uint32_t addr_reg) { static const uint32_t blk_mov_pat[] = { @@ -300,7 +312,7 @@ static const uint32_t stld_pat = 0x3d800000; /* str q, [xn|sp], offset */ x10=mem[x19,]; res_reg=mem[x10]; ... pop x19, x30; ret x30. */ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, size_t nargs, - _MIR_arg_desc_t *arg_descs, size_t arg_vars_num) { + _MIR_arg_desc_t *arg_descs, size_t arg_vars_num MIR_UNUSED) { static const uint32_t prolog[] = { 0xa9bf7bf3, /* stp x19,x30,[sp, -16]! */ 0xd10003ff, /* sub sp,sp, */ @@ -359,12 +371,12 @@ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, s pat = ld_pat | offset_imm | addr_reg; push_insns (code, &pat, sizeof (pat)); if (n_xregs + qwords <= 8) { - for (int n = 0; n < qwords; n++) { + for (uint32_t n = 0; n < qwords; n++) { pat = gen_ld_pat | (((n * 8) >> scale) << 10) | (n_xregs + n) | (addr_reg << 5); push_insns (code, &pat, sizeof (pat)); } } else { - for (int n = 0; n < qwords; n++) { + for (uint32_t n = 0; n < qwords; n++) { pat = gen_ld_pat | (((n * 8) >> scale) << 10) | temp_reg | (addr_reg << 5); push_insns (code, &pat, sizeof (pat)); pat = st_pat | ((sp_offset >> scale) << 10) | temp_reg | (sp << 5); @@ -450,7 +462,7 @@ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, s res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); #if 0 if (getenv ("MIR_code_dump") != NULL) - _MIR_dump_code ("ffi:", 0, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); + _MIR_dump_code ("ffi:", VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); #endif VARR_DESTROY (uint8_t, code); return res; @@ -542,7 +554,7 @@ void *_MIR_get_interp_shim (MIR_context_t ctx, MIR_item_t func_item, void *handl /* passing by one or two qwords */ if (n_xregs + qwords <= 8) { /* passed by hard regs: str xreg, offset[sp]; str xreg, offset+8[sp] */ - for (int n = 0; n < qwords; n++) { + for (uint32_t n = 0; n < qwords; n++) { pat = st_pat | ((sp_offset >> scale) << 10) | n_xregs++ | (sp << 5); sp_offset += 8; push_insns (code, &pat, sizeof (pat)); @@ -656,44 +668,172 @@ void *_MIR_get_interp_shim (MIR_context_t ctx, MIR_item_t func_item, void *handl res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); #if 0 if (getenv ("MIR_code_dump") != NULL) - _MIR_dump_code (func->name, 0, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); + _MIR_dump_code (func->name, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); #endif VARR_DESTROY (uint8_t, code); return res; } -/* Save regs x8, x0-x7, q0-q7; x9 = call hook_address (ctx, called_func); restore regs; br x9 */ +/* Save x0,x1; x0=ctx; x1=called_func; x10=hook_address;goto wrap_end. */ void *_MIR_get_wrapper (MIR_context_t ctx, MIR_item_t called_func, void *hook_address) { - static const uint32_t jmp_insn = 0xd61f0120; /* br x9 */ - static const uint32_t move_insn = 0xaa0003e9; /* mov x9, x0 */ - static const uint32_t save_fplr = 0xa9bf7bfd; /* stp R29, R30, [SP, #-16]! */ - static const uint32_t restore_fplr = 0xa8c17bfd; /* ldp R29, R30, SP, #16 */ + static const uint32_t save_insn = 0xa9bf07e0; /* stp R0, R1, [SP, #-16]! */ + static const uint32_t jmp_pat = 0x14000000; /* jmp */ + uint32_t insn; uint8_t *base_addr, *curr_addr, *res_code = NULL; - size_t len = sizeof (save_insns) + sizeof (restore_insns); /* initial code length */ + size_t len = 5 * 4; /* initial len */ VARR (uint8_t) * code; - mir_mutex_lock (&code_mutex); VARR_CREATE (uint8_t, code, 128); - for (;;) { /* dealing with moving code to another page */ + for (;;) { /* dealing with moving code to another page as the immediate call is pc relative */ curr_addr = base_addr = _MIR_get_new_code_addr (ctx, len); if (curr_addr == NULL) break; VARR_TRUNC (uint8_t, code, 0); - push_insns (code, &save_fplr, sizeof (save_fplr)); + push_insns (code, &save_insn, sizeof (save_insn)); curr_addr += 4; - push_insns (code, save_insns, sizeof (save_insns)); - curr_addr += sizeof (save_insns); - curr_addr += gen_mov_addr (code, 0, ctx); /*mov x0,ctx */ - curr_addr += gen_mov_addr (code, 1, called_func); /*mov x1,called_func */ - gen_call_addr (code, curr_addr, 10, hook_address); /*call , use x10 as temp */ - push_insns (code, &move_insn, sizeof (move_insn)); - push_insns (code, restore_insns, sizeof (restore_insns)); - push_insns (code, &restore_fplr, sizeof (restore_fplr)); - push_insns (code, &jmp_insn, sizeof (jmp_insn)); + curr_addr += gen_mov_addr (code, 0, ctx); /*mov x0,ctx */ + curr_addr += gen_mov_addr (code, 1, called_func); /*mov x1,called_func */ + curr_addr += gen_mov_addr (code, 10, hook_address); /*mov x10,hook_address */ + int64_t offset = (uint32_t *) wrapper_end_addr - (uint32_t *) curr_addr; + mir_assert (-(int64_t) MAX_BR_OFFSET <= offset && offset < (int64_t) MAX_BR_OFFSET); + insn = jmp_pat | ((uint32_t) offset & BR_OFFSET_MASK); + push_insns (code, &insn, sizeof (insn)); len = VARR_LENGTH (uint8_t, code); res_code = _MIR_publish_code_by_addr (ctx, base_addr, VARR_ADDR (uint8_t, code), len); if (res_code != NULL) break; } VARR_DESTROY (uint8_t, code); - mir_mutex_unlock (&code_mutex); return res_code; } + +void *_MIR_get_wrapper_end (MIR_context_t ctx) { + static const uint32_t wrap_end[] = { + 0xa9bf7bfd, /* stp R29, R30, [SP, #-16]! */ + 0xa9bf1fe6, /* stp R6, R7, [SP, #-16]! */ + 0xa9bf17e4, /* stp R4, R5, [SP, #-16]! */ + 0xa9bf0fe2, /* stp R2, R3, [SP, #-16]! */ + 0xd10043ff, /* sub SP, SP, #16 */ + 0xf90007e8, /* str x8, [SP, #8] */ + 0xadbf1fe6, /* stp Q6, Q7, [SP, #-32]! */ + 0xadbf17e4, /* stp Q4, Q5, [SP, #-32]! */ + 0xadbf0fe2, /* stp Q2, Q3, [SP, #-32]! */ + 0xadbf07e0, /* stp Q0, Q1, [SP, #-32]! */ + 0xd63f0140, /* call *x10 */ + 0xaa0003e9, /* mov x9, x0 */ + 0xacc107e0, /* ldp Q0, Q1, SP, #32 */ + 0xacc10fe2, /* ldp Q2, Q3, SP, #32 */ + 0xacc117e4, /* ldp Q4, Q5, SP, #32 */ + 0xacc11fe6, /* ldp Q6, Q7, SP, #32 */ + 0xf94007e8, /* ldr x8, [SP, #8] */ + 0x910043ff, /* add SP, SP, #16 */ + 0xa8c10fe2, /* ldp R2, R3, SP, #16 */ + 0xa8c117e4, /* ldp R4, R5, SP, #16 */ + 0xa8c11fe6, /* ldp R6, R7, SP, #16 */ + 0xa8c17bfd, /* ldp R29, R30, SP, #16 */ + 0xa8c107e0, /* ldp R0, R1, SP, #16 */ + 0xd61f0120, /* br x9 */ + }; + uint8_t *res_code = NULL; + VARR (uint8_t) * code; + size_t len; + + VARR_CREATE (uint8_t, code, 128); + push_insns (code, wrap_end, sizeof (wrap_end)); + len = VARR_LENGTH (uint8_t, code); + res_code = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), len); + VARR_DESTROY (uint8_t, code); + return res_code; +} + +/* r9=; (b|br) handler ??? mutex free */ +void *_MIR_get_bb_thunk (MIR_context_t ctx, void *bb_version, void *handler) { + /* maximal size thunk -- see _MIR_redirect_thunk */ + uint32_t pat[5] = {TARGET_NOP, TARGET_NOP, TARGET_NOP, TARGET_NOP, TARGET_NOP}; + void *res; + size_t offset; + VARR (uint8_t) * code; + + VARR_CREATE (uint8_t, code, 64); + offset = gen_mov_addr (code, 9, bb_version); /* x9 = bb_version */ + push_insns (code, pat, sizeof (pat)); + res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); + _MIR_redirect_thunk (ctx, (uint8_t *) res + offset, handler); +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("bb thunk:", res, VARR_LENGTH (uint8_t, code)); +#endif + VARR_DESTROY (uint8_t, code); + return res; +} + +/* change to (b|br) to */ +void _MIR_replace_bb_thunk (MIR_context_t ctx, void *thunk, void *to) { + _MIR_redirect_thunk (ctx, thunk, to); +} + +static const uint32_t save_fplr = 0xa9bf7bfd; /* stp R29, R30, [SP, #-16]! */ +static const uint32_t restore_fplr = 0xa8c17bfd; /* ldp R29, R30, SP, #16 */ + +static const uint32_t save_insns2[] = { + /* save r10-r18,v16-v31: should be used only right after save_insn */ + 0xf90043ea, /* str R10, [SP, #128] */ + 0xa9bf4bf1, /* stp R17, R18, [SP, #-16]! */ + 0xa9bf43ef, /* stp R15, R16, [SP, #-16]! */ + 0xa9bf3bed, /* stp R13, R14, [SP, #-16]! */ + 0xa9bf33eb, /* stp R11, R12, [SP, #-16]! */ + 0xadbf7ffe, /* stp Q30, Q31, [SP, #-32]! */ + 0xadbf77fc, /* stp Q28, Q29, [SP, #-32]! */ + 0xadbf6ffa, /* stp Q26, Q27, [SP, #-32]! */ + 0xadbf67f8, /* stp Q24, Q25, [SP, #-32]! */ + 0xadbf5ff6, /* stp Q22, Q23, [SP, #-32]! */ + 0xadbf57f4, /* stp Q20, Q21, [SP, #-32]! */ + 0xadbf4ff2, /* stp Q18, Q19, [SP, #-32]! */ + 0xadbf47f0, /* stp Q16, Q17, [SP, #-32]! */ +}; +static const uint32_t restore_insns2[] = { + /* restore r10-r18,v16-v32: should be used only right before restore_insns */ + 0xacc147f0, /* ldp Q16, Q17, SP, #32 */ + 0xacc14ff2, /* ldp Q18, Q19, SP, #32 */ + 0xacc157f4, /* ldp Q20, Q21, SP, #32 */ + 0xacc15ff6, /* ldp Q22, Q23, SP, #32 */ + 0xacc167f8, /* ldp Q24, Q25, SP, #32 */ + 0xacc16ffa, /* ldp Q26, Q27, SP, #32 */ + 0xacc177fc, /* ldp Q28, Q29, SP, #32 */ + 0xacc17ffe, /* ldp Q30, Q31, SP, #32 */ + 0xa8c133eb, /* ldp R11, R12, SP, #16 */ + 0xa8c13bed, /* ldp R13, R14, SP, #16 */ + 0xa8c143ef, /* ldp R15, R16, SP, #16 */ + 0xa8c14bf1, /* ldp R17, R18, SP, #16 */ + 0xf94043ea, /* ldr R10, [SP, #128] */ +}; + +/* save all clobbered regs but 9; r9 = call hook_address (data, r9); restore regs; br r9 + r9 is a generator temp reg which is not used across bb borders. */ +void *_MIR_get_bb_wrapper (MIR_context_t ctx, void *data, void *hook_address) { + static const uint32_t wrap_end = 0xd61f0120; /* br x9 */ + static const uint32_t call_pat[] = { + 0xaa0903e1, /* mov x1,x9 */ + 0xd63f0140, /* blr x10 */ + 0xaa0003e9, /* mov x9,x0 */ + }; + void *res; + VARR (uint8_t) * code; + + VARR_CREATE (uint8_t, code, 128); + push_insns (code, &save_fplr, sizeof (save_fplr)); + push_insns (code, save_insns, sizeof (save_insns)); + push_insns (code, save_insns2, sizeof (save_insns2)); + gen_mov_addr (code, 10, hook_address); /* x10 = hook_address */ + gen_mov_addr (code, 0, data); /* x0 = data */ + push_insns (code, call_pat, sizeof (call_pat)); + push_insns (code, restore_insns2, sizeof (restore_insns2)); + push_insns (code, restore_insns, sizeof (restore_insns)); + push_insns (code, &restore_fplr, sizeof (restore_fplr)); + push_insns (code, &wrap_end, sizeof (wrap_end)); + res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("bb wrapper:", VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); +#endif + VARR_DESTROY (uint8_t, code); + return res; +} diff --git a/mir/mir-aarch64.h b/mir/mir-aarch64.h new file mode 100644 index 00000000..a50f995c --- /dev/null +++ b/mir/mir-aarch64.h @@ -0,0 +1,61 @@ +/* This file is a part of MIR project. + Copyright (C) 2018-2024 Vladimir Makarov . + A common include file for mir-aarch64.c and mir-gen-aarch64.c +*/ + +#include "mir.h" + +#define TARGET_NOP 0xd503201f + +#define HREG_EL(h) h##_HARD_REG +#define REP_SEP , +enum { + REP8 (HREG_EL, R0, R1, R2, R3, R4, R5, R6, R7), + REP8 (HREG_EL, R8, R9, R10, R11, R12, R13, R14, R15), + REP8 (HREG_EL, R16, R17, R18, R19, R20, R21, R22, R23), + REP8 (HREG_EL, R24, R25, R26, R27, R28, R29, R30, SP), + ZR_HARD_REG = SP_HARD_REG, + REP8 (HREG_EL, V0, V1, V2, V3, V4, V5, V6, V7), + REP8 (HREG_EL, V8, V9, V10, V11, V12, V13, V14, V15), + REP8 (HREG_EL, V16, V17, V18, V19, V20, V21, V22, V23), + REP8 (HREG_EL, V24, V25, V26, V27, V28, V29, V30, V31), +}; +#undef REP_SEP + +static const char *const target_hard_reg_names[] = { + "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12", + "r13", "r14", "r15", "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23", "r24", "r25", + "r26", "r27", "r28", "r29", "r30", "sp", "v0", "v1", "v2", "v3", "v4", "v5", "v6", + "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", + "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", +}; + +#define MAX_HARD_REG V31_HARD_REG + +/* Hard regs not used in machinized code, preferably call used ones. */ +static const MIR_reg_t TEMP_INT_HARD_REG1 = R9_HARD_REG, TEMP_INT_HARD_REG2 = R10_HARD_REG; +static const MIR_reg_t TEMP_FLOAT_HARD_REG1 = V16_HARD_REG, TEMP_FLOAT_HARD_REG2 = V17_HARD_REG; +static const MIR_reg_t TEMP_DOUBLE_HARD_REG1 = V16_HARD_REG, TEMP_DOUBLE_HARD_REG2 = V17_HARD_REG; +static const MIR_reg_t TEMP_LDOUBLE_HARD_REG1 = V16_HARD_REG; +static const MIR_reg_t TEMP_LDOUBLE_HARD_REG2 = V17_HARD_REG; + +static inline int target_hard_reg_type_ok_p (MIR_reg_t hard_reg, MIR_type_t type) { + assert (hard_reg <= MAX_HARD_REG); + return MIR_fp_type_p (type) ? hard_reg >= V0_HARD_REG : hard_reg < V0_HARD_REG; +} + +static inline int target_fixed_hard_reg_p (MIR_reg_t hard_reg) { + assert (hard_reg <= MAX_HARD_REG); +#if defined(__APPLE__) + if (hard_reg == R18_HARD_REG) return TRUE; +#endif + return (hard_reg == R29_HARD_REG /*FP*/ || hard_reg == SP_HARD_REG + || hard_reg == TEMP_INT_HARD_REG1 || hard_reg == TEMP_INT_HARD_REG2 + || hard_reg == TEMP_FLOAT_HARD_REG1 || hard_reg == TEMP_FLOAT_HARD_REG2 + || hard_reg == TEMP_DOUBLE_HARD_REG1 || hard_reg == TEMP_DOUBLE_HARD_REG2 + || hard_reg == TEMP_LDOUBLE_HARD_REG1 || hard_reg == TEMP_LDOUBLE_HARD_REG2); +} + +static int target_locs_num (MIR_reg_t loc, MIR_type_t type) { + return loc > MAX_HARD_REG && type == MIR_T_LD ? 2 : 1; +} diff --git a/mir/mir-bitmap.h b/mir/mir-bitmap.h index aab22622..76d10dd0 100644 --- a/mir/mir-bitmap.h +++ b/mir/mir-bitmap.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ #ifndef MIR_BITMAP_H @@ -54,6 +54,10 @@ static inline bitmap_t bitmap_create (void) { return bitmap_create2 (0); } static inline void bitmap_destroy (bitmap_t bm) { VARR_DESTROY (bitmap_el_t, bm); } +static inline size_t bitmap_size (bitmap_t bm) { + return VARR_CAPACITY (bitmap_el_t, bm) * sizeof (bitmap_el_t); +} + static inline void bitmap_clear (bitmap_t bm) { VARR_TRUNC (bitmap_el_t, bm, 0); } static inline void bitmap_expand (bitmap_t bm, size_t nb) { @@ -213,6 +217,38 @@ static inline size_t bitmap_bit_count (const_bitmap_t bm) { return count; } +/* Return min bit number in BM. Return 0 for empty bitmap. */ +static inline size_t bitmap_bit_min (const_bitmap_t bm) { + size_t i, len = VARR_LENGTH (bitmap_el_t, bm); + bitmap_el_t el, *addr = VARR_ADDR (bitmap_el_t, bm); + int count; + + for (i = 0; i < len; i++) { + if ((el = addr[i]) != 0) { + for (count = 0; el != 0; el >>= 1, count++) + if (el & 1) return i * BITMAP_WORD_BITS + count; + } + } + return 0; +} + +/* Return max bit number in BM. Return 0 for empty bitmap. */ +static inline size_t bitmap_bit_max (const_bitmap_t bm) { + size_t i, len = VARR_LENGTH (bitmap_el_t, bm); + bitmap_el_t el, *addr = VARR_ADDR (bitmap_el_t, bm); + int count; + + if (len == 0) return 0; + for (i = len - 1;; i--) { + if ((el = addr[i]) != 0) { + for (count = BITMAP_WORD_BITS - 1; count >= 0; count--) + if ((el >> count) & 1) return i * BITMAP_WORD_BITS + count; + } + if (i == 0) break; + } + return 0; +} + static inline int bitmap_op2 (bitmap_t dst, const_bitmap_t src1, const_bitmap_t src2, bitmap_el_t (*op) (bitmap_el_t, bitmap_el_t)) { size_t i, len, bound, src1_len, src2_len; diff --git a/mir/mir-dlist.h b/mir/mir-dlist.h index f34d1365..39bc6887 100644 --- a/mir/mir-dlist.h +++ b/mir/mir-dlist.h @@ -1,5 +1,5 @@ /* This file is part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ /* Typed doubly linked lists. */ diff --git a/mir/mir-gen-aarch64.c b/mir/mir-gen-aarch64.c index f123f601..e0ad7caf 100644 --- a/mir/mir-gen-aarch64.c +++ b/mir/mir-gen-aarch64.c @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ static void fancy_abort (int code) { @@ -10,53 +10,13 @@ static void fancy_abort (int code) { #include -#define HREG_EL(h) h##_HARD_REG -#define REP_SEP , -enum { - REP8 (HREG_EL, R0, R1, R2, R3, R4, R5, R6, R7), - REP8 (HREG_EL, R8, R9, R10, R11, R12, R13, R14, R15), - REP8 (HREG_EL, R16, R17, R18, R19, R20, R21, R22, R23), - REP8 (HREG_EL, R24, R25, R26, R27, R28, R29, R30, SP), - ZR_HARD_REG = SP_HARD_REG, - REP8 (HREG_EL, V0, V1, V2, V3, V4, V5, V6, V7), - REP8 (HREG_EL, V8, V9, V10, V11, V12, V13, V14, V15), - REP8 (HREG_EL, V16, V17, V18, V19, V20, V21, V22, V23), - REP8 (HREG_EL, V24, V25, V26, V27, V28, V29, V30, V31), -}; -#undef REP_SEP +#include "mir-aarch64.h" -static const MIR_reg_t MAX_HARD_REG = V31_HARD_REG; static const MIR_reg_t FP_HARD_REG = R29_HARD_REG; static const MIR_reg_t LINK_HARD_REG = R30_HARD_REG; -static int target_locs_num (MIR_reg_t loc, MIR_type_t type) { - return loc > MAX_HARD_REG && type == MIR_T_LD ? 2 : 1; -} - -static inline MIR_reg_t target_nth_loc (MIR_reg_t loc, MIR_type_t type, int n) { return loc + n; } - -/* Hard regs not used in machinized code, preferably call used ones. */ -const MIR_reg_t TEMP_INT_HARD_REG1 = R9_HARD_REG, TEMP_INT_HARD_REG2 = R10_HARD_REG; -const MIR_reg_t TEMP_FLOAT_HARD_REG1 = V16_HARD_REG, TEMP_FLOAT_HARD_REG2 = V17_HARD_REG; -const MIR_reg_t TEMP_DOUBLE_HARD_REG1 = V16_HARD_REG, TEMP_DOUBLE_HARD_REG2 = V17_HARD_REG; -const MIR_reg_t TEMP_LDOUBLE_HARD_REG1 = V16_HARD_REG; -const MIR_reg_t TEMP_LDOUBLE_HARD_REG2 = V17_HARD_REG; - -static inline int target_hard_reg_type_ok_p (MIR_reg_t hard_reg, MIR_type_t type) { - assert (hard_reg <= MAX_HARD_REG); - return MIR_fp_type_p (type) ? hard_reg >= V0_HARD_REG : hard_reg < V0_HARD_REG; -} - -static inline int target_fixed_hard_reg_p (MIR_reg_t hard_reg) { - assert (hard_reg <= MAX_HARD_REG); -#if defined(__APPLE__) - if (hard_reg == R18_HARD_REG) return TRUE; -#endif - return (hard_reg == FP_HARD_REG || hard_reg == SP_HARD_REG || hard_reg == TEMP_INT_HARD_REG1 - || hard_reg == TEMP_INT_HARD_REG2 || hard_reg == TEMP_FLOAT_HARD_REG1 - || hard_reg == TEMP_FLOAT_HARD_REG2 || hard_reg == TEMP_DOUBLE_HARD_REG1 - || hard_reg == TEMP_DOUBLE_HARD_REG2 || hard_reg == TEMP_LDOUBLE_HARD_REG1 - || hard_reg == TEMP_LDOUBLE_HARD_REG2); +static inline MIR_reg_t target_nth_loc (MIR_reg_t loc, MIR_type_t type MIR_UNUSED, int n) { + return loc + n; } static inline int target_call_used_hard_reg_p (MIR_reg_t hard_reg, MIR_type_t type) { @@ -133,7 +93,7 @@ static MIR_reg_t get_arg_reg (MIR_type_t arg_type, size_t *int_arg_num, size_t * case 5: case 6: case 7: arg_reg = V0_HARD_REG + *fp_arg_num; break; - default: arg_reg = MIR_NON_HARD_REG; break; + default: arg_reg = MIR_NON_VAR; break; } (*fp_arg_num)++; *mov_code = arg_type == MIR_T_F ? MIR_FMOV : arg_type == MIR_T_D ? MIR_DMOV : MIR_LDMOV; @@ -147,7 +107,7 @@ static MIR_reg_t get_arg_reg (MIR_type_t arg_type, size_t *int_arg_num, size_t * case 5: case 6: case 7: arg_reg = R0_HARD_REG + *int_arg_num; break; - default: arg_reg = MIR_NON_HARD_REG; break; + default: arg_reg = MIR_NON_VAR; break; } (*int_arg_num)++; *mov_code = MIR_MOV; @@ -166,20 +126,24 @@ static MIR_insn_t gen_mov (gen_ctx_t gen_ctx, MIR_insn_t anchor, MIR_insn_code_t return insn; } +static MIR_reg_t target_get_stack_slot_base_reg (gen_ctx_t gen_ctx MIR_UNUSED) { + return FP_HARD_REG; +} + static int target_valid_mem_offset_p (gen_ctx_t gen_ctx, MIR_type_t type, MIR_disp_t offset); static MIR_op_t new_mem_op (gen_ctx_t gen_ctx, MIR_insn_t anchor, MIR_type_t type, MIR_disp_t disp, MIR_reg_t base) { MIR_context_t ctx = gen_ctx->ctx; if (target_valid_mem_offset_p (gen_ctx, type, disp)) - return MIR_new_mem_op (ctx, type, disp, base, 0, 1); + return _MIR_new_var_mem_op (ctx, type, disp, base, MIR_NON_VAR, 1); MIR_reg_t temp_reg = gen_new_temp_reg (gen_ctx, MIR_T_I64, curr_func_item->u.func); - MIR_op_t temp_reg_op = MIR_new_reg_op (ctx, temp_reg); + MIR_op_t temp_reg_op = _MIR_new_var_op (ctx, temp_reg); gen_mov (gen_ctx, anchor, MIR_MOV, temp_reg_op, MIR_new_int_op (ctx, disp)); gen_add_insn_before (gen_ctx, anchor, MIR_new_insn (ctx, MIR_ADD, temp_reg_op, temp_reg_op, - MIR_new_reg_op (ctx, base))); - return MIR_new_mem_op (ctx, type, 0, temp_reg, 0, 1); + _MIR_new_var_op (ctx, base))); + return _MIR_new_var_mem_op (ctx, type, 0, temp_reg, MIR_NON_VAR, 1); } static MIR_op_t get_new_hard_reg_mem_op (gen_ctx_t gen_ctx, MIR_type_t type, MIR_disp_t disp, @@ -187,11 +151,11 @@ static MIR_op_t get_new_hard_reg_mem_op (gen_ctx_t gen_ctx, MIR_type_t type, MIR MIR_context_t ctx = gen_ctx->ctx; *insn1 = *insn2 = NULL; if (target_valid_mem_offset_p (gen_ctx, type, disp)) - return _MIR_new_hard_reg_mem_op (ctx, type, disp, base, MIR_NON_HARD_REG, 1); - MIR_op_t temp_reg_op = _MIR_new_hard_reg_op (ctx, TEMP_INT_HARD_REG2); + return _MIR_new_var_mem_op (ctx, type, disp, base, MIR_NON_VAR, 1); + MIR_op_t temp_reg_op = _MIR_new_var_op (ctx, TEMP_INT_HARD_REG2); *insn1 = MIR_new_insn (ctx, MIR_MOV, temp_reg_op, MIR_new_int_op (ctx, disp)); - *insn2 = MIR_new_insn (ctx, MIR_ADD, temp_reg_op, temp_reg_op, _MIR_new_hard_reg_op (ctx, base)); - return _MIR_new_hard_reg_mem_op (ctx, type, 0, TEMP_INT_HARD_REG2, MIR_NON_HARD_REG, 1); + *insn2 = MIR_new_insn (ctx, MIR_ADD, temp_reg_op, temp_reg_op, _MIR_new_var_op (ctx, base)); + return _MIR_new_var_mem_op (ctx, type, 0, TEMP_INT_HARD_REG2, MIR_NON_VAR, 1); } static MIR_op_t new_hard_reg_mem_op (gen_ctx_t gen_ctx, MIR_insn_t anchor, MIR_type_t type, @@ -215,7 +179,7 @@ static void gen_blk_mov (gen_ctx_t gen_ctx, MIR_insn_t anchor, size_t to_disp, MIR_insn_t new_insn; MIR_op_t ops[5], freg_op, treg_op, treg_op2, treg_op3; - treg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + treg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); if (qwords <= 16) { for (; qwords > 0; qwords--, to_disp += 8, from_disp += 8) { gen_mov (gen_ctx, anchor, MIR_MOV, treg_op, @@ -226,46 +190,46 @@ static void gen_blk_mov (gen_ctx_t gen_ctx, MIR_insn_t anchor, size_t to_disp, } return; } - treg_op2 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - treg_op3 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + treg_op2 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + treg_op3 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); /* Save arg regs: */ if (save_regs > 0) - gen_mov (gen_ctx, anchor, MIR_MOV, treg_op, _MIR_new_hard_reg_op (ctx, R0_HARD_REG)); + gen_mov (gen_ctx, anchor, MIR_MOV, treg_op, _MIR_new_var_op (ctx, R0_HARD_REG)); if (save_regs > 1) - gen_mov (gen_ctx, anchor, MIR_MOV, treg_op2, _MIR_new_hard_reg_op (ctx, R1_HARD_REG)); + gen_mov (gen_ctx, anchor, MIR_MOV, treg_op2, _MIR_new_var_op (ctx, R1_HARD_REG)); if (save_regs > 2) - gen_mov (gen_ctx, anchor, MIR_MOV, treg_op3, _MIR_new_hard_reg_op (ctx, R2_HARD_REG)); + gen_mov (gen_ctx, anchor, MIR_MOV, treg_op3, _MIR_new_var_op (ctx, R2_HARD_REG)); /* call blk move: */ proto_item = _MIR_builtin_proto (ctx, curr_func_item->module, BLK_MOV_P, 0, NULL, 3, MIR_T_I64, "to", MIR_T_I64, "from", MIR_T_I64, "nwords"); func_import_item = _MIR_builtin_func (ctx, curr_func_item->module, BLK_MOV, mir_blk_mov); - freg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + freg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); new_insn = MIR_new_insn (ctx, MIR_MOV, freg_op, MIR_new_ref_op (ctx, func_import_item)); gen_add_insn_before (gen_ctx, anchor, new_insn); gen_add_insn_before (gen_ctx, anchor, - MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_hard_reg_op (ctx, R0_HARD_REG), - _MIR_new_hard_reg_op (ctx, to_base_hard_reg), + MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_var_op (ctx, R0_HARD_REG), + _MIR_new_var_op (ctx, to_base_hard_reg), MIR_new_int_op (ctx, to_disp))); gen_add_insn_before (gen_ctx, anchor, - MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_hard_reg_op (ctx, R1_HARD_REG), - MIR_new_reg_op (ctx, from_base_reg), + MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_var_op (ctx, R1_HARD_REG), + _MIR_new_var_op (ctx, from_base_reg), MIR_new_int_op (ctx, from_disp))); - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, R2_HARD_REG), + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, R2_HARD_REG), MIR_new_int_op (ctx, qwords)); ops[0] = MIR_new_ref_op (ctx, proto_item); ops[1] = freg_op; - ops[2] = _MIR_new_hard_reg_op (ctx, R0_HARD_REG); - ops[3] = _MIR_new_hard_reg_op (ctx, R1_HARD_REG); - ops[4] = _MIR_new_hard_reg_op (ctx, R2_HARD_REG); + ops[2] = _MIR_new_var_op (ctx, R0_HARD_REG); + ops[3] = _MIR_new_var_op (ctx, R1_HARD_REG); + ops[4] = _MIR_new_var_op (ctx, R2_HARD_REG); new_insn = MIR_new_insn_arr (ctx, MIR_CALL, 5, ops); gen_add_insn_before (gen_ctx, anchor, new_insn); /* Restore arg regs: */ if (save_regs > 0) - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, R0_HARD_REG), treg_op); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, R0_HARD_REG), treg_op); if (save_regs > 1) - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, R1_HARD_REG), treg_op2); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, R1_HARD_REG), treg_op2); if (save_regs > 2) - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, R2_HARD_REG), treg_op3); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, R2_HARD_REG), treg_op3); } static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { @@ -294,9 +258,9 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { nargs = VARR_LENGTH (MIR_var_t, proto->args); arg_vars = VARR_ADDR (MIR_var_t, proto->args); } - if (call_insn->ops[1].mode != MIR_OP_REG && call_insn->ops[1].mode != MIR_OP_HARD_REG) { + if (call_insn->ops[1].mode != MIR_OP_VAR) { // ??? to optimize (can be immediate operand for func call) - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); new_insn = MIR_new_insn (ctx, MIR_MOV, temp_op, call_insn->ops[1]); call_insn->ops[1] = temp_op; gen_add_insn_before (gen_ctx, call_insn, new_insn); @@ -304,8 +268,8 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { for (size_t i = start; i < nops; i++) { /* calculate offset for blk params */ if (i - start < nargs) { type = arg_vars[i - start].type; - } else if (call_insn->ops[i].mode == MIR_OP_MEM) { - type = call_insn->ops[i].u.mem.type; + } else if (call_insn->ops[i].mode == MIR_OP_VAR_MEM) { + type = call_insn->ops[i].u.var_mem.type; gen_assert (MIR_all_blk_type_p (type)); } else { mode = call_insn->ops[i].value_mode; // ??? smaller ints @@ -317,15 +281,15 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { if (mode == MIR_OP_LDOUBLE && __SIZEOF_LONG_DOUBLE__ == 8) mode = MIR_OP_DOUBLE; type = mode == MIR_OP_DOUBLE ? MIR_T_D : mode == MIR_OP_LDOUBLE ? MIR_T_LD : MIR_T_I64; } - gen_assert (!MIR_all_blk_type_p (type) || call_insn->ops[i].mode == MIR_OP_MEM); + gen_assert (!MIR_all_blk_type_p (type) || call_insn->ops[i].mode == MIR_OP_VAR_MEM); if (type == MIR_T_RBLK && i == start) continue; /* hidden arg */ #if defined(__APPLE__) /* all varargs are passed on stack */ if (i - start == nargs) int_arg_num = fp_arg_num = 8; #endif - if (MIR_blk_type_p (type) && (qwords = (call_insn->ops[i].u.mem.disp + 7) / 8) <= 2) { + if (MIR_blk_type_p (type) && (qwords = (call_insn->ops[i].u.var_mem.disp + 7) / 8) <= 2) { if (int_arg_num + qwords > 8) blk_offset += qwords * 8; int_arg_num += qwords; - } else if (get_arg_reg (type, &int_arg_num, &fp_arg_num, &new_insn_code) == MIR_NON_HARD_REG) { + } else if (get_arg_reg (type, &int_arg_num, &fp_arg_num, &new_insn_code) == MIR_NON_VAR) { if (type == MIR_T_LD && __SIZEOF_LONG_DOUBLE__ == 16 && blk_offset % 16 != 0) blk_offset = (blk_offset + 15) / 16 * 16; blk_offset += type == MIR_T_LD && __SIZEOF_LONG_DOUBLE__ == 16 ? 16 : 8; @@ -338,12 +302,12 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { if (i - start == nargs) int_arg_num = fp_arg_num = 8; #endif arg_op = call_insn->ops[i]; - gen_assert (arg_op.mode == MIR_OP_REG || arg_op.mode == MIR_OP_HARD_REG - || (arg_op.mode == MIR_OP_MEM && MIR_all_blk_type_p (arg_op.u.mem.type))); + gen_assert (arg_op.mode == MIR_OP_VAR + || (arg_op.mode == MIR_OP_VAR_MEM && MIR_all_blk_type_p (arg_op.u.var_mem.type))); if (i - start < nargs) { type = arg_vars[i - start].type; - } else if (call_insn->ops[i].mode == MIR_OP_MEM) { - type = call_insn->ops[i].u.mem.type; + } else if (call_insn->ops[i].mode == MIR_OP_VAR_MEM) { + type = call_insn->ops[i].u.var_mem.type; gen_assert (MIR_all_blk_type_p (type)); } else { mode = call_insn->ops[i].value_mode; // ??? smaller ints @@ -352,77 +316,77 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { } ext_insn = NULL; if ((ext_code = get_ext_code (type)) != MIR_INVALID_INSN) { /* extend arg if necessary */ - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); ext_insn = MIR_new_insn (ctx, ext_code, temp_op, arg_op); call_insn->ops[i] = arg_op = temp_op; } - gen_assert ( - !MIR_all_blk_type_p (type) - || (arg_op.mode == MIR_OP_MEM && arg_op.u.mem.disp >= 0 && arg_op.u.mem.index == 0)); + gen_assert (!MIR_all_blk_type_p (type) + || (arg_op.mode == MIR_OP_VAR_MEM && arg_op.u.var_mem.disp >= 0 + && arg_op.u.var_mem.index == MIR_NON_VAR)); if (type == MIR_T_RBLK && i == start) { /* hidden arg */ - arg_reg_op = _MIR_new_hard_reg_op (ctx, R8_HARD_REG); - gen_mov (gen_ctx, call_insn, MIR_MOV, arg_reg_op, MIR_new_reg_op (ctx, arg_op.u.mem.base)); + arg_reg_op = _MIR_new_var_op (ctx, R8_HARD_REG); + gen_mov (gen_ctx, call_insn, MIR_MOV, arg_reg_op, + _MIR_new_var_op (ctx, arg_op.u.var_mem.base)); call_insn->ops[i] = arg_reg_op; continue; } else if (MIR_blk_type_p (type)) { - qwords = (arg_op.u.mem.disp + 7) / 8; + qwords = (arg_op.u.var_mem.disp + 7) / 8; if (qwords <= 2) { arg_reg = R0_HARD_REG + int_arg_num; if (int_arg_num + qwords <= 8) { /* A trick to keep arg regs live: */ - call_insn->ops[i] - = _MIR_new_hard_reg_mem_op (ctx, MIR_T_UNDEF, 0, int_arg_num, - qwords < 2 ? MIR_NON_HARD_REG : int_arg_num + 1, 1); + call_insn->ops[i] = _MIR_new_var_mem_op (ctx, MIR_T_UNDEF, 0, int_arg_num, + qwords < 2 ? MIR_NON_VAR : int_arg_num + 1, 1); if (qwords == 0) continue; - new_insn - = MIR_new_insn (ctx, MIR_MOV, _MIR_new_hard_reg_op (ctx, R0_HARD_REG + int_arg_num++), - MIR_new_mem_op (ctx, MIR_T_I64, 0, arg_op.u.mem.base, 0, 1)); + new_insn = MIR_new_insn (ctx, MIR_MOV, _MIR_new_var_op (ctx, R0_HARD_REG + int_arg_num++), + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, arg_op.u.var_mem.base, + MIR_NON_VAR, 1)); gen_add_insn_before (gen_ctx, call_insn, new_insn); if (qwords == 2) { new_insn - = MIR_new_insn (ctx, MIR_MOV, _MIR_new_hard_reg_op (ctx, R0_HARD_REG + int_arg_num++), - MIR_new_mem_op (ctx, MIR_T_I64, 8, arg_op.u.mem.base, 0, 1)); + = MIR_new_insn (ctx, MIR_MOV, _MIR_new_var_op (ctx, R0_HARD_REG + int_arg_num++), + _MIR_new_var_mem_op (ctx, MIR_T_I64, 8, arg_op.u.var_mem.base, + MIR_NON_VAR, 1)); gen_add_insn_before (gen_ctx, call_insn, new_insn); } } else { /* pass on stack w/o address: */ - gen_blk_mov (gen_ctx, call_insn, mem_size, SP_HARD_REG, 0, arg_op.u.mem.base, qwords, + gen_blk_mov (gen_ctx, call_insn, mem_size, SP_HARD_REG, 0, arg_op.u.var_mem.base, qwords, int_arg_num); call_insn->ops[i] - = _MIR_new_hard_reg_mem_op (ctx, MIR_T_UNDEF, - mem_size, /* we don't care about valid mem disp here */ - SP_HARD_REG, MIR_NON_HARD_REG, 1); + = _MIR_new_var_mem_op (ctx, MIR_T_UNDEF, + mem_size, /* we don't care about valid mem disp here */ + SP_HARD_REG, MIR_NON_VAR, 1); mem_size += qwords * 8; blk_offset += qwords * 8; int_arg_num += qwords; } continue; } - gen_blk_mov (gen_ctx, call_insn, blk_offset, SP_HARD_REG, 0, arg_op.u.mem.base, qwords, + gen_blk_mov (gen_ctx, call_insn, blk_offset, SP_HARD_REG, 0, arg_op.u.var_mem.base, qwords, int_arg_num); - arg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + arg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); gen_assert (curr_prev_call_insn != NULL); /* call_insn should not be 1st after simplification */ - new_insn - = MIR_new_insn (gen_ctx->ctx, MIR_ADD, arg_op, _MIR_new_hard_reg_op (ctx, SP_HARD_REG), - MIR_new_int_op (ctx, blk_offset)); + new_insn = MIR_new_insn (gen_ctx->ctx, MIR_ADD, arg_op, _MIR_new_var_op (ctx, SP_HARD_REG), + MIR_new_int_op (ctx, blk_offset)); gen_add_insn_after (gen_ctx, curr_prev_call_insn, new_insn); curr_prev_call_insn = DLIST_NEXT (MIR_insn_t, new_insn); blk_offset += qwords * 8; } - if ((arg_reg = get_arg_reg (type, &int_arg_num, &fp_arg_num, &new_insn_code)) - != MIR_NON_HARD_REG) { + if ((arg_reg = get_arg_reg (type, &int_arg_num, &fp_arg_num, &new_insn_code)) != MIR_NON_VAR) { /* put arguments to argument hard regs */ if (ext_insn != NULL) gen_add_insn_before (gen_ctx, call_insn, ext_insn); - arg_reg_op = _MIR_new_hard_reg_op (ctx, arg_reg); + arg_reg_op = _MIR_new_var_op (ctx, arg_reg); if (type != MIR_T_RBLK) { new_insn = MIR_new_insn (ctx, new_insn_code, arg_reg_op, arg_op); } else { - assert (arg_op.mode == MIR_OP_MEM); - new_insn = MIR_new_insn (ctx, MIR_MOV, arg_reg_op, MIR_new_reg_op (ctx, arg_op.u.mem.base)); + assert (arg_op.mode == MIR_OP_VAR_MEM); + new_insn + = MIR_new_insn (ctx, MIR_MOV, arg_reg_op, _MIR_new_var_op (ctx, arg_op.u.var_mem.base)); arg_reg_op - = _MIR_new_hard_reg_mem_op (ctx, MIR_T_RBLK, - arg_op.u.mem.disp, /* we don't care about valid disp here */ - arg_reg, MIR_NON_HARD_REG, 1); + = _MIR_new_var_mem_op (ctx, MIR_T_RBLK, + arg_op.u.var_mem.disp, /* we don't care about valid disp here */ + arg_reg, MIR_NON_VAR, 1); } gen_add_insn_before (gen_ctx, call_insn, new_insn); call_insn->ops[i] = arg_reg_op; @@ -438,9 +402,9 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { if (type != MIR_T_RBLK) { new_insn = MIR_new_insn (ctx, new_insn_code, mem_op, arg_op); } else { - assert (arg_op.mode == MIR_OP_MEM); + assert (arg_op.mode == MIR_OP_VAR_MEM); new_insn - = MIR_new_insn (ctx, new_insn_code, mem_op, MIR_new_reg_op (ctx, arg_op.u.mem.base)); + = MIR_new_insn (ctx, new_insn_code, mem_op, _MIR_new_var_op (ctx, arg_op.u.var_mem.base)); } gen_assert (curr_prev_call_insn != NULL); /* call_insn should not be 1st after simplification */ @@ -463,7 +427,7 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { int float_p; ret_reg_op = call_insn->ops[i + 2]; - gen_assert (ret_reg_op.mode == MIR_OP_REG || ret_reg_op.mode == MIR_OP_HARD_REG); + gen_assert (ret_reg_op.mode == MIR_OP_VAR); type = proto->res_types[i]; float_p = type == MIR_T_F || type == MIR_T_D || type == MIR_T_LD; if (float_p && n_vregs < 8) { @@ -471,11 +435,11 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { type == MIR_T_F ? MIR_FMOV : type == MIR_T_D ? MIR_DMOV : MIR_LDMOV, - ret_reg_op, _MIR_new_hard_reg_op (ctx, V0_HARD_REG + n_vregs)); + ret_reg_op, _MIR_new_var_op (ctx, V0_HARD_REG + n_vregs)); n_vregs++; } else if (!float_p && n_iregs < 8) { - new_insn = MIR_new_insn (ctx, MIR_MOV, ret_reg_op, - _MIR_new_hard_reg_op (ctx, R0_HARD_REG + n_iregs)); + new_insn + = MIR_new_insn (ctx, MIR_MOV, ret_reg_op, _MIR_new_var_op (ctx, R0_HARD_REG + n_iregs)); n_iregs++; } else { (*MIR_get_error_func (ctx)) (MIR_ret_error, @@ -492,15 +456,13 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { } if (mem_size != 0) { /* allocate/deallocate stack for args passed on stack */ mem_size = (mem_size + 15) / 16 * 16; /* make it of several 16 bytes */ - new_insn - = MIR_new_insn (ctx, MIR_SUB, _MIR_new_hard_reg_op (ctx, SP_HARD_REG), - _MIR_new_hard_reg_op (ctx, SP_HARD_REG), MIR_new_int_op (ctx, mem_size)); + new_insn = MIR_new_insn (ctx, MIR_SUB, _MIR_new_var_op (ctx, SP_HARD_REG), + _MIR_new_var_op (ctx, SP_HARD_REG), MIR_new_int_op (ctx, mem_size)); MIR_insert_insn_after (ctx, curr_func_item, prev_call_insn, new_insn); next_insn = DLIST_NEXT (MIR_insn_t, new_insn); create_new_bb_insns (gen_ctx, prev_call_insn, next_insn, call_insn); - new_insn - = MIR_new_insn (ctx, MIR_ADD, _MIR_new_hard_reg_op (ctx, SP_HARD_REG), - _MIR_new_hard_reg_op (ctx, SP_HARD_REG), MIR_new_int_op (ctx, mem_size)); + new_insn = MIR_new_insn (ctx, MIR_ADD, _MIR_new_var_op (ctx, SP_HARD_REG), + _MIR_new_var_op (ctx, SP_HARD_REG), MIR_new_int_op (ctx, mem_size)); MIR_insert_insn_after (ctx, curr_func_item, call_insn, new_insn); next_insn = DLIST_NEXT (MIR_insn_t, new_insn); create_new_bb_insns (gen_ctx, call_insn, next_insn, call_insn); @@ -716,10 +678,6 @@ static int get_builtin (gen_ctx_t gen_ctx, MIR_insn_code_t code, MIR_item_t *pro } } -DEF_VARR (int); -DEF_VARR (uint8_t); -DEF_VARR (uint64_t); - struct insn_pattern_info { int start, num; }; @@ -730,17 +688,20 @@ DEF_VARR (insn_pattern_info_t); struct label_ref { int abs_addr_p, short_p; size_t label_val_disp; - MIR_label_t label; + union { + MIR_label_t label; + void *jump_addr; /* absolute addr for BBV */ + } u; }; typedef struct label_ref label_ref_t; DEF_VARR (label_ref_t); -DEF_VARR (MIR_code_reloc_t); - struct target_ctx { - unsigned char alloca_p, block_arg_func_p, leaf_p; + unsigned char alloca_p, block_arg_func_p, leaf_p, short_bb_branch_p; size_t small_aggregate_save_area; + MIR_insn_t temp_jump; + const char *temp_jump_replacement; VARR (int) * pattern_indexes; VARR (insn_pattern_info_t) * insn_pattern_info; VARR (uint8_t) * result_code; @@ -752,7 +713,10 @@ struct target_ctx { #define alloca_p gen_ctx->target_ctx->alloca_p #define block_arg_func_p gen_ctx->target_ctx->block_arg_func_p #define leaf_p gen_ctx->target_ctx->leaf_p +#define short_bb_branch_p gen_ctx->target_ctx->short_bb_branch_p #define small_aggregate_save_area gen_ctx->target_ctx->small_aggregate_save_area +#define temp_jump gen_ctx->target_ctx->temp_jump +#define temp_jump_replacement gen_ctx->target_ctx->temp_jump_replacement #define pattern_indexes gen_ctx->target_ctx->pattern_indexes #define insn_pattern_info gen_ctx->target_ctx->insn_pattern_info #define result_code gen_ctx->target_ctx->result_code @@ -760,7 +724,7 @@ struct target_ctx { #define abs_address_locs gen_ctx->target_ctx->abs_address_locs #define relocs gen_ctx->target_ctx->relocs -static MIR_disp_t target_get_stack_slot_offset (gen_ctx_t gen_ctx, MIR_type_t type, +static MIR_disp_t target_get_stack_slot_offset (gen_ctx_t gen_ctx, MIR_type_t type MIR_UNUSED, MIR_reg_t slot) { /* slot is 0, 1, ... */ size_t offset = curr_func_item->u.func->vararg_p || block_arg_func_p ? 32 : 16; @@ -768,7 +732,8 @@ static MIR_disp_t target_get_stack_slot_offset (gen_ctx_t gen_ctx, MIR_type_t ty return ((MIR_disp_t) slot * 8 + offset); } -static int target_valid_mem_offset_p (gen_ctx_t gen_ctx, MIR_type_t type, MIR_disp_t offset) { +static int target_valid_mem_offset_p (gen_ctx_t gen_ctx MIR_UNUSED, MIR_type_t type, + MIR_disp_t offset) { int scale; switch (type) { case MIR_T_I8: @@ -787,13 +752,6 @@ static int target_valid_mem_offset_p (gen_ctx_t gen_ctx, MIR_type_t type, MIR_di return offset >= 0 && offset % scale == 0 && offset / scale < (1 << 12); } -#define SWAP(v1, v2, t) \ - do { \ - t = v1; \ - v1 = v2; \ - v2 = t; \ - } while (0) - static void target_machinize (gen_ctx_t gen_ctx) { MIR_context_t ctx = gen_ctx->ctx; MIR_func_t func; @@ -816,31 +774,34 @@ static void target_machinize (gen_ctx_t gen_ctx) { var = VARR_GET (MIR_var_t, func->vars, i); type = var.type; if (type == MIR_T_RBLK && i == 0) { /* hidden arg */ - arg_reg_op = _MIR_new_hard_reg_op (ctx, R8_HARD_REG); - gen_mov (gen_ctx, anchor, MIR_MOV, MIR_new_reg_op (ctx, i + 1), arg_reg_op); + arg_reg_op = _MIR_new_var_op (ctx, R8_HARD_REG); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, i + MAX_HARD_REG + 1), arg_reg_op); continue; } else if (MIR_blk_type_p (type) && (qwords = (var.size + 7) / 8) <= 2) { if (int_arg_num + qwords <= 8) { small_aggregate_save_area += qwords * 8; - new_insn = MIR_new_insn (ctx, MIR_SUB, MIR_new_reg_op (ctx, i + 1), - _MIR_new_hard_reg_op (ctx, FP_HARD_REG), + new_insn = MIR_new_insn (ctx, MIR_SUB, _MIR_new_var_op (ctx, i + MAX_HARD_REG + 1), + _MIR_new_var_op (ctx, FP_HARD_REG), MIR_new_int_op (ctx, small_aggregate_save_area)); gen_add_insn_before (gen_ctx, anchor, new_insn); if (qwords == 0) continue; - gen_mov (gen_ctx, anchor, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 0, i + 1, 0, 1), - _MIR_new_hard_reg_op (ctx, int_arg_num)); + gen_mov (gen_ctx, anchor, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, i + MAX_HARD_REG + 1, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, int_arg_num)); if (qwords == 2) - gen_mov (gen_ctx, anchor, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 8, i + 1, 0, 1), - _MIR_new_hard_reg_op (ctx, int_arg_num + 1)); + gen_mov (gen_ctx, anchor, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 8, i + MAX_HARD_REG + 1, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, int_arg_num + 1)); } else { /* pass on stack w/o address: */ if (!block_arg_func_p) { block_arg_func_p = TRUE; - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, R8_HARD_REG), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 16, FP_HARD_REG, MIR_NON_HARD_REG, 1)); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, R8_HARD_REG), + _MIR_new_var_mem_op (ctx, MIR_T_I64, 16, FP_HARD_REG, MIR_NON_VAR, 1)); } gen_add_insn_before (gen_ctx, anchor, - MIR_new_insn (ctx, MIR_ADD, MIR_new_reg_op (ctx, i + 1), - _MIR_new_hard_reg_op (ctx, R8_HARD_REG), + MIR_new_insn (ctx, MIR_ADD, + _MIR_new_var_op (ctx, i + MAX_HARD_REG + 1), + _MIR_new_var_op (ctx, R8_HARD_REG), MIR_new_int_op (ctx, mem_size))); mem_size += qwords * 8; } @@ -848,15 +809,16 @@ static void target_machinize (gen_ctx_t gen_ctx) { continue; } arg_reg = get_arg_reg (type, &int_arg_num, &fp_arg_num, &new_insn_code); - if (arg_reg != MIR_NON_HARD_REG) { - arg_reg_op = _MIR_new_hard_reg_op (ctx, arg_reg); - gen_mov (gen_ctx, anchor, new_insn_code, MIR_new_reg_op (ctx, i + 1), arg_reg_op); + if (arg_reg != MIR_NON_VAR) { + arg_reg_op = _MIR_new_var_op (ctx, arg_reg); + gen_mov (gen_ctx, anchor, new_insn_code, _MIR_new_var_op (ctx, i + MAX_HARD_REG + 1), + arg_reg_op); } else { /* arg is on the stack */ if (!block_arg_func_p) { block_arg_func_p = TRUE; - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, R8_HARD_REG), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 16, FP_HARD_REG, MIR_NON_HARD_REG, 1)); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, R8_HARD_REG), + _MIR_new_var_mem_op (ctx, MIR_T_I64, 16, FP_HARD_REG, MIR_NON_VAR, 1)); } mem_type = type == MIR_T_F || type == MIR_T_D || type == MIR_T_LD ? type : MIR_T_I64; if (type == MIR_T_LD) mem_size = (mem_size + 15) / 16 * 16; @@ -865,7 +827,7 @@ static void target_machinize (gen_ctx_t gen_ctx) { : type == MIR_T_LD ? MIR_LDMOV : MIR_MOV); mem_op = new_hard_reg_mem_op (gen_ctx, anchor, mem_type, mem_size, R8_HARD_REG); - gen_mov (gen_ctx, anchor, new_insn_code, MIR_new_reg_op (ctx, i + 1), mem_op); + gen_mov (gen_ctx, anchor, new_insn_code, _MIR_new_var_op (ctx, i + MAX_HARD_REG + 1), mem_op); mem_size += type == MIR_T_LD ? 16 : 8; } } @@ -879,7 +841,7 @@ static void target_machinize (gen_ctx_t gen_ctx) { code = insn->code; if (code == MIR_LDBEQ || code == MIR_LDBNE || code == MIR_LDBLT || code == MIR_LDBGE || code == MIR_LDBGT || code == MIR_LDBLE) { - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); code = (code == MIR_LDBEQ ? MIR_LDEQ : code == MIR_LDBNE ? MIR_LDNE : code == MIR_LDBLT ? MIR_LDLT @@ -901,16 +863,16 @@ static void target_machinize (gen_ctx_t gen_ctx) { MIR_op_t ops[6], func_reg_op, reg_op3; MIR_op_t res_reg_op = insn->ops[0], va_reg_op = insn->ops[1], op3 = insn->ops[2]; - assert (res_reg_op.mode == MIR_OP_REG && va_reg_op.mode == MIR_OP_REG - && op3.mode == (code == MIR_VA_ARG ? MIR_OP_MEM : MIR_OP_REG)); - func_reg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - reg_op3 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + assert (res_reg_op.mode == MIR_OP_VAR && va_reg_op.mode == MIR_OP_VAR + && op3.mode == (code == MIR_VA_ARG ? MIR_OP_VAR_MEM : MIR_OP_VAR)); + func_reg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + reg_op3 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); next_insn = new_insn = MIR_new_insn (ctx, MIR_MOV, func_reg_op, MIR_new_ref_op (ctx, func_import_item)); gen_add_insn_before (gen_ctx, insn, new_insn); if (code == MIR_VA_ARG) { - new_insn - = MIR_new_insn (ctx, MIR_MOV, reg_op3, MIR_new_int_op (ctx, (int64_t) op3.u.mem.type)); + new_insn = MIR_new_insn (ctx, MIR_MOV, reg_op3, + MIR_new_int_op (ctx, (int64_t) op3.u.var_mem.type)); op3 = reg_op3; gen_add_insn_before (gen_ctx, insn, new_insn); } @@ -926,8 +888,8 @@ static void target_machinize (gen_ctx_t gen_ctx) { } else { /* Use builtin: mov freg, func ref; call proto, freg, res_reg, op_reg[, op_reg2] */ MIR_op_t freg_op, res_reg_op = insn->ops[0], op_reg_op = insn->ops[1], ops[5]; - assert (res_reg_op.mode == MIR_OP_REG && op_reg_op.mode == MIR_OP_REG); - freg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + assert (res_reg_op.mode == MIR_OP_VAR && op_reg_op.mode == MIR_OP_VAR); + freg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); next_insn = new_insn = MIR_new_insn (ctx, MIR_MOV, freg_op, MIR_new_ref_op (ctx, func_import_item)); gen_add_insn_before (gen_ctx, insn, new_insn); @@ -942,44 +904,48 @@ static void target_machinize (gen_ctx_t gen_ctx) { } } else if (code == MIR_VA_START) { #if !defined(__APPLE__) - MIR_op_t treg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + MIR_op_t treg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); #endif - MIR_op_t prev_sp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + MIR_op_t prev_sp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); MIR_op_t va_op = insn->ops[0]; MIR_reg_t va_reg; int gp_offset, fp_offset; - assert (func->vararg_p && va_op.mode == MIR_OP_REG); + assert (func->vararg_p && va_op.mode == MIR_OP_VAR); gp_offset = (int_arg_num >= 8 ? 0 : 8 * int_arg_num - 64); fp_offset = (fp_arg_num >= 8 ? 0 : 16 * fp_arg_num - 128); - va_reg = va_op.u.reg; + va_reg = va_op.u.var; /* Insns can be not simplified as soon as they match a machine insn. */ #if !defined(__APPLE__) /* mem32[va_reg].__gr_offset = gp_offset; mem32[va_reg].__vr_offset = fp_offset */ gen_mov (gen_ctx, insn, MIR_MOV, treg_op, MIR_new_int_op (ctx, gp_offset)); - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_U32, 24, va_reg, 0, 1), treg_op); + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_U32, 24, va_reg, MIR_NON_VAR, 1), treg_op); gen_mov (gen_ctx, insn, MIR_MOV, treg_op, MIR_new_int_op (ctx, fp_offset)); - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_U32, 28, va_reg, 0, 1), treg_op); + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_U32, 28, va_reg, MIR_NON_VAR, 1), treg_op); #endif /* __stack: prev_sp = mem64[fp + 16] */ gen_mov (gen_ctx, insn, MIR_MOV, prev_sp_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 16, FP_HARD_REG, MIR_NON_HARD_REG, 1)); + _MIR_new_var_mem_op (ctx, MIR_T_I64, 16, FP_HARD_REG, MIR_NON_VAR, 1)); #if defined(__APPLE__) - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 0, va_reg, 0, 1), - prev_sp_op); + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, va_reg, MIR_NON_VAR, 1), prev_sp_op); #else /* mem64[va_reg].__stack = prev_sp + mem_size */ new_insn = MIR_new_insn (ctx, MIR_ADD, treg_op, prev_sp_op, MIR_new_int_op (ctx, mem_size)); gen_add_insn_before (gen_ctx, insn, new_insn); - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 0, va_reg, 0, 1), treg_op); + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, va_reg, MIR_NON_VAR, 1), treg_op); /* __gr_top: mem64[va_reg].__gr_top = prev_sp */ - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 8, va_reg, 0, 1), - prev_sp_op); + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 8, va_reg, MIR_NON_VAR, 1), prev_sp_op); /* __vr_top: treg = prev_sp - int_reg_save_area; mem64[va_reg].__vr_top = treg */ new_insn = MIR_new_insn (ctx, MIR_SUB, treg_op, prev_sp_op, MIR_new_int_op (ctx, int_reg_save_area_size)); gen_add_insn_before (gen_ctx, insn, new_insn); - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 16, va_reg, 0, 1), treg_op); + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 16, va_reg, MIR_NON_VAR, 1), treg_op); #endif gen_delete_insn (gen_ctx, insn); } else if (code == MIR_VA_END) { /* do nothing */ @@ -1007,8 +973,8 @@ static void target_machinize (gen_ctx_t gen_ctx) { uint32_t n_iregs = 0, n_vregs = 0; assert (func->nres == MIR_insn_nops (ctx, insn)); - for (size_t i = 0; i < func->nres; i++) { - assert (insn->ops[i].mode == MIR_OP_REG); + for (i = 0; i < func->nres; i++) { + assert (insn->ops[i].mode == MIR_OP_VAR); res_type = func->res_types[i]; if ((res_type == MIR_T_F || res_type == MIR_T_D || res_type == MIR_T_LD) && n_vregs < 8) { new_insn_code = res_type == MIR_T_F ? MIR_FMOV @@ -1022,7 +988,7 @@ static void target_machinize (gen_ctx_t gen_ctx) { (*MIR_get_error_func (ctx)) (MIR_ret_error, "aarch64 can not handle this combination of return values"); } - ret_reg_op = _MIR_new_hard_reg_op (ctx, ret_reg); + ret_reg_op = _MIR_new_var_op (ctx, ret_reg); gen_mov (gen_ctx, insn, new_insn_code, ret_reg_op, insn->ops[i]); insn->ops[i] = ret_reg_op; } @@ -1034,13 +1000,13 @@ static void target_machinize (gen_ctx_t gen_ctx) { static void isave (gen_ctx_t gen_ctx, MIR_insn_t anchor, int disp, MIR_reg_t base, MIR_reg_t hard_reg) { gen_mov (gen_ctx, anchor, MIR_MOV, new_hard_reg_mem_op (gen_ctx, anchor, MIR_T_I64, disp, base), - _MIR_new_hard_reg_op (gen_ctx->ctx, hard_reg)); + _MIR_new_var_op (gen_ctx->ctx, hard_reg)); } static void fsave (gen_ctx_t gen_ctx, MIR_insn_t anchor, int disp, MIR_reg_t base, MIR_reg_t hard_reg) { gen_mov (gen_ctx, anchor, MIR_LDMOV, new_hard_reg_mem_op (gen_ctx, anchor, MIR_T_LD, disp, base), - _MIR_new_hard_reg_op (gen_ctx->ctx, hard_reg)); + _MIR_new_var_op (gen_ctx->ctx, hard_reg)); } #endif @@ -1065,8 +1031,8 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg if (leaf_p && !alloca_p && saved_iregs_num == 0 && saved_fregs_num == 0 && !func->vararg_p && stack_slots_num == 0 && !block_arg_func_p && small_aggregate_save_area == 0) return; - sp_reg_op = _MIR_new_hard_reg_op (ctx, SP_HARD_REG); - fp_reg_op = _MIR_new_hard_reg_op (ctx, FP_HARD_REG); + sp_reg_op = _MIR_new_var_op (ctx, SP_HARD_REG); + fp_reg_op = _MIR_new_var_op (ctx, FP_HARD_REG); /* Prologue: */ anchor = DLIST_HEAD (MIR_insn_t, func->insns); #if defined(__APPLE__) @@ -1088,13 +1054,13 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg frame_size += stack_slots_num * 8; if (frame_size % 16 != 0) frame_size = (frame_size + 15) / 16 * 16; save_prev_stack_p = func->vararg_p || block_arg_func_p; - treg_op = _MIR_new_hard_reg_op (ctx, R9_HARD_REG); + treg_op = _MIR_new_var_op (ctx, R9_HARD_REG); if (save_prev_stack_p) { /* prev stack pointer */ gen_mov (gen_ctx, anchor, MIR_MOV, treg_op, sp_reg_op); frame_size += 16; } frame_size += 16; /* lr/fp */ - treg_op2 = _MIR_new_hard_reg_op (ctx, R10_HARD_REG); + treg_op2 = _MIR_new_var_op (ctx, R10_HARD_REG); if (frame_size < (1 << 12)) { new_insn = MIR_new_insn (ctx, MIR_SUB, sp_reg_op, sp_reg_op, MIR_new_int_op (ctx, frame_size)); } else { @@ -1105,14 +1071,15 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg gen_add_insn_before (gen_ctx, anchor, new_insn); /* sp = sp - (frame_size|t) */ if (save_prev_stack_p) gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 16, SP_HARD_REG, MIR_NON_HARD_REG, 1), + _MIR_new_var_mem_op (ctx, MIR_T_I64, 16, SP_HARD_REG, MIR_NON_VAR, 1), treg_op); /* mem[sp + 16] = treg */ + if (!func->jret_p) + gen_mov (gen_ctx, anchor, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 8, SP_HARD_REG, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, LINK_HARD_REG)); /* mem[sp + 8] = lr */ gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 8, SP_HARD_REG, MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (ctx, LINK_HARD_REG)); /* mem[sp + 8] = lr */ - gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 0, SP_HARD_REG, MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (ctx, FP_HARD_REG)); /* mem[sp] = fp */ + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, SP_HARD_REG, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, FP_HARD_REG)); /* mem[sp] = fp */ gen_mov (gen_ctx, anchor, MIR_MOV, fp_reg_op, sp_reg_op); /* fp = sp */ #if !defined(__APPLE__) if (func->vararg_p) { // ??? saving only regs corresponding to ... @@ -1151,13 +1118,13 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg if (i < V0_HARD_REG) { gen_mov (gen_ctx, anchor, MIR_MOV, new_hard_reg_mem_op (gen_ctx, anchor, MIR_T_I64, offset, FP_HARD_REG), - _MIR_new_hard_reg_op (ctx, i)); + _MIR_new_var_op (ctx, i)); offset += 8; } else { if (offset % 16 != 0) offset = (offset + 15) / 16 * 16; new_insn = gen_mov (gen_ctx, anchor, MIR_LDMOV, new_hard_reg_mem_op (gen_ctx, anchor, MIR_T_LD, offset, FP_HARD_REG), - _MIR_new_hard_reg_op (ctx, i)); + _MIR_new_var_op (ctx, i)); #if defined(__APPLE__) /* MIR API can change insn code - change it back as we need to generate code to save all * vreg. */ @@ -1174,21 +1141,21 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg gen_add_insn_before (gen_ctx, anchor, new_insn); /* sp -= */ } /* Epilogue: */ - anchor = DLIST_TAIL (MIR_insn_t, func->insns); - /* It might be infinite loop after CCP with dead code elimination: */ - if (anchor->code == MIR_JMP) return; - assert (anchor->code == MIR_RET); + for (anchor = DLIST_TAIL (MIR_insn_t, func->insns); anchor != NULL; + anchor = DLIST_PREV (MIR_insn_t, anchor)) + if (anchor->code == MIR_RET || anchor->code == MIR_JRET) break; + if (anchor == NULL) return; /* Restoring hard registers: */ offset = frame_size - frame_size_after_saved_regs; for (i = 0; i <= MAX_HARD_REG; i++) if (!target_call_used_hard_reg_p (i, MIR_T_UNDEF) && bitmap_bit_p (used_hard_regs, i)) { if (i < V0_HARD_REG) { - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, i), + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, i), new_hard_reg_mem_op (gen_ctx, anchor, MIR_T_I64, offset, FP_HARD_REG)); offset += 8; } else { if (offset % 16 != 0) offset = (offset + 15) / 16 * 16; - new_insn = gen_mov (gen_ctx, anchor, MIR_LDMOV, _MIR_new_hard_reg_op (ctx, i), + new_insn = gen_mov (gen_ctx, anchor, MIR_LDMOV, _MIR_new_var_op (ctx, i), new_hard_reg_mem_op (gen_ctx, anchor, MIR_T_LD, offset, FP_HARD_REG)); #if defined(__APPLE__) if (new_insn->code == MIR_DMOV) new_insn->code = MIR_LDMOV; @@ -1197,11 +1164,12 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg } } /* Restore lr, sp, fp */ - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, LINK_HARD_REG), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 8, FP_HARD_REG, MIR_NON_HARD_REG, 1)); + if (!func->jret_p) + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, LINK_HARD_REG), + _MIR_new_var_mem_op (ctx, MIR_T_I64, 8, FP_HARD_REG, MIR_NON_VAR, 1)); gen_mov (gen_ctx, anchor, MIR_MOV, treg_op2, fp_reg_op); /* r10 = fp */ gen_mov (gen_ctx, anchor, MIR_MOV, fp_reg_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 0, FP_HARD_REG, MIR_NON_HARD_REG, 1)); + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, FP_HARD_REG, MIR_NON_VAR, 1)); if (frame_size < (1 << 12)) { new_insn = MIR_new_insn (ctx, MIR_ADD, sp_reg_op, treg_op2, MIR_new_int_op (ctx, frame_size)); } else { @@ -1257,9 +1225,9 @@ struct pattern { Sr -- any immediate for right 32-bit shift (0-31) SL -- any immediate for left 64-bit shift (0-63) Sl -- any immediate for left 32-bit shift (0-31) - L - reference or label as the 1st or 2nd op which can be present by signed 26-bit pc offset (in - 4 insn bytes) l - label as the 1st op which can be present by signed 19-bit pc offset (in 4 - insn bytes) + L - reference or label as the 1st or 2nd op which can be present by signed 26-bit pc offset + (in 4 insn bytes) l - label as the 1st op which can be present by signed 19-bit pc offset (in + 4 insn bytes) Remember we have no float or (long) double immediate at this stage. They are represented by a reference to data item. */ @@ -1287,10 +1255,18 @@ struct pattern { L -- operand-label as 26-bit offset l -- operand-label as 19-bit offset T -- pc-relative address [5..23] + i -- shift value in [10..15] + I -- shift value in [16..21] */ const char *replacement; }; +#define SUB_UBO MIR_INSN_BOUND +#define SUB_UBNO (SUB_UBO + 1) +#define MUL_BO (SUB_UBNO + 1) +#define MUL_BNO (MUL_BO + 1) +#define ARM_INSN_BOUND (MUL_BNO + 1) + static const struct pattern patterns[] = { {MIR_MOV, "r h31", "91000000:fffffc00 rd0 hn1f"}, /* mov Rd,SP */ {MIR_MOV, "h31 r", "91000000:fffffc00 hd1f rn1"}, /* mov SP,Rn */ @@ -1367,20 +1343,24 @@ static const struct pattern patterns[] = { {MIR_UEXT16, "r r", "53003c00:fffffc00 rd0 rn1"}, /* uxth wd, wn */ {MIR_UEXT32, "r r", "2a0003e0:7fe0ffe0 rd0 rm1"}, /* mov wd, wm */ - {MIR_ADD, "r r r", "8b206000:ffe0fc00 rd0 rn1 rm2"}, /* extended add Rd,Rn,Rm*/ - {MIR_ADD, "r r I", "91000000:ff000000 rd0 rn1 I"}, /* add Rd,Rn,I,shift */ - {MIR_ADDS, "r r r", "0b000000:ff200000 rd0 rn1 rm2"}, /* add Wd,Wn,Wm*/ - {MIR_ADDS, "r r I", "11000000:ff000000 rd0 rn1 I"}, /* add Wd,Wn,I,shift */ - {MIR_FADD, "r r r", "1e202800:ffe0fc00 vd0 vn1 vm2"}, /* fadd Sd,Sn,Sm*/ - {MIR_DADD, "r r r", "1e602800:ffe0fc00 vd0 vn1 vm2"}, /* fadd Dd,Dn,Dm*/ +#define IOP(icode, rop, iop, rops, iops) \ + {icode, "r r r", rop ":ffe0fc00 rd0 rn1 rm2"}, /* extended op Rd,Rn,Rm*/ \ + {icode, "r r I", iop ":ff000000 rd0 rn1 I"}, /* op Rd,Rn,I,shift */ \ + {icode##S, "r r r", rops ":ff200000 rd0 rn1 rm2"}, /* op Wd,Wn,Wm*/ \ + {icode##S, "r r I", iops ":ff000000 rd0 rn1 I"}, /* op Wd,Wn,I,shift */ + + IOP (MIR_ADD, "8b206000", "91000000", "0b000000", "11000000") + IOP (MIR_ADDO, "ab206000", "b1000000", "2b000000", "31000000") + + {MIR_FADD, "r r r", "1e202800:ffe0fc00 vd0 vn1 vm2"}, /* fadd Sd,Sn,Sm*/ + {MIR_DADD, "r r r", "1e602800:ffe0fc00 vd0 vn1 vm2"}, /* fadd Dd,Dn,Dm*/ // ldadd is implemented through builtin - {MIR_SUB, "r r r", "cb206000:ffe0fc00 rd0 rn1 rm2"}, /* extended sub Rd,Rn,Rm*/ - {MIR_SUB, "r r I", "d1000000:ff000000 rd0 rn1 I"}, /* sub Rd,Rn,I,shift */ - {MIR_SUBS, "r r r", "4b000000:ff200000 rd0 rn1 rm2"}, /* sub Wd,Wn,Wm*/ - {MIR_SUBS, "r r I", "51000000:ff000000 rd0 rn1 I"}, /* sub Wd,Wn,I,shift */ - {MIR_FSUB, "r r r", "1e203800:ffe0fc00 vd0 vn1 vm2"}, /* fsub Sd,Sn,Sm*/ - {MIR_DSUB, "r r r", "1e603800:ffe0fc00 vd0 vn1 vm2"}, /* fsub Dd,Dn,Dm*/ + IOP (MIR_SUB, "cb206000", "d1000000", "4b000000", "51000000") + IOP (MIR_SUBO, "eb206000", "f1000000", "6b000000", "71000000") + + {MIR_FSUB, "r r r", "1e203800:ffe0fc00 vd0 vn1 vm2"}, /* fsub Sd,Sn,Sm*/ + {MIR_DSUB, "r r r", "1e603800:ffe0fc00 vd0 vn1 vm2"}, /* fsub Dd,Dn,Dm*/ // ldsub is implemented through builtin {MIR_MUL, "r r r", "9b007c00:ffe0fc00 rd0 rn1 rm2"}, /* mul Rd,Rn,Rm*/ @@ -1457,7 +1437,7 @@ static const struct pattern patterns[] = { #define RLTS "1a9fa7e0:ffffffe0 rd0" #define RULT "9a9f27e0:ffffffe0 rd0" #define RULTS "1a9f27e0:ffffffe0 rd0" -#define FLT "9a9f57e0:ffffffe0 rd0" +#define FLTC "9a9f57e0:ffffffe0 rd0" /* cmp Rn,Rm; cset Rd,lt */ {MIR_LT, "r r r", CMPR "; " RLT}, /* cmp Rn,I,shift ; cset Wd,lt */ @@ -1475,13 +1455,13 @@ static const struct pattern patterns[] = { /* cmp Wn,I,shift; cset Wd,cc */ {MIR_ULTS, "r r I", SCMPI "; " RULTS}, /* fcmpe Sn,Sm; cset Rd,mi */ - {MIR_FLT, "r r r", FCMP "; " FLT}, + {MIR_FLT, "r r r", FCMP "; " FLTC}, /* fcmpe Dn,Dm; cset Rd,mi */ - {MIR_DLT, "r r r", DCMP "; " FLT}, + {MIR_DLT, "r r r", DCMP "; " FLTC}, /* fcmpe Sn,0.0; cset Rd,mi */ - {MIR_FLT, "r r Zf", "1e202018:fffffc1f vn1 vm2; " FLT}, + {MIR_FLT, "r r Zf", "1e202018:fffffc1f vn1 vm2; " FLTC}, /* fcmpe Dn,0.0; cset Rd,mi */ - {MIR_DLT, "r r Zd", "1e602018:fffffc1f vn1 vm2; " FLT}, + {MIR_DLT, "r r Zd", "1e602018:fffffc1f vn1 vm2; " FLTC}, #define RGE "9a9fb7e0:ffffffe0 rd0" #define RGES "1a9fb7e0:ffffffe0 rd0" @@ -1545,7 +1525,7 @@ static const struct pattern patterns[] = { #define RLES "1a9fc7e0:ffffffe0 rd0" #define RULE "9a9f87e0:ffffffe0 rd0" #define RULES "1a9f87e0:ffffffe0 rd0" -#define FLE "9a9f87e0:ffffffe0 rd0" +#define FLEC "9a9f87e0:ffffffe0 rd0" /* cmp Rn,Rm; cset Rd,le */ {MIR_LE, "r r r", CMPR "; " RLE}, /* cmp Rn,I,shift ; cset Wd,le */ @@ -1563,21 +1543,30 @@ static const struct pattern patterns[] = { /* cmp Wn,I,shift; cset Wd,ls */ {MIR_ULES, "r r I", SCMPI "; " RULES}, /* fcmpe Sn,Sm; cset Rd,ls */ - {MIR_FLE, "r r r", FCMP "; " FLE}, + {MIR_FLE, "r r r", FCMP "; " FLEC}, /* fcmpe Dn,Dm; cset Rd,ls */ - {MIR_DLE, "r r r", DCMP "; " FLE}, + {MIR_DLE, "r r r", DCMP "; " FLEC}, /* fcmpe Sn,0.0; cset Rd,ls */ - {MIR_FLE, "r r Zf", "1e202018:fffffc1f vn1 vm2; " FLE}, + {MIR_FLE, "r r Zf", "1e202018:fffffc1f vn1 vm2; " FLEC}, /* fcmpe Dn,0.0; cset Rd,ls */ - {MIR_DLE, "r r Zd", "1e602018:fffffc1f vn1 vm2; " FLE}, + {MIR_DLE, "r r Zd", "1e602018:fffffc1f vn1 vm2; " FLEC}, {MIR_JMP, "L", "14000000:fc000000 L"}, /* 26-bit offset jmp */ + {MIR_LADDR, "r l", "10000000:ff000000 rd0 l"}, /* adr r, L ip-relative address */ + {MIR_JMPI, "r", "d61f0000:fffffc00 rn0"}, /* jmp *r */ + {MIR_BT, "l r", "b5000000:ff000000 rd1 l"}, /* cbnz rd,l */ {MIR_BTS, "l r", "35000000:ff000000 rd1 l"}, /* cbnz wd,l */ {MIR_BF, "l r", "b4000000:ff000000 rd1 l"}, /* cbz rd,l */ {MIR_BFS, "l r", "34000000:ff000000 rd1 l"}, /* cbz wd,l */ + {MIR_BO, "l", "54000006:ff00001f l"}, /* b.vs */ + {MIR_UBO, "l", "54000002:ff00001f l"}, /* b.cs */ + + {MIR_BNO, "l", "54000007:ff00001f l"}, /* b.vc */ + {MIR_UBNO, "l", "54000003:ff00001f l"}, /* b.cc */ + #define BEQ "54000000:ff00001f l" // ??? add extended reg cmp insns: // all ld insn are changed to builtins and bt/bts @@ -1757,6 +1746,9 @@ static const struct pattern patterns[] = { {MIR_INLINE, "X L $", "94000000:fc000000 rn1"}, /* bl address */ {MIR_RET, "$", "d65f0000:fffffc1f hn1e"}, /* ret R30 */ + {MIR_JCALL, "X r $", "d61f0000:fffffc00 rn1"}, /* br r1 */ + {MIR_JRET, "r $", "d61f0000:fffffc00 rn0"}, /* br r0 */ + /* add r0, r1, 15; and r0, r0, -16; sub sp, sp, r0; mov r0, sp: */ {MIR_ALLOCA, "r r", "91003c00:fffffc00 rd0 rn1; 927cec00:fffffc00 rd0 rn0;" /* add r0,r1,15;and r0,r0,-16 @@ -1773,13 +1765,39 @@ static const struct pattern patterns[] = { {MIR_SWITCH, "r $", "10000000:ff000000 hda T; f8607800:ffe0fc00 hda hna rm0; d61f0000:fffffc00 hna;"}, + /* Used only during machine code generation. Should have the same format as branch on overflow + insns */ + /* unsigned sub sets up carry flag when there is no overflow: */ + {SUB_UBO, "l", "54000003:ff00001f l"}, /* b.cc */ + {SUB_UBNO, "l", "54000002:ff00001f l"}, /* b.cs */ + + /* MULOS:smull Rd,Wn,Wm; asr r10,Rd,32; cmp W10,Wd,asr 31 */ + {MIR_MULOS, "r r r", + "9b207c00:ffe0fc00 rd0 rn1 rm2; 9340fc00:ffc0fc00 hda rn0 I20; " + "6b80001f:ffe0001f hna rm0 i1f"}, + /* UMULOS:umull Rd,Wn,Wm; cmp xzr,Rd,lsr 32 */ + {MIR_UMULOS, "r r r", "9ba07c00:ffe0fc00 rd0 rn1 rm2; eb40001f:ffe0001f hn1f rm0 i20"}, + /* MULO:smulh h11,Rn,Rm; mul Rd,Rn,Rm; cmp h11,Rd,asr 63 (r11 is a scratch reg) */ + {MIR_MULO, "r r r", + "9b407c00:ffe0fc00 hdb rn1 rm2; 9b007c00:ffe0fc00 rd0 rn1 rm2; " + "eb80001f:ffe0001f hnb rm0 i3f"}, + /* UMULO:umulh h11,Rn,Rm; mul Rd,Rn,Rm; cmp xzr,h11 (r11 is a scratch reg) */ + {MIR_UMULO, "r r r", + "9bc07c00:ffe0fc00 hdb rn1 rm2; 9b007c00:ffe0fc00 rd0 rn1 rm2; " + "eb00001f:ff20001f hn1f hmb"}, + + /* [u]mulo[s] insns uses zero flag to check overflow: */ + {MUL_BO, "l", BNE}, /* b.ne */ + {MUL_BNO, "l", BEQ}, /* b.eq */ }; static void target_get_early_clobbered_hard_regs (MIR_insn_t insn, MIR_reg_t *hr1, MIR_reg_t *hr2) { - *hr1 = *hr2 = MIR_NON_HARD_REG; + *hr1 = *hr2 = MIR_NON_VAR; if (insn->code == MIR_MOD || insn->code == MIR_MODS || insn->code == MIR_UMOD || insn->code == MIR_UMODS) *hr1 = R8_HARD_REG; + else if (insn->code == MIR_MULO || insn->code == MIR_UMULO) + *hr1 = R11_HARD_REG; } static int pattern_index_cmp (const void *a1, const void *a2) { @@ -1799,9 +1817,9 @@ static void patterns_init (gen_ctx_t gen_ctx) { for (i = 0; i < n; i++) VARR_PUSH (int, pattern_indexes, i); qsort (VARR_ADDR (int, pattern_indexes), n, sizeof (int), pattern_index_cmp); VARR_CREATE (insn_pattern_info_t, insn_pattern_info, 0); - for (i = 0; i < MIR_INSN_BOUND; i++) VARR_PUSH (insn_pattern_info_t, insn_pattern_info, pinfo); + for (i = 0; i < ARM_INSN_BOUND; i++) VARR_PUSH (insn_pattern_info_t, insn_pattern_info, pinfo); info_addr = VARR_ADDR (insn_pattern_info_t, insn_pattern_info); - for (prev_code = MIR_INSN_BOUND, i = 0; i < n; i++) { + for (prev_code = ARM_INSN_BOUND, i = 0; i < n; i++) { ind = VARR_GET (int, pattern_indexes, i); if ((code = patterns[ind].code) != prev_code) { if (i != 0) info_addr[prev_code].num = i - info_addr[prev_code].start; @@ -1809,7 +1827,7 @@ static void patterns_init (gen_ctx_t gen_ctx) { prev_code = code; } } - assert (prev_code != MIR_INSN_BOUND); + assert (prev_code != ARM_INSN_BOUND); info_addr[prev_code].num = n - info_addr[prev_code].start; } @@ -1880,8 +1898,7 @@ static int lshift_const_p (int64_t v, int short_p, int *immr, int *imms) { static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_insn_t insn) { MIR_context_t ctx = gen_ctx->ctx; - int nop; - size_t nops = MIR_insn_nops (ctx, insn); + size_t nop, nops = MIR_insn_nops (ctx, insn); const char *p; char ch, start_ch; MIR_op_t op; @@ -1896,10 +1913,10 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in switch (start_ch = *p) { case 'X': break; case 'r': - if (op.mode != MIR_OP_HARD_REG) return FALSE; + if (op.mode != MIR_OP_VAR) return FALSE; break; case 'h': - if (op.mode != MIR_OP_HARD_REG) return FALSE; + if (op.mode != MIR_OP_VAR) return FALSE; ch = *++p; gen_assert ('0' <= ch && ch <= '9'); hr = ch - '0'; @@ -1909,14 +1926,14 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in else --p; gen_assert (hr <= MAX_HARD_REG); - if (op.u.hard_reg != hr) return FALSE; + if (op.u.var != hr) return FALSE; break; case 'm': case 'M': { MIR_type_t type, type2, type3 = MIR_T_BOUND; int scale, u_p, s_p; - if (op.mode != MIR_OP_HARD_REG_MEM) return FALSE; + if (op.mode != MIR_OP_VAR_MEM) return FALSE; u_p = s_p = TRUE; ch = *++p; switch (ch) { @@ -1942,7 +1959,7 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in u_p = ch == 'u'; s_p = ch == 's'; ch = *++p; - /* Fall through: */ + /* fall through */ default: gen_assert ('0' <= ch && ch <= '3'); scale = 1 << (ch - '0'); @@ -1966,18 +1983,16 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in #endif } } - if (op.u.hard_reg_mem.type != type && op.u.hard_reg_mem.type != type2 - && op.u.hard_reg_mem.type != type3) + if (op.u.var_mem.type != type && op.u.var_mem.type != type2 && op.u.var_mem.type != type3) return FALSE; if (start_ch == 'm' - && (op.u.hard_reg_mem.disp != 0 - || (op.u.hard_reg_mem.index != MIR_NON_HARD_REG && op.u.hard_reg_mem.scale != 1 - && op.u.hard_reg_mem.scale != scale))) + && (op.u.var_mem.disp != 0 + || (op.u.var_mem.index != MIR_NON_VAR && op.u.var_mem.scale != 1 + && op.u.var_mem.scale != scale))) return FALSE; if (start_ch == 'M' - && (op.u.hard_reg_mem.index != MIR_NON_HARD_REG || op.u.hard_reg_mem.disp < 0 - || op.u.hard_reg_mem.disp % scale != 0 - || op.u.hard_reg_mem.disp / scale >= (1 << 12))) + && (op.u.var_mem.index != MIR_NON_VAR || op.u.var_mem.disp < 0 + || op.u.var_mem.disp % scale != 0 || op.u.var_mem.disp / scale >= (1 << 12))) return FALSE; break; } @@ -2055,8 +2070,28 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in static const char *find_insn_pattern_replacement (gen_ctx_t gen_ctx, MIR_insn_t insn) { int i; const struct pattern *pat; - insn_pattern_info_t info = VARR_GET (insn_pattern_info_t, insn_pattern_info, insn->code); - + insn_pattern_info_t info; + int code = insn->code; + + if (code == MIR_BO || code == MIR_BNO || code == MIR_UBO || code == MIR_UBNO) { + for (MIR_insn_t prev_insn = DLIST_PREV (MIR_insn_t, insn); prev_insn != NULL; + prev_insn = DLIST_PREV (MIR_insn_t, prev_insn)) { + if (prev_insn->code == MIR_SUBOS || prev_insn->code == MIR_SUBO) { + /* unsigned sub sets up carry flag when there is no overflow: */ + if (code == MIR_UBO || code == MIR_UBNO) code = code == MIR_UBO ? SUB_UBO : SUB_UBNO; + break; + } else if (prev_insn->code == MIR_MULOS || prev_insn->code == MIR_MULO + || prev_insn->code == MIR_UMULOS || prev_insn->code == MIR_UMULO) { + /* [u]mulo[s] insns uses zero flag to check overflow: */ + code = code == MIR_BO || code == MIR_UBO ? MUL_BO : MUL_BNO; + break; + } else if (prev_insn->code == MIR_ADDOS || prev_insn->code == MIR_ADDO + || prev_insn->code == MIR_LABEL || MIR_branch_code_p (prev_insn->code)) { + break; + } + } + } + info = VARR_GET (insn_pattern_info_t, insn_pattern_info, code); for (i = 0; i < info.num; i++) { pat = &patterns[VARR_GET (int, pattern_indexes, info.start + i)]; if (pattern_match_p (gen_ctx, pat, insn)) return pat->replacement; @@ -2120,7 +2155,8 @@ static uint32_t check_and_set_mask (uint32_t opcode_mask, uint32_t mask) { return opcode_mask | mask; } -static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacement) { +static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacement, + void **jump_addrs) { MIR_context_t ctx = gen_ctx->ctx; size_t offset; const char *p, *insn_str; @@ -2173,10 +2209,10 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen if (start_ch == 'h') { reg = read_hex (&p); } else { - gen_assert ('0' <= ch && ch <= '2' && ch - '0' < insn->nops); + gen_assert ('0' <= ch && ch <= '2' && ch - '0' < (int) insn->nops); op = insn->ops[ch - '0']; - gen_assert (op.mode == MIR_OP_HARD_REG); - reg = op.u.hard_reg; + gen_assert (op.mode == MIR_OP_VAR); + reg = op.u.var; if (start_ch != 'v') { gen_assert (reg < V0_HARD_REG); } else { @@ -2196,38 +2232,38 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen break; } case 'm': - op = insn->ops[0].mode == MIR_OP_HARD_REG_MEM ? insn->ops[0] : insn->ops[1]; - rn = op.u.hard_reg_mem.base; - rm = op.u.hard_reg_mem.index == MIR_NON_HARD_REG ? ZR_HARD_REG : op.u.hard_reg_mem.index; - scale = op.u.hard_reg_mem.scale; + op = insn->ops[0].mode == MIR_OP_VAR_MEM ? insn->ops[0] : insn->ops[1]; + rn = op.u.var_mem.base; + rm = op.u.var_mem.index == MIR_NON_VAR ? ZR_HARD_REG : op.u.var_mem.index; + scale = op.u.var_mem.scale; break; case 'M': { - int scale = 1; + int dsize = 1; - op = insn->ops[0].mode == MIR_OP_HARD_REG_MEM ? insn->ops[0] : insn->ops[1]; - switch (op.u.hard_reg_mem.type) { + op = insn->ops[0].mode == MIR_OP_VAR_MEM ? insn->ops[0] : insn->ops[1]; + switch (op.u.var_mem.type) { case MIR_T_I8: - case MIR_T_U8: scale = 1; break; + case MIR_T_U8: dsize = 1; break; case MIR_T_I16: - case MIR_T_U16: scale = 2; break; + case MIR_T_U16: dsize = 2; break; #if MIR_PTR32 case MIR_T_P: #endif case MIR_T_I32: case MIR_T_U32: - case MIR_T_F: scale = 4; break; + case MIR_T_F: dsize = 4; break; #if MIR_PTR64 case MIR_T_P: #endif case MIR_T_I64: case MIR_T_U64: - case MIR_T_D: scale = 8; break; - case MIR_T_LD: scale = 16; break; + case MIR_T_D: dsize = 8; break; + case MIR_T_LD: dsize = 16; break; default: assert (FALSE); } - gen_assert (op.u.hard_reg_mem.disp % scale == 0); - rn = op.u.hard_reg_mem.base; - disp = op.u.hard_reg_mem.disp / scale; + gen_assert (op.u.var_mem.disp % dsize == 0); + rn = op.u.var_mem.base; + disp = op.u.var_mem.disp / dsize; gen_assert (disp < (1 << 12)); break; } @@ -2251,7 +2287,7 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen case 'Z': { int n, n2; uint64_t v; - struct imm imms[4]; + struct imm immediates[4]; ch = *++p; gen_assert ('0' <= ch && ch <= '3'); @@ -2265,26 +2301,34 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen } else { v = (uint64_t) op.u.ref->addr; } - n2 = movnzk_const (v, start_ch == 'N', imms); + n2 = movnzk_const (v, start_ch == 'N', immediates); gen_assert (n < n2); - imm16 = imms[n].v; - imm16_shift = imms[n].shift >> 4; + imm16 = immediates[n].v; + imm16_shift = immediates[n].shift >> 4; break; } case 'I': { ch = *++p; - if (ch != 'u') { /* I */ + if (ch == 'u') { /* Iu */ + op = insn->ops[1]; + gen_assert (op.mode == MIR_OP_INT || op.mode == MIR_OP_UINT); + imm12_shift = arithm_roundup_const (op.u.u, &imm12); + } else if (hex_value (ch) >= 0) { + immr = read_hex (&p); + } else { /* I */ op = insn->ops[2]; gen_assert (op.mode == MIR_OP_INT || op.mode == MIR_OP_UINT); imm12_shift = arithm_const (op.u.u, &imm12); p--; - } else { - op = insn->ops[1]; - gen_assert (op.mode == MIR_OP_INT || op.mode == MIR_OP_UINT); - imm12_shift = arithm_roundup_const (op.u.u, &imm12); } break; } + case 'i': { + p++; + gen_assert (hex_value (*p) >= 0); + imms = read_hex (&p); + break; + } case 'T': { gen_assert (!switch_table_addr_p && switch_table_adr_insn_start < 0); switch_table_addr_p = TRUE; @@ -2292,13 +2336,17 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen } case 'l': case 'L': { - op = insn->ops[start_ch == 'l' || (insn->code != MIR_CALL && insn->code != MIR_INLINE) ? 0 - : 1]; + int nop = 0; + if (insn->code == MIR_LADDR || insn->code == MIR_CALL || insn->code == MIR_INLINE) nop = 1; + op = insn->ops[nop]; gen_assert (op.mode == MIR_OP_LABEL || (start_ch == 'L' && op.mode == MIR_OP_REF)); lr.abs_addr_p = FALSE; lr.short_p = start_ch == 'l'; lr.label_val_disp = 0; - lr.label = op.u.label; + if (jump_addrs == NULL) + lr.u.label = op.u.label; + else + lr.u.jump_addr = jump_addrs[0]; label_ref_num = VARR_LENGTH (label_ref_t, label_refs); VARR_PUSH (label_ref_t, label_refs, lr); break; @@ -2384,16 +2432,37 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen lr.abs_addr_p = TRUE; lr.short_p = FALSE; lr.label_val_disp = VARR_LENGTH (uint8_t, result_code); - lr.label = insn->ops[i].u.label; + if (jump_addrs == NULL) + lr.u.label = insn->ops[i].u.label; + else + lr.u.jump_addr = jump_addrs[i - 1]; VARR_PUSH (label_ref_t, label_refs, lr); put_uint64 (gen_ctx, 0, 8); } } +static int target_memory_ok_p (gen_ctx_t gen_ctx, MIR_op_t *op_ref) { + gen_assert (op_ref->mode == MIR_OP_VAR_MEM); + MIR_context_t ctx = gen_ctx->ctx; + size_t size = _MIR_type_size (ctx, op_ref->u.var_mem.type); + int scale = gen_int_log2 ((int64_t) size); + + if (op_ref->u.var_mem.disp == 0 + && ((op_ref->u.var_mem.index == MIR_NON_VAR || op_ref->u.var_mem.scale == 1 + || op_ref->u.var_mem.scale == scale))) + return TRUE; + if (op_ref->u.var_mem.index == MIR_NON_VAR && op_ref->u.var_mem.disp >= 0 + && op_ref->u.var_mem.disp % scale == 0 && op_ref->u.var_mem.disp / scale < (1 << 12)) + return TRUE; + return FALSE; +} + static int target_insn_ok_p (gen_ctx_t gen_ctx, MIR_insn_t insn) { return find_insn_pattern_replacement (gen_ctx, insn) != NULL; } +static void target_split_insns (gen_ctx_t gen_ctx MIR_UNUSED) {} + static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { MIR_context_t ctx = gen_ctx->ctx; size_t i; @@ -2408,7 +2477,7 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { insn = DLIST_NEXT (MIR_insn_t, insn)) { if (insn->code == MIR_LABEL) { set_label_disp (gen_ctx, insn, VARR_LENGTH (uint8_t, result_code)); - } else { + } else if (insn->code != MIR_USE) { replacement = find_insn_pattern_replacement (gen_ctx, insn); if (replacement == NULL) { fprintf (stderr, "fatal failure in matching insn:"); @@ -2416,7 +2485,7 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { exit (1); } else { gen_assert (replacement != NULL); - out_insn (gen_ctx, insn, replacement); + out_insn (gen_ctx, insn, replacement, NULL); } } } @@ -2425,7 +2494,7 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { label_ref_t lr = VARR_GET (label_ref_t, label_refs, i); if (!lr.abs_addr_p) { - int64_t offset = (int64_t) get_label_disp (gen_ctx, lr.label) + int64_t offset = (int64_t) get_label_disp (gen_ctx, lr.u.label) - (int64_t) lr.label_val_disp; /* pc offset */ gen_assert ((offset & 0x3) == 0); if (lr.short_p) @@ -2436,7 +2505,7 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { |= (offset / 4) & 0x3ffffff; /* 26-bit */ } else { set_int64 (&VARR_ADDR (uint8_t, result_code)[lr.label_val_disp], - (int64_t) get_label_disp (gen_ctx, lr.label), 8); + (int64_t) get_label_disp (gen_ctx, lr.u.label), 8); VARR_PUSH (uint64_t, abs_address_locs, lr.label_val_disp); } } @@ -2457,19 +2526,146 @@ static void target_rebase (gen_ctx_t gen_ctx, uint8_t *base) { } _MIR_update_code_arr (gen_ctx->ctx, base, VARR_LENGTH (MIR_code_reloc_t, relocs), VARR_ADDR (MIR_code_reloc_t, relocs)); + gen_setup_lrefs (gen_ctx, base); +} + +static void target_change_to_direct_calls (MIR_context_t ctx MIR_UNUSED) {} + +struct target_bb_version { + uint8_t *base; + label_ref_t branch_ref; /* label cand used for jump to this bb version */ +}; + +static void target_init_bb_version_data (target_bb_version_t data) { + data->base = NULL; /* we don't know origin branch */ +} + +static void target_bb_translate_start (gen_ctx_t gen_ctx) { + short_bb_branch_p = FALSE; + VARR_TRUNC (uint8_t, result_code, 0); + VARR_TRUNC (label_ref_t, label_refs, 0); + VARR_TRUNC (uint64_t, abs_address_locs, 0); +} + +static void target_bb_insn_translate (gen_ctx_t gen_ctx, MIR_insn_t insn, void **jump_addrs) { + const char *replacement; + if (insn->code == MIR_LABEL) return; + replacement = find_insn_pattern_replacement (gen_ctx, insn); + gen_assert (replacement != NULL); + out_insn (gen_ctx, insn, replacement, jump_addrs); + if (MIR_branch_code_p (insn->code) && insn->code != MIR_JMP) short_bb_branch_p = TRUE; +} + +static void target_output_jump (gen_ctx_t gen_ctx, void **jump_addrs) { + out_insn (gen_ctx, temp_jump, temp_jump_replacement, jump_addrs); +} + +static uint8_t *target_bb_translate_finish (gen_ctx_t gen_ctx, size_t *len) { + /* add nop for possible conversion short branch to branch and jump */ + if (short_bb_branch_p) put_uint64 (gen_ctx, TARGET_NOP, 4); + while (VARR_LENGTH (uint8_t, result_code) % 16 != 0) /* Align the pool */ + VARR_PUSH (uint8_t, result_code, 0); + *len = VARR_LENGTH (uint8_t, result_code); + return VARR_ADDR (uint8_t, result_code); +} + +static void setup_rel (gen_ctx_t gen_ctx, label_ref_t *lr, uint8_t *base, void *addr) { + MIR_context_t ctx = gen_ctx->ctx; + int64_t offset = (int64_t) addr - (int64_t) (base + lr->label_val_disp); + + gen_assert ((offset & 0x3) == 0); + offset >>= 2; + /* check max 26-bit offset with possible branch conversion (see offset - 2): */ + if (lr->abs_addr_p || !(-(1 << 25) <= (offset - 2) && offset < (1 << 25))) { + fprintf (stderr, "too big offset (%lld) in setup_rel", (long long) offset); + exit (1); + } + /* ??? thread safe: */ + uint32_t *insn_ptr = (uint32_t *) (base + lr->label_val_disp), insn = *insn_ptr; + if (!lr->short_p) { + insn = (insn & ~0x3ffffff) | (offset & 0x3ffffff); + } else if (-(1 << 18) <= offset && offset < (1 << 18)) { /* 19 bit offset */ + insn = (insn & ~(0x7ffff << 5)) | ((offset & 0x7ffff) << 5); + } else { + insn = (insn & ~(0x7ffff << 5)) | (2 << 5); /* skip jump */ + uint32_t *nop_ptr = (uint32_t *) (base + lr->label_val_disp + 8); + gen_assert (TARGET_NOP == *nop_ptr || (*nop_ptr & ~0x3ffffff) == 0x14000000); /* nop or jump */ + uint32_t jump_insn = 0x14000000 | ((offset - 2) & 0x3ffffff); + _MIR_change_code (ctx, (uint8_t *) nop_ptr, (uint8_t *) &jump_insn, 4); + lr->short_p = FALSE; + lr->label_val_disp += 8; + } + _MIR_change_code (ctx, (uint8_t *) insn_ptr, (uint8_t *) &insn, 4); +} + +static void target_bb_rebase (gen_ctx_t gen_ctx, uint8_t *base) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_code_reloc_t reloc; + + /* Setting up relative labels */ + for (size_t i = 0; i < VARR_LENGTH (label_ref_t, label_refs); i++) { + label_ref_t lr = VARR_GET (label_ref_t, label_refs, i); + if (lr.abs_addr_p) { + _MIR_change_code (ctx, (uint8_t *) base + lr.label_val_disp, (uint8_t *) &lr.u.jump_addr, 8); + } else { + setup_rel (gen_ctx, &lr, base, lr.u.jump_addr); + } + } + VARR_TRUNC (MIR_code_reloc_t, relocs, 0); + for (size_t i = 0; i < VARR_LENGTH (uint64_t, abs_address_locs); i++) { + reloc.offset = VARR_GET (uint64_t, abs_address_locs, i); + reloc.value = base + get_int64 (base + reloc.offset, 8); + VARR_PUSH (MIR_code_reloc_t, relocs, reloc); + } + _MIR_update_code_arr (gen_ctx->ctx, base, VARR_LENGTH (MIR_code_reloc_t, relocs), + VARR_ADDR (MIR_code_reloc_t, relocs)); +} + +static void target_setup_succ_bb_version_data (gen_ctx_t gen_ctx, uint8_t *base) { + if (VARR_LENGTH (label_ref_t, label_refs) + != VARR_LENGTH (target_bb_version_t, target_succ_bb_versions)) + /* We can have more one possible branch from original insn + (e.g. SWITCH, FBNE). If it is so, we will make jumps only + through BB thunk. */ + return; + for (size_t i = 0; i < VARR_LENGTH (target_bb_version_t, target_succ_bb_versions); i++) { + target_bb_version_t data = VARR_GET (target_bb_version_t, target_succ_bb_versions, i); + if (data == NULL) continue; + data->branch_ref = VARR_GET (label_ref_t, label_refs, i); + data->base = base; + } +} + +static void target_redirect_bb_origin_branch (gen_ctx_t gen_ctx, target_bb_version_t data, + void *addr) { + MIR_context_t ctx = gen_ctx->ctx; + + if (data->base == NULL) return; + if (data->branch_ref.abs_addr_p) { + _MIR_change_code (ctx, (uint8_t *) data->base + data->branch_ref.label_val_disp, + (uint8_t *) &addr, 8); + } else { + setup_rel (gen_ctx, &data->branch_ref, data->base, addr); + } + data->base = NULL; } static void target_init (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; + gen_ctx->target_ctx = gen_malloc (gen_ctx, sizeof (struct target_ctx)); VARR_CREATE (uint8_t, result_code, 0); VARR_CREATE (label_ref_t, label_refs, 0); VARR_CREATE (uint64_t, abs_address_locs, 0); VARR_CREATE (MIR_code_reloc_t, relocs, 0); patterns_init (gen_ctx); + temp_jump = MIR_new_insn (ctx, MIR_JMP, MIR_new_label_op (ctx, NULL)); + temp_jump_replacement = find_insn_pattern_replacement (gen_ctx, temp_jump); } static void target_finish (gen_ctx_t gen_ctx) { patterns_finish (gen_ctx); + _MIR_free_insn (gen_ctx->ctx, temp_jump); VARR_DESTROY (uint8_t, result_code); VARR_DESTROY (label_ref_t, label_refs); VARR_DESTROY (uint64_t, abs_address_locs); diff --git a/mir/mir-gen-ppc64.c b/mir/mir-gen-ppc64.c index 4454cbdf..091220ad 100644 --- a/mir/mir-gen-ppc64.c +++ b/mir/mir-gen-ppc64.c @@ -1,7 +1,11 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ +#include "mir-ppc64.h" + +#include + /* We don't use TOC. So r2 is not necessary for the generated code. */ static void fancy_abort (int code) { if (!code) abort (); @@ -9,66 +13,31 @@ static void fancy_abort (int code) { #undef gen_assert #define gen_assert(c) fancy_abort (c) -#include - -#define HREG_EL(h) h##_HARD_REG -#define REP_SEP , -enum { - REP8 (HREG_EL, R0, R1, R2, R3, R4, R5, R6, R7), - REP8 (HREG_EL, R8, R9, R10, R11, R12, R13, R14, R15), - REP8 (HREG_EL, R16, R17, R18, R19, R20, R21, R22, R23), - REP8 (HREG_EL, R24, R25, R26, R27, R28, R29, R30, R31), - REP8 (HREG_EL, F0, F1, F2, F3, F4, F5, F6, F7), - REP8 (HREG_EL, F8, F9, F10, F11, F12, F13, F14, F15), - REP8 (HREG_EL, F16, F17, F18, F19, F20, F21, F22, F23), - REP8 (HREG_EL, F24, F25, F26, F27, F28, F29, F30, F31), - HREG_EL (LR), -}; -#undef REP_SEP +#define TARGET_EXPAND_ADDO +#define TARGET_EXPAND_ADDOS +#define TARGET_EXPAND_UADDO +#define TARGET_EXPAND_UADDOS +#define TARGET_EXPAND_MULO +#define TARGET_EXPAND_MULOS +#define TARGET_EXPAND_UMULO +#define TARGET_EXPAND_UMULOS -static const MIR_reg_t MAX_HARD_REG = LR_HARD_REG; -static const MIR_reg_t SP_HARD_REG = R1_HARD_REG; -static const MIR_reg_t FP_HARD_REG = R31_HARD_REG; static const MIR_reg_t LINK_HARD_REG = LR_HARD_REG; -static int target_locs_num (MIR_reg_t loc, MIR_type_t type) { - return /*loc > MAX_HARD_REG && */ type == MIR_T_LD ? 2 : 1; -} - -static inline MIR_reg_t target_nth_loc (MIR_reg_t loc, MIR_type_t type, int n) { return loc + n; } - -/* Hard regs not used in machinized code, preferably call used ones. */ -const MIR_reg_t TEMP_INT_HARD_REG1 = R11_HARD_REG, TEMP_INT_HARD_REG2 = R12_HARD_REG; -const MIR_reg_t TEMP_FLOAT_HARD_REG1 = F11_HARD_REG, TEMP_FLOAT_HARD_REG2 = F12_HARD_REG; -const MIR_reg_t TEMP_DOUBLE_HARD_REG1 = F11_HARD_REG, TEMP_DOUBLE_HARD_REG2 = F12_HARD_REG; -const MIR_reg_t TEMP_LDOUBLE_HARD_REG1 = F11_HARD_REG; //??? -const MIR_reg_t TEMP_LDOUBLE_HARD_REG2 = F12_HARD_REG; - -static inline int target_hard_reg_type_ok_p (MIR_reg_t hard_reg, MIR_type_t type) { - assert (hard_reg <= MAX_HARD_REG); - if (type == MIR_T_LD) return FALSE; - return MIR_fp_type_p (type) ? F0_HARD_REG <= hard_reg && hard_reg <= F31_HARD_REG - : hard_reg < F0_HARD_REG; -} - -static inline int target_fixed_hard_reg_p (MIR_reg_t hard_reg) { - assert (hard_reg <= MAX_HARD_REG); - return (hard_reg == FP_HARD_REG || hard_reg == SP_HARD_REG - || hard_reg == LR_HARD_REG - /* don't bother to allocate R0 as it has special meaning for base reg and of addi: */ - || hard_reg == R0_HARD_REG || hard_reg == R2_HARD_REG || hard_reg == R13_HARD_REG - || hard_reg == TEMP_INT_HARD_REG1 || hard_reg == TEMP_INT_HARD_REG2 - || hard_reg == TEMP_FLOAT_HARD_REG1 || hard_reg == TEMP_FLOAT_HARD_REG2 - || hard_reg == TEMP_DOUBLE_HARD_REG1 || hard_reg == TEMP_DOUBLE_HARD_REG2 - || hard_reg == TEMP_LDOUBLE_HARD_REG1 || hard_reg == TEMP_LDOUBLE_HARD_REG2); +static inline MIR_reg_t target_nth_loc (MIR_reg_t loc, MIR_type_t type MIR_UNUSED, int n) { + return loc + n; } -static inline int target_call_used_hard_reg_p (MIR_reg_t hard_reg, MIR_type_t type) { +static inline int target_call_used_hard_reg_p (MIR_reg_t hard_reg, MIR_type_t type MIR_UNUSED) { assert (hard_reg <= MAX_HARD_REG); return ((R0_HARD_REG <= hard_reg && hard_reg <= R13_HARD_REG) || (F0_HARD_REG <= hard_reg && hard_reg <= F13_HARD_REG)); } +static MIR_reg_t target_get_stack_slot_base_reg (gen_ctx_t gen_ctx MIR_UNUSED) { + return FP_HARD_REG; +} + /* Stack layout (r1(sp) refers to the last reserved stack slot address) from higher address to lower address memory: @@ -90,16 +59,6 @@ SP,R31->+-- Back chain (SP + 0) (SP + Originally SP(r1) and FP (r31) are the same but r1 can be changed by alloca */ -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define PPC64_STACK_HEADER_SIZE 32 -#define PPC64_TOC_OFFSET 24 -#define PPC64_FUNC_DESC_LEN 0 -#else -#define PPC64_STACK_HEADER_SIZE 48 -#define PPC64_TOC_OFFSET 40 -#define PPC64_FUNC_DESC_LEN 24 -#endif - /* ppc64 has 3-ops insns */ static const MIR_insn_code_t target_io_dup_op_insn_codes[] = {MIR_INSN_BOUND}; @@ -115,10 +74,6 @@ static MIR_insn_code_t get_ext_code (MIR_type_t type) { } } -DEF_VARR (int); -DEF_VARR (uint8_t); -DEF_VARR (uint64_t); - struct insn_pattern_info { int start, num; }; @@ -126,20 +81,25 @@ struct insn_pattern_info { typedef struct insn_pattern_info insn_pattern_info_t; DEF_VARR (insn_pattern_info_t); +enum branch_type { BRCOND, JUMP, LADDR, BCTR }; struct label_ref { - int abs_addr_p, short_addr_p; + int abs_addr_p; + enum branch_type branch_type; size_t label_val_disp; - MIR_label_t label; + union { + MIR_label_t label; + void *jump_addr; /* absolute addr for BBV */ + } u; }; typedef struct label_ref label_ref_t; DEF_VARR (label_ref_t); -DEF_VARR (MIR_code_reloc_t); - struct target_ctx { - unsigned char alloca_p, block_arg_func_p, leaf_p, switch_p; + unsigned char alloca_p, block_arg_func_p, leaf_p, switch_p, laddr_p, short_bb_branch_p; size_t param_save_area_size; + MIR_insn_t temp_jump; + const char *temp_jump_replacement; VARR (int) * pattern_indexes; VARR (insn_pattern_info_t) * insn_pattern_info; VARR (uint8_t) * result_code; @@ -152,7 +112,11 @@ struct target_ctx { #define block_arg_func_p gen_ctx->target_ctx->block_arg_func_p #define leaf_p gen_ctx->target_ctx->leaf_p #define switch_p gen_ctx->target_ctx->switch_p +#define laddr_p gen_ctx->target_ctx->laddr_p +#define short_bb_branch_p gen_ctx->target_ctx->short_bb_branch_p #define param_save_area_size gen_ctx->target_ctx->param_save_area_size +#define temp_jump gen_ctx->target_ctx->temp_jump +#define temp_jump_replacement gen_ctx->target_ctx->temp_jump_replacement #define pattern_indexes gen_ctx->target_ctx->pattern_indexes #define insn_pattern_info gen_ctx->target_ctx->insn_pattern_info #define result_code gen_ctx->target_ctx->result_code @@ -181,51 +145,50 @@ static void gen_blk_mov (gen_ctx_t gen_ctx, MIR_insn_t anchor, size_t to_disp, MIR_insn_t new_insn; MIR_op_t ops[5], freg_op, treg_op, treg_op2, treg_op3; - treg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + treg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); if (qwords <= 16) { for (; qwords > 0; qwords--, to_disp += 8, from_disp += 8) { gen_mov (gen_ctx, anchor, MIR_MOV, treg_op, - MIR_new_mem_op (ctx, MIR_T_I64, from_disp, from_base_reg, 0, 1)); + _MIR_new_var_mem_op (ctx, MIR_T_I64, from_disp, from_base_reg, MIR_NON_VAR, 1)); gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, to_disp, to_base_hard_reg, - MIR_NON_HARD_REG, 1), + _MIR_new_var_mem_op (ctx, MIR_T_I64, to_disp, to_base_hard_reg, MIR_NON_VAR, 1), treg_op); } return; } - treg_op2 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - treg_op3 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + treg_op2 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + treg_op3 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); /* Save arg regs: */ - if (save_regs > 0) gen_mov (gen_ctx, anchor, MIR_MOV, treg_op, _MIR_new_hard_reg_op (ctx, 3)); - if (save_regs > 1) gen_mov (gen_ctx, anchor, MIR_MOV, treg_op2, _MIR_new_hard_reg_op (ctx, 4)); - if (save_regs > 2) gen_mov (gen_ctx, anchor, MIR_MOV, treg_op3, _MIR_new_hard_reg_op (ctx, 5)); + if (save_regs > 0) gen_mov (gen_ctx, anchor, MIR_MOV, treg_op, _MIR_new_var_op (ctx, 3)); + if (save_regs > 1) gen_mov (gen_ctx, anchor, MIR_MOV, treg_op2, _MIR_new_var_op (ctx, 4)); + if (save_regs > 2) gen_mov (gen_ctx, anchor, MIR_MOV, treg_op3, _MIR_new_var_op (ctx, 5)); /* call blk move: */ proto_item = _MIR_builtin_proto (ctx, curr_func_item->module, BLK_MOV_P, 0, NULL, 3, MIR_T_I64, "to", MIR_T_I64, "from", MIR_T_I64, "nwords"); func_import_item = _MIR_builtin_func (ctx, curr_func_item->module, BLK_MOV, mir_blk_mov); - freg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + freg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); new_insn = MIR_new_insn (ctx, MIR_MOV, freg_op, MIR_new_ref_op (ctx, func_import_item)); gen_add_insn_before (gen_ctx, anchor, new_insn); gen_add_insn_before (gen_ctx, anchor, - MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_hard_reg_op (ctx, 3), - _MIR_new_hard_reg_op (ctx, to_base_hard_reg), + MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_var_op (ctx, 3), + _MIR_new_var_op (ctx, to_base_hard_reg), MIR_new_int_op (ctx, to_disp))); gen_add_insn_before (gen_ctx, anchor, - MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_hard_reg_op (ctx, 4), - MIR_new_reg_op (ctx, from_base_reg), + MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_var_op (ctx, 4), + _MIR_new_var_op (ctx, from_base_reg), MIR_new_int_op (ctx, from_disp))); - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, 5), MIR_new_int_op (ctx, qwords)); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, 5), MIR_new_int_op (ctx, qwords)); ops[0] = MIR_new_ref_op (ctx, proto_item); ops[1] = freg_op; - ops[2] = _MIR_new_hard_reg_op (ctx, 3); - ops[3] = _MIR_new_hard_reg_op (ctx, 4); - ops[4] = _MIR_new_hard_reg_op (ctx, 5); + ops[2] = _MIR_new_var_op (ctx, 3); + ops[3] = _MIR_new_var_op (ctx, 4); + ops[4] = _MIR_new_var_op (ctx, 5); new_insn = MIR_new_insn_arr (ctx, MIR_CALL, 5, ops); gen_add_insn_before (gen_ctx, anchor, new_insn); /* Restore arg regs: */ - if (save_regs > 0) gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, 3), treg_op); - if (save_regs > 1) gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, 4), treg_op2); - if (save_regs > 2) gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, 5), treg_op3); + if (save_regs > 0) gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, 3), treg_op); + if (save_regs > 1) gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, 4), treg_op2); + if (save_regs > 2) gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, 5), treg_op3); } static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { @@ -252,20 +215,20 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { nargs = VARR_LENGTH (MIR_var_t, proto->args); arg_vars = VARR_ADDR (MIR_var_t, proto->args); } - if (call_insn->ops[1].mode != MIR_OP_REG && call_insn->ops[1].mode != MIR_OP_HARD_REG) { + if (call_insn->ops[1].mode != MIR_OP_VAR) { // ??? to optimize (can be immediate operand for func call) - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); new_insn = MIR_new_insn (ctx, MIR_MOV, temp_op, call_insn->ops[1]); call_insn->ops[1] = temp_op; gen_add_insn_before (gen_ctx, call_insn, new_insn); } for (size_t i = start; i < nops; i++) { arg_op = call_insn->ops[i]; - gen_assert (arg_op.mode == MIR_OP_REG || arg_op.mode == MIR_OP_HARD_REG - || (arg_op.mode == MIR_OP_MEM && MIR_all_blk_type_p (arg_op.u.mem.type))); + gen_assert (arg_op.mode == MIR_OP_VAR + || (arg_op.mode == MIR_OP_VAR_MEM && MIR_all_blk_type_p (arg_op.u.mem.type))); if (i - start < nargs) { type = arg_vars[i - start].type; - } else if (call_insn->ops[i].mode == MIR_OP_MEM) { + } else if (call_insn->ops[i].mode == MIR_OP_VAR_MEM) { type = arg_op.u.mem.type; gen_assert (MIR_all_blk_type_p (type)); } else { @@ -279,7 +242,7 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { } ext_insn = NULL; if ((ext_code = get_ext_code (type)) != MIR_INVALID_INSN) { /* extend arg if necessary */ - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); ext_insn = MIR_new_insn (ctx, ext_code, temp_op, arg_op); call_insn->ops[i] = arg_op = temp_op; } @@ -287,41 +250,44 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { if ((type == MIR_T_F || type == MIR_T_D || type == MIR_T_LD) && n_fregs < 13) { /* put arguments to argument hard regs */ if (ext_insn != NULL) gen_add_insn_before (gen_ctx, call_insn, ext_insn); - arg_reg_op = _MIR_new_hard_reg_op (ctx, F1_HARD_REG + n_fregs); + arg_reg_op = _MIR_new_var_op (ctx, F1_HARD_REG + n_fregs); gen_mov (gen_ctx, call_insn, - type == MIR_T_F ? MIR_FMOV : type == MIR_T_D ? MIR_DMOV : MIR_LDMOV, // ??? + type == MIR_T_F ? MIR_FMOV + : type == MIR_T_D ? MIR_DMOV + : MIR_LDMOV, // ??? arg_reg_op, arg_op); call_insn->ops[i] = arg_reg_op; if (vararg_p) { // ??? dead insns if (n_iregs >= 8 || (type == MIR_T_LD && n_iregs == 7)) { /* store in memory too */ - mem_op = _MIR_new_hard_reg_mem_op (ctx, mem_type, mem_size + PPC64_STACK_HEADER_SIZE, - SP_HARD_REG, MIR_NON_HARD_REG, 1); + mem_op = _MIR_new_var_mem_op (ctx, mem_type, mem_size + PPC64_STACK_HEADER_SIZE, + SP_HARD_REG, MIR_NON_VAR, 1); gen_assert (n_fregs < 12); gen_mov (gen_ctx, call_insn, type == MIR_T_LD ? MIR_LDMOV : MIR_DMOV, mem_op, arg_reg_op); } if (n_iregs < 8) { /* load into gp reg too */ - mem_op = _MIR_new_hard_reg_mem_op (ctx, mem_type, -16, SP_HARD_REG, MIR_NON_HARD_REG, 1); + mem_op = _MIR_new_var_mem_op (ctx, mem_type, -16, SP_HARD_REG, MIR_NON_VAR, 1); gen_mov (gen_ctx, call_insn, type == MIR_T_LD ? MIR_LDMOV : MIR_DMOV, mem_op, arg_reg_op); mem_type = mem_type == MIR_T_F ? MIR_T_I32 : MIR_T_I64; // ??? - mem_op = _MIR_new_hard_reg_mem_op (ctx, mem_type, -16, SP_HARD_REG, MIR_NON_HARD_REG, 1); - arg_reg_op = _MIR_new_hard_reg_op (ctx, R3_HARD_REG + n_iregs); + mem_op = _MIR_new_var_mem_op (ctx, mem_type, -16, SP_HARD_REG, MIR_NON_VAR, 1); + arg_reg_op = _MIR_new_var_op (ctx, R3_HARD_REG + n_iregs); gen_mov (gen_ctx, call_insn, MIR_MOV, arg_reg_op, mem_op); call_insn->ops[i] = arg_reg_op; /* keep it alive */ if (type == MIR_T_LD && n_iregs + 1 < 8) { - mem_op = _MIR_new_hard_reg_mem_op (ctx, mem_type, -8, SP_HARD_REG, MIR_NON_HARD_REG, 1); - gen_mov (gen_ctx, call_insn, MIR_MOV, - _MIR_new_hard_reg_op (ctx, R3_HARD_REG + n_iregs + 1), mem_op); + mem_op = _MIR_new_var_mem_op (ctx, mem_type, -8, SP_HARD_REG, MIR_NON_VAR, 1); + gen_mov (gen_ctx, call_insn, MIR_MOV, _MIR_new_var_op (ctx, R3_HARD_REG + n_iregs + 1), + mem_op); } } } n_fregs += type == MIR_T_LD ? 2 : 1; } else if (MIR_blk_type_p (type)) { - gen_assert (arg_op.mode == MIR_OP_MEM && arg_op.u.mem.disp >= 0 && arg_op.u.mem.index == 0); + gen_assert (arg_op.mode == MIR_OP_VAR_MEM && arg_op.u.mem.disp >= 0 + && arg_op.u.mem.index == MIR_NON_VAR); qwords = (arg_op.u.mem.disp + 7) / 8; for (disp = 0; qwords > 0 && n_iregs < 8; qwords--, n_iregs++, mem_size += 8, disp += 8) { - arg_reg_op = _MIR_new_hard_reg_op (ctx, R3_HARD_REG + n_iregs); + arg_reg_op = _MIR_new_var_op (ctx, R3_HARD_REG + n_iregs); gen_mov (gen_ctx, call_insn, MIR_MOV, arg_reg_op, - MIR_new_mem_op (ctx, MIR_T_I64, disp, arg_op.u.mem.base, 0, 1)); + _MIR_new_var_mem_op (ctx, MIR_T_I64, disp, arg_op.u.mem.base, MIR_NON_VAR, 1)); setup_call_hard_reg_args (gen_ctx, call_insn, R3_HARD_REG + n_iregs); } if (qwords > 0) @@ -332,45 +298,42 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { continue; } else if (type != MIR_T_F && type != MIR_T_D && type != MIR_T_LD && n_iregs < 8) { if (ext_insn != NULL) gen_add_insn_before (gen_ctx, call_insn, ext_insn); - arg_reg_op = _MIR_new_hard_reg_op (ctx, R3_HARD_REG + n_iregs); + arg_reg_op = _MIR_new_var_op (ctx, R3_HARD_REG + n_iregs); if (type != MIR_T_RBLK) { gen_mov (gen_ctx, call_insn, MIR_MOV, arg_reg_op, arg_op); } else { - assert (arg_op.mode == MIR_OP_MEM); - gen_mov (gen_ctx, call_insn, MIR_MOV, arg_reg_op, MIR_new_reg_op (ctx, arg_op.u.mem.base)); - arg_reg_op = _MIR_new_hard_reg_mem_op (ctx, MIR_T_RBLK, arg_op.u.mem.disp, - R3_HARD_REG + n_iregs, MIR_NON_HARD_REG, 1); + assert (arg_op.mode == MIR_OP_VAR_MEM); + gen_mov (gen_ctx, call_insn, MIR_MOV, arg_reg_op, _MIR_new_var_op (ctx, arg_op.u.mem.base)); + arg_reg_op = _MIR_new_var_mem_op (ctx, MIR_T_RBLK, arg_op.u.mem.disp, R3_HARD_REG + n_iregs, + MIR_NON_VAR, 1); } call_insn->ops[i] = arg_reg_op; } else { /* put arguments on the stack */ if (ext_insn != NULL) gen_add_insn_before (gen_ctx, call_insn, ext_insn); - new_insn_code - = (type == MIR_T_F ? MIR_FMOV - : type == MIR_T_D ? MIR_DMOV : type == MIR_T_LD ? MIR_LDMOV : MIR_MOV); - mem_op = _MIR_new_hard_reg_mem_op (ctx, mem_type, mem_size + PPC64_STACK_HEADER_SIZE, - SP_HARD_REG, MIR_NON_HARD_REG, 1); + new_insn_code = (type == MIR_T_F ? MIR_FMOV + : type == MIR_T_D ? MIR_DMOV + : type == MIR_T_LD ? MIR_LDMOV + : MIR_MOV); + mem_op = _MIR_new_var_mem_op (ctx, mem_type, mem_size + PPC64_STACK_HEADER_SIZE, SP_HARD_REG, + MIR_NON_VAR, 1); if (type != MIR_T_RBLK) { gen_mov (gen_ctx, call_insn, new_insn_code, mem_op, arg_op); } else { - assert (arg_op.mode == MIR_OP_MEM); + assert (arg_op.mode == MIR_OP_VAR_MEM); gen_mov (gen_ctx, call_insn, new_insn_code, mem_op, - MIR_new_reg_op (ctx, arg_op.u.mem.base)); + _MIR_new_var_op (ctx, arg_op.u.mem.base)); } call_insn->ops[i] = mem_op; } mem_size += type == MIR_T_LD ? 16 : 8; n_iregs += type == MIR_T_LD ? 2 : 1; } -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ if (vararg_p && mem_size < 64) mem_size = 64; /* to save all arg gprs */ -#else - if (mem_size < 64) mem_size = 64; /* minimal param save area */ -#endif if (param_save_area_size < mem_size) param_save_area_size = mem_size; n_iregs = n_fregs = 0; for (size_t i = 0; i < proto->nres; i++) { ret_reg_op = call_insn->ops[i + 2]; - gen_assert (ret_reg_op.mode == MIR_OP_REG || ret_reg_op.mode == MIR_OP_HARD_REG); + gen_assert (ret_reg_op.mode == MIR_OP_VAR); type = proto->res_types[i]; if (((type == MIR_T_F || type == MIR_T_D) && n_fregs < 4) || (type == MIR_T_LD && n_fregs < 3)) { @@ -383,7 +346,7 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { (*MIR_get_error_func (ctx)) (MIR_ret_error, "ppc64 can not handle this combination of return values"); } - new_insn = MIR_new_insn (ctx, new_insn_code, ret_reg_op, _MIR_new_hard_reg_op (ctx, ret_reg)); + new_insn = MIR_new_insn (ctx, new_insn_code, ret_reg_op, _MIR_new_var_op (ctx, ret_reg)); MIR_insert_insn_after (ctx, curr_func_item, call_insn, new_insn); call_insn->ops[i + 2] = new_insn->ops[1]; if ((ext_code = get_ext_code (type)) != MIR_INVALID_INSN) { @@ -604,7 +567,7 @@ static int get_builtin (gen_ctx_t gen_ctx, MIR_insn_code_t code, MIR_item_t *pro } } -static MIR_disp_t target_get_stack_slot_offset (gen_ctx_t gen_ctx, MIR_type_t type, +static MIR_disp_t target_get_stack_slot_offset (gen_ctx_t gen_ctx, MIR_type_t type MIR_UNUSED, MIR_reg_t slot) { /* slot is 0, 1, ... */ return ((MIR_disp_t) slot * 8 + PPC64_STACK_HEADER_SIZE + param_save_area_size); @@ -614,14 +577,14 @@ static void set_prev_sp_op (gen_ctx_t gen_ctx, MIR_insn_t anchor, MIR_op_t *prev if (!block_arg_func_p) { /* don't use r11 as we can have spilled param<-mem in param set up which needs r11 as a temp */ block_arg_func_p = TRUE; - *prev_sp_op = _MIR_new_hard_reg_op (gen_ctx->ctx, R12_HARD_REG); + *prev_sp_op = _MIR_new_var_op (gen_ctx->ctx, R12_HARD_REG); gen_mov (gen_ctx, anchor, MIR_MOV, *prev_sp_op, - _MIR_new_hard_reg_mem_op (gen_ctx->ctx, MIR_T_I64, 0, SP_HARD_REG, MIR_NON_HARD_REG, - 1)); + _MIR_new_var_mem_op (gen_ctx->ctx, MIR_T_I64, 0, SP_HARD_REG, MIR_NON_VAR, 1)); } } -static int target_valid_mem_offset_p (gen_ctx_t gen_ctx, MIR_type_t type, MIR_disp_t offset) { +static int target_valid_mem_offset_p (gen_ctx_t gen_ctx MIR_UNUSED, MIR_type_t type MIR_UNUSED, + MIR_disp_t offset MIR_UNUSED) { return TRUE; } @@ -647,25 +610,29 @@ static void target_machinize (gen_ctx_t gen_ctx) { /* Argument extensions is already done in simplify */ /* Prologue: generate arg_var = hard_reg|stack mem ... */ type = VARR_GET (MIR_var_t, func->vars, i).type; - arg_var_op = MIR_new_reg_op (ctx, i + 1); + arg_var_op = _MIR_new_var_op (ctx, i + MAX_HARD_REG + 1); if ((type == MIR_T_F || type == MIR_T_D || type == MIR_T_LD) && fp_arg_num < 13) { if (type == MIR_T_LD && fp_arg_num == 12) { /* dmov f14,disp(r1) -> lfd f14,disp(r1): */ set_prev_sp_op (gen_ctx, anchor, &prev_sp_op); - arg_reg_op = _MIR_new_hard_reg_op (ctx, F14_HARD_REG); + arg_reg_op = _MIR_new_var_op (ctx, F14_HARD_REG); gen_mov (gen_ctx, anchor, MIR_DMOV, arg_reg_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_D, disp + 8, R12_HARD_REG, MIR_NON_HARD_REG, - 1)); + _MIR_new_var_mem_op (ctx, MIR_T_D, disp + 8, R12_HARD_REG, MIR_NON_VAR, 1)); } - arg_reg_op = _MIR_new_hard_reg_op (ctx, F1_HARD_REG + fp_arg_num); - gen_mov (gen_ctx, anchor, type == MIR_T_F ? MIR_FMOV : type == MIR_T_D ? MIR_DMOV : MIR_LDMOV, + arg_reg_op = _MIR_new_var_op (ctx, F1_HARD_REG + fp_arg_num); + gen_mov (gen_ctx, anchor, + type == MIR_T_F ? MIR_FMOV + : type == MIR_T_D ? MIR_DMOV + : MIR_LDMOV, arg_var_op, arg_reg_op); /* (f|d|ld|)mov arg, arg_hard_reg */ fp_arg_num += type == MIR_T_LD ? 2 : 1; } else if (type == MIR_T_F || type == MIR_T_D || type == MIR_T_LD) { /* (f|d|ld|)mov arg, arg_memory */ set_prev_sp_op (gen_ctx, anchor, &prev_sp_op); - gen_mov (gen_ctx, anchor, type == MIR_T_F ? MIR_FMOV : type == MIR_T_D ? MIR_DMOV : MIR_LDMOV, - arg_var_op, - _MIR_new_hard_reg_mem_op (ctx, type, disp, R12_HARD_REG, MIR_NON_HARD_REG, 1)); + gen_mov (gen_ctx, anchor, + type == MIR_T_F ? MIR_FMOV + : type == MIR_T_D ? MIR_DMOV + : MIR_LDMOV, + arg_var_op, _MIR_new_var_mem_op (ctx, type, disp, R12_HARD_REG, MIR_NON_VAR, 1)); } else if (MIR_blk_type_p (type)) { qwords = (VARR_GET (MIR_var_t, func->vars, i).size + 7) / 8; offset = int_arg_num < 8 ? PPC64_STACK_HEADER_SIZE + int_arg_num * 8 : disp; @@ -673,31 +640,30 @@ static void target_machinize (gen_ctx_t gen_ctx) { for (; qwords > 0 && int_arg_num < 8; qwords--, int_arg_num++, disp += 8) { if (!func->vararg_p) gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, - PPC64_STACK_HEADER_SIZE + int_arg_num * 8, - R12_HARD_REG, MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (ctx, R3_HARD_REG + int_arg_num)); + _MIR_new_var_mem_op (ctx, MIR_T_I64, PPC64_STACK_HEADER_SIZE + int_arg_num * 8, + R12_HARD_REG, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, R3_HARD_REG + int_arg_num)); } gen_add_insn_before (gen_ctx, anchor, MIR_new_insn (ctx, MIR_ADD, arg_var_op, - _MIR_new_hard_reg_op (ctx, R12_HARD_REG), + _MIR_new_var_op (ctx, R12_HARD_REG), MIR_new_int_op (ctx, offset))); disp += qwords * 8; int_arg_num += qwords; continue; } else if (int_arg_num < 8) { /* mov arg, arg_hard_reg */ - arg_reg_op = _MIR_new_hard_reg_op (ctx, R3_HARD_REG + int_arg_num); + arg_reg_op = _MIR_new_var_op (ctx, R3_HARD_REG + int_arg_num); gen_mov (gen_ctx, anchor, MIR_MOV, arg_var_op, arg_reg_op); } else { /* mov arg, arg_memory */ set_prev_sp_op (gen_ctx, anchor, &prev_sp_op); gen_mov (gen_ctx, anchor, MIR_MOV, arg_var_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, disp, R12_HARD_REG, MIR_NON_HARD_REG, 1)); + _MIR_new_var_mem_op (ctx, MIR_T_I64, disp, R12_HARD_REG, MIR_NON_VAR, 1)); } disp += type == MIR_T_LD ? 16 : 8; int_arg_num += type == MIR_T_LD ? 2 : 1; } var_args_start = disp; - switch_p = alloca_p = FALSE; + switch_p = laddr_p = alloca_p = FALSE; leaf_p = TRUE; for (insn = DLIST_HEAD (MIR_insn_t, func->insns); insn != NULL; insn = next_insn) { MIR_item_t proto_item, func_import_item; @@ -707,14 +673,13 @@ static void target_machinize (gen_ctx_t gen_ctx) { code = insn->code; if (code == MIR_LDBEQ || code == MIR_LDBNE || code == MIR_LDBLT || code == MIR_LDBGE || code == MIR_LDBGT || code == MIR_LDBLE) { /* split to cmp and branch */ - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - code = (code == MIR_LDBEQ - ? MIR_LDEQ - : code == MIR_LDBNE - ? MIR_LDNE - : code == MIR_LDBLT - ? MIR_LDLT - : code == MIR_LDBGE ? MIR_LDGE : code == MIR_LDBGT ? MIR_LDGT : MIR_LDLE); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + code = (code == MIR_LDBEQ ? MIR_LDEQ + : code == MIR_LDBNE ? MIR_LDNE + : code == MIR_LDBLT ? MIR_LDLT + : code == MIR_LDBGE ? MIR_LDGE + : code == MIR_LDBGT ? MIR_LDGT + : MIR_LDLE); new_insn = MIR_new_insn (ctx, code, temp_op, insn->ops[1], insn->ops[2]); gen_add_insn_before (gen_ctx, insn, new_insn); next_insn = MIR_new_insn (ctx, MIR_BT, insn->ops[0], temp_op); @@ -730,10 +695,10 @@ static void target_machinize (gen_ctx_t gen_ctx) { MIR_op_t ops[6], func_reg_op, reg_op3; MIR_op_t res_reg_op = insn->ops[0], va_reg_op = insn->ops[1], op3 = insn->ops[2]; - assert (res_reg_op.mode == MIR_OP_REG && va_reg_op.mode == MIR_OP_REG - && op3.mode == (code == MIR_VA_ARG ? MIR_OP_MEM : MIR_OP_REG)); - func_reg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - reg_op3 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + assert (res_reg_op.mode == MIR_OP_VAR && va_reg_op.mode == MIR_OP_VAR + && op3.mode == (code == MIR_VA_ARG ? MIR_OP_VAR_MEM : MIR_OP_VAR)); + func_reg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + reg_op3 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); next_insn = new_insn = MIR_new_insn (ctx, MIR_MOV, func_reg_op, MIR_new_ref_op (ctx, func_import_item)); gen_add_insn_before (gen_ctx, insn, new_insn); @@ -755,8 +720,8 @@ static void target_machinize (gen_ctx_t gen_ctx) { } else { /* Use builtin: mov freg, func ref; call proto, freg, res_reg, op_reg[, op_reg2] */ MIR_op_t freg_op, res_reg_op = insn->ops[0], op_reg_op = insn->ops[1], ops[5]; - assert (res_reg_op.mode == MIR_OP_REG && op_reg_op.mode == MIR_OP_REG); - freg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + assert (res_reg_op.mode == MIR_OP_VAR && op_reg_op.mode == MIR_OP_VAR); + freg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); next_insn = new_insn = MIR_new_insn (ctx, MIR_MOV, freg_op, MIR_new_ref_op (ctx, func_import_item)); gen_add_insn_before (gen_ctx, insn, new_insn); @@ -770,22 +735,23 @@ static void target_machinize (gen_ctx_t gen_ctx) { gen_delete_insn (gen_ctx, insn); } } else if (code == MIR_VA_START) { - MIR_op_t treg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - MIR_op_t treg_op2 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + MIR_op_t treg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + MIR_op_t treg_op2 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); MIR_op_t va_op = insn->ops[0]; MIR_reg_t va_reg; - assert (func->vararg_p && va_op.mode == MIR_OP_REG); + assert (func->vararg_p && va_op.mode == MIR_OP_VAR); va_reg = va_op.u.reg; /* Insns can be non-simplified as soon as they match a machine insn. */ /* treg = mem64[r1]; treg = treg + var_args_start; mem64[va_reg] = treg */ gen_mov (gen_ctx, insn, MIR_MOV, treg_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 0, R1_HARD_REG, MIR_NON_HARD_REG, 1)); + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, R1_HARD_REG, MIR_NON_VAR, 1)); gen_mov (gen_ctx, insn, MIR_MOV, treg_op2, MIR_new_int_op (ctx, var_args_start)); /* don't use immediate in ADD as treg_op can become r0: */ new_insn = MIR_new_insn (ctx, MIR_ADD, treg_op, treg_op, treg_op2); gen_add_insn_before (gen_ctx, insn, new_insn); - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 0, va_reg, 0, 1), treg_op); + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, va_reg, MIR_NON_VAR, 1), treg_op); gen_delete_insn (gen_ctx, insn); } else if (code == MIR_VA_END) { /* do nothing */ gen_delete_insn (gen_ctx, insn); @@ -796,19 +762,22 @@ static void target_machinize (gen_ctx_t gen_ctx) { alloca_p = TRUE; } else if (code == MIR_SWITCH) { switch_p = TRUE; + } else if (code == MIR_LADDR) { + laddr_p = TRUE; } else if (code == MIR_RET) { /* In simplify we already transformed code for one return insn and added extension insn (if any). */ uint32_t n_gpregs = 0, n_fregs = 0; assert (func->nres == MIR_insn_nops (ctx, insn)); - for (size_t i = 0; i < func->nres; i++) { - assert (insn->ops[i].mode == MIR_OP_REG); + for (i = 0; i < func->nres; i++) { + assert (insn->ops[i].mode == MIR_OP_VAR); res_type = func->res_types[i]; if (((res_type == MIR_T_F || res_type == MIR_T_D) && n_fregs < 4) || (res_type == MIR_T_LD && n_fregs < 3)) { - new_insn_code - = res_type == MIR_T_F ? MIR_FMOV : res_type == MIR_T_D ? MIR_DMOV : MIR_LDMOV; + new_insn_code = res_type == MIR_T_F ? MIR_FMOV + : res_type == MIR_T_D ? MIR_DMOV + : MIR_LDMOV; ret_reg = F1_HARD_REG + n_fregs++; } else if (n_gpregs < 8) { new_insn_code = MIR_MOV; @@ -817,7 +786,7 @@ static void target_machinize (gen_ctx_t gen_ctx) { (*MIR_get_error_func (ctx)) (MIR_ret_error, "ppc64 can not handle this combination of return values"); } - ret_reg_op = _MIR_new_hard_reg_op (ctx, ret_reg); + ret_reg_op = _MIR_new_var_op (ctx, ret_reg); gen_mov (gen_ctx, insn, new_insn_code, ret_reg_op, insn->ops[i]); insn->ops[i] = ret_reg_op; } @@ -827,15 +796,14 @@ static void target_machinize (gen_ctx_t gen_ctx) { static void isave (gen_ctx_t gen_ctx, MIR_insn_t anchor, int disp, MIR_reg_t hard_reg) { gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (gen_ctx->ctx, MIR_T_I64, disp, R1_HARD_REG, MIR_NON_HARD_REG, - 1), - _MIR_new_hard_reg_op (gen_ctx->ctx, hard_reg)); + _MIR_new_var_mem_op (gen_ctx->ctx, MIR_T_I64, disp, R1_HARD_REG, MIR_NON_VAR, 1), + _MIR_new_var_op (gen_ctx->ctx, hard_reg)); } static void fsave (gen_ctx_t gen_ctx, MIR_insn_t anchor, int disp, MIR_reg_t hard_reg) { gen_mov (gen_ctx, anchor, MIR_DMOV, - _MIR_new_hard_reg_mem_op (gen_ctx->ctx, MIR_T_D, disp, R1_HARD_REG, MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (gen_ctx->ctx, hard_reg)); + _MIR_new_var_mem_op (gen_ctx->ctx, MIR_T_D, disp, R1_HARD_REG, MIR_NON_VAR, 1), + _MIR_new_var_op (gen_ctx->ctx, hard_reg)); } static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_regs, @@ -861,33 +829,34 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg else saved_fregs_num++; } - if (leaf_p && !alloca_p && !switch_p /* switch changes LR */ + if (leaf_p && !alloca_p && !switch_p && !laddr_p /* switch and laddr changes LR */ && saved_iregs_num == 0 && saved_fregs_num == 0 && stack_slots_num == 0) return; saved_iregs_num++; /* for fp (R31) ??? only alloca_p */ - r0_reg_op = _MIR_new_hard_reg_op (ctx, R0_HARD_REG); - lr_reg_op = _MIR_new_hard_reg_op (ctx, LR_HARD_REG); - sp_reg_op = _MIR_new_hard_reg_op (ctx, R1_HARD_REG); - fp_reg_op = _MIR_new_hard_reg_op (ctx, R31_HARD_REG); + r0_reg_op = _MIR_new_var_op (ctx, R0_HARD_REG); + lr_reg_op = _MIR_new_var_op (ctx, LR_HARD_REG); + sp_reg_op = _MIR_new_var_op (ctx, R1_HARD_REG); + fp_reg_op = _MIR_new_var_op (ctx, R31_HARD_REG); /* Prologue: */ frame_size = param_save_area_size + PPC64_STACK_HEADER_SIZE + stack_slots_num * 8; start_save_regs_offset = frame_size; frame_size += (saved_iregs_num + saved_fregs_num) * 8; if (frame_size % 16 != 0) frame_size = (frame_size + 15) / 16 * 16; - gen_mov (gen_ctx, anchor, MIR_MOV, r0_reg_op, lr_reg_op); /* r0 = lr */ - gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 16, R1_HARD_REG, MIR_NON_HARD_REG, 1), - r0_reg_op); /* mem[r1] = r0 */ + if (!func->jret_p) { + gen_mov (gen_ctx, anchor, MIR_MOV, r0_reg_op, lr_reg_op); /* r0 = lr */ + gen_mov (gen_ctx, anchor, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 16, R1_HARD_REG, MIR_NON_VAR, 1), + r0_reg_op); /* mem[r1] = r0 */ + } gen_mov (gen_ctx, anchor, MIR_MOV, r0_reg_op, sp_reg_op); new_insn = MIR_new_insn (ctx, MIR_ADD, sp_reg_op, sp_reg_op, MIR_new_int_op (ctx, -frame_size)); gen_add_insn_before (gen_ctx, anchor, new_insn); /* r1 -= frame_size */ gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 0, R1_HARD_REG, MIR_NON_HARD_REG, 1), + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, R1_HARD_REG, MIR_NON_VAR, 1), r0_reg_op); /* mem[r1] = r0 */ gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, PPC64_TOC_OFFSET, R1_HARD_REG, - MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (ctx, R2_HARD_REG)); /* mem[r1+toc_off] = r2 */ + _MIR_new_var_mem_op (ctx, MIR_T_I64, PPC64_TOC_OFFSET, R1_HARD_REG, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, R2_HARD_REG)); /* mem[r1+toc_off] = r2 */ for (n = i = 0; i <= MAX_HARD_REG; i++) if (!target_call_used_hard_reg_p (i, MIR_T_UNDEF) && bitmap_bit_p (used_hard_regs, i)) { if (i < F0_HARD_REG) @@ -898,33 +867,35 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg isave (gen_ctx, anchor, start_save_regs_offset + n * 8, R31_HARD_REG); /* save R31 */ gen_mov (gen_ctx, anchor, MIR_MOV, fp_reg_op, sp_reg_op); /* r31 = r1 */ /* Epilogue: */ - anchor = DLIST_TAIL (MIR_insn_t, func->insns); - /* It might be infinite loop after CCP with dead code elimination: */ - if (anchor->code == MIR_JMP) return; - assert (anchor->code == MIR_RET); + for (anchor = DLIST_TAIL (MIR_insn_t, func->insns); anchor != NULL; + anchor = DLIST_PREV (MIR_insn_t, anchor)) + if (anchor->code == MIR_RET || anchor->code == MIR_JRET) break; + if (anchor == NULL) return; /* Restoring hard registers: */ for (i = n = 0; i <= MAX_HARD_REG; i++) if (!target_call_used_hard_reg_p (i, MIR_T_UNDEF) && bitmap_bit_p (used_hard_regs, i)) { if (i < F0_HARD_REG) { - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, i), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, start_save_regs_offset + (n++) * 8, - FP_HARD_REG, MIR_NON_HARD_REG, 1)); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, i), + _MIR_new_var_mem_op (ctx, MIR_T_I64, start_save_regs_offset + (n++) * 8, + FP_HARD_REG, MIR_NON_VAR, 1)); } else { - gen_mov (gen_ctx, anchor, MIR_DMOV, _MIR_new_hard_reg_op (ctx, i), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_D, start_save_regs_offset + (n++) * 8, - FP_HARD_REG, MIR_NON_HARD_REG, 1)); + gen_mov (gen_ctx, anchor, MIR_DMOV, _MIR_new_var_op (ctx, i), + _MIR_new_var_mem_op (ctx, MIR_T_D, start_save_regs_offset + (n++) * 8, FP_HARD_REG, + MIR_NON_VAR, 1)); } } /* Restore sp, fp, lr */ new_insn = MIR_new_insn (ctx, MIR_ADD, sp_reg_op, fp_reg_op, MIR_new_int_op (ctx, frame_size)); gen_add_insn_before (gen_ctx, anchor, new_insn); /* sp = fp + frame_size */ gen_mov (gen_ctx, anchor, MIR_MOV, fp_reg_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, start_save_regs_offset + n * 8, FP_HARD_REG, - MIR_NON_HARD_REG, 1)); /* restore fp */ - gen_mov (gen_ctx, anchor, MIR_MOV, r0_reg_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 16, R1_HARD_REG, MIR_NON_HARD_REG, - 1)); /* r0 = 16(sp) */ - gen_mov (gen_ctx, anchor, MIR_MOV, lr_reg_op, r0_reg_op); /* lr = r0 */ + _MIR_new_var_mem_op (ctx, MIR_T_I64, start_save_regs_offset + n * 8, FP_HARD_REG, + MIR_NON_VAR, 1)); /* restore fp */ + if (!func->jret_p) { + gen_mov (gen_ctx, anchor, MIR_MOV, r0_reg_op, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 16, R1_HARD_REG, MIR_NON_VAR, + 1)); /* r0 = 16(sp) */ + gen_mov (gen_ctx, anchor, MIR_MOV, lr_reg_op, r0_reg_op); /* lr = r0 */ + } } struct pattern { @@ -1036,6 +1007,7 @@ struct pattern { BI - field [11..15] for bcond LK - field [31..31] + W - LADDR label which is a 32-bit signed offset from previous insn at - address disp PPC64_TOC_OFFSET T - switch table displacement */ @@ -1293,6 +1265,10 @@ static const struct pattern patterns[] = { {MIR_JMP, "L", "o18 L"}, /* 24-bit offset word jmp */ + /* bl l4; mflr r0; addis r0,r0,I; addi r0,r0,i: */ + {MIR_LADDR, "r W", "o18 l4 LK1; o31 O339 rt0 sr8; o15 rt0 ra0 W; o14 rt0 ra0"}, + {MIR_JMPI, "r", "o31 O467 rs0 sr9; o19 O528 BO20 BI0"}, /* mtctr r; bcctr */ + #define BRC(o, i) "o16 BO" #o " BI" #i " l" #define BRCL(o, i) "o16 BO" #o " BI" #i " l8; o18 L" #define BRLOG(CODE, CMP, BI, COND, NEG_COND) \ @@ -1443,53 +1419,31 @@ static const struct pattern patterns[] = { {MIR_D2I, "r r", "o63 O815 ht32 rb1; o54 hs32 mt; o58 rt0 mt"}, {MIR_F2D, "r r", "o63 O72 rt0 rb1"}, /* fmr rt,rb */ {MIR_D2F, "r r", "o63 O12 rt0 rb1"}, /* frsp rt,rb */ -// i2ld, ui2ld, ld2i, f2ld, d2ld, ld2f, ld2d are builtins + // i2ld, ui2ld, ld2i, f2ld, d2ld, ld2f, ld2d are builtins -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ {MIR_CALL, "X h12 $", "o31 O467 rs1 sr9; o19 O528 BO20 BI0 LK1"}, /* mtctr r12; bcctrl */ {MIR_CALL, "X r $", "o31 O444 ha12 rs1 rb1; o31 O467 rs1 sr9; o19 O528 BO20 BI0 LK1"}, /* mr r12,r; mtctr r; bcctrl */ -#else - /* mr r2,r0; ld r0,0(r2); ld r2,8(r2); mtctr r0; bcctrl */ - {MIR_CALL, "X h0 $", - "o31 O444 ha2 hs0 hb0; o58 ht0 ha2; o58 ht2 ha2 i8; o31 O467 hs0 sr9; o19 O528 BO20 BI0 LK1; " - "o31 O444 ha0 hs0 hb0"}, - /* ld r0,0(r); ld r2,8(r); mtctr r0; bcctrl */ - {MIR_CALL, "X r $", - "o58 ht0 ra1; o58 ht2 ra1 i8; o31 O467 hs0 sr9; o19 O528 BO20 BI0 LK1; o31 O444 ha0 hs0 hb0"}, -#endif {MIR_RET, "$", "o19 O16 BO20 BI0"}, /* bclr */ -#if 0 - /* ld r10,16(r1); subf r1,rt,r1; ldx r0,(r1,rt); std r10,16(r1); std r0,0(r1); - std r2,PPC64_TOC_OFFSET(r1); add rt,r1,PPC64_STACK_HEADER_SIZE+PARAM_AREA_SIZE: */ -#define ALLOCA_END \ - "o58 ht10 ha1 i16; o31 O40 ht1 ra0 hb1; o31 O21 ht0 ha1 rb0; " \ - "o62 hs10 ha1 i16; o62 hs0 ha1;o62 hs2 ha1 at; o14 rt0 ha1 ih" -#else + {MIR_JCALL, "X r $", "o31 O467 rs1 sr9; o19 O528 BO20 BI0"}, /* mtctr r; bcctr */ + {MIR_JRET, "r $", "o31 O467 rs0 sr9; o19 O528 BO20 BI0"}, /* mtctr r; bcctr */ + /* subf r1,rt,r1; ldx r0,(r1,rt); std r0,0(r1); add rt,r1,PPC64_STACK_HEADER_SIZE+PARAM_AREA_SIZE: */ #define ALLOCA_END \ "o31 O40 ht1 ra0 hb1; o31 O21 ht0 ha1 rb0; " \ "o62 hs0 ha1; o14 rt0 ha1 ih" -#endif /* addi rt,ra,15;rldicr rt,rt,0,59; ... : */ {MIR_ALLOCA, "r r", "o14 rt0 ra1 i15; o30 ra0 rs0 Sh0 Me59; " ALLOCA_END}, /* mov rt,ia; ...: */ {MIR_ALLOCA, "r ia", "o14 rt0 ha0 ia; " ALLOCA_END}, {MIR_BSTART, "r", "o31 O444 ra0 hs1 hb1"}, /* or ra,r1,r1 */ -#if 0 - /* ld r0,0(r1);ld r2,PPC64_TOC_OFFSET(r1);ld r10,16(sp);or r1,rs,rs - std r0,0(r1);std r2,PPC64_TOC_OFFSET(r1);std r10,16(sp): */ - {MIR_BEND, "r", "o58 hs0 ha1;o58 hs2 ha1 at;o58 hs10 ha1 i16;o31 O444 ha1 rs0 rb0;" - "o62 hs0 ha1;o62 hs2 ha1 at;o62 hs10 ha1 i16"}, -#else /* ld r0,0(r1);or r1,rs,rs; std r0,0(r1): */ {MIR_BEND, "r", "o58 hs0 ha1;o31 O444 ha1 rs0 rb0; o62 hs0 ha1"}, -#endif /* bl l4; mflr r0; rldicr r10,rt,3,60; add r10,r0,r10; ld r0,table-disp(r10); mtctr r0; bcctr; TableContent: */ @@ -1501,7 +1455,7 @@ static const struct pattern patterns[] = { static void target_get_early_clobbered_hard_regs (MIR_insn_t insn, MIR_reg_t *hr1, MIR_reg_t *hr2) { MIR_insn_code_t code = insn->code; - *hr1 = *hr2 = MIR_NON_HARD_REG; + *hr1 = *hr2 = MIR_NON_VAR; if (code == MIR_MOD || code == MIR_MODS || code == MIR_UMOD || code == MIR_UMODS) { *hr1 = R10_HARD_REG; } else if (code == MIR_I2F || code == MIR_I2D || code == MIR_UI2F || code == MIR_UI2D @@ -1510,15 +1464,7 @@ static void target_get_early_clobbered_hard_regs (MIR_insn_t insn, MIR_reg_t *hr } else if (code == MIR_LDMOV) { /* if mem base reg is R0 */ *hr1 = R11_HARD_REG; /* don't use arg regs as ldmov can be used in param passing part */ } else if (code == MIR_CALL || code == MIR_INLINE) { -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ *hr1 = R12_HARD_REG; -#else - *hr1 = R2_HARD_REG; -#endif -#if 0 - } else if (code == MIR_ALLOCA || code == MIR_BEND) { - *hr1 = R10_HARD_REG; -#endif } else if (code == MIR_SWITCH) { *hr1 = R10_HARD_REG; } @@ -1575,8 +1521,7 @@ static int negative32_p (uint64_t u, uint64_t *n) { static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_insn_t insn, int use_short_label_p) { MIR_context_t ctx = gen_ctx->ctx; - int nop; - size_t nops = MIR_insn_nops (ctx, insn); + size_t nop, nops = MIR_insn_nops (ctx, insn); const char *p; char ch, start_ch; MIR_op_t op; @@ -1591,15 +1536,13 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in switch (start_ch = *p) { case 'X': break; case 'r': - if (op.mode != MIR_OP_HARD_REG || op.u.hard_reg == LR_HARD_REG) return FALSE; + if (op.mode != MIR_OP_VAR || op.u.var == LR_HARD_REG) return FALSE; break; case 'R': - if (op.mode != MIR_OP_HARD_REG || op.u.hard_reg == R0_HARD_REG - || op.u.hard_reg == LR_HARD_REG) - return FALSE; + if (op.mode != MIR_OP_VAR || op.u.var == R0_HARD_REG || op.u.var == LR_HARD_REG) return FALSE; break; case 'h': - if (op.mode != MIR_OP_HARD_REG) return FALSE; + if (op.mode != MIR_OP_VAR) return FALSE; ch = *++p; gen_assert ('0' <= ch && ch <= '9'); hr = ch - '0'; @@ -1609,14 +1552,14 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in else --p; gen_assert (hr <= MAX_HARD_REG); - if (op.u.hard_reg != hr) return FALSE; + if (op.u.var != hr) return FALSE; break; case 'm': case 'M': { MIR_type_t type, type2, type3 = MIR_T_BOUND; int ds_p = FALSE, l_p = FALSE, br0_p = FALSE, u_p = TRUE, s_p = TRUE; - if (op.mode != MIR_OP_HARD_REG_MEM) return FALSE; + if (op.mode != MIR_OP_VAR_MEM) return FALSE; ch = *++p; switch (ch) { case 'f': @@ -1657,7 +1600,7 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in u_p = ch == 'u'; s_p = ch == 's'; ch = *++p; - /* Fall through: */ + /* fall through */ default: gen_assert ('0' <= ch && ch <= '3'); if (ch == '0') { @@ -1680,25 +1623,21 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in #endif } } - if (op.u.hard_reg_mem.type != type && op.u.hard_reg_mem.type != type2 - && op.u.hard_reg_mem.type != type3) + if (op.u.var_mem.type != type && op.u.var_mem.type != type2 && op.u.var_mem.type != type3) return FALSE; if (ds_p - && (op.u.hard_reg_mem.index != MIR_NON_HARD_REG || op.u.hard_reg_mem.base == R0_HARD_REG - || op.u.hard_reg_mem.disp % 4 != 0 || !int16_p (op.u.hard_reg_mem.disp))) + && (op.u.var_mem.index != MIR_NON_VAR || op.u.var_mem.base == R0_HARD_REG + || op.u.var_mem.disp % 4 != 0 || !int16_p (op.u.var_mem.disp))) return FALSE; if (!ds_p && start_ch == 'm' - && (op.u.hard_reg_mem.index != MIR_NON_HARD_REG - || (!br0_p && op.u.hard_reg_mem.base == R0_HARD_REG) - || (br0_p && op.u.hard_reg_mem.base != R0_HARD_REG) - || !int16_p (op.u.hard_reg_mem.disp) - || (l_p && !int16_p (op.u.hard_reg_mem.disp + 8)))) + && (op.u.var_mem.index != MIR_NON_VAR || (!br0_p && op.u.var_mem.base == R0_HARD_REG) + || (br0_p && op.u.var_mem.base != R0_HARD_REG) || !int16_p (op.u.var_mem.disp) + || (l_p && !int16_p (op.u.var_mem.disp + 8)))) return FALSE; if (!ds_p && start_ch == 'M' - && (op.u.hard_reg_mem.disp != 0 - || (op.u.hard_reg_mem.index != MIR_NON_HARD_REG && op.u.hard_reg_mem.scale != 1) - || (op.u.hard_reg_mem.base == R0_HARD_REG - && op.u.hard_reg_mem.index != MIR_NON_HARD_REG))) + && (op.u.var_mem.disp != 0 + || (op.u.var_mem.index != MIR_NON_VAR && op.u.var_mem.scale != 1) + || (op.u.var_mem.base == R0_HARD_REG && op.u.var_mem.index != MIR_NON_VAR))) return FALSE; break; } @@ -1764,6 +1703,9 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in case 'L': if (op.mode != MIR_OP_LABEL && op.mode != MIR_OP_REF) return FALSE; break; + case 'W': + if (op.mode != MIR_OP_LABEL) return FALSE; + break; default: gen_assert (FALSE); } } @@ -1825,7 +1767,8 @@ static void set_int64 (uint8_t *addr, int64_t v) { *(int64_t *) addr = v; } static int64_t get_int64 (uint8_t *addr) { return *(int64_t *) addr; } -static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacement) { +static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacement, + void **jump_addrs) { MIR_context_t ctx = gen_ctx->ctx; size_t nops = MIR_insn_nops (ctx, insn); size_t offset; @@ -1887,10 +1830,10 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen if (start_ch == 'h') { reg = read_dec (&p); } else { - gen_assert ('0' <= ch && ch <= '2' && ch - '0' < insn->nops); + gen_assert ('0' <= ch && ch <= '2' && ch - '0' < (int) insn->nops); op = insn->ops[ch - '0']; - gen_assert (op.mode == MIR_OP_HARD_REG); - reg = op.u.hard_reg + (start_ch == 'n' ? 1 : 0); + gen_assert (op.mode == MIR_OP_VAR); + reg = op.u.var + (start_ch == 'n' ? 1 : 0); } if (reg > R31_HARD_REG) reg -= F0_HARD_REG; gen_assert (reg <= 31); @@ -1973,24 +1916,24 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen Me = ((Me & 0x1f) << 1) | ((Me >> 5) & 1); } } else { - op = insn->ops[0].mode == MIR_OP_HARD_REG_MEM ? insn->ops[0] : insn->ops[1]; - gen_assert (op.mode == MIR_OP_HARD_REG_MEM); + op = insn->ops[0].mode == MIR_OP_VAR_MEM ? insn->ops[0] : insn->ops[1]; + gen_assert (op.mode == MIR_OP_VAR_MEM); if (ch2 == 'd') { ch2 = *++p; gen_assert (ch2 == 's' && ra < 0 && disp4 < 0); - ra = op.u.hard_reg_mem.base; - if (ra == MIR_NON_HARD_REG) ra = R0_HARD_REG; - disp4 = op.u.hard_reg_mem.disp & 0xffff; + ra = (int) op.u.var_mem.base; + if (op.u.var_mem.base == MIR_NON_VAR) ra = R0_HARD_REG; + disp4 = op.u.var_mem.disp & 0xffff; gen_assert ((disp4 & 0x3) == 0); } else { --p; gen_assert (ra < 0 && rb < 0); - ra = op.u.hard_reg_mem.base; - rb = op.u.hard_reg_mem.index; - if (rb == MIR_NON_HARD_REG) { + ra = (int) op.u.var_mem.base; + rb = (int) op.u.var_mem.index; + if (op.u.var_mem.index == MIR_NON_VAR) { rb = ra; ra = R0_HARD_REG; - } else if (ra == MIR_NON_HARD_REG) { + } else if (op.u.var_mem.base == MIR_NON_VAR) { ra = R0_HARD_REG; } else if (ra == R0_HARD_REG) { ra = rb; @@ -2043,21 +1986,21 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen } } } else { - op = insn->ops[0].mode == MIR_OP_HARD_REG_MEM ? insn->ops[0] : insn->ops[1]; - gen_assert (op.mode == MIR_OP_HARD_REG_MEM); + op = insn->ops[0].mode == MIR_OP_VAR_MEM ? insn->ops[0] : insn->ops[1]; + gen_assert (op.mode == MIR_OP_VAR_MEM); if (ch2 == 'd') { ch2 = *++p; gen_assert (ch2 == 's' && ra < 0 && disp4 < 0); - ra = op.u.hard_reg_mem.base; - if (ra == MIR_NON_HARD_REG) ra = R0_HARD_REG; - disp4 = op.u.hard_reg_mem.disp & 0xffff; + ra = (int) op.u.var_mem.base; + if (op.u.var_mem.base == MIR_NON_VAR) ra = R0_HARD_REG; + disp4 = op.u.var_mem.disp & 0xffff; gen_assert ((disp4 & 0x3) == 0); } else { if (ch2 != 'n') --p; gen_assert (ra < 0 && disp < 0); - ra = op.u.hard_reg_mem.base; - if (ra == MIR_NON_HARD_REG) ra = R0_HARD_REG; - disp = (op.u.hard_reg_mem.disp + (ch2 != 'n' ? 0 : 8)) & 0xffff; + ra = (int) op.u.var_mem.base; + if (op.u.var_mem.base == MIR_NON_VAR) ra = R0_HARD_REG; + disp = (op.u.var_mem.disp + (ch2 != 'n' ? 0 : 8)) & 0xffff; } } break; @@ -2085,7 +2028,7 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen break; } p--; - /* fall through: */ + /* fall through */ case 'u': case 'I': case 'U': @@ -2097,7 +2040,7 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen case 'x': case 'z': { int ok_p; - uint64_t v, n; + uint64_t v; op = insn->ops[nops - 1]; gen_assert (op.mode == MIR_OP_INT || op.mode == MIR_OP_UINT || op.mode == MIR_OP_REF); @@ -2110,10 +2053,11 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen v = (uint64_t) op.u.ref->addr; } if (start_ch == 'x') { - ok_p = negative32_p (v, &n); - n = 32 - n; + uint64_t num; + ok_p = negative32_p (v, &num); + num = 32 - num; gen_assert (Mb < 0 && ok_p); - Mb = ((n & 0x1f) << 1) | ((n >> 5) & 1); + Mb = ((num & 0x1f) << 1) | ((num >> 5) & 1); } else { gen_assert (imm < 0); n = dec_value (*++p); @@ -2158,9 +2102,12 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen op = insn->ops[0]; gen_assert (op.mode == MIR_OP_LABEL); lr.abs_addr_p = FALSE; - lr.short_addr_p = TRUE; + lr.branch_type = BRCOND; lr.label_val_disp = 0; - lr.label = op.u.label; + if (jump_addrs == NULL) + lr.u.label = op.u.label; + else + lr.u.jump_addr = jump_addrs[0]; label_ref_num = VARR_LENGTH (label_ref_t, label_refs); VARR_PUSH (label_ref_t, label_refs, lr); } @@ -2180,14 +2127,32 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen --p; op = insn->ops[insn->code != MIR_CALL ? 0 : 1]; gen_assert (op.mode == MIR_OP_LABEL); - lr.abs_addr_p = lr.short_addr_p = FALSE; + lr.abs_addr_p = FALSE; + lr.branch_type = JUMP; lr.label_val_disp = 0; - lr.label = op.u.label; + if (jump_addrs == NULL) + lr.u.label = op.u.label; + else + lr.u.jump_addr = jump_addrs[0]; label_ref_num = VARR_LENGTH (label_ref_t, label_refs); VARR_PUSH (label_ref_t, label_refs, lr); } break; } + case 'W': { + op = insn->ops[1]; + gen_assert (insn->code == MIR_LADDR && op.mode == MIR_OP_LABEL); + lr.abs_addr_p = FALSE; + lr.branch_type = LADDR; + lr.label_val_disp = 0; + if (jump_addrs == NULL) + lr.u.label = op.u.label; + else + lr.u.jump_addr = jump_addrs[0]; + label_ref_num = VARR_LENGTH (label_ref_t, label_refs); + VARR_PUSH (label_ref_t, label_refs, lr); + break; + } case 'a': gen_assert (imm < 0); ch2 = *++p; @@ -2334,10 +2299,8 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen binsn |= lb << (32 - 11); binsn_mask = check_and_set_mask (binsn_mask, 1 << (32 - 11)); } - if (label_ref_num >= 0) - VARR_ADDR (label_ref_t, label_refs) - [label_ref_num].label_val_disp - = VARR_LENGTH (uint8_t, result_code); + if (label_ref_num >= 0) VARR_ADDR (label_ref_t, label_refs) + [label_ref_num].label_val_disp = VARR_LENGTH (uint8_t, result_code); if (switch_table_addr_p) switch_table_addr_insn_start = VARR_LENGTH (uint8_t, result_code); put_uint32 (gen_ctx, binsn); /* output the machine insn */ @@ -2355,23 +2318,76 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen for (size_t i = 1; i < insn->nops; i++) { gen_assert (insn->ops[i].mode == MIR_OP_LABEL); lr.abs_addr_p = TRUE; - lr.short_addr_p = FALSE; + lr.branch_type = BCTR; /* the value does not matter */ lr.label_val_disp = VARR_LENGTH (uint8_t, result_code); - lr.label = insn->ops[i].u.label; + if (jump_addrs == NULL) + lr.u.label = insn->ops[i].u.label; + else + lr.u.jump_addr = jump_addrs[i - 1]; VARR_PUSH (label_ref_t, label_refs, lr); put_uint64 (gen_ctx, 0); /* reserve mem for label address */ } } +static int target_memory_ok_p (gen_ctx_t gen_ctx, MIR_op_t *op_ref) { + MIR_context_t ctx = gen_ctx->ctx; + if (op_ref->mode != MIR_OP_VAR_MEM) return FALSE; + if (op_ref->u.var_mem.index == MIR_NON_VAR && int16_p (op_ref->u.var_mem.disp)) return TRUE; + size_t size = _MIR_type_size (ctx, op_ref->u.var_mem.type); + if (op_ref->u.var_mem.index != MIR_NON_VAR && op_ref->u.var_mem.disp == 0 + && op_ref->u.var_mem.scale == size) + return TRUE; + if (op_ref->u.var_mem.index == MIR_NON_VAR && op_ref->u.var_mem.disp % 4 == 0 + && (size == 4 || size == 8) && int16_p (op_ref->u.var_mem.disp)) + return TRUE; + return FALSE; +} + static int target_insn_ok_p (gen_ctx_t gen_ctx, MIR_insn_t insn) { return find_insn_pattern_replacement (gen_ctx, insn, TRUE) != NULL; } +static void target_split_insns (gen_ctx_t gen_ctx) { + for (MIR_insn_t insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); insn != NULL; + insn = DLIST_NEXT (MIR_insn_t, insn)) { + MIR_insn_code_t code = insn->code; + + if ((code != MIR_RSH && code != MIR_LSH && code != MIR_URSH && code != MIR_RSHS + && code != MIR_LSHS && code != MIR_URSHS) + || (insn->ops[2].mode != MIR_OP_INT && insn->ops[2].mode != MIR_OP_UINT)) + continue; + if (insn->ops[2].u.i == 0) { + gen_mov (gen_ctx, insn, MIR_MOV, insn->ops[0], insn->ops[1]); + MIR_insn_t old_insn = insn; + insn = DLIST_PREV (MIR_insn_t, insn); + gen_delete_insn (gen_ctx, old_insn); + } else { + if (insn->ops[2].mode == MIR_OP_INT && insn->ops[2].u.i < 0) { + switch (code) { + case MIR_RSH: insn->code = MIR_LSH; break; + case MIR_URSH: insn->code = MIR_LSH; break; + case MIR_LSH: insn->code = MIR_RSH; break; + case MIR_RSHS: insn->code = MIR_LSHS; break; + case MIR_URSHS: insn->code = MIR_LSHS; break; + case MIR_LSHS: insn->code = MIR_RSHS; break; + default: gen_assert (FALSE); break; + } + insn->ops[2].u.i = -insn->ops[2].u.i; + } + if (code == MIR_RSH || code == MIR_LSH || code == MIR_URSH) { + if (insn->ops[2].u.i > 64) insn->ops[2].u.i = 64; + } else if (insn->ops[2].u.i > 32) { + insn->ops[2].u.i = 32; + } + } + } +} + static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { MIR_context_t ctx = gen_ctx->ctx; size_t i; int short_label_disp_fail_p, n_iter = 0; - MIR_insn_t insn, old_insn; + MIR_insn_t insn; const char *replacement; gen_assert (curr_func_item->item_type == MIR_func_item); @@ -2382,42 +2398,12 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { short_label_disp_fail_p = FALSE; for (insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); insn != NULL; insn = DLIST_NEXT (MIR_insn_t, insn)) { - MIR_insn_code_t code = insn->code; - - if ((code == MIR_RSH || code == MIR_LSH || code == MIR_URSH || code == MIR_RSHS - || code == MIR_LSHS || code == MIR_URSHS) - && (insn->ops[2].mode == MIR_OP_INT || insn->ops[2].mode == MIR_OP_UINT)) { - if (insn->ops[2].u.i == 0) { - gen_mov (gen_ctx, insn, MIR_MOV, insn->ops[0], insn->ops[1]); - old_insn = insn; - insn = DLIST_PREV (MIR_insn_t, insn); - gen_delete_insn (gen_ctx, old_insn); - } else { - if (insn->ops[2].mode == MIR_OP_INT && insn->ops[2].u.i < 0) { - switch (code) { - case MIR_RSH: insn->code = MIR_LSH; break; - case MIR_URSH: insn->code = MIR_LSH; break; - case MIR_LSH: insn->code = MIR_RSH; break; - case MIR_RSHS: insn->code = MIR_LSHS; break; - case MIR_URSHS: insn->code = MIR_LSHS; break; - case MIR_LSHS: insn->code = MIR_RSHS; break; - default: gen_assert (FALSE); break; - } - insn->ops[2].u.i = -insn->ops[2].u.i; - } - if (code == MIR_RSH || code == MIR_LSH || code == MIR_URSH) { - if (insn->ops[2].u.i > 64) insn->ops[2].u.i = 64; - } else if (insn->ops[2].u.i > 32) { - insn->ops[2].u.i = 32; - } - } - } if (insn->code == MIR_LABEL) { set_label_disp (gen_ctx, insn, VARR_LENGTH (uint8_t, result_code)); - } else { + } else if (insn->code != MIR_USE) { int use_short_label_p = TRUE; - if (n_iter > 0 && MIR_branch_code_p (code)) { + if (n_iter > 0 && MIR_branch_code_p (insn->code)) { MIR_label_t label = insn->ops[0].u.label; int64_t offset = (int64_t) get_label_disp (gen_ctx, label) - (int64_t) VARR_LENGTH (uint8_t, result_code); @@ -2431,7 +2417,7 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { exit (1); } else { gen_assert (replacement != NULL); - out_insn (gen_ctx, insn, replacement); + out_insn (gen_ctx, insn, replacement, NULL); } } } @@ -2441,10 +2427,18 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { if (lr.abs_addr_p) { set_int64 (&VARR_ADDR (uint8_t, result_code)[lr.label_val_disp], - (int64_t) get_label_disp (gen_ctx, lr.label)); + (int64_t) get_label_disp (gen_ctx, lr.u.label)); VARR_PUSH (uint64_t, abs_address_locs, lr.label_val_disp); - } else if (lr.short_addr_p) { /* 14-bit relative addressing */ - int64_t offset = (int64_t) get_label_disp (gen_ctx, lr.label) - (int64_t) lr.label_val_disp; + } else if (lr.branch_type == LADDR) { + int64_t offset + = (int64_t) get_label_disp (gen_ctx, lr.u.label) - (int64_t) lr.label_val_disp + 4; + int hi = offset >> 16, low = offset & 0xffff; + if ((low & 0x8000) != 0) hi++; + *(uint32_t *) (VARR_ADDR (uint8_t, result_code) + lr.label_val_disp) |= hi & 0xffff; + *(uint32_t *) (VARR_ADDR (uint8_t, result_code) + lr.label_val_disp + 4) |= low; + } else if (lr.branch_type == BRCOND) { /* 14-bit relative addressing */ + int64_t offset + = (int64_t) get_label_disp (gen_ctx, lr.u.label) - (int64_t) lr.label_val_disp; gen_assert ((offset & 0x3) == 0); if (((offset < 0 ? -offset : offset) & ~(int64_t) 0x7fff) != 0) { @@ -2454,7 +2448,8 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { |= ((offset / 4) & 0x3fff) << 2; } } else { /* 24-bit relative address */ - int64_t offset = (int64_t) get_label_disp (gen_ctx, lr.label) - (int64_t) lr.label_val_disp; + int64_t offset + = (int64_t) get_label_disp (gen_ctx, lr.u.label) - (int64_t) lr.label_val_disp; gen_assert ((offset & 0x3) == 0 && ((offset < 0 ? -offset : offset) & ~(int64_t) 0x1ffffff) == 0); *(uint32_t *) (VARR_ADDR (uint8_t, result_code) + lr.label_val_disp) @@ -2480,19 +2475,167 @@ static void target_rebase (gen_ctx_t gen_ctx, uint8_t *base) { } _MIR_update_code_arr (gen_ctx->ctx, base, VARR_LENGTH (MIR_code_reloc_t, relocs), VARR_ADDR (MIR_code_reloc_t, relocs)); + gen_setup_lrefs (gen_ctx, base); +} + +static void target_change_to_direct_calls (MIR_context_t ctx MIR_UNUSED) {} + +struct target_bb_version { + uint8_t *base; + label_ref_t branch_ref; /* label cand used for jump to this bb version */ +}; + +static void target_init_bb_version_data (target_bb_version_t data) { + data->base = NULL; /* we don't know origin branch */ +} + +static void target_bb_translate_start (gen_ctx_t gen_ctx) { + short_bb_branch_p = FALSE; + VARR_TRUNC (uint8_t, result_code, 0); + VARR_TRUNC (label_ref_t, label_refs, 0); + VARR_TRUNC (uint64_t, abs_address_locs, 0); +} + +static void target_bb_insn_translate (gen_ctx_t gen_ctx, MIR_insn_t insn, void **jump_addrs) { + const char *replacement; + if (insn->code == MIR_LABEL) return; + replacement = find_insn_pattern_replacement (gen_ctx, insn, TRUE); + gen_assert (replacement != NULL); + out_insn (gen_ctx, insn, replacement, jump_addrs); + if (MIR_branch_code_p (insn->code) && insn->code != MIR_JMP) short_bb_branch_p = TRUE; +} + +static void target_output_jump (gen_ctx_t gen_ctx, void **jump_addrs) { + out_insn (gen_ctx, temp_jump, temp_jump_replacement, jump_addrs); +} + +static uint8_t *target_bb_translate_finish (gen_ctx_t gen_ctx, size_t *len) { + /* add nops for possible conversion short branch or jump to branch and bctr */ + for (int i = 0; i < (short_bb_branch_p ? 13 : 6); i++) put_uint32 (gen_ctx, TARGET_NOP); + while (VARR_LENGTH (uint8_t, result_code) % 16 != 0) /* Align the pool */ + VARR_PUSH (uint8_t, result_code, 0); + *len = VARR_LENGTH (uint8_t, result_code); + return VARR_ADDR (uint8_t, result_code); +} + +static void setup_rel (gen_ctx_t gen_ctx, label_ref_t *lr, uint8_t *base, void *addr) { + MIR_context_t ctx = gen_ctx->ctx; + int64_t offset = (int64_t) addr - (int64_t) (base + lr->label_val_disp); + + gen_assert ((offset & 0x3) == 0 && !lr->abs_addr_p); + /* ??? thread safe: */ + uint32_t *insn_ptr = (uint32_t *) (base + lr->label_val_disp), insn = *insn_ptr; + if (lr->branch_type == BRCOND) { + if (((offset < 0 ? -offset : offset) & ~(int64_t) 0x7fff) == 0) { /* a valid branch offset */ + insn = (insn & ~0xffff) | (((offset / 4) & 0x3fff) << 2); + _MIR_change_code (ctx, (uint8_t *) insn_ptr, (uint8_t *) &insn, 4); + return; + } + insn = (insn & ~0xffff) | (4 * 8); /* skip next jump and 6 nops for it */ + _MIR_change_code (ctx, (uint8_t *) insn_ptr, (uint8_t *) &insn, 4); + insn = PPC_JUMP_OPCODE << (32 - 6); + insn_ptr += 8; + lr->branch_type = JUMP; + lr->label_val_disp += 4 * 8; + offset -= 4 * 8; + } + if (lr->branch_type == LADDR) { + offset += 4; + int hi = offset >> 16, low = offset & 0xffff; + if ((low & 0x8000) != 0) hi++; + insn |= hi & 0xffff; + _MIR_change_code (ctx, (uint8_t *) insn_ptr, (uint8_t *) &insn, 4); + insn_ptr++; + insn = *insn_ptr | low; + _MIR_change_code (ctx, (uint8_t *) insn_ptr, (uint8_t *) &insn, 4); + return; + } else if (lr->branch_type == JUMP) { + if (((offset < 0 ? -offset : offset) & ~(int64_t) 0x1ffffff) == 0) { /* a valid jump offset */ + insn = (insn & ~0x3ffffff) | (((offset / 4) & 0xffffff) << 2); + _MIR_change_code (ctx, (uint8_t *) insn_ptr, (uint8_t *) &insn, 4); + return; + } + lr->branch_type = BCTR; + } + gen_assert (lr->branch_type == BCTR); + VARR_TRUNC (uint8_t, result_code, 0); + ppc64_gen_address (result_code, 12, addr); /* r12 = addr */ + insn = 0x7d8903a6; /* mtctr r12 */ + put_uint32 (gen_ctx, insn); + insn = 0x4e800420; /* bctr */ + put_uint32 (gen_ctx, insn); + _MIR_change_code (ctx, (uint8_t *) insn_ptr, VARR_ADDR (uint8_t, result_code), + VARR_LENGTH (uint8_t, result_code)); +} + +static void target_bb_rebase (gen_ctx_t gen_ctx, uint8_t *base) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_code_reloc_t reloc; + + /* Setting up relative labels */ + for (size_t i = 0; i < VARR_LENGTH (label_ref_t, label_refs); i++) { + label_ref_t lr = VARR_GET (label_ref_t, label_refs, i); + if (lr.abs_addr_p) { + _MIR_change_code (ctx, (uint8_t *) base + lr.label_val_disp, (uint8_t *) &lr.u.jump_addr, 8); + } else { + setup_rel (gen_ctx, &lr, base, lr.u.jump_addr); + } + } + VARR_TRUNC (MIR_code_reloc_t, relocs, 0); + for (size_t i = 0; i < VARR_LENGTH (uint64_t, abs_address_locs); i++) { + reloc.offset = VARR_GET (uint64_t, abs_address_locs, i); + reloc.value = base + get_int64 (base + reloc.offset); + VARR_PUSH (MIR_code_reloc_t, relocs, reloc); + } + _MIR_update_code_arr (gen_ctx->ctx, base, VARR_LENGTH (MIR_code_reloc_t, relocs), + VARR_ADDR (MIR_code_reloc_t, relocs)); +} + +static void target_setup_succ_bb_version_data (gen_ctx_t gen_ctx, uint8_t *base) { + if (VARR_LENGTH (label_ref_t, label_refs) + != VARR_LENGTH (target_bb_version_t, target_succ_bb_versions)) + /* We can have more one possible branch from original insn + (e.g. SWITCH, FBNE). If it is so, we will make jumps only + through BB thunk. */ + return; + for (size_t i = 0; i < VARR_LENGTH (target_bb_version_t, target_succ_bb_versions); i++) { + target_bb_version_t data = VARR_GET (target_bb_version_t, target_succ_bb_versions, i); + if (data == NULL) continue; + data->branch_ref = VARR_GET (label_ref_t, label_refs, i); + data->base = base; + } +} + +static void target_redirect_bb_origin_branch (gen_ctx_t gen_ctx, target_bb_version_t data, + void *addr) { + MIR_context_t ctx = gen_ctx->ctx; + + if (data->base == NULL) return; + if (data->branch_ref.abs_addr_p) { + _MIR_change_code (ctx, (uint8_t *) data->base + data->branch_ref.label_val_disp, + (uint8_t *) &addr, 8); + } else { + setup_rel (gen_ctx, &data->branch_ref, data->base, addr); + } + data->base = NULL; } static void target_init (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; + gen_ctx->target_ctx = gen_malloc (gen_ctx, sizeof (struct target_ctx)); VARR_CREATE (uint8_t, result_code, 0); VARR_CREATE (label_ref_t, label_refs, 0); VARR_CREATE (uint64_t, abs_address_locs, 0); VARR_CREATE (MIR_code_reloc_t, relocs, 0); patterns_init (gen_ctx); + temp_jump = MIR_new_insn (ctx, MIR_JMP, MIR_new_label_op (ctx, NULL)); + temp_jump_replacement = find_insn_pattern_replacement (gen_ctx, temp_jump, FALSE); } static void target_finish (gen_ctx_t gen_ctx) { patterns_finish (gen_ctx); + _MIR_free_insn (gen_ctx->ctx, temp_jump); VARR_DESTROY (uint8_t, result_code); VARR_DESTROY (label_ref_t, label_refs); VARR_DESTROY (uint64_t, abs_address_locs); diff --git a/mir/mir-gen-riscv64.c b/mir/mir-gen-riscv64.c index 53321533..5c587b58 100644 --- a/mir/mir-gen-riscv64.c +++ b/mir/mir-gen-riscv64.c @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ /* In MIR generated code unsigned 32-bit values are zero extended but @@ -26,32 +26,20 @@ static void fancy_abort (int code) { #undef gen_assert #define gen_assert(c) fancy_abort (c) -#include +#define TARGET_EXPAND_ADDO +#define TARGET_EXPAND_ADDOS +#define TARGET_EXPAND_UADDO +#define TARGET_EXPAND_UADDOS +#define TARGET_EXPAND_MULO +#define TARGET_EXPAND_MULOS +#define TARGET_EXPAND_UMULO +#define TARGET_EXPAND_UMULOS -#define HREG_EL(h) h##_HARD_REG -#define REP_SEP , -enum { - REP8 (HREG_EL, R0, R1, R2, R3, R4, R5, R6, R7), - REP8 (HREG_EL, R8, R9, R10, R11, R12, R13, R14, R15), - REP8 (HREG_EL, R16, R17, R18, R19, R20, R21, R22, R23), - REP8 (HREG_EL, R24, R25, R26, R27, R28, R29, R30, R31), - /*Aliases: */ ZERO_HARD_REG = R0_HARD_REG, - REP7 (HREG_EL, RA, SP, GP, TP, T0, T1, T2), - REP8 (HREG_EL, FP, S1, A0, A1, A2, A3, A4, A5), - REP8 (HREG_EL, A6, A7, S2, S3, S4, S5, S6, S7), - REP8 (HREG_EL, S8, S9, S10, S11, T3, T4, T5, T6), +#include - REP8 (HREG_EL, F0, F1, F2, F3, F4, F5, F6, F7), - REP8 (HREG_EL, F8, F9, F10, F11, F12, F13, F14, F15), - REP8 (HREG_EL, F16, F17, F18, F19, F20, F21, F22, F23), - REP8 (HREG_EL, F24, F25, F26, F27, F28, F29, F30, F31), - /* Aliases: */ FT0_HARD_REG = F0_HARD_REG, - REP7 (HREG_EL, FT1, FT2, FT3, FT4, FT5, FT6, FT7), - REP8 (HREG_EL, FS0, FS1, FA0, FA1, FA2, FA3, FA4, FA5), - REP8 (HREG_EL, FA6, FA7, FS2, FS3, FS4, FS5, FS6, FS7), - REP8 (HREG_EL, FS8, FS9, FS10, FS11, FT8, FT9, FT10, FT11), -}; +#include "mir-riscv64.h" +#define REP_SEP , static const MIR_reg_t hard_reg_alloc_order[] = { REP8 (HREG_EL, R8, R9, R10, R11, R12, R13, R14, R15), REP8 (HREG_EL, F8, F9, F10, F11, F12, F13, F14, F15), @@ -66,7 +54,6 @@ static const MIR_reg_t hard_reg_alloc_order[] = { }; #undef REP_SEP -static const MIR_reg_t MAX_HARD_REG = F31_HARD_REG; static const MIR_reg_t LINK_HARD_REG = RA_HARD_REG; #define TARGET_HARD_REG_ALLOC_ORDER(n) hard_reg_alloc_order[n] @@ -85,34 +72,8 @@ static void check_hard_reg_alloc_order (void) { for (i = 0; i <= MAX_HARD_REG; i++) gen_assert (check_p[i]); } -static int target_locs_num (MIR_reg_t loc, MIR_type_t type) { - return loc > MAX_HARD_REG && type == MIR_T_LD ? 2 : 1; -} - -static inline MIR_reg_t target_nth_loc (MIR_reg_t loc, MIR_type_t type, int n) { return loc + n; } - -/* Hard regs not used in machinized code, preferably call used ones. */ -const MIR_reg_t TEMP_INT_HARD_REG1 = T5_HARD_REG, TEMP_INT_HARD_REG2 = T6_HARD_REG; -const MIR_reg_t TEMP_FLOAT_HARD_REG1 = FT10_HARD_REG, TEMP_FLOAT_HARD_REG2 = FT11_HARD_REG; -const MIR_reg_t TEMP_DOUBLE_HARD_REG1 = FT10_HARD_REG, TEMP_DOUBLE_HARD_REG2 = FT11_HARD_REG; -/* we use only builtins for long double ops: */ -const MIR_reg_t TEMP_LDOUBLE_HARD_REG1 = MIR_NON_HARD_REG; -const MIR_reg_t TEMP_LDOUBLE_HARD_REG2 = MIR_NON_HARD_REG; - -static inline int target_hard_reg_type_ok_p (MIR_reg_t hard_reg, MIR_type_t type) { - assert (hard_reg <= MAX_HARD_REG); - if (type == MIR_T_LD) return FALSE; /* long double can be in hard regs only for arg passing */ - return MIR_fp_type_p (type) ? hard_reg >= F0_HARD_REG : hard_reg < F0_HARD_REG; -} - -static inline int target_fixed_hard_reg_p (MIR_reg_t hard_reg) { - assert (hard_reg <= MAX_HARD_REG); - return (hard_reg == ZERO_HARD_REG || hard_reg == FP_HARD_REG || hard_reg == SP_HARD_REG - || hard_reg == GP_HARD_REG || hard_reg == TP_HARD_REG // ??? - || hard_reg == TEMP_INT_HARD_REG1 || hard_reg == TEMP_INT_HARD_REG2 - || hard_reg == TEMP_FLOAT_HARD_REG1 || hard_reg == TEMP_FLOAT_HARD_REG2 - || hard_reg == TEMP_DOUBLE_HARD_REG1 || hard_reg == TEMP_DOUBLE_HARD_REG2 - || hard_reg == TEMP_LDOUBLE_HARD_REG1 || hard_reg == TEMP_LDOUBLE_HARD_REG2); +static inline MIR_reg_t target_nth_loc (MIR_reg_t loc, MIR_type_t type MIR_UNUSED, int n) { + return loc + n; } static inline int target_call_used_hard_reg_p (MIR_reg_t hard_reg, MIR_type_t type) { @@ -188,7 +149,7 @@ static MIR_reg_t get_arg_reg (MIR_type_t arg_type, int vararg_p, size_t *int_arg case 5: case 6: case 7: arg_reg = FA0_HARD_REG + *fp_arg_num; break; - default: arg_reg = MIR_NON_HARD_REG; break; + default: arg_reg = MIR_NON_VAR; break; } (*fp_arg_num)++; *mov_code = arg_type == MIR_T_F ? MIR_FMOV : MIR_DMOV; @@ -203,7 +164,7 @@ static MIR_reg_t get_arg_reg (MIR_type_t arg_type, int vararg_p, size_t *int_arg case 5: case 6: case 7: arg_reg = A0_HARD_REG + *int_arg_num; break; - default: arg_reg = MIR_NON_HARD_REG; break; + default: arg_reg = MIR_NON_VAR; break; } (*int_arg_num)++; if (arg_type != MIR_T_LD) { @@ -240,63 +201,62 @@ static void gen_blk_mov (gen_ctx_t gen_ctx, MIR_insn_t anchor, size_t to_disp, MIR_insn_t new_insn; MIR_op_t ops[5], freg_op, treg_op, treg_op2, treg_op3, treg_op4; - treg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - treg_op2 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + treg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + treg_op2 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); if (qwords <= 16) { gen_mov (gen_ctx, anchor, MIR_MOV, treg_op2, MIR_new_int_op (ctx, to_disp)); gen_add_insn_before (gen_ctx, anchor, MIR_new_insn (gen_ctx->ctx, MIR_ADD, treg_op2, treg_op2, - _MIR_new_hard_reg_op (ctx, to_base_hard_reg))); + _MIR_new_var_op (ctx, to_base_hard_reg))); for (; qwords > 0; qwords--, to_disp += 8, from_disp += 8) { gen_mov (gen_ctx, anchor, MIR_MOV, treg_op, - MIR_new_mem_op (ctx, MIR_T_I64, from_disp, from_base_reg, 0, 1)); + _MIR_new_var_mem_op (ctx, MIR_T_I64, from_disp, from_base_reg, MIR_NON_VAR, 1)); gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, to_disp, to_base_hard_reg, - MIR_NON_HARD_REG, 1), + _MIR_new_var_mem_op (ctx, MIR_T_I64, to_disp, to_base_hard_reg, MIR_NON_VAR, 1), treg_op); } return; } - treg_op3 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + treg_op3 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); /* Save arg regs: */ if (save_regs > 0) - gen_mov (gen_ctx, anchor, MIR_MOV, treg_op, _MIR_new_hard_reg_op (ctx, A0_HARD_REG)); + gen_mov (gen_ctx, anchor, MIR_MOV, treg_op, _MIR_new_var_op (ctx, A0_HARD_REG)); if (save_regs > 1) - gen_mov (gen_ctx, anchor, MIR_MOV, treg_op2, _MIR_new_hard_reg_op (ctx, A1_HARD_REG)); + gen_mov (gen_ctx, anchor, MIR_MOV, treg_op2, _MIR_new_var_op (ctx, A1_HARD_REG)); if (save_regs > 2) - gen_mov (gen_ctx, anchor, MIR_MOV, treg_op3, _MIR_new_hard_reg_op (ctx, A2_HARD_REG)); + gen_mov (gen_ctx, anchor, MIR_MOV, treg_op3, _MIR_new_var_op (ctx, A2_HARD_REG)); /* call blk move: */ proto_item = _MIR_builtin_proto (ctx, curr_func_item->module, BLK_MOV_P, 0, NULL, 3, MIR_T_I64, "to", MIR_T_I64, "from", MIR_T_I64, "nwords"); func_import_item = _MIR_builtin_func (ctx, curr_func_item->module, BLK_MOV, mir_blk_mov); - freg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + freg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); new_insn = MIR_new_insn (ctx, MIR_MOV, freg_op, MIR_new_ref_op (ctx, func_import_item)); gen_add_insn_before (gen_ctx, anchor, new_insn); - treg_op4 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + treg_op4 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); gen_mov (gen_ctx, anchor, MIR_MOV, treg_op4, MIR_new_int_op (ctx, to_disp)); gen_add_insn_before (gen_ctx, anchor, - MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_hard_reg_op (ctx, A0_HARD_REG), - _MIR_new_hard_reg_op (ctx, to_base_hard_reg), treg_op4)); + MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_var_op (ctx, A0_HARD_REG), + _MIR_new_var_op (ctx, to_base_hard_reg), treg_op4)); gen_add_insn_before (gen_ctx, anchor, - MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_hard_reg_op (ctx, A1_HARD_REG), - MIR_new_reg_op (ctx, from_base_reg), + MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_var_op (ctx, A1_HARD_REG), + _MIR_new_var_op (ctx, from_base_reg), MIR_new_int_op (ctx, from_disp))); - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, A2_HARD_REG), + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, A2_HARD_REG), MIR_new_int_op (ctx, qwords)); ops[0] = MIR_new_ref_op (ctx, proto_item); ops[1] = freg_op; - ops[2] = _MIR_new_hard_reg_op (ctx, A0_HARD_REG); - ops[3] = _MIR_new_hard_reg_op (ctx, A1_HARD_REG); - ops[4] = _MIR_new_hard_reg_op (ctx, A2_HARD_REG); + ops[2] = _MIR_new_var_op (ctx, A0_HARD_REG); + ops[3] = _MIR_new_var_op (ctx, A1_HARD_REG); + ops[4] = _MIR_new_var_op (ctx, A2_HARD_REG); new_insn = MIR_new_insn_arr (ctx, MIR_CALL, 5, ops); gen_add_insn_before (gen_ctx, anchor, new_insn); /* Restore arg regs: */ if (save_regs > 0) - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, A0_HARD_REG), treg_op); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, A0_HARD_REG), treg_op); if (save_regs > 1) - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, A1_HARD_REG), treg_op2); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, A1_HARD_REG), treg_op2); if (save_regs > 2) - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, R2_HARD_REG), treg_op3); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, R2_HARD_REG), treg_op3); } #define FMVXW_CODE 0 @@ -329,9 +289,9 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { nargs = VARR_LENGTH (MIR_var_t, proto->args); arg_vars = VARR_ADDR (MIR_var_t, proto->args); } - if (call_insn->ops[1].mode != MIR_OP_REG && call_insn->ops[1].mode != MIR_OP_HARD_REG) { + if (call_insn->ops[1].mode != MIR_OP_VAR) { // ??? to optimize (can be immediate operand for func call) - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); new_insn = MIR_new_insn (ctx, MIR_MOV, temp_op, call_insn->ops[1]); call_insn->ops[1] = temp_op; gen_add_insn_before (gen_ctx, call_insn, new_insn); @@ -339,7 +299,7 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { for (size_t i = start; i < nops; i++) { /* calculate offset for blk params */ if (i - start < nargs) { type = arg_vars[i - start].type; - } else if (call_insn->ops[i].mode == MIR_OP_MEM) { + } else if (call_insn->ops[i].mode == MIR_OP_VAR_MEM) { type = call_insn->ops[i].u.mem.type; gen_assert (MIR_all_blk_type_p (type)); } else { @@ -351,7 +311,7 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { "passing float variadic arg (should be passed as double)"); type = mode == MIR_OP_DOUBLE ? MIR_T_D : mode == MIR_OP_LDOUBLE ? MIR_T_LD : MIR_T_I64; } - gen_assert (!MIR_all_blk_type_p (type) || call_insn->ops[i].mode == MIR_OP_MEM); + gen_assert (!MIR_all_blk_type_p (type) || call_insn->ops[i].mode == MIR_OP_VAR_MEM); if ((MIR_T_I8 <= type && type <= MIR_T_U64) || type == MIR_T_P || type == MIR_T_LD || MIR_all_blk_type_p (type)) { if (MIR_blk_type_p (type) && (qwords = (call_insn->ops[i].u.mem.disp + 7) / 8) <= 2) { @@ -381,11 +341,11 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { int_arg_num = fp_arg_num = 0; for (size_t i = start; i < nops; i++) { arg_op = call_insn->ops[i]; - gen_assert (arg_op.mode == MIR_OP_REG || arg_op.mode == MIR_OP_HARD_REG - || (arg_op.mode == MIR_OP_MEM && MIR_all_blk_type_p (arg_op.u.mem.type))); + gen_assert (arg_op.mode == MIR_OP_VAR + || (arg_op.mode == MIR_OP_VAR_MEM && MIR_all_blk_type_p (arg_op.u.mem.type))); if (i - start < nargs) { type = arg_vars[i - start].type; - } else if (call_insn->ops[i].mode == MIR_OP_MEM) { + } else if (call_insn->ops[i].mode == MIR_OP_VAR_MEM) { type = call_insn->ops[i].u.mem.type; gen_assert (MIR_all_blk_type_p (type)); } else { @@ -394,34 +354,33 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { } ext_insn = NULL; if ((ext_code = get_ext_code (type, TRUE)) != MIR_INVALID_INSN) { /* extend arg if necessary */ - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); ext_insn = MIR_new_insn (ctx, ext_code, temp_op, arg_op); call_insn->ops[i] = arg_op = temp_op; } - gen_assert ( - !MIR_all_blk_type_p (type) - || (arg_op.mode == MIR_OP_MEM && arg_op.u.mem.disp >= 0 && arg_op.u.mem.index == 0)); + gen_assert (!MIR_all_blk_type_p (type) + || (arg_op.mode == MIR_OP_VAR_MEM && arg_op.u.mem.disp >= 0 + && arg_op.u.mem.index == MIR_NON_VAR)); if (MIR_blk_type_p (type)) { qwords = (arg_op.u.mem.disp + 7) / 8; if (qwords <= 2) { arg_reg = A0_HARD_REG + int_arg_num; if (type == MIR_T_BLK + 1) int_arg_num = (int_arg_num + 1) / 2 * 2; /* Make even */ - for (int n = 0; n < qwords; n++) { + for (size_t n = 0; n < qwords; n++) { if (int_arg_num < 8) { - new_insn - = MIR_new_insn (ctx, MIR_MOV, _MIR_new_hard_reg_op (ctx, A0_HARD_REG + int_arg_num), - MIR_new_mem_op (ctx, MIR_T_I64, n * 8, arg_op.u.mem.base, 0, 1)); + new_insn = MIR_new_insn (ctx, MIR_MOV, _MIR_new_var_op (ctx, A0_HARD_REG + int_arg_num), + _MIR_new_var_mem_op (ctx, MIR_T_I64, n * 8, arg_op.u.mem.base, + MIR_NON_VAR, 1)); gen_add_insn_before (gen_ctx, call_insn, new_insn); setup_call_hard_reg_args (gen_ctx, call_insn, A0_HARD_REG + int_arg_num); int_arg_num++; } else { /* put word on stack */ - treg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - new_insn - = MIR_new_insn (ctx, MIR_MOV, treg_op, - MIR_new_mem_op (ctx, MIR_T_I64, n * 8, arg_op.u.mem.base, 0, 1)); + treg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + new_insn = MIR_new_insn (ctx, MIR_MOV, treg_op, + _MIR_new_var_mem_op (ctx, MIR_T_I64, n * 8, arg_op.u.mem.base, + MIR_NON_VAR, 1)); gen_add_insn_before (gen_ctx, call_insn, new_insn); - mem_op = _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, mem_size, SP_HARD_REG, - MIR_NON_HARD_REG, 1); + mem_op = _MIR_new_var_mem_op (ctx, MIR_T_I64, mem_size, SP_HARD_REG, MIR_NON_VAR, 1); new_insn = MIR_new_insn (ctx, MIR_MOV, mem_op, treg_op); gen_add_insn_before (gen_ctx, call_insn, new_insn); mem_size += 8; @@ -432,22 +391,21 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { /* Put on stack and pass the address: */ gen_blk_mov (gen_ctx, call_insn, blk_offset, SP_HARD_REG, arg_op.u.mem.base, qwords, int_arg_num); - arg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + arg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); gen_assert (curr_prev_call_insn != NULL); /* call_insn should not be 1st after simplification */ - new_insn - = MIR_new_insn (gen_ctx->ctx, MIR_ADD, arg_op, _MIR_new_hard_reg_op (ctx, SP_HARD_REG), - MIR_new_int_op (ctx, blk_offset)); + new_insn = MIR_new_insn (gen_ctx->ctx, MIR_ADD, arg_op, _MIR_new_var_op (ctx, SP_HARD_REG), + MIR_new_int_op (ctx, blk_offset)); gen_add_insn_after (gen_ctx, curr_prev_call_insn, new_insn); curr_prev_call_insn = DLIST_NEXT (MIR_insn_t, new_insn); blk_offset += qwords * 8; } if ((arg_reg = get_arg_reg (type, i - start >= nargs, &int_arg_num, &fp_arg_num, &new_insn_code)) - != MIR_NON_HARD_REG) { + != MIR_NON_VAR) { /* put arguments to argument hard regs */ if (ext_insn != NULL) gen_add_insn_before (gen_ctx, call_insn, ext_insn); - arg_reg_op = _MIR_new_hard_reg_op (ctx, arg_reg); + arg_reg_op = _MIR_new_var_op (ctx, arg_reg); if (type != MIR_T_RBLK) { if (new_insn_code == MIR_MOV && (type == MIR_T_F || type == MIR_T_D)) { new_insn @@ -458,10 +416,11 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { new_insn = MIR_new_insn (ctx, new_insn_code, arg_reg_op, arg_op); } } else { - assert (arg_op.mode == MIR_OP_MEM); - new_insn = MIR_new_insn (ctx, MIR_MOV, arg_reg_op, MIR_new_reg_op (ctx, arg_op.u.mem.base)); - arg_reg_op = _MIR_new_hard_reg_mem_op (ctx, MIR_T_RBLK, arg_op.u.mem.disp, arg_reg, - MIR_NON_HARD_REG, 1); + assert (arg_op.mode == MIR_OP_VAR_MEM); + new_insn + = MIR_new_insn (ctx, MIR_MOV, arg_reg_op, _MIR_new_var_op (ctx, arg_op.u.mem.base)); + arg_reg_op + = _MIR_new_var_mem_op (ctx, MIR_T_RBLK, arg_op.u.mem.disp, arg_reg, MIR_NON_VAR, 1); } gen_add_insn_before (gen_ctx, call_insn, new_insn); call_insn->ops[i] = arg_reg_op; @@ -473,13 +432,13 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { : type == MIR_T_D ? MIR_DMOV : type == MIR_T_LD ? MIR_LDMOV : MIR_MOV); - mem_op = _MIR_new_hard_reg_mem_op (ctx, mem_type, mem_size, SP_HARD_REG, MIR_NON_HARD_REG, 1); + mem_op = _MIR_new_var_mem_op (ctx, mem_type, mem_size, SP_HARD_REG, MIR_NON_VAR, 1); if (type != MIR_T_RBLK) { new_insn = MIR_new_insn (ctx, new_insn_code, mem_op, arg_op); } else { - assert (arg_op.mode == MIR_OP_MEM); + assert (arg_op.mode == MIR_OP_VAR_MEM); new_insn - = MIR_new_insn (ctx, new_insn_code, mem_op, MIR_new_reg_op (ctx, arg_op.u.mem.base)); + = MIR_new_insn (ctx, new_insn_code, mem_op, _MIR_new_var_op (ctx, arg_op.u.mem.base)); } gen_assert (curr_prev_call_insn != NULL); /* call should not be 1st after simplification */ MIR_insert_insn_after (ctx, curr_func_item, curr_prev_call_insn, new_insn); @@ -496,20 +455,20 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { int_arg_num = fp_arg_num = 0; for (size_t i = 0; i < proto->nres; i++) { ret_reg_op = call_insn->ops[i + 2]; - gen_assert (ret_reg_op.mode == MIR_OP_REG || ret_reg_op.mode == MIR_OP_HARD_REG); + gen_assert (ret_reg_op.mode == MIR_OP_VAR); type = proto->res_types[i]; float_p = type == MIR_T_F || type == MIR_T_D; if (float_p && fp_arg_num < 2) { new_insn = MIR_new_insn (ctx, type == MIR_T_F ? MIR_FMOV : MIR_DMOV, ret_reg_op, - _MIR_new_hard_reg_op (ctx, FA0_HARD_REG + fp_arg_num)); + _MIR_new_var_op (ctx, FA0_HARD_REG + fp_arg_num)); fp_arg_num++; } else if (type == MIR_T_LD && int_arg_num < 2) { new_insn = MIR_new_insn (ctx, MIR_LDMOV, ret_reg_op, - _MIR_new_hard_reg_op (ctx, A0_HARD_REG + int_arg_num)); + _MIR_new_var_op (ctx, A0_HARD_REG + int_arg_num)); int_arg_num += 2; } else if (!float_p && int_arg_num < 2) { - new_insn = MIR_new_insn (ctx, MIR_MOV, ret_reg_op, - _MIR_new_hard_reg_op (ctx, A0_HARD_REG + int_arg_num)); + new_insn + = MIR_new_insn (ctx, MIR_MOV, ret_reg_op, _MIR_new_var_op (ctx, A0_HARD_REG + int_arg_num)); int_arg_num++; } else { (*MIR_get_error_func (ctx)) (MIR_ret_error, @@ -525,21 +484,21 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { create_new_bb_insns (gen_ctx, call_insn, DLIST_NEXT (MIR_insn_t, new_insn), call_insn); } if (mem_size != 0) { /* allocate/deallocate stack for args passed on stack */ - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); mem_size = (mem_size + 15) / 16 * 16; /* make it of several 16 bytes */ - new_insn = MIR_new_insn (ctx, MIR_ADD, _MIR_new_hard_reg_op (ctx, SP_HARD_REG), - _MIR_new_hard_reg_op (ctx, SP_HARD_REG), temp_op); + new_insn = MIR_new_insn (ctx, MIR_ADD, _MIR_new_var_op (ctx, SP_HARD_REG), + _MIR_new_var_op (ctx, SP_HARD_REG), temp_op); MIR_insert_insn_after (ctx, curr_func_item, prev_call_insn, new_insn); next_insn = DLIST_NEXT (MIR_insn_t, new_insn); new_insn = MIR_new_insn (ctx, MIR_MOV, temp_op, MIR_new_int_op (ctx, -(int64_t) mem_size)); MIR_insert_insn_after (ctx, curr_func_item, prev_call_insn, new_insn); create_new_bb_insns (gen_ctx, prev_call_insn, next_insn, call_insn); - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); new_insn = MIR_new_insn (ctx, MIR_MOV, temp_op, MIR_new_int_op (ctx, mem_size)); MIR_insert_insn_after (ctx, curr_func_item, call_insn, new_insn); next_insn = DLIST_NEXT (MIR_insn_t, new_insn); - new_insn = MIR_new_insn (ctx, MIR_ADD, _MIR_new_hard_reg_op (ctx, SP_HARD_REG), - _MIR_new_hard_reg_op (ctx, SP_HARD_REG), temp_op); + new_insn = MIR_new_insn (ctx, MIR_ADD, _MIR_new_var_op (ctx, SP_HARD_REG), + _MIR_new_var_op (ctx, SP_HARD_REG), temp_op); MIR_insert_insn_before (ctx, curr_func_item, next_insn, new_insn); create_new_bb_insns (gen_ctx, call_insn, next_insn, call_insn); } @@ -754,10 +713,6 @@ static int get_builtin (gen_ctx_t gen_ctx, MIR_insn_code_t code, MIR_item_t *pro } } -DEF_VARR (int); -DEF_VARR (uint8_t); -DEF_VARR (uint64_t); - struct insn_pattern_info { int start, num; }; @@ -765,10 +720,15 @@ struct insn_pattern_info { typedef struct insn_pattern_info insn_pattern_info_t; DEF_VARR (insn_pattern_info_t); +enum branch_type { BRANCH, JAL, AUIPC, AUIPC_JALR }; struct label_ref { - int abs_addr_p, short_p; + int abs_addr_p; + enum branch_type branch_type; size_t label_val_disp; - MIR_label_t label; + union { + MIR_label_t label; + void *jump_addr; /* absolute addr for BBV */ + } u; }; typedef struct label_ref label_ref_t; @@ -782,12 +742,12 @@ struct const_ref { typedef struct const_ref const_ref_t; DEF_VARR (const_ref_t); -DEF_VARR (MIR_code_reloc_t); - struct target_ctx { - unsigned char alloca_p, block_arg_func_p, leaf_p; + unsigned char alloca_p, block_arg_func_p, leaf_p, add_nops; uint32_t non_vararg_int_args_num; size_t small_aggregate_save_area; + MIR_insn_t temp_jump; + const char *temp_jump_replacement; VARR (int) * pattern_indexes; VARR (insn_pattern_info_t) * insn_pattern_info; VARR (uint8_t) * result_code; @@ -800,8 +760,11 @@ struct target_ctx { #define alloca_p gen_ctx->target_ctx->alloca_p #define block_arg_func_p gen_ctx->target_ctx->block_arg_func_p #define leaf_p gen_ctx->target_ctx->leaf_p +#define add_nops gen_ctx->target_ctx->add_nops #define non_vararg_int_args_num gen_ctx->target_ctx->non_vararg_int_args_num #define small_aggregate_save_area gen_ctx->target_ctx->small_aggregate_save_area +#define temp_jump gen_ctx->target_ctx->temp_jump +#define temp_jump_replacement gen_ctx->target_ctx->temp_jump_replacement #define pattern_indexes gen_ctx->target_ctx->pattern_indexes #define insn_pattern_info gen_ctx->target_ctx->insn_pattern_info #define result_code gen_ctx->target_ctx->result_code @@ -810,7 +773,7 @@ struct target_ctx { #define abs_address_locs gen_ctx->target_ctx->abs_address_locs #define relocs gen_ctx->target_ctx->relocs -static MIR_disp_t target_get_stack_slot_offset (gen_ctx_t gen_ctx, MIR_type_t type, +static MIR_disp_t target_get_stack_slot_offset (gen_ctx_t gen_ctx, MIR_type_t type MIR_UNUSED, MIR_reg_t slot) { /* slot is 0, 1, ... */ size_t offset = curr_func_item->u.func->vararg_p || block_arg_func_p ? 32 : 16; @@ -818,7 +781,12 @@ static MIR_disp_t target_get_stack_slot_offset (gen_ctx_t gen_ctx, MIR_type_t ty return ((MIR_disp_t) slot * 8 + offset); } -static int target_valid_mem_offset_p (gen_ctx_t gen_ctx, MIR_type_t type, MIR_disp_t offset) { +static MIR_reg_t target_get_stack_slot_base_reg (gen_ctx_t gen_ctx MIR_UNUSED) { + return FP_HARD_REG; +} + +static int target_valid_mem_offset_p (gen_ctx_t gen_ctx MIR_UNUSED, MIR_type_t type, + MIR_disp_t offset) { MIR_disp_t offset2 = type == MIR_T_LD ? offset + 8 : offset; return -(1 << 11) <= offset && offset2 < (1 << 11); } @@ -854,32 +822,35 @@ static void target_machinize (gen_ctx_t gen_ctx) { small_aggregate_save_area += qwords * 8; gen_assert (small_aggregate_save_area < (1 << 11)); - new_insn = MIR_new_insn (ctx, MIR_ADD, MIR_new_reg_op (ctx, i + 1), - _MIR_new_hard_reg_op (ctx, FP_HARD_REG), + new_insn = MIR_new_insn (ctx, MIR_ADD, _MIR_new_var_op (ctx, i + MAX_HARD_REG + 1), + _MIR_new_var_op (ctx, FP_HARD_REG), MIR_new_int_op (ctx, -(int64_t) small_aggregate_save_area)); gen_add_insn_before (gen_ctx, anchor, new_insn); if (qwords == 0) continue; - gen_mov (gen_ctx, anchor, mov_code1, MIR_new_mem_op (ctx, mem_type1, 0, i + 1, 0, 1), - _MIR_new_hard_reg_op (ctx, base_arg_reg + arg_reg_num)); + gen_mov (gen_ctx, anchor, mov_code1, + _MIR_new_var_mem_op (ctx, mem_type1, 0, i + MAX_HARD_REG + 1, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, base_arg_reg + arg_reg_num)); if (qwords == 2) { MIR_insn_code_t mov_code2 = MIR_MOV; MIR_type_t mem_type2 = MIR_T_I64; MIR_disp_t disp = 8; if (arg_reg_num < 7) { - gen_mov (gen_ctx, anchor, mov_code2, MIR_new_mem_op (ctx, mem_type2, disp, i + 1, 0, 1), - _MIR_new_hard_reg_op (ctx, base_arg_reg + arg_reg_num + 1)); + gen_mov (gen_ctx, anchor, mov_code2, + _MIR_new_var_mem_op (ctx, mem_type2, disp, i + MAX_HARD_REG + 1, MIR_NON_VAR, + 1), + _MIR_new_var_op (ctx, base_arg_reg + arg_reg_num + 1)); } else { if (!block_arg_func_p) { /* t0 = prev sp */ block_arg_func_p = TRUE; - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, T0_HARD_REG), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 16, FP_HARD_REG, MIR_NON_HARD_REG, - 1)); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, T0_HARD_REG), + _MIR_new_var_mem_op (ctx, MIR_T_I64, 16, FP_HARD_REG, MIR_NON_VAR, 1)); } - treg_op = _MIR_new_hard_reg_op (ctx, T1_HARD_REG); + treg_op = _MIR_new_var_op (ctx, T1_HARD_REG); gen_mov (gen_ctx, anchor, mov_code2, treg_op, - _MIR_new_hard_reg_mem_op (ctx, mem_type2, mem_size, T0_HARD_REG, - MIR_NON_HARD_REG, 1)); - gen_mov (gen_ctx, anchor, mov_code2, MIR_new_mem_op (ctx, mem_type2, disp, i + 1, 0, 1), + _MIR_new_var_mem_op (ctx, mem_type2, mem_size, T0_HARD_REG, MIR_NON_VAR, 1)); + gen_mov (gen_ctx, anchor, mov_code2, + _MIR_new_var_mem_op (ctx, mem_type2, disp, i + MAX_HARD_REG + 1, MIR_NON_VAR, + 1), treg_op); mem_size += 8; } @@ -888,34 +859,35 @@ static void target_machinize (gen_ctx_t gen_ctx) { } else { /* fully on stack -- use the address: */ if (!block_arg_func_p) { /* t0 = prev sp */ block_arg_func_p = TRUE; - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, T0_HARD_REG), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 16, FP_HARD_REG, MIR_NON_HARD_REG, 1)); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, T0_HARD_REG), + _MIR_new_var_mem_op (ctx, MIR_T_I64, 16, FP_HARD_REG, MIR_NON_VAR, 1)); } new_insn - = MIR_new_insn (ctx, MIR_ADD, MIR_new_reg_op (ctx, i + 1), - _MIR_new_hard_reg_op (ctx, T0_HARD_REG), MIR_new_int_op (ctx, mem_size)); + = MIR_new_insn (ctx, MIR_ADD, _MIR_new_var_op (ctx, i + MAX_HARD_REG + 1), + _MIR_new_var_op (ctx, T0_HARD_REG), MIR_new_int_op (ctx, mem_size)); gen_add_insn_before (gen_ctx, anchor, new_insn); mem_size += qwords * 8; } continue; } arg_reg = get_arg_reg (type, FALSE, &int_arg_num, &fp_arg_num, &new_insn_code); - if (arg_reg != MIR_NON_HARD_REG) { - arg_reg_op = _MIR_new_hard_reg_op (ctx, arg_reg); - gen_mov (gen_ctx, anchor, new_insn_code, MIR_new_reg_op (ctx, i + 1), arg_reg_op); + if (arg_reg != MIR_NON_VAR) { + arg_reg_op = _MIR_new_var_op (ctx, arg_reg); + gen_mov (gen_ctx, anchor, new_insn_code, _MIR_new_var_op (ctx, i + MAX_HARD_REG + 1), + arg_reg_op); } else { /* arg is on the stack or blk address is on the stack: */ if (!block_arg_func_p) { block_arg_func_p = TRUE; - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, T0_HARD_REG), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 16, FP_HARD_REG, MIR_NON_HARD_REG, 1)); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, T0_HARD_REG), + _MIR_new_var_mem_op (ctx, MIR_T_I64, 16, FP_HARD_REG, MIR_NON_VAR, 1)); } mem_type = type == MIR_T_F || type == MIR_T_D || type == MIR_T_LD ? type : MIR_T_I64; new_insn_code = (type == MIR_T_F ? MIR_FMOV : type == MIR_T_D ? MIR_DMOV : type == MIR_T_LD ? MIR_LDMOV : MIR_MOV); - mem_op = _MIR_new_hard_reg_mem_op (ctx, mem_type, mem_size, T0_HARD_REG, MIR_NON_HARD_REG, 1); - gen_mov (gen_ctx, anchor, new_insn_code, MIR_new_reg_op (ctx, i + 1), mem_op); + mem_op = _MIR_new_var_mem_op (ctx, mem_type, mem_size, T0_HARD_REG, MIR_NON_VAR, 1); + gen_mov (gen_ctx, anchor, new_insn_code, _MIR_new_var_op (ctx, i + MAX_HARD_REG + 1), mem_op); mem_size += type == MIR_T_LD ? 16 : 8; } } @@ -969,10 +941,10 @@ static void target_machinize (gen_ctx_t gen_ctx) { case MIR_UBGES: ext_code = MIR_UEXT32; short_cmp: - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); new_insn = MIR_new_insn (ctx, ext_code, temp_op, insn->ops[1]); gen_add_insn_before (gen_ctx, insn, new_insn); - treg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + treg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); new_insn = MIR_new_insn (ctx, ext_code, treg_op, insn->ops[2]); gen_add_insn_before (gen_ctx, insn, new_insn); insn->ops[1] = temp_op; @@ -981,7 +953,7 @@ static void target_machinize (gen_ctx_t gen_ctx) { default: break; } if (code != insn->code) { - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); new_insn = MIR_new_insn (ctx, code, temp_op, insn->ops[1], insn->ops[2]); gen_add_insn_before (gen_ctx, insn, new_insn); next_insn = MIR_new_insn (ctx, MIR_BT, insn->ops[0], temp_op); @@ -997,10 +969,10 @@ static void target_machinize (gen_ctx_t gen_ctx) { MIR_op_t ops[6], func_reg_op, reg_op3; MIR_op_t res_reg_op = insn->ops[0], va_reg_op = insn->ops[1], op3 = insn->ops[2]; - assert (res_reg_op.mode == MIR_OP_REG && va_reg_op.mode == MIR_OP_REG - && op3.mode == (code == MIR_VA_ARG ? MIR_OP_MEM : MIR_OP_REG)); - func_reg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - reg_op3 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + assert (res_reg_op.mode == MIR_OP_VAR && va_reg_op.mode == MIR_OP_VAR + && op3.mode == (code == MIR_VA_ARG ? MIR_OP_VAR_MEM : MIR_OP_VAR)); + func_reg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + reg_op3 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); next_insn = new_insn = MIR_new_insn (ctx, MIR_MOV, func_reg_op, MIR_new_ref_op (ctx, func_import_item)); gen_add_insn_before (gen_ctx, insn, new_insn); @@ -1022,8 +994,8 @@ static void target_machinize (gen_ctx_t gen_ctx) { } else { /* Use builtin: mov freg, func ref; call proto, freg, res_reg, op_reg[, op_reg2] */ MIR_op_t freg_op, res_reg_op = insn->ops[0], op_reg_op = insn->ops[1], ops[5]; - assert (res_reg_op.mode == MIR_OP_REG && op_reg_op.mode == MIR_OP_REG); - freg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + assert (res_reg_op.mode == MIR_OP_VAR && op_reg_op.mode == MIR_OP_VAR); + freg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); next_insn = new_insn = MIR_new_insn (ctx, MIR_MOV, freg_op, MIR_new_ref_op (ctx, func_import_item)); gen_add_insn_before (gen_ctx, insn, new_insn); @@ -1037,24 +1009,24 @@ static void target_machinize (gen_ctx_t gen_ctx) { gen_delete_insn (gen_ctx, insn); } } else if (code == MIR_VA_START) { - MIR_op_t prev_sp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + MIR_op_t prev_sp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); MIR_op_t va_op = insn->ops[0]; MIR_reg_t va_reg; - assert (func->vararg_p && va_op.mode == MIR_OP_REG); + assert (func->vararg_p && va_op.mode == MIR_OP_VAR); va_reg = va_op.u.reg; /* Insns can be not simplified as soon as they match a machine insn. */ /* __stack: prev_sp = mem64[fp + 16] */ gen_mov (gen_ctx, insn, MIR_MOV, prev_sp_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 16, FP_HARD_REG, MIR_NON_HARD_REG, 1)); + _MIR_new_var_mem_op (ctx, MIR_T_I64, 16, FP_HARD_REG, MIR_NON_VAR, 1)); if (non_vararg_int_args_num != 8) gen_add_insn_before (gen_ctx, insn, MIR_new_insn (ctx, MIR_ADD, prev_sp_op, prev_sp_op, MIR_new_int_op (ctx, ((uint64_t) non_vararg_int_args_num - 8) * 8))); - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 0, va_reg, 0, 1), - prev_sp_op); + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, va_reg, MIR_NON_VAR, 1), prev_sp_op); gen_delete_insn (gen_ctx, insn); } else if (code == MIR_VA_END) { /* do nothing */ gen_delete_insn (gen_ctx, insn); @@ -1069,8 +1041,8 @@ static void target_machinize (gen_ctx_t gen_ctx) { uint32_t n_xregs = 0, n_fpregs = 0; assert (func->nres == MIR_insn_nops (ctx, insn)); - for (size_t i = 0; i < func->nres; i++) { - assert (insn->ops[i].mode == MIR_OP_REG); + for (i = 0; i < func->nres; i++) { + assert (insn->ops[i].mode == MIR_OP_VAR); res_type = func->res_types[i]; if ((res_type == MIR_T_F || res_type == MIR_T_D) && n_fpregs < 2) { new_insn_code = res_type == MIR_T_F ? MIR_FMOV : MIR_DMOV; @@ -1083,7 +1055,7 @@ static void target_machinize (gen_ctx_t gen_ctx) { (*MIR_get_error_func (ctx)) (MIR_ret_error, "riscv can not handle this combination of return values"); } - ret_reg_op = _MIR_new_hard_reg_op (ctx, ret_reg); + ret_reg_op = _MIR_new_var_op (ctx, ret_reg); /* We should return unsigned 32-bit integer with sign extension according to ABI: */ gen_mov (gen_ctx, insn, res_type == MIR_T_U32 ? MIR_EXT32 : new_insn_code, ret_reg_op, insn->ops[i]); @@ -1096,8 +1068,8 @@ static void target_machinize (gen_ctx_t gen_ctx) { static void isave (gen_ctx_t gen_ctx, MIR_insn_t anchor, int disp, MIR_reg_t base, MIR_reg_t hard_reg) { gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (gen_ctx->ctx, MIR_T_I64, disp, base, MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (gen_ctx->ctx, hard_reg)); + _MIR_new_var_mem_op (gen_ctx->ctx, MIR_T_I64, disp, base, MIR_NON_VAR, 1), + _MIR_new_var_op (gen_ctx->ctx, hard_reg)); } static MIR_reg_t get_base_reg_offset_for_saved_regs (gen_ctx_t gen_ctx, MIR_insn_t anchor, @@ -1108,11 +1080,10 @@ static MIR_reg_t get_base_reg_offset_for_saved_regs (gen_ctx_t gen_ctx, MIR_insn if (*offset + MAX_HARD_REG * 8 < (1 << 11)) return FP_HARD_REG; base_reg = T2_HARD_REG; - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, base_reg), + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, base_reg), MIR_new_int_op (ctx, *offset)); - new_insn - = MIR_new_insn (ctx, MIR_ADD, _MIR_new_hard_reg_op (ctx, base_reg), - _MIR_new_hard_reg_op (ctx, base_reg), _MIR_new_hard_reg_op (ctx, FP_HARD_REG)); + new_insn = MIR_new_insn (ctx, MIR_ADD, _MIR_new_var_op (ctx, base_reg), + _MIR_new_var_op (ctx, base_reg), _MIR_new_var_op (ctx, FP_HARD_REG)); gen_add_insn_before (gen_ctx, anchor, new_insn); *offset = 0; return base_reg; @@ -1143,8 +1114,8 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg && stack_slots_num == 0 && !block_arg_func_p && small_aggregate_save_area == 0 && !bitmap_bit_p (used_hard_regs, RA_HARD_REG)) return; - sp_reg_op = _MIR_new_hard_reg_op (ctx, SP_HARD_REG); - fp_reg_op = _MIR_new_hard_reg_op (ctx, FP_HARD_REG); + sp_reg_op = _MIR_new_var_op (ctx, SP_HARD_REG); + fp_reg_op = _MIR_new_var_op (ctx, FP_HARD_REG); /* Prologue: */ anchor = DLIST_HEAD (MIR_insn_t, func->insns); frame_size = 0; @@ -1158,7 +1129,7 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg frame_size += stack_slots_num * 8; if (frame_size % 16 != 0) frame_size = (frame_size + 15) / 16 * 16; save_prev_stack_p = func->vararg_p || block_arg_func_p; - treg_op = _MIR_new_hard_reg_op (ctx, T1_HARD_REG); + treg_op = _MIR_new_var_op (ctx, T1_HARD_REG); if (save_prev_stack_p) { /* the 1st insn: putting stack pointer into T1: */ gen_mov (gen_ctx, anchor, MIR_MOV, treg_op, sp_reg_op); frame_size += 16; @@ -1168,7 +1139,7 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg new_insn = MIR_new_insn (ctx, MIR_ADD, sp_reg_op, sp_reg_op, MIR_new_int_op (ctx, -(int64_t) frame_size)); } else { - treg_op2 = _MIR_new_hard_reg_op (ctx, T2_HARD_REG); + treg_op2 = _MIR_new_var_op (ctx, T2_HARD_REG); new_insn = MIR_new_insn (ctx, MIR_MOV, treg_op2, MIR_new_int_op (ctx, -(int64_t) frame_size)); gen_add_insn_before (gen_ctx, anchor, new_insn); /* t = -frame_size */ new_insn = MIR_new_insn (ctx, MIR_ADD, sp_reg_op, sp_reg_op, treg_op2); @@ -1176,14 +1147,15 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg gen_add_insn_before (gen_ctx, anchor, new_insn); /* sp = sp - (frame_size|t) */ if (save_prev_stack_p) /* save prev sp value which is in T1: */ gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 16, SP_HARD_REG, MIR_NON_HARD_REG, 1), + _MIR_new_var_mem_op (ctx, MIR_T_I64, 16, SP_HARD_REG, MIR_NON_VAR, 1), treg_op); /* mem[sp + 16] = t1 */ + if (!func->jret_p) + gen_mov (gen_ctx, anchor, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 8, SP_HARD_REG, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, LINK_HARD_REG)); /* mem[sp + 8] = ra */ gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 8, SP_HARD_REG, MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (ctx, LINK_HARD_REG)); /* mem[sp + 8] = ra */ - gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 0, SP_HARD_REG, MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (ctx, FP_HARD_REG)); /* mem[sp] = fp */ + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, SP_HARD_REG, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, FP_HARD_REG)); /* mem[sp] = fp */ gen_mov (gen_ctx, anchor, MIR_MOV, fp_reg_op, sp_reg_op); /* fp = sp */ if (func->vararg_p && non_vararg_int_args_num < 8) { /* save vararg int regs: */ MIR_reg_t base = SP_HARD_REG; @@ -1208,16 +1180,15 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg if (i < F0_HARD_REG) { gen_assert (offset < (1 << 11)); gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, offset, base_reg, MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (ctx, i)); + _MIR_new_var_mem_op (ctx, MIR_T_I64, offset, base_reg, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, i)); offset += 8; } else { // if (offset % 16 != 0) offset = (offset + 15) / 16 * 16; gen_assert (offset < (1 << 11)); - new_insn - = gen_mov (gen_ctx, anchor, MIR_DMOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_D, offset, base_reg, MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (ctx, i)); + new_insn = gen_mov (gen_ctx, anchor, MIR_DMOV, + _MIR_new_var_mem_op (ctx, MIR_T_D, offset, base_reg, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, i)); offset += 8; } } @@ -1229,10 +1200,10 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg gen_add_insn_before (gen_ctx, anchor, new_insn); /* sp -= */ } /* Epilogue: */ - anchor = DLIST_TAIL (MIR_insn_t, func->insns); - /* It might be infinite loop after CCP with dead code elimination: */ - if (anchor->code == MIR_JMP) return; - assert (anchor->code == MIR_RET); + for (anchor = DLIST_TAIL (MIR_insn_t, func->insns); anchor != NULL; + anchor = DLIST_PREV (MIR_insn_t, anchor)) + if (anchor->code == MIR_RET || anchor->code == MIR_JRET) break; + if (anchor == NULL) return; /* Restoring hard registers: */ offset = frame_size - frame_size_after_saved_regs; base_reg = get_base_reg_offset_for_saved_regs (gen_ctx, anchor, &offset); @@ -1241,20 +1212,20 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg && i != FP_HARD_REG) { if (i < F0_HARD_REG) { gen_assert (offset < (1 << 11)); - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, i), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, offset, base_reg, MIR_NON_HARD_REG, 1)); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, i), + _MIR_new_var_mem_op (ctx, MIR_T_I64, offset, base_reg, MIR_NON_VAR, 1)); offset += 8; } else { gen_assert (offset < (1 << 11)); - new_insn = gen_mov (gen_ctx, anchor, MIR_DMOV, _MIR_new_hard_reg_op (ctx, i), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_D, offset, base_reg, - MIR_NON_HARD_REG, 1)); + new_insn = gen_mov (gen_ctx, anchor, MIR_DMOV, _MIR_new_var_op (ctx, i), + _MIR_new_var_mem_op (ctx, MIR_T_D, offset, base_reg, MIR_NON_VAR, 1)); offset += 8; } } /* Restore ra, sp, fp */ - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, LINK_HARD_REG), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 8, FP_HARD_REG, MIR_NON_HARD_REG, 1)); + if (!func->jret_p) + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, LINK_HARD_REG), + _MIR_new_var_mem_op (ctx, MIR_T_I64, 8, FP_HARD_REG, MIR_NON_VAR, 1)); if (frame_size < (1 << 11)) { new_insn = MIR_new_insn (ctx, MIR_ADD, sp_reg_op, fp_reg_op, MIR_new_int_op (ctx, frame_size)); } else { @@ -1264,7 +1235,7 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg } gen_add_insn_before (gen_ctx, anchor, new_insn); /* sp = fp + (frame_size|t) */ gen_mov (gen_ctx, anchor, MIR_MOV, fp_reg_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 0, FP_HARD_REG, MIR_NON_HARD_REG, 1)); + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, FP_HARD_REG, MIR_NON_VAR, 1)); } /* 32-bit insn formats: @@ -1337,6 +1308,8 @@ struct pattern { S -- immediate shift (6 bits) as 3th op Sp -- nonzero immediate shift (6 bits) as 3th op l -- label as the 1st or 2nd op which can be present by signed 13-bit pc offset + L -- label as the 1st or 2nd op which can be present by unsigned 20-bit pc offset + U -- label used in LADDR as 2nd op which can be present by signed 32-bit pc offset k -- 2nd or 3rd immediate op for arithmetic insn (6-bit signed) kp -- nonzero 2nd or 3rd immediate op for arithmetic insn (6-bit signed) @@ -1410,6 +1383,7 @@ struct pattern { l -- operand-label as signed 13-bit offset ([12|10:5] as [31:25] and [4:1|11] as [11:7]), remember address of any insn is even L -- operand-label as signed 21-bit offset ([20|10:1|11|19:12] as [31:12]) + U -- operand-label as signed 32-bit offset in auipc,addi lc -- operand-label as signed 9-bit offset ([12..10,6-2]), 16-bit insn Lc -- operand-label as signed 12-bit offset ([12-2]), 16-bit insn @@ -1649,6 +1623,9 @@ static const struct pattern patterns[] = { {MIR_JMP, "L", "O6f hd0 L"}, /* jal: 20-bit offset (w/o 1 bit) jmp */ + {MIR_LADDR, "r U", "O17 rd0 U; O13 F0 rd0 rs0"}, /* auipc r,hi(l);addi r,r,low(L) */ + {MIR_JMPI, "r", "O67 F0 hd0 rs0 i0"}, /* jmp *r: jalr zero,r,0 */ + {MIR_BT, "l r", "O63 F1 rs1 hS0 l"}, /* bne rs1,zero,l */ {MIR_BTS, "l r", "O63 F1 rs1 hS0 l"}, /* bne rs1,zero,l */ {MIR_BF, "l r", "O63 F0 rs1 hS0 l"}, /* beq rs1,zero,l */ @@ -1770,6 +1747,9 @@ static const struct pattern patterns[] = { {MIR_INLINE, "X r $", "O67 F0 hd1 rs1 i0"}, /* jalr rd,rs1 */ {MIR_RET, "$", "O67 F0 hd0 hs1 i0"}, /* jalr hr0,hr1,0 */ + {MIR_JCALL, "X r $", "O67 F0 hd0 rs1 i0"}, /* jmp *r: jalr zero,r,0 */ + {MIR_JRET, "r $", "O67 F0 hd0 rs0 i0"}, /* jmp *r: jalr zero,r,0 */ + #if COMPRESS_INSNS /* addi r0,r0,15; andi r0,r0,-16; c.sub sp,sp,r0; c.mov r0,sp: */ {MIR_ALLOCA, "C 0", @@ -1803,7 +1783,7 @@ static const struct pattern patterns[] = { {MIR_BSTART, "r", "O13 F0 rd0 hs2 i0"}, /* r = sp: addi rd,rs1,0 */ {MIR_BEND, "r", "O13 F0 hd2 rs0 i0"}, /* sp = r: addi rd,rs1,0 */ #endif - /* slli t5,r,3; auipc t6,16; add t6,t6,t5;ld t6,T(t6);jalr zero,t6,0; + /* slli t5,r,3; auipc t6,0; add t6,t6,t5;ld t6,T(t6);jalr zero,t6,0; 8-byte aligned TableContent. Remember r can be t5 can be if switch operand is memory. */ {MIR_SWITCH, "r $", "O13 F1 hd1e rs0 S3; O17 hd1f iu0; O33 F0 hd1f hs1f hS1e; O3 F3 hd1f hs1f T; O67 F0 hd0 hs1f " @@ -1812,7 +1792,7 @@ static const struct pattern patterns[] = { }; static void target_get_early_clobbered_hard_regs (MIR_insn_t insn, MIR_reg_t *hr1, MIR_reg_t *hr2) { - *hr1 = *hr2 = MIR_NON_HARD_REG; + *hr1 = *hr2 = MIR_NON_VAR; if (insn->code == MIR_MOD || insn->code == MIR_MODS || insn->code == MIR_UMOD || insn->code == MIR_UMODS) *hr1 = R8_HARD_REG; @@ -1873,8 +1853,8 @@ static int compressed_reg_p (MIR_reg_t reg, int int_only_p) { static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_insn_t insn) { MIR_context_t ctx = gen_ctx->ctx; - int n, nop; - size_t nops = MIR_insn_nops (ctx, insn); + int n; + size_t nop, nops = MIR_insn_nops (ctx, insn); const char *p; char ch, start_ch; MIR_op_t op, original; @@ -1892,35 +1872,34 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in ch = *++p; if (ch != 'p') { p--; - if (op.mode != MIR_OP_HARD_REG || op.u.hard_reg == R0_HARD_REG) return FALSE; + if (op.mode != MIR_OP_VAR || op.u.var == R0_HARD_REG) return FALSE; } else { - if (op.mode != MIR_OP_HARD_REG || op.u.hard_reg == R0_HARD_REG - || op.u.hard_reg == SP_HARD_REG) + if (op.mode != MIR_OP_VAR || op.u.var == R0_HARD_REG || op.u.var == SP_HARD_REG) return FALSE; } break; case 'h': { - uint64_t n; + uint64_t num; p++; - n = read_dec (&p); - if (op.mode != MIR_OP_HARD_REG || op.u.hard_reg != n) return FALSE; + num = read_dec (&p); + if (op.mode != MIR_OP_VAR || op.u.var != num) return FALSE; break; } case 'C': - if (op.mode != MIR_OP_HARD_REG || !compressed_reg_p (op.u.hard_reg, FALSE)) return FALSE; + if (op.mode != MIR_OP_VAR || !compressed_reg_p (op.u.var, FALSE)) return FALSE; break; case 'c': { - uint64_t n; + uint64_t num; p++; - n = read_dec (&p); - if ((op.mode != MIR_OP_INT && op.mode != MIR_OP_UINT) || op.u.u != n) return FALSE; + num = read_dec (&p); + if ((op.mode != MIR_OP_INT && op.mode != MIR_OP_UINT) || op.u.u != num) return FALSE; break; } case 'm': { MIR_type_t type, type2, type3 = MIR_T_BOUND; int scale, u_p, s_p, compressed_p = FALSE; - if (op.mode != MIR_OP_HARD_REG_MEM) return FALSE; + if (op.mode != MIR_OP_VAR_MEM) return FALSE; u_p = s_p = TRUE; ch = *++p; if (ch == 'c') { @@ -1946,12 +1925,14 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in type2 = MIR_T_BOUND; scale = 16; break; - case 'u': gen_assert (!compressed_p); + case 'u': + gen_assert (!compressed_p); + /* fall through */ case 's': u_p = ch == 'u'; s_p = ch == 's'; ch = *++p; - /* Fall through: */ + /* fall through */ default: gen_assert ('0' <= ch && ch <= '3'); gen_assert (!compressed_p || '2' <= ch); @@ -1976,23 +1957,22 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in #endif } } - if (op.u.hard_reg_mem.type != type && op.u.hard_reg_mem.type != type2 - && op.u.hard_reg_mem.type != type3) + if (op.u.var_mem.type != type && op.u.var_mem.type != type2 && op.u.var_mem.type != type3) return FALSE; - if (op.u.hard_reg_mem.index != MIR_NON_HARD_REG || op.u.hard_reg_mem.disp < -(1 << 11) - || op.u.hard_reg_mem.disp >= (1 << 11) - || (type == MIR_T_LD && op.u.hard_reg_mem.disp + 8 >= (1 << 11))) + if (op.u.var_mem.index != MIR_NON_VAR || op.u.var_mem.disp < -(1 << 11) + || op.u.var_mem.disp >= (1 << 11) + || (type == MIR_T_LD && op.u.var_mem.disp + 8 >= (1 << 11))) return FALSE; if (compressed_p) { - if (op.u.hard_reg_mem.disp < 0 || op.u.hard_reg_mem.disp % scale != 0) return FALSE; + if (op.u.var_mem.disp < 0 || op.u.var_mem.disp % scale != 0) return FALSE; ch = *++p; if (ch == 's') { - if (op.u.hard_reg_mem.base != SP_HARD_REG) return FALSE; - if (op.u.hard_reg_mem.disp / scale >= (1 << 6)) return FALSE; + if (op.u.var_mem.base != SP_HARD_REG) return FALSE; + if (op.u.var_mem.disp / scale >= (1 << 6)) return FALSE; } else { p--; - if (!compressed_reg_p (op.u.hard_reg_mem.base, TRUE)) return FALSE; - if (op.u.hard_reg_mem.disp / scale >= (1 << 5)) return FALSE; + if (!compressed_reg_p (op.u.var_mem.base, TRUE)) return FALSE; + if (op.u.var_mem.disp / scale >= (1 << 5)) return FALSE; } } break; @@ -2083,22 +2063,23 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in break; case 'l': case 'L': + case 'U': if (op.mode != MIR_OP_LABEL) return FALSE; break; case '0': case '1': case '2': n = start_ch - '0'; - gen_assert (n < nop); + gen_assert (n < (int) nop); original = insn->ops[n]; mode = op.mode; if (mode == MIR_OP_UINT) mode = MIR_OP_INT; if (original.mode != mode && (original.mode != MIR_OP_UINT || mode != MIR_OP_INT)) return FALSE; - gen_assert (mode == MIR_OP_HARD_REG || mode == MIR_OP_INT || mode == MIR_OP_FLOAT - || mode == MIR_OP_DOUBLE || mode == MIR_OP_LDOUBLE || mode == MIR_OP_HARD_REG_MEM + gen_assert (mode == MIR_OP_VAR || mode == MIR_OP_INT || mode == MIR_OP_FLOAT + || mode == MIR_OP_DOUBLE || mode == MIR_OP_LDOUBLE || mode == MIR_OP_VAR_MEM || mode == MIR_OP_LABEL); - if (mode == MIR_OP_HARD_REG && op.u.hard_reg != original.u.hard_reg) + if (mode == MIR_OP_VAR && op.u.var != original.u.var) return FALSE; else if (mode == MIR_OP_INT && op.u.i != original.u.i) return FALSE; @@ -2110,11 +2091,12 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in return FALSE; else if (mode == MIR_OP_LABEL && op.u.label != original.u.label) return FALSE; - else if (mode == MIR_OP_HARD_REG_MEM && op.u.hard_reg_mem.type != original.u.hard_reg_mem.type - && op.u.hard_reg_mem.scale != original.u.hard_reg_mem.scale - && op.u.hard_reg_mem.base != original.u.hard_reg_mem.base - && op.u.hard_reg_mem.index != original.u.hard_reg_mem.index - && op.u.hard_reg_mem.disp != original.u.hard_reg_mem.disp) + else if (mode == MIR_OP_VAR_MEM + && (op.u.var_mem.type != original.u.var_mem.type + || op.u.var_mem.scale != original.u.var_mem.scale + || op.u.var_mem.base != original.u.var_mem.base + || op.u.var_mem.index != original.u.var_mem.index + || op.u.var_mem.disp != original.u.var_mem.disp)) return FALSE; break; default: gen_assert (FALSE); @@ -2192,7 +2174,8 @@ static uint32_t check_and_set_mask (uint32_t opcode_mask, uint32_t mask) { return opcode_mask | mask; } -static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacement) { +static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacement, + void **jump_addrs) { MIR_context_t ctx = gen_ctx->ctx; const char *p, *insn_str; label_ref_t lr; @@ -2307,10 +2290,10 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen if (start_ch == 'h') { reg = read_hex (&p); } else { - gen_assert ('0' <= ch && ch <= '2' && ch - '0' < nops); + gen_assert ('0' <= ch && ch <= '2' && ch - '0' < (int) nops); op = insn->ops[ch - '0']; - gen_assert (op.mode == MIR_OP_HARD_REG); - reg = op.u.hard_reg; + gen_assert (op.mode == MIR_OP_VAR); + reg = op.u.var; } if (reg >= F0_HARD_REG) reg -= F0_HARD_REG; gen_assert (reg <= 31); @@ -2343,17 +2326,17 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen ch = *++p; if (ch == 'c') { op = insn->ops[0]; - if (op.mode == MIR_OP_HARD_REG_MEM) { /* store */ - gen_assert (insn->ops[1].mode == MIR_OP_HARD_REG); + if (op.mode == MIR_OP_VAR_MEM) { /* store */ + gen_assert (insn->ops[1].mode == MIR_OP_VAR); } else { op = insn->ops[1]; - gen_assert (op.mode == MIR_OP_HARD_REG_MEM && insn->ops[0].mode == MIR_OP_HARD_REG); + gen_assert (op.mode == MIR_OP_VAR_MEM && insn->ops[0].mode == MIR_OP_VAR); } ch = *++p; gen_assert (ch == '2' || ch == '3' || ch == 'd'); - d = op.u.hard_reg_mem.disp >> (ch == '2' ? 2 : 3); + d = op.u.var_mem.disp >> (ch == '2' ? 2 : 3); if (*++p == 's') { - gen_assert (d < (1 << 6) && op.u.hard_reg_mem.base == SP_HARD_REG); + gen_assert (d < (1 << 6) && op.u.var_mem.base == SP_HARD_REG); if (*++p == 's') { if (ch == '2') { unsign_sp_store_disp4 = d; @@ -2371,8 +2354,8 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen el_mask = 0x107c; } } else { - gen_assert (compressed_reg_p (op.u.hard_reg_mem.base, TRUE)); - rdc = op.u.hard_reg_mem.base - R8_HARD_REG; + gen_assert (compressed_reg_p (op.u.var_mem.base, TRUE)); + rdc = op.u.var_mem.base - R8_HARD_REG; p--; gen_assert (d < (1 << 5)); if (ch == '2') { @@ -2384,21 +2367,20 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen } } else { if (ch == 's') { /* store */ - gen_assert (insn->ops[0].mode == MIR_OP_HARD_REG_MEM); + gen_assert (insn->ops[0].mode == MIR_OP_VAR_MEM); op = insn->ops[0]; - st_disp - = ((op.u.hard_reg_mem.disp << 13) & 0x01fc0000) | (op.u.hard_reg_mem.disp & 0x1f); + st_disp = ((op.u.var_mem.disp << 13) & 0x01fc0000) | (op.u.var_mem.disp & 0x1f); el_mask = 0xfe000f80; st_disp_p = TRUE; } else { /* load */ - gen_assert (ch == 'l' && insn->ops[1].mode == MIR_OP_HARD_REG_MEM); + gen_assert (ch == 'l' && insn->ops[1].mode == MIR_OP_VAR_MEM); op = insn->ops[1]; - imm12 = op.u.hard_reg_mem.disp; + imm12 = op.u.var_mem.disp; imm12_p = TRUE; el_mask = 0xfff00000; } el_mask |= 0xf8000; - rs1 = op.u.hard_reg_mem.base; + rs1 = op.u.var_mem.base; } break; case 's': @@ -2566,19 +2548,23 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen } break; case 'l': - case 'L': { - op = insn->ops[start_ch == 'l' || (insn->code != MIR_CALL && insn->code != MIR_INLINE) ? 0 - : 1]; + case 'L': + case 'U': + n = 0; + if (insn->code == MIR_CALL || insn->code == MIR_INLINE || insn->code == MIR_LADDR) n = 1; + op = insn->ops[n]; gen_assert (op.mode == MIR_OP_LABEL || op.mode == MIR_OP_REF); lr.abs_addr_p = FALSE; - lr.short_p = start_ch == 'l'; + lr.branch_type = start_ch == 'l' ? BRANCH : start_ch == 'L' ? JAL : AUIPC; lr.label_val_disp = 0; - lr.label = op.u.label; + if (jump_addrs == NULL) + lr.u.label = op.u.label; + else + lr.u.jump_addr = jump_addrs[0]; label_ref_num = VARR_LENGTH (label_ref_t, label_refs); VARR_PUSH (label_ref_t, label_refs, lr); el_mask = start_ch == 'l' ? 0xfe000f80 : 0xfffff000; break; - } default: gen_assert (FALSE); } } @@ -2656,33 +2642,100 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen gen_assert (insn->ops[i].mode == MIR_OP_LABEL); lr.abs_addr_p = TRUE; lr.label_val_disp = VARR_LENGTH (uint8_t, result_code); - lr.label = insn->ops[i].u.label; + if (jump_addrs == NULL) + lr.u.label = insn->ops[i].u.label; + else + lr.u.jump_addr = jump_addrs[i - 1]; VARR_PUSH (label_ref_t, label_refs, lr); put_uint64 (gen_ctx, 0, 8); } } +static int target_memory_ok_p (gen_ctx_t gen_ctx MIR_UNUSED, MIR_op_t *op_ref) { + if (op_ref->mode != MIR_OP_VAR_MEM) return FALSE; + if (op_ref->u.var_mem.index == MIR_NON_VAR && op_ref->u.var_mem.disp >= -(1 << 11) + && op_ref->u.var_mem.disp < (1 << 11) + && (op_ref->u.var_mem.type != MIR_T_LD || op_ref->u.var_mem.disp + 8 < (1 << 11))) + return TRUE; + return FALSE; +} + static int target_insn_ok_p (gen_ctx_t gen_ctx, MIR_insn_t insn) { return find_insn_pattern_replacement (gen_ctx, insn) != NULL; } -static uint32_t get_b_format_imm (int offset) { +static const uint32_t b_imm_mask = ((0x7f << 25) | (0x1f << 7)); +static uint32_t get_b_format_imm (int32_t offset) { int d = offset >> 1; /* scale */ gen_assert (-(1 << 11) <= d && d < (1 << 11)); return ((((d >> 5) & 0x40) | ((d >> 4) & 0x3f)) << 25) | ((((d & 0xf) << 1) | ((d >> 10) & 0x1)) << 7); } -static uint32_t get_j_format_imm (int offset) { - int d = offset >> 1; /* scale */ - gen_assert (-(1 << 19) <= d && d < (1 << 19)); - return ((d & 0x80000) | ((d & 0x3ff) << 9) | (((d >> 10) & 0x1) << 8) | ((d >> 11) & 0xff)) << 12; +static void add_consts (gen_ctx_t gen_ctx) { + /* Setting up 64-bit const addresses */ + for (size_t i = 0; i < VARR_LENGTH (const_ref_t, const_refs); i++) { + const_ref_t cr = VARR_GET (const_ref_t, const_refs, i); + uint32_t disp, carry; + gen_assert (VARR_LENGTH (uint8_t, result_code) > cr.const_addr_disp + && VARR_LENGTH (uint8_t, result_code) - cr.const_addr_disp < (1l << 31)); + disp = (uint32_t) (VARR_LENGTH (uint8_t, result_code) - cr.const_addr_disp); + carry = (disp & 0x800) << 1; + *(uint32_t *) (&VARR_ADDR (uint8_t, result_code)[cr.const_addr_disp]) + |= (disp + carry) & 0xfffff000; + *(uint32_t *) (&VARR_ADDR (uint8_t, result_code)[cr.const_addr_disp + 4]) |= disp << 20; + put_uint64 (gen_ctx, cr.val, 8); + } +} + +static void target_split_insns (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_insn_t insn, next_insn; + MIR_op_t op; + + for (insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); insn != NULL; + insn = next_insn) { + next_insn = DLIST_NEXT (MIR_insn_t, insn); + if (insn->code != MIR_LDMOV) continue; + + if (insn->ops[0].mode == MIR_OP_VAR) { + gen_assert (insn->ops[0].u.var + 1 < F0_HARD_REG && insn->ops[1].mode == MIR_OP_VAR_MEM); + op = insn->ops[1]; + op.u.var_mem.type = MIR_T_I64; + next_insn = gen_mov (gen_ctx, insn, MIR_MOV, insn->ops[0], op); + op.u.var_mem.disp += 8; + gen_mov (gen_ctx, insn, MIR_MOV, _MIR_new_var_op (ctx, insn->ops[0].u.var + 1), op); + gen_delete_insn (gen_ctx, insn); + } else if (insn->ops[1].mode == MIR_OP_VAR) { + gen_assert (insn->ops[1].u.var + 1 < F0_HARD_REG && insn->ops[0].mode == MIR_OP_VAR_MEM); + op = insn->ops[0]; + op.u.var_mem.type = MIR_T_I64; + next_insn = gen_mov (gen_ctx, insn, MIR_MOV, op, insn->ops[1]); + op.u.var_mem.disp += 8; + gen_mov (gen_ctx, insn, MIR_MOV, op, _MIR_new_var_op (ctx, insn->ops[1].u.var + 1)); + gen_delete_insn (gen_ctx, insn); + } else { + gen_assert (insn->ops[0].mode == MIR_OP_VAR_MEM && insn->ops[1].mode == MIR_OP_VAR_MEM); + op = insn->ops[1]; + op.u.var_mem.type = MIR_T_D; + next_insn + = gen_mov (gen_ctx, insn, MIR_DMOV, _MIR_new_var_op (ctx, TEMP_DOUBLE_HARD_REG1), op); + op.u.var_mem.disp += 8; + gen_mov (gen_ctx, insn, MIR_DMOV, _MIR_new_var_op (ctx, TEMP_DOUBLE_HARD_REG2), op); + op = insn->ops[0]; + op.u.var_mem.type = MIR_T_D; + gen_mov (gen_ctx, insn, MIR_DMOV, op, _MIR_new_var_op (ctx, TEMP_DOUBLE_HARD_REG1)); + op.u.var_mem.disp += 8; + gen_mov (gen_ctx, insn, MIR_DMOV, op, _MIR_new_var_op (ctx, TEMP_DOUBLE_HARD_REG2)); + gen_delete_insn (gen_ctx, insn); + } + } } static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { MIR_context_t ctx = gen_ctx->ctx; size_t i; - MIR_insn_t insn, next_insn; + MIR_insn_t insn; const char *replacement; gen_assert (curr_func_item->item_type == MIR_func_item); @@ -2691,50 +2744,10 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { VARR_TRUNC (const_ref_t, const_refs, 0); VARR_TRUNC (uint64_t, abs_address_locs, 0); for (insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); insn != NULL; - insn = next_insn) { - next_insn = DLIST_NEXT (MIR_insn_t, insn); - if (insn->code == MIR_LDMOV) { /* split ld move: */ - MIR_op_t op; - - if (insn->ops[0].mode == MIR_OP_HARD_REG) { - gen_assert (insn->ops[0].u.hard_reg + 1 < F0_HARD_REG - && insn->ops[1].mode == MIR_OP_HARD_REG_MEM); - op = insn->ops[1]; - op.u.hard_reg_mem.type = MIR_T_I64; - next_insn = gen_mov (gen_ctx, insn, MIR_MOV, insn->ops[0], op); - op.u.hard_reg_mem.disp += 8; - gen_mov (gen_ctx, insn, MIR_MOV, _MIR_new_hard_reg_op (ctx, insn->ops[0].u.hard_reg + 1), - op); - gen_delete_insn (gen_ctx, insn); - } else if (insn->ops[1].mode == MIR_OP_HARD_REG) { - gen_assert (insn->ops[1].u.hard_reg + 1 < F0_HARD_REG - && insn->ops[0].mode == MIR_OP_HARD_REG_MEM); - op = insn->ops[0]; - op.u.hard_reg_mem.type = MIR_T_I64; - next_insn = gen_mov (gen_ctx, insn, MIR_MOV, op, insn->ops[1]); - op.u.hard_reg_mem.disp += 8; - gen_mov (gen_ctx, insn, MIR_MOV, op, - _MIR_new_hard_reg_op (ctx, insn->ops[1].u.hard_reg + 1)); - gen_delete_insn (gen_ctx, insn); - } else { - gen_assert (insn->ops[0].mode == MIR_OP_HARD_REG_MEM - && insn->ops[1].mode == MIR_OP_HARD_REG_MEM); - op = insn->ops[1]; - op.u.hard_reg_mem.type = MIR_T_D; - next_insn = gen_mov (gen_ctx, insn, MIR_DMOV, - _MIR_new_hard_reg_op (ctx, TEMP_DOUBLE_HARD_REG1), op); - op.u.hard_reg_mem.disp += 8; - gen_mov (gen_ctx, insn, MIR_DMOV, _MIR_new_hard_reg_op (ctx, TEMP_DOUBLE_HARD_REG2), op); - op = insn->ops[0]; - op.u.hard_reg_mem.type = MIR_T_D; - gen_mov (gen_ctx, insn, MIR_DMOV, op, _MIR_new_hard_reg_op (ctx, TEMP_DOUBLE_HARD_REG1)); - op.u.hard_reg_mem.disp += 8; - gen_mov (gen_ctx, insn, MIR_DMOV, op, _MIR_new_hard_reg_op (ctx, TEMP_DOUBLE_HARD_REG2)); - gen_delete_insn (gen_ctx, insn); - } - } else if (insn->code == MIR_LABEL) { + insn = DLIST_NEXT (MIR_insn_t, insn)) { + if (insn->code == MIR_LABEL) { set_label_disp (gen_ctx, insn, VARR_LENGTH (uint8_t, result_code)); - } else { + } else if (insn->code != MIR_USE) { replacement = find_insn_pattern_replacement (gen_ctx, insn); if (replacement == NULL) { fprintf (stderr, "fatal failure in matching insn:"); @@ -2742,7 +2755,7 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { exit (1); } else { gen_assert (replacement != NULL); - out_insn (gen_ctx, insn, replacement); + out_insn (gen_ctx, insn, replacement, NULL); } } } @@ -2751,10 +2764,10 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { label_ref_t lr = VARR_GET (label_ref_t, label_refs, i); if (!lr.abs_addr_p) { - int64_t offset = (int64_t) get_label_disp (gen_ctx, lr.label) - (int64_t) lr.label_val_disp; + int64_t offset = (int64_t) get_label_disp (gen_ctx, lr.u.label) - (int64_t) lr.label_val_disp; uint32_t bin_insn; gen_assert ((offset & 0x1) == 0); - if (lr.short_p && (offset < -(1 << 12) || offset >= (1 << 12))) { + if (lr.branch_type == BRANCH && (offset < -(1 << 12) || offset >= (1 << 12))) { /* BL:br L => BL:jmp NBL; ... NBL: br TL;jmp BL+4;TL:jmp L: */ bin_insn = *(uint32_t *) (VARR_ADDR (uint8_t, result_code) + lr.label_val_disp); offset = (int64_t) VARR_LENGTH (uint8_t, result_code) - (int64_t) lr.label_val_disp; @@ -2765,36 +2778,29 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { offset = (int64_t) lr.label_val_disp - (int64_t) VARR_LENGTH (uint8_t, result_code) + 4; bin_insn = 0x6f | get_j_format_imm (offset); put_uint64 (gen_ctx, bin_insn, 4); - offset = (int64_t) get_label_disp (gen_ctx, lr.label) + offset = (int64_t) get_label_disp (gen_ctx, lr.u.label) - (int64_t) VARR_LENGTH (uint8_t, result_code); bin_insn = 0x6f | get_j_format_imm (offset); put_uint64 (gen_ctx, bin_insn, 4); - + } else if (lr.branch_type == AUIPC) { + int hi = offset >> 12, low = offset & 0xfff; + if ((low & 0x800) != 0) hi++; + *(uint32_t *) (VARR_ADDR (uint8_t, result_code) + lr.label_val_disp) |= hi << 12; + *(uint32_t *) (VARR_ADDR (uint8_t, result_code) + lr.label_val_disp + 4) |= low << 20; } else { + gen_assert (lr.branch_type != AUIPC_JALR); *(uint32_t *) (VARR_ADDR (uint8_t, result_code) + lr.label_val_disp) - |= (lr.short_p ? get_b_format_imm (offset) : get_j_format_imm (offset)); + |= (lr.branch_type == BRANCH ? get_b_format_imm (offset) : get_j_format_imm (offset)); } } else { set_int64 (&VARR_ADDR (uint8_t, result_code)[lr.label_val_disp], - (int64_t) get_label_disp (gen_ctx, lr.label), 8); + (int64_t) get_label_disp (gen_ctx, lr.u.label), 8); VARR_PUSH (uint64_t, abs_address_locs, lr.label_val_disp); } } while (VARR_LENGTH (uint8_t, result_code) % 8 != 0) /* Align the pool */ VARR_PUSH (uint8_t, result_code, 0); - /* Setting up 64-bit const addresses */ - for (i = 0; i < VARR_LENGTH (const_ref_t, const_refs); i++) { - const_ref_t cr = VARR_GET (const_ref_t, const_refs, i); - uint32_t disp, carry; - gen_assert (VARR_LENGTH (uint8_t, result_code) > cr.const_addr_disp - && VARR_LENGTH (uint8_t, result_code) - cr.const_addr_disp < (1l << 31)); - disp = (uint32_t) (VARR_LENGTH (uint8_t, result_code) - cr.const_addr_disp); - carry = (disp & 0x800) << 1; - *(uint32_t *) (&VARR_ADDR (uint8_t, result_code)[cr.const_addr_disp]) - |= (disp + carry) & 0xfffff000; - *(uint32_t *) (&VARR_ADDR (uint8_t, result_code)[cr.const_addr_disp + 4]) |= disp << 20; - put_uint64 (gen_ctx, cr.val, 8); - } + add_consts (gen_ctx); while (VARR_LENGTH (uint8_t, result_code) % 16 != 0) /* Align the pool */ VARR_PUSH (uint8_t, result_code, 0); *len = VARR_LENGTH (uint8_t, result_code); @@ -2812,9 +2818,156 @@ static void target_rebase (gen_ctx_t gen_ctx, uint8_t *base) { } _MIR_update_code_arr (gen_ctx->ctx, base, VARR_LENGTH (MIR_code_reloc_t, relocs), VARR_ADDR (MIR_code_reloc_t, relocs)); + gen_setup_lrefs (gen_ctx, base); +} + +static void target_change_to_direct_calls (MIR_context_t ctx MIR_UNUSED) {} + +struct target_bb_version { + uint8_t *base; + label_ref_t branch_ref; /* label cand used for jump to this bb version */ +}; + +static void target_init_bb_version_data (target_bb_version_t data) { + data->base = NULL; /* we don't know origin branch */ +} + +static void target_bb_translate_start (gen_ctx_t gen_ctx) { + add_nops = 0; + VARR_TRUNC (uint8_t, result_code, 0); + VARR_TRUNC (label_ref_t, label_refs, 0); + VARR_TRUNC (const_ref_t, const_refs, 0); + VARR_TRUNC (uint64_t, abs_address_locs, 0); +} + +static void target_bb_insn_translate (gen_ctx_t gen_ctx, MIR_insn_t insn, void **jump_addrs) { + const char *replacement; + + if (insn->code == MIR_LABEL) return; + replacement = find_insn_pattern_replacement (gen_ctx, insn); + gen_assert (replacement != NULL); + out_insn (gen_ctx, insn, replacement, jump_addrs); + if (MIR_branch_code_p (insn->code)) add_nops = insn->code == MIR_JMP ? 1 : 3; +} + +static void target_output_jump (gen_ctx_t gen_ctx, void **jump_addrs) { + out_insn (gen_ctx, temp_jump, temp_jump_replacement, jump_addrs); + put_uint64 (gen_ctx, TARGET_NOP, 4); /* add space for transformation to auipc;jar */ +} + +static uint8_t *target_bb_translate_finish (gen_ctx_t gen_ctx, size_t *len) { + /* add nops for conversion jmp->lui+jalr and br->jmp|lui+jalr */ + for (int i = 0; i < add_nops; i++) put_uint64 (gen_ctx, TARGET_NOP, 4); + while (VARR_LENGTH (uint8_t, result_code) % 8 != 0) /* Align the pool */ + VARR_PUSH (uint8_t, result_code, 0); + add_consts (gen_ctx); + *len = VARR_LENGTH (uint8_t, result_code); + return VARR_ADDR (uint8_t, result_code); +} + +static void setup_rel (gen_ctx_t gen_ctx, label_ref_t *lr, uint8_t *base, void *addr) { + MIR_context_t ctx = gen_ctx->ctx; + int64_t offset = (int64_t) addr - (int64_t) (base + lr->label_val_disp); + + gen_assert ((offset & 0x1) == 0); + /* check max 32-bit offset with possible branch conversion (see offset - 3): */ + if (lr->abs_addr_p || !(-(1l << 31) <= (offset / 2 - 3) && offset / 2 < (1l << 31))) { + fprintf (stderr, "too big offset (%lld) in setup_rel", (long long) offset); + exit (1); + } + /* ??? thread safe: */ + uint32_t *insn_ptr = (uint32_t *) (base + lr->label_val_disp), insn = *insn_ptr; + if (lr->branch_type == BRANCH) { + if (-(1 << 12) <= offset && offset < (1 << 12)) { /* a valid branch offset*/ + insn = (insn & ~b_imm_mask) | get_b_format_imm (offset); + } else { + insn = (insn & ~b_imm_mask) | get_b_format_imm (12); /* skip next jump and nop */ + _MIR_change_code (ctx, (uint8_t *) insn_ptr, (uint8_t *) &insn, 4); + insn_ptr += 3; + lr->branch_type = JAL; + lr->label_val_disp += 12; + offset -= 12; + } + } + if (lr->branch_type == JAL) { + if (-(1 << 20) <= offset && offset < (1 << 20)) { /* a valid jal offset*/ + insn = 0x6f | get_j_format_imm (offset); + } else { + lr->branch_type = AUIPC_JALR; + } + } + if (lr->branch_type == AUIPC) { + int hi = offset >> 12, low = offset & 0xfff; + if ((low & 0x800) != 0) hi++; + insn |= hi << 12; + _MIR_change_code (ctx, (uint8_t *) insn_ptr, (uint8_t *) &insn, 4); + insn_ptr += 1; + insn = *insn_ptr | (low << 20); + } else if (lr->branch_type == AUIPC_JALR) { + uint32_t carry = (offset & 0x800) << 1; + insn = 0x17 | (TEMP_INT_HARD_REG1 << 7) + | (((uint32_t) offset + carry) & 0xfffff000); /* auipc t5 */ + _MIR_change_code (ctx, (uint8_t *) insn_ptr, (uint8_t *) &insn, 4); + insn = 0x67 | (TEMP_INT_HARD_REG1 << 15) | ((offset & 0xfff) << 20); /* jr t5 */ + insn_ptr += 1; + } + _MIR_change_code (ctx, (uint8_t *) insn_ptr, (uint8_t *) &insn, 4); +} + +static void target_bb_rebase (gen_ctx_t gen_ctx, uint8_t *base) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_code_reloc_t reloc; + + /* Setting up relative labels */ + for (size_t i = 0; i < VARR_LENGTH (label_ref_t, label_refs); i++) { + label_ref_t lr = VARR_GET (label_ref_t, label_refs, i); + if (lr.abs_addr_p) { + _MIR_change_code (ctx, (uint8_t *) base + lr.label_val_disp, (uint8_t *) &lr.u.jump_addr, 8); + } else { + setup_rel (gen_ctx, &lr, base, lr.u.jump_addr); + } + } + VARR_TRUNC (MIR_code_reloc_t, relocs, 0); + for (size_t i = 0; i < VARR_LENGTH (uint64_t, abs_address_locs); i++) { + reloc.offset = VARR_GET (uint64_t, abs_address_locs, i); + reloc.value = base + get_int64 (base + reloc.offset, 8); + VARR_PUSH (MIR_code_reloc_t, relocs, reloc); + } + _MIR_update_code_arr (gen_ctx->ctx, base, VARR_LENGTH (MIR_code_reloc_t, relocs), + VARR_ADDR (MIR_code_reloc_t, relocs)); +} + +static void target_setup_succ_bb_version_data (gen_ctx_t gen_ctx, uint8_t *base) { + if (VARR_LENGTH (label_ref_t, label_refs) + != VARR_LENGTH (target_bb_version_t, target_succ_bb_versions)) + /* We can have more one possible branch from original insn + (e.g. SWITCH, FBNE). If it is so, we will make jumps only + through BB thunk. */ + return; + for (size_t i = 0; i < VARR_LENGTH (target_bb_version_t, target_succ_bb_versions); i++) { + target_bb_version_t data = VARR_GET (target_bb_version_t, target_succ_bb_versions, i); + if (data == NULL) continue; + data->branch_ref = VARR_GET (label_ref_t, label_refs, i); + data->base = base; + } +} + +static void target_redirect_bb_origin_branch (gen_ctx_t gen_ctx, target_bb_version_t data, + void *addr) { + MIR_context_t ctx = gen_ctx->ctx; + + if (data->base == NULL) return; + if (data->branch_ref.abs_addr_p) { + _MIR_change_code (ctx, (uint8_t *) data->base + data->branch_ref.label_val_disp, + (uint8_t *) &addr, 8); + } else { + setup_rel (gen_ctx, &data->branch_ref, data->base, addr); + } + data->base = NULL; } static void target_init (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; check_hard_reg_alloc_order (); gen_ctx->target_ctx = gen_malloc (gen_ctx, sizeof (struct target_ctx)); VARR_CREATE (uint8_t, result_code, 0); @@ -2823,14 +2976,18 @@ static void target_init (gen_ctx_t gen_ctx) { VARR_CREATE (uint64_t, abs_address_locs, 0); VARR_CREATE (MIR_code_reloc_t, relocs, 0); MIR_type_t res = MIR_T_I64; - MIR_var_t args1[] = {{MIR_T_F, "src"}}, args2[] = {{MIR_T_D, "src"}}; - _MIR_register_unspec_insn (gen_ctx->ctx, FMVXW_CODE, "fmv.x.w", 1, &res, 1, FALSE, args1); - _MIR_register_unspec_insn (gen_ctx->ctx, FMVXD_CODE, "fmv.x.d", 1, &res, 1, FALSE, args2); + MIR_var_t args1[] = {{MIR_T_F, "src", 0}}; + MIR_var_t args2[] = {{MIR_T_D, "src", 0}}; + _MIR_register_unspec_insn (ctx, FMVXW_CODE, "fmv.x.w", 1, &res, 1, FALSE, args1); + _MIR_register_unspec_insn (ctx, FMVXD_CODE, "fmv.x.d", 1, &res, 1, FALSE, args2); patterns_init (gen_ctx); + temp_jump = MIR_new_insn (ctx, MIR_JMP, MIR_new_label_op (ctx, NULL)); + temp_jump_replacement = find_insn_pattern_replacement (gen_ctx, temp_jump); } static void target_finish (gen_ctx_t gen_ctx) { patterns_finish (gen_ctx); + _MIR_free_insn (gen_ctx->ctx, temp_jump); VARR_DESTROY (uint8_t, result_code); VARR_DESTROY (label_ref_t, label_refs); VARR_DESTROY (const_ref_t, const_refs); diff --git a/mir/mir-gen-s390x.c b/mir/mir-gen-s390x.c index 66a22c6d..d8e80def 100644 --- a/mir/mir-gen-s390x.c +++ b/mir/mir-gen-s390x.c @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2020-2023 Vladimir Makarov . + Copyright (C) 2020-2024 Vladimir Makarov . */ // ??? More patterns (ult, ugt, ule, uge w/o branches, multi-insn combining). @@ -11,38 +11,16 @@ static void fancy_abort (int code) { #undef gen_assert #define gen_assert(c) fancy_abort (c) -#include - -#define HREG_EL(h) h##_HARD_REG -#define REP_SEP , -enum { - REP8 (HREG_EL, R0, R1, R2, R3, R4, R5, R6, R7), - REP8 (HREG_EL, R8, R9, R10, R11, R12, R13, R14, R15), - REP8 (HREG_EL, F0, F1, F2, F3, F4, F5, F6, F7), - REP8 (HREG_EL, F8, F9, F10, F11, F12, F13, F14, F15), -}; -#undef REP_SEP - -static const MIR_reg_t MAX_HARD_REG = F15_HARD_REG; -static const MIR_reg_t SP_HARD_REG = R15_HARD_REG; -static const MIR_reg_t FP_HARD_REG = R11_HARD_REG; +#define TARGET_EXPAND_UADDO +#define TARGET_EXPAND_UADDOS +#define TARGET_EXPAND_MULO +#define TARGET_EXPAND_MULOS +#define TARGET_EXPAND_UMULO +#define TARGET_EXPAND_UMULOS -static int target_locs_num (MIR_reg_t loc, MIR_type_t type) { return type == MIR_T_LD ? 2 : 1; } - -/* Hard regs not used in machinized code and for passing args, preferably call saved ones. */ -const MIR_reg_t TEMP_INT_HARD_REG1 = R8_HARD_REG, TEMP_INT_HARD_REG2 = R9_HARD_REG; -const MIR_reg_t TEMP_FLOAT_HARD_REG1 = F8_HARD_REG, TEMP_FLOAT_HARD_REG2 = F10_HARD_REG; -const MIR_reg_t TEMP_DOUBLE_HARD_REG1 = F8_HARD_REG, TEMP_DOUBLE_HARD_REG2 = F10_HARD_REG; -const MIR_reg_t TEMP_LDOUBLE_HARD_REG1 = F8_HARD_REG; //??? -const MIR_reg_t TEMP_LDOUBLE_HARD_REG2 = F10_HARD_REG; +#include -static inline int target_hard_reg_type_ok_p (MIR_reg_t hard_reg, MIR_type_t type) { - gen_assert (hard_reg <= MAX_HARD_REG); - if (type == MIR_T_LD) /* f0,f1,f4,f5,f8,f9,f12,f13 - pair starts */ - return hard_reg >= F0_HARD_REG && (hard_reg - F0_HARD_REG) % 4 <= 1; - return MIR_fp_type_p (type) ? F0_HARD_REG <= hard_reg && hard_reg <= F15_HARD_REG - : hard_reg < F0_HARD_REG; -} +#include "mir-s390x.h" static inline MIR_reg_t target_nth_loc (MIR_reg_t loc, MIR_type_t type, int n) { gen_assert (n == 0 || (type == MIR_T_LD && loc >= F0_HARD_REG && n == 1)); @@ -50,19 +28,7 @@ static inline MIR_reg_t target_nth_loc (MIR_reg_t loc, MIR_type_t type, int n) { return loc >= F15_HARD_REG ? loc + 1 : loc + 2; /* coupled fp reg */ } -static inline int target_fixed_hard_reg_p (MIR_reg_t hard_reg) { - gen_assert (hard_reg <= MAX_HARD_REG); - return (hard_reg == FP_HARD_REG - || hard_reg == SP_HARD_REG - /* don't bother to allocate R0 as it has special meaning for base and index reg: */ - || hard_reg == R0_HARD_REG || hard_reg == TEMP_INT_HARD_REG1 - || hard_reg == TEMP_INT_HARD_REG2 || hard_reg == TEMP_FLOAT_HARD_REG1 - || hard_reg == TEMP_FLOAT_HARD_REG2 || hard_reg == TEMP_DOUBLE_HARD_REG1 - || hard_reg == TEMP_DOUBLE_HARD_REG2 || hard_reg == TEMP_LDOUBLE_HARD_REG1 - || hard_reg == TEMP_LDOUBLE_HARD_REG2); -} - -static inline int target_call_used_hard_reg_p (MIR_reg_t hard_reg, MIR_type_t type) { +static inline int target_call_used_hard_reg_p (MIR_reg_t hard_reg, MIR_type_t type MIR_UNUSED) { gen_assert (hard_reg <= MAX_HARD_REG); return ((R0_HARD_REG <= hard_reg && hard_reg <= R5_HARD_REG) || hard_reg == R14_HARD_REG || (F0_HARD_REG <= hard_reg && hard_reg <= F7_HARD_REG)); @@ -93,11 +59,11 @@ Originally SP(r15) and FP (r11) are the same but r15 can be changed by alloca */ /* s390x has 3-ops insns */ static const MIR_insn_code_t target_io_dup_op_insn_codes[] - = {MIR_ADD, MIR_ADDS, MIR_FADD, MIR_DADD, MIR_SUB, MIR_SUBS, MIR_FSUB, MIR_DSUB, - MIR_MUL, MIR_MULS, MIR_FMUL, MIR_DMUL, MIR_DIV, MIR_DIVS, MIR_UDIV, MIR_UDIVS, - MIR_FDIV, MIR_DDIV, MIR_MOD, MIR_MODS, MIR_UMOD, MIR_UMODS, MIR_EQ, MIR_EQS, - MIR_NE, MIR_NES, MIR_LSHS, MIR_RSHS, MIR_URSHS, MIR_AND, MIR_ANDS, MIR_OR, - MIR_ORS, MIR_XOR, MIR_XORS, MIR_INSN_BOUND}; + = {MIR_ADD, MIR_ADDS, MIR_FADD, MIR_DADD, MIR_SUB, MIR_SUBS, MIR_SUBO, MIR_SUBOS, + MIR_ADDO, MIR_ADDOS, MIR_FSUB, MIR_DSUB, MIR_MUL, MIR_MULS, MIR_FMUL, MIR_DMUL, + MIR_DIV, MIR_DIVS, MIR_UDIV, MIR_UDIVS, MIR_FDIV, MIR_DDIV, MIR_MOD, MIR_MODS, + MIR_UMOD, MIR_UMODS, MIR_EQ, MIR_EQS, MIR_NE, MIR_NES, MIR_LSHS, MIR_RSHS, + MIR_URSHS, MIR_AND, MIR_ANDS, MIR_OR, MIR_ORS, MIR_XOR, MIR_XORS, MIR_INSN_BOUND}; static MIR_insn_code_t get_ext_code (MIR_type_t type) { switch (type) { @@ -111,10 +77,6 @@ static MIR_insn_code_t get_ext_code (MIR_type_t type) { } } -DEF_VARR (int); -DEF_VARR (uint8_t); -DEF_VARR (uint64_t); - struct insn_pattern_info { int start, num; }; @@ -133,17 +95,20 @@ DEF_VARR (const_ref_t); struct label_ref { int abs_addr_p; size_t label_val_disp; - MIR_label_t label; + union { + MIR_label_t label; + void *jump_addr; /* absolute addr for BBV */ + } u; }; typedef struct label_ref label_ref_t; DEF_VARR (label_ref_t); -DEF_VARR (MIR_code_reloc_t); - struct target_ctx { unsigned char alloca_p, leaf_p, stack_param_p, switch_p; size_t param_save_area_size, blk_ld_value_save_area_size; + MIR_insn_t temp_jump; + const char *temp_jump_replacement; VARR (int) * pattern_indexes; VARR (insn_pattern_info_t) * insn_pattern_info; VARR (uint8_t) * result_code; @@ -161,6 +126,8 @@ struct target_ctx { #define switch_p gen_ctx->target_ctx->switch_p #define param_save_area_size gen_ctx->target_ctx->param_save_area_size #define blk_ld_value_save_area_size gen_ctx->target_ctx->blk_ld_value_save_area_size +#define temp_jump gen_ctx->target_ctx->temp_jump +#define temp_jump_replacement gen_ctx->target_ctx->temp_jump_replacement #define pattern_indexes gen_ctx->target_ctx->pattern_indexes #define insn_pattern_info gen_ctx->target_ctx->insn_pattern_info #define result_code gen_ctx->target_ctx->result_code @@ -192,58 +159,57 @@ static void gen_blk_mov (gen_ctx_t gen_ctx, MIR_insn_t anchor, size_t to_disp, MIR_insn_t new_insn; MIR_op_t ops[5], freg_op, treg_op, treg_op2, treg_op3; - treg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + treg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); if (qwords <= 16) { for (; qwords > 0; qwords--, to_disp += 8, from_disp += 8) { gen_mov (gen_ctx, anchor, MIR_MOV, treg_op, - MIR_new_mem_op (ctx, MIR_T_I64, from_disp, from_base_reg, 0, 1)); + _MIR_new_var_mem_op (ctx, MIR_T_I64, from_disp, from_base_reg, MIR_NON_VAR, 1)); gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, to_disp, to_base_hard_reg, - MIR_NON_HARD_REG, 1), + _MIR_new_var_mem_op (ctx, MIR_T_I64, to_disp, to_base_hard_reg, MIR_NON_VAR, 1), treg_op); } return; } - treg_op2 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - treg_op3 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + treg_op2 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + treg_op3 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); /* Save arg regs: */ if (save_regs > 0) - gen_mov (gen_ctx, anchor, MIR_MOV, treg_op, _MIR_new_hard_reg_op (ctx, R2_HARD_REG)); + gen_mov (gen_ctx, anchor, MIR_MOV, treg_op, _MIR_new_var_op (ctx, R2_HARD_REG)); if (save_regs > 1) - gen_mov (gen_ctx, anchor, MIR_MOV, treg_op2, _MIR_new_hard_reg_op (ctx, R3_HARD_REG)); + gen_mov (gen_ctx, anchor, MIR_MOV, treg_op2, _MIR_new_var_op (ctx, R3_HARD_REG)); if (save_regs > 2) - gen_mov (gen_ctx, anchor, MIR_MOV, treg_op3, _MIR_new_hard_reg_op (ctx, R4_HARD_REG)); + gen_mov (gen_ctx, anchor, MIR_MOV, treg_op3, _MIR_new_var_op (ctx, R4_HARD_REG)); /* call blk move: */ proto_item = _MIR_builtin_proto (ctx, curr_func_item->module, BLK_MOV_P, 0, NULL, 3, MIR_T_I64, "to", MIR_T_I64, "from", MIR_T_I64, "nwords"); func_import_item = _MIR_builtin_func (ctx, curr_func_item->module, BLK_MOV, mir_blk_mov); - freg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + freg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); new_insn = MIR_new_insn (ctx, MIR_MOV, freg_op, MIR_new_ref_op (ctx, func_import_item)); gen_add_insn_before (gen_ctx, anchor, new_insn); gen_add_insn_before (gen_ctx, anchor, - MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_hard_reg_op (ctx, R2_HARD_REG), - _MIR_new_hard_reg_op (ctx, to_base_hard_reg), + MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_var_op (ctx, R2_HARD_REG), + _MIR_new_var_op (ctx, to_base_hard_reg), MIR_new_int_op (ctx, to_disp))); gen_add_insn_before (gen_ctx, anchor, - MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_hard_reg_op (ctx, R3_HARD_REG), - MIR_new_reg_op (ctx, from_base_reg), + MIR_new_insn (gen_ctx->ctx, MIR_ADD, _MIR_new_var_op (ctx, R3_HARD_REG), + _MIR_new_var_op (ctx, from_base_reg), MIR_new_int_op (ctx, from_disp))); - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, R4_HARD_REG), + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, R4_HARD_REG), MIR_new_int_op (ctx, qwords)); ops[0] = MIR_new_ref_op (ctx, proto_item); ops[1] = freg_op; - ops[2] = _MIR_new_hard_reg_op (ctx, R2_HARD_REG); - ops[3] = _MIR_new_hard_reg_op (ctx, R3_HARD_REG); - ops[4] = _MIR_new_hard_reg_op (ctx, R4_HARD_REG); + ops[2] = _MIR_new_var_op (ctx, R2_HARD_REG); + ops[3] = _MIR_new_var_op (ctx, R3_HARD_REG); + ops[4] = _MIR_new_var_op (ctx, R4_HARD_REG); new_insn = MIR_new_insn_arr (ctx, MIR_CALL, 5, ops); gen_add_insn_before (gen_ctx, anchor, new_insn); /* Restore arg regs: */ if (save_regs > 0) - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, R2_HARD_REG), treg_op); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, R2_HARD_REG), treg_op); if (save_regs > 1) - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, R3_HARD_REG), treg_op2); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, R3_HARD_REG), treg_op2); if (save_regs > 2) - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, R4_HARD_REG), treg_op3); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, R4_HARD_REG), treg_op3); } static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { @@ -270,9 +236,9 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { nargs = VARR_LENGTH (MIR_var_t, proto->args); arg_vars = VARR_ADDR (MIR_var_t, proto->args); } - if (call_insn->ops[1].mode != MIR_OP_REG && call_insn->ops[1].mode != MIR_OP_HARD_REG) { + if (call_insn->ops[1].mode != MIR_OP_VAR) { // ??? to optimize (can be immediate operand for func call) - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); new_insn = MIR_new_insn (ctx, MIR_MOV, temp_op, call_insn->ops[1]); call_insn->ops[1] = temp_op; gen_add_insn_before (gen_ctx, call_insn, new_insn); @@ -285,7 +251,7 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { type = proto->res_types[i - 2]; } else if (i - start < nargs) { type = arg_vars[i - start].type; - } else if (arg_op.mode == MIR_OP_MEM) { + } else if (arg_op.mode == MIR_OP_VAR_MEM) { type = arg_op.u.mem.type; gen_assert (MIR_all_blk_type_p (type)); } else { @@ -301,7 +267,8 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { if (type == MIR_T_LD) call_blk_ld_value_area_size += 16; else if (MIR_blk_type_p (type)) { - gen_assert (arg_op.mode == MIR_OP_MEM && arg_op.u.mem.disp >= 0 && arg_op.u.mem.index == 0); + gen_assert (arg_op.mode == MIR_OP_VAR_MEM && arg_op.u.mem.disp >= 0 + && arg_op.u.mem.index == MIR_NON_VAR); call_blk_ld_value_area_size += (arg_op.u.mem.disp + 7) / 8 * 8; } if ((type == MIR_T_F || type == MIR_T_D) && n_fregs < 4) { @@ -320,13 +287,13 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { param_mem_size = n_fregs = n_iregs = 0; for (size_t i = 2; i < nops; i++) { /* process args and ???long double results: */ arg_op = call_insn->ops[i]; - gen_assert (arg_op.mode == MIR_OP_REG || arg_op.mode == MIR_OP_HARD_REG - || (arg_op.mode == MIR_OP_MEM && MIR_all_blk_type_p (arg_op.u.mem.type))); + gen_assert (arg_op.mode == MIR_OP_VAR + || (arg_op.mode == MIR_OP_VAR_MEM && MIR_all_blk_type_p (arg_op.u.mem.type))); if (i < start) { type = proto->res_types[i - 2]; } else if (i - start < nargs) { type = arg_vars[i - start].type; - } else if (call_insn->ops[i].mode == MIR_OP_MEM) { + } else if (call_insn->ops[i].mode == MIR_OP_VAR_MEM) { type = call_insn->ops[i].u.mem.type; gen_assert (MIR_all_blk_type_p (type)); } else { @@ -338,25 +305,24 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { if (type != MIR_T_LD && i < start) continue; ext_insn = NULL; if ((ext_code = get_ext_code (type)) != MIR_INVALID_INSN) { /* extend arg if necessary */ - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); ext_insn = MIR_new_insn (ctx, ext_code, temp_op, arg_op); call_insn->ops[i] = arg_op = temp_op; } if (type == MIR_T_LD || MIR_blk_type_p (type)) { if (i >= start) { /* put arg value in saved blk/ld value area: */ if (type == MIR_T_LD) { - mem_op - = _MIR_new_hard_reg_mem_op (ctx, MIR_T_LD, blk_ld_value_disp + S390X_STACK_HEADER_SIZE, - FP_HARD_REG, MIR_NON_HARD_REG, 1); + mem_op = _MIR_new_var_mem_op (ctx, MIR_T_LD, blk_ld_value_disp + S390X_STACK_HEADER_SIZE, + SP_HARD_REG, MIR_NON_VAR, 1); gen_mov (gen_ctx, call_insn, MIR_LDMOV, mem_op, arg_op); } else { qwords = (arg_op.u.mem.disp + 7) / 8; - gen_blk_mov (gen_ctx, call_insn, S390X_STACK_HEADER_SIZE + blk_ld_value_disp, FP_HARD_REG, + gen_blk_mov (gen_ctx, call_insn, S390X_STACK_HEADER_SIZE + blk_ld_value_disp, SP_HARD_REG, 0, arg_op.u.mem.base, qwords, n_iregs); } } - arg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - new_insn = MIR_new_insn (ctx, MIR_ADD, arg_op, _MIR_new_hard_reg_op (ctx, FP_HARD_REG), + arg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + new_insn = MIR_new_insn (ctx, MIR_ADD, arg_op, _MIR_new_var_op (ctx, SP_HARD_REG), MIR_new_int_op (ctx, S390X_STACK_HEADER_SIZE + blk_ld_value_disp)); gen_add_insn_before (gen_ctx, call_insn, new_insn); blk_ld_value_disp += type == MIR_T_LD ? 16 : qwords * 8; @@ -365,34 +331,34 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { if ((type == MIR_T_F || type == MIR_T_D) && n_fregs < 4) { /* put arguments to argument hard regs: */ if (ext_insn != NULL) gen_add_insn_before (gen_ctx, call_insn, ext_insn); - arg_reg_op = _MIR_new_hard_reg_op (ctx, F0_HARD_REG + n_fregs * 2); + arg_reg_op = _MIR_new_var_op (ctx, F0_HARD_REG + n_fregs * 2); gen_mov (gen_ctx, call_insn, type == MIR_T_F ? MIR_FMOV : MIR_DMOV, arg_reg_op, arg_op); call_insn->ops[i] = arg_reg_op; n_fregs++; } else if (type != MIR_T_F && type != MIR_T_D && n_iregs < 5) { if (ext_insn != NULL) gen_add_insn_before (gen_ctx, call_insn, ext_insn); - arg_reg_op = _MIR_new_hard_reg_op (ctx, R2_HARD_REG + n_iregs); + arg_reg_op = _MIR_new_var_op (ctx, R2_HARD_REG + n_iregs); if (type != MIR_T_RBLK) { gen_mov (gen_ctx, call_insn, MIR_MOV, arg_reg_op, arg_op); } else { - assert (arg_op.mode == MIR_OP_MEM); - gen_mov (gen_ctx, call_insn, MIR_MOV, arg_reg_op, MIR_new_reg_op (ctx, arg_op.u.mem.base)); - arg_reg_op = _MIR_new_hard_reg_mem_op (ctx, MIR_T_RBLK, arg_op.u.mem.disp, - R2_HARD_REG + n_iregs, MIR_NON_HARD_REG, 1); + assert (arg_op.mode == MIR_OP_VAR_MEM); + gen_mov (gen_ctx, call_insn, MIR_MOV, arg_reg_op, _MIR_new_var_op (ctx, arg_op.u.mem.base)); + arg_reg_op = _MIR_new_var_mem_op (ctx, MIR_T_RBLK, arg_op.u.mem.disp, R2_HARD_REG + n_iregs, + MIR_NON_VAR, 1); } if (i >= start) call_insn->ops[i] = arg_reg_op; /* don't change LD return yet */ n_iregs++; } else { /* put arguments on the stack: */ if (ext_insn != NULL) gen_add_insn_before (gen_ctx, call_insn, ext_insn); new_insn_code = (type == MIR_T_F ? MIR_FMOV : type == MIR_T_D ? MIR_DMOV : MIR_MOV); - mem_op = _MIR_new_hard_reg_mem_op (ctx, mem_type, param_mem_size + S390X_STACK_HEADER_SIZE, - SP_HARD_REG, MIR_NON_HARD_REG, 1); + mem_op = _MIR_new_var_mem_op (ctx, mem_type, param_mem_size + S390X_STACK_HEADER_SIZE, + SP_HARD_REG, MIR_NON_VAR, 1); if (type != MIR_T_RBLK) { gen_mov (gen_ctx, call_insn, new_insn_code, mem_op, arg_op); } else { - assert (arg_op.mode == MIR_OP_MEM); + assert (arg_op.mode == MIR_OP_VAR_MEM); gen_mov (gen_ctx, call_insn, new_insn_code, mem_op, - MIR_new_reg_op (ctx, arg_op.u.mem.base)); + _MIR_new_var_op (ctx, arg_op.u.mem.base)); } if (i >= start) call_insn->ops[i] = mem_op; param_mem_size += 8; @@ -402,26 +368,26 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { blk_ld_value_disp = param_mem_size; for (size_t i = 0; i < proto->nres; i++) { ret_op = call_insn->ops[i + 2]; - gen_assert (ret_op.mode == MIR_OP_REG || ret_op.mode == MIR_OP_HARD_REG); + gen_assert (ret_op.mode == MIR_OP_VAR); type = proto->res_types[i]; if (type == MIR_T_LD) { /* returned by address */ new_insn_code = MIR_LDMOV; call_res_op = ret_val_op - = _MIR_new_hard_reg_mem_op (ctx, MIR_T_LD, S390X_STACK_HEADER_SIZE + blk_ld_value_disp, - FP_HARD_REG, MIR_NON_HARD_REG, 1); + = _MIR_new_var_mem_op (ctx, MIR_T_LD, S390X_STACK_HEADER_SIZE + blk_ld_value_disp, + SP_HARD_REG, MIR_NON_VAR, 1); if (n_iregs < 5) { /* use it as a call result to keep assignment to ld_n_iregs: */ - call_res_op = _MIR_new_hard_reg_mem_op (ctx, MIR_T_LD, 0, R2_HARD_REG + ld_n_iregs, - MIR_NON_HARD_REG, 1); + call_res_op + = _MIR_new_var_mem_op (ctx, MIR_T_LD, 0, R2_HARD_REG + ld_n_iregs, MIR_NON_VAR, 1); ld_n_iregs++; } blk_ld_value_disp += 16; } else if ((type == MIR_T_F || type == MIR_T_D) && n_fregs < 4) { new_insn_code = type == MIR_T_F ? MIR_FMOV : MIR_DMOV; - call_res_op = ret_val_op = _MIR_new_hard_reg_op (ctx, F0_HARD_REG + n_fregs * 2); + call_res_op = ret_val_op = _MIR_new_var_op (ctx, F0_HARD_REG + n_fregs * 2); n_fregs++; } else if (type != MIR_T_F && type != MIR_T_D && n_iregs < 1) { new_insn_code = MIR_MOV; - call_res_op = ret_val_op = _MIR_new_hard_reg_op (ctx, R2_HARD_REG + n_iregs); + call_res_op = ret_val_op = _MIR_new_var_op (ctx, R2_HARD_REG + n_iregs); n_iregs++; } else { (*MIR_get_error_func (ctx)) (MIR_ret_error, @@ -649,7 +615,7 @@ static int get_builtin (gen_ctx_t gen_ctx, MIR_insn_code_t code, MIR_item_t *pro } } -static MIR_disp_t target_get_stack_slot_offset (gen_ctx_t gen_ctx, MIR_type_t type, +static MIR_disp_t target_get_stack_slot_offset (gen_ctx_t gen_ctx, MIR_type_t type MIR_UNUSED, MIR_reg_t slot) { /* slot is 0, 1, ... */ return ((MIR_disp_t) slot * 8 + S390X_STACK_HEADER_SIZE + param_save_area_size @@ -664,12 +630,17 @@ static void set_prev_sp_reg (gen_ctx_t gen_ctx, MIR_insn_t anchor, int *prev_sp_ if (!*prev_sp_set_p) { *prev_sp_set_p = TRUE; *prev_sp_reg = gen_new_temp_reg (gen_ctx, MIR_T_I64, func); - gen_mov (gen_ctx, anchor, MIR_MOV, MIR_new_reg_op (ctx, *prev_sp_reg), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 0, SP_HARD_REG, MIR_NON_HARD_REG, 1)); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, *prev_sp_reg), + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, SP_HARD_REG, MIR_NON_VAR, 1)); } } -static int target_valid_mem_offset_p (gen_ctx_t gen_ctx, MIR_type_t type, MIR_disp_t offset) { +static MIR_reg_t target_get_stack_slot_base_reg (gen_ctx_t gen_ctx MIR_UNUSED) { + return FP_HARD_REG; +} + +static int target_valid_mem_offset_p (gen_ctx_t gen_ctx MIR_UNUSED, MIR_type_t type MIR_UNUSED, + MIR_disp_t offset MIR_UNUSED) { return TRUE; } @@ -694,49 +665,49 @@ static void target_machinize (gen_ctx_t gen_ctx) { ld_addr_reg = gen_new_temp_reg (gen_ctx, MIR_T_I64, func); VARR_PUSH (uint64_t, ld_addr_regs, ld_addr_reg); if (int_arg_num < 5) { - gen_mov (gen_ctx, anchor, MIR_MOV, MIR_new_reg_op (ctx, ld_addr_reg), - _MIR_new_hard_reg_op (ctx, R2_HARD_REG + int_arg_num)); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, ld_addr_reg), + _MIR_new_var_op (ctx, R2_HARD_REG + int_arg_num)); int_arg_num++; } else { set_prev_sp_reg (gen_ctx, anchor, &prev_sp_set_p, &prev_sp_reg); - gen_mov (gen_ctx, anchor, MIR_MOV, MIR_new_reg_op (ctx, ld_addr_reg), - MIR_new_mem_op (ctx, MIR_T_I64, disp, prev_sp_reg, 0, 1)); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, ld_addr_reg), + _MIR_new_var_mem_op (ctx, MIR_T_I64, disp, prev_sp_reg, MIR_NON_VAR, 1)); disp += 8; } } for (i = 0; i < func->nargs; i++) { /* Prologue: generate arg_var = hard_reg|stack mem ... */ /* Argument extensions is already done in simplify */ type = VARR_GET (MIR_var_t, func->vars, i).type; - arg_var_op = MIR_new_reg_op (ctx, i + 1); + arg_var_op = _MIR_new_var_op (ctx, i + MAX_HARD_REG + 1); if ((type == MIR_T_F || type == MIR_T_D) && fp_arg_num < 4) { - arg_reg_op = _MIR_new_hard_reg_op (ctx, F0_HARD_REG + fp_arg_num * 2); + arg_reg_op = _MIR_new_var_op (ctx, F0_HARD_REG + fp_arg_num * 2); /* (f|d)mov arg, arg_hard_reg: */ gen_mov (gen_ctx, anchor, type == MIR_T_F ? MIR_FMOV : MIR_DMOV, arg_var_op, arg_reg_op); fp_arg_num++; } else if (type == MIR_T_F || type == MIR_T_D) { /* (f|d)mov arg, arg_memory */ set_prev_sp_reg (gen_ctx, anchor, &prev_sp_set_p, &prev_sp_reg); gen_mov (gen_ctx, anchor, type == MIR_T_F ? MIR_FMOV : MIR_DMOV, arg_var_op, - MIR_new_mem_op (ctx, type, disp + (type == MIR_T_F ? 4 : 0), prev_sp_reg, 0, 1)); + _MIR_new_var_mem_op (ctx, type, disp + (type == MIR_T_F ? 4 : 0), prev_sp_reg, + MIR_NON_VAR, 1)); disp += 8; } else if (int_arg_num < 5) { /* (ld)mov arg, arg_hard_reg */ if (type != MIR_T_LD) gen_mov (gen_ctx, anchor, MIR_MOV, arg_var_op, - _MIR_new_hard_reg_op (ctx, R2_HARD_REG + int_arg_num)); + _MIR_new_var_op (ctx, R2_HARD_REG + int_arg_num)); else gen_mov (gen_ctx, anchor, MIR_LDMOV, arg_var_op, - _MIR_new_hard_reg_mem_op (ctx, type, 0, R2_HARD_REG + int_arg_num, - MIR_NON_HARD_REG, 1)); + _MIR_new_var_mem_op (ctx, type, 0, R2_HARD_REG + int_arg_num, MIR_NON_VAR, 1)); int_arg_num++; } else { /* (ld)mov arg, arg_memory */ set_prev_sp_reg (gen_ctx, anchor, &prev_sp_set_p, &prev_sp_reg); if (type != MIR_T_LD) { gen_mov (gen_ctx, anchor, MIR_MOV, arg_var_op, - MIR_new_mem_op (ctx, MIR_T_I64, disp, prev_sp_reg, 0, 1)); + _MIR_new_var_mem_op (ctx, MIR_T_I64, disp, prev_sp_reg, MIR_NON_VAR, 1)); } else { - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, R1_HARD_REG), - MIR_new_mem_op (ctx, MIR_T_I64, disp, prev_sp_reg, 0, 1)); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, R1_HARD_REG), + _MIR_new_var_mem_op (ctx, MIR_T_I64, disp, prev_sp_reg, MIR_NON_VAR, 1)); gen_mov (gen_ctx, anchor, MIR_MOV, arg_var_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_LD, 0, R1_HARD_REG, MIR_NON_HARD_REG, 1)); + _MIR_new_var_mem_op (ctx, MIR_T_LD, 0, R1_HARD_REG, MIR_NON_VAR, 1)); } disp += 8; } @@ -753,14 +724,13 @@ static void target_machinize (gen_ctx_t gen_ctx) { code = insn->code; if (code == MIR_LDBEQ || code == MIR_LDBNE || code == MIR_LDBLT || code == MIR_LDBGE || code == MIR_LDBGT || code == MIR_LDBLE) { /* split to cmp and branch */ - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - code = (code == MIR_LDBEQ - ? MIR_LDEQ - : code == MIR_LDBNE - ? MIR_LDNE - : code == MIR_LDBLT - ? MIR_LDLT - : code == MIR_LDBGE ? MIR_LDGE : code == MIR_LDBGT ? MIR_LDGT : MIR_LDLE); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + code = (code == MIR_LDBEQ ? MIR_LDEQ + : code == MIR_LDBNE ? MIR_LDNE + : code == MIR_LDBLT ? MIR_LDLT + : code == MIR_LDBGE ? MIR_LDGE + : code == MIR_LDBGT ? MIR_LDGT + : MIR_LDLE); new_insn = MIR_new_insn (ctx, code, temp_op, insn->ops[1], insn->ops[2]); gen_add_insn_before (gen_ctx, insn, new_insn); next_insn = MIR_new_insn (ctx, MIR_BT, insn->ops[0], temp_op); @@ -776,10 +746,10 @@ static void target_machinize (gen_ctx_t gen_ctx) { MIR_op_t ops[6], func_reg_op, reg_op3; MIR_op_t res_reg_op = insn->ops[0], va_reg_op = insn->ops[1], op3 = insn->ops[2]; - assert (res_reg_op.mode == MIR_OP_REG && va_reg_op.mode == MIR_OP_REG - && op3.mode == (code == MIR_VA_ARG ? MIR_OP_MEM : MIR_OP_REG)); - func_reg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - reg_op3 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + assert (res_reg_op.mode == MIR_OP_VAR && va_reg_op.mode == MIR_OP_VAR + && op3.mode == (code == MIR_VA_ARG ? MIR_OP_VAR_MEM : MIR_OP_VAR)); + func_reg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + reg_op3 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); next_insn = new_insn = MIR_new_insn (ctx, MIR_MOV, func_reg_op, MIR_new_ref_op (ctx, func_import_item)); gen_add_insn_before (gen_ctx, insn, new_insn); @@ -801,8 +771,8 @@ static void target_machinize (gen_ctx_t gen_ctx) { } else { /* Use builtin: mov freg, func ref; call proto, freg, res_reg, op_reg[, op_reg2] */ MIR_op_t freg_op, res_reg_op = insn->ops[0], op_reg_op = insn->ops[1], ops[5]; - gen_assert (res_reg_op.mode == MIR_OP_REG && op_reg_op.mode == MIR_OP_REG); - freg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + gen_assert (res_reg_op.mode == MIR_OP_VAR && op_reg_op.mode == MIR_OP_VAR); + freg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); next_insn = new_insn = MIR_new_insn (ctx, MIR_MOV, freg_op, MIR_new_ref_op (ctx, func_import_item)); gen_add_insn_before (gen_ctx, insn, new_insn); @@ -817,20 +787,20 @@ static void target_machinize (gen_ctx_t gen_ctx) { } } else if (code == MIR_VA_START) { MIR_op_t treg_op - = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, curr_func_item->u.func)); + = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, curr_func_item->u.func)); MIR_op_t va_op = insn->ops[0]; MIR_reg_t va_reg; int gpr_val = 0, fpr_val = 0; - size_t disp = 0; MIR_var_t var; - assert (func->vararg_p && (va_op.mode == MIR_OP_REG || va_op.mode == MIR_OP_HARD_REG)); - for (uint32_t i = 0; i < func->nargs; i++) + disp = 0; + assert (func->vararg_p && (va_op.mode == MIR_OP_VAR)); + for (i = 0; i < func->nargs; i++) if (func->res_types[i] == MIR_T_LD) { if (gpr_val > 5) disp += 8; gpr_val++; } - for (uint32_t i = 0; i < func->nargs; i++) { + for (i = 0; i < func->nargs; i++) { var = VARR_GET (MIR_var_t, func->vars, i); if (var.type == MIR_T_F || var.type == MIR_T_D) { if (fpr_val > 4) disp += 8; @@ -840,24 +810,28 @@ static void target_machinize (gen_ctx_t gen_ctx) { gpr_val++; } } - va_reg = va_op.mode == MIR_OP_REG ? va_op.u.reg : va_op.u.hard_reg; + va_reg = va_op.u.var; /* Insns can be not simplified as soon as they match a machine insn. */ /* mem64[va_reg] = gpr_val; mem64[va_reg + 8] = fpr_val */ - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 0, va_reg, 0, 1), + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, va_reg, MIR_NON_VAR, 1), MIR_new_int_op (ctx, gpr_val)); next_insn = DLIST_PREV (MIR_insn_t, insn); - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 8, va_reg, 0, 1), + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 8, va_reg, MIR_NON_VAR, 1), MIR_new_int_op (ctx, fpr_val)); /* reg_save_area: treg = mem64[sp]; mem64[va_reg+24] = treg: */ gen_mov (gen_ctx, insn, MIR_MOV, treg_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 0, SP_HARD_REG, MIR_NON_HARD_REG, 1)); - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 24, va_reg, 0, 1), treg_op); + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, SP_HARD_REG, MIR_NON_VAR, 1)); + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 24, va_reg, MIR_NON_VAR, 1), treg_op); /* overflow_arg_area_reg: treg = treg_op+S390X_STACK_HEADER_SIZE + disp; mem64[va_reg+16] = treg: */ new_insn = MIR_new_insn (ctx, MIR_ADD, treg_op, treg_op, MIR_new_int_op (ctx, S390X_STACK_HEADER_SIZE + disp)); gen_add_insn_before (gen_ctx, insn, new_insn); - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 16, va_reg, 0, 1), treg_op); + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 16, va_reg, MIR_NON_VAR, 1), treg_op); gen_delete_insn (gen_ctx, insn); } else if (code == MIR_VA_END) { /* do nothing */ gen_delete_insn (gen_ctx, insn); @@ -873,14 +847,14 @@ static void target_machinize (gen_ctx_t gen_ctx) { uint32_t n_gpregs = 0, n_fregs = 0, ld_addr_n = 0; gen_assert (func->nres == MIR_insn_nops (ctx, insn)); - for (size_t i = 0; i < func->nres; i++) { - gen_assert (insn->ops[i].mode == MIR_OP_REG); + for (i = 0; i < func->nres; i++) { + gen_assert (insn->ops[i].mode == MIR_OP_VAR); res_type = func->res_types[i]; if (res_type == MIR_T_LD) { /* ldmov f1,0(addr_reg);std f1,0(r2);std f3,8(r2): */ ld_addr_reg = VARR_GET (uint64_t, ld_addr_regs, ld_addr_n); - gen_mov (gen_ctx, insn, MIR_LDMOV, _MIR_new_hard_reg_op (ctx, F1_HARD_REG), insn->ops[i]); - insn->ops[i] = MIR_new_mem_op (ctx, MIR_T_LD, 0, ld_addr_reg, 0, 1); - gen_mov (gen_ctx, insn, MIR_LDMOV, insn->ops[i], _MIR_new_hard_reg_op (ctx, F1_HARD_REG)); + gen_mov (gen_ctx, insn, MIR_LDMOV, _MIR_new_var_op (ctx, F1_HARD_REG), insn->ops[i]); + insn->ops[i] = _MIR_new_var_mem_op (ctx, MIR_T_LD, 0, ld_addr_reg, MIR_NON_VAR, 1); + gen_mov (gen_ctx, insn, MIR_LDMOV, insn->ops[i], _MIR_new_var_op (ctx, F1_HARD_REG)); ld_addr_n++; continue; } @@ -895,7 +869,7 @@ static void target_machinize (gen_ctx_t gen_ctx) { (*MIR_get_error_func (ctx)) (MIR_ret_error, "s390x can not handle this combination of return values"); } - ret_reg_op = _MIR_new_hard_reg_op (ctx, ret_reg); + ret_reg_op = _MIR_new_var_op (ctx, ret_reg); gen_mov (gen_ctx, insn, new_insn_code, ret_reg_op, insn->ops[i]); insn->ops[i] = ret_reg_op; } @@ -905,15 +879,14 @@ static void target_machinize (gen_ctx_t gen_ctx) { static void isave (gen_ctx_t gen_ctx, MIR_insn_t anchor, int disp, MIR_reg_t hard_reg) { gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (gen_ctx->ctx, MIR_T_I64, disp, SP_HARD_REG, MIR_NON_HARD_REG, - 1), - _MIR_new_hard_reg_op (gen_ctx->ctx, hard_reg)); + _MIR_new_var_mem_op (gen_ctx->ctx, MIR_T_I64, disp, SP_HARD_REG, MIR_NON_VAR, 1), + _MIR_new_var_op (gen_ctx->ctx, hard_reg)); } static void fsave (gen_ctx_t gen_ctx, MIR_insn_t anchor, int disp, MIR_reg_t hard_reg) { gen_mov (gen_ctx, anchor, MIR_DMOV, - _MIR_new_hard_reg_mem_op (gen_ctx->ctx, MIR_T_D, disp, SP_HARD_REG, MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (gen_ctx->ctx, hard_reg)); + _MIR_new_var_mem_op (gen_ctx->ctx, MIR_T_D, disp, SP_HARD_REG, MIR_NON_VAR, 1), + _MIR_new_var_op (gen_ctx->ctx, hard_reg)); } static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_regs, @@ -941,23 +914,24 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg if (i >= F0_HARD_REG) saved_fregs_num++; } if (leaf_p && !stack_param_p && !alloca_p && saved_regs_p == 0 && stack_slots_num == 0) return; - r0_reg_op = _MIR_new_hard_reg_op (ctx, R0_HARD_REG); - r11_reg_op = _MIR_new_hard_reg_op (ctx, R11_HARD_REG); - r14_reg_op = _MIR_new_hard_reg_op (ctx, R14_HARD_REG); - r15_reg_op = _MIR_new_hard_reg_op (ctx, R15_HARD_REG); + r0_reg_op = _MIR_new_var_op (ctx, R0_HARD_REG); + r11_reg_op = _MIR_new_var_op (ctx, R11_HARD_REG); + r14_reg_op = _MIR_new_var_op (ctx, R14_HARD_REG); + r15_reg_op = _MIR_new_var_op (ctx, R15_HARD_REG); /* Prologue: */ frame_size = (param_save_area_size + S390X_STACK_HEADER_SIZE + blk_ld_value_save_area_size + stack_slots_num * 8); start_saved_fregs_offset = frame_size; frame_size += saved_fregs_num * 8; gen_assert (frame_size % 8 == 0); + if (!func->jret_p) + gen_mov (gen_ctx, anchor, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, S390X_GP_REG_RSAVE_AREA_START + (14 - 2) * 8, + R15_HARD_REG, MIR_NON_VAR, 1), + r14_reg_op); /* mem[r15+112] = r14 */ gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, S390X_GP_REG_RSAVE_AREA_START + (14 - 2) * 8, - R15_HARD_REG, MIR_NON_HARD_REG, 1), - r14_reg_op); /* mem[r15+112] = r14 */ - gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, S390X_GP_REG_RSAVE_AREA_START + (11 - 2) * 8, - R15_HARD_REG, MIR_NON_HARD_REG, 1), + _MIR_new_var_mem_op (ctx, MIR_T_I64, S390X_GP_REG_RSAVE_AREA_START + (11 - 2) * 8, + R15_HARD_REG, MIR_NON_VAR, 1), r11_reg_op); /* mem[r15+76] = r11 */ for (i = R2_HARD_REG; i < R15_HARD_REG; i++) /* exclude r15 */ if (!target_call_used_hard_reg_p (i, MIR_T_UNDEF) && bitmap_bit_p (used_hard_regs, i) @@ -967,38 +941,39 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg new_insn = MIR_new_insn (ctx, MIR_ADD, r15_reg_op, r15_reg_op, MIR_new_int_op (ctx, -frame_size)); gen_add_insn_before (gen_ctx, anchor, new_insn); /* r15 -= frame_size */ gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 0, R15_HARD_REG, MIR_NON_HARD_REG, 1), + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, R15_HARD_REG, MIR_NON_VAR, 1), r0_reg_op); /* mem[r15] = r0 */ for (n = 0, i = F0_HARD_REG; i <= MAX_HARD_REG; i++) if (!target_call_used_hard_reg_p (i, MIR_T_UNDEF) && bitmap_bit_p (used_hard_regs, i)) fsave (gen_ctx, anchor, start_saved_fregs_offset + (n++) * 8, i); gen_mov (gen_ctx, anchor, MIR_MOV, r11_reg_op, r15_reg_op); /* r11 = r15 */ /* Epilogue: */ - anchor = DLIST_TAIL (MIR_insn_t, func->insns); - /* It might be infinite loop after CCP with dead code elimination: */ - if (anchor->code == MIR_JMP) return; - gen_assert (anchor->code == MIR_RET); + for (anchor = DLIST_TAIL (MIR_insn_t, func->insns); anchor != NULL; + anchor = DLIST_PREV (MIR_insn_t, anchor)) + if (anchor->code == MIR_RET || anchor->code == MIR_JRET) break; + if (anchor == NULL) return; /* Restoring fp hard registers: */ for (n = 0, i = F0_HARD_REG; i <= MAX_HARD_REG; i++) if (!target_call_used_hard_reg_p (i, MIR_T_UNDEF) && bitmap_bit_p (used_hard_regs, i)) - gen_mov (gen_ctx, anchor, MIR_DMOV, _MIR_new_hard_reg_op (ctx, i), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_D, start_saved_fregs_offset + (n++) * 8, - R11_HARD_REG, MIR_NON_HARD_REG, 1)); + gen_mov (gen_ctx, anchor, MIR_DMOV, _MIR_new_var_op (ctx, i), + _MIR_new_var_mem_op (ctx, MIR_T_D, start_saved_fregs_offset + (n++) * 8, + R11_HARD_REG, MIR_NON_VAR, 1)); new_insn = MIR_new_insn (ctx, MIR_ADD, r15_reg_op, r11_reg_op, MIR_new_int_op (ctx, frame_size)); gen_add_insn_before (gen_ctx, anchor, new_insn); /* r15 = r11 + frame_size */ /* Restore saved gp regs (including r11 and excluding r15) and r14 */ for (i = R2_HARD_REG; i < R15_HARD_REG; i++) if (!target_call_used_hard_reg_p (i, MIR_T_UNDEF) && bitmap_bit_p (used_hard_regs, i)) - gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_hard_reg_op (ctx, i), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, - S390X_GP_REG_RSAVE_AREA_START + (i - R2_HARD_REG) * 8, - SP_HARD_REG, MIR_NON_HARD_REG, 1)); + gen_mov (gen_ctx, anchor, MIR_MOV, _MIR_new_var_op (ctx, i), + _MIR_new_var_mem_op (ctx, MIR_T_I64, + S390X_GP_REG_RSAVE_AREA_START + (i - R2_HARD_REG) * 8, + SP_HARD_REG, MIR_NON_VAR, 1)); gen_mov (gen_ctx, anchor, MIR_MOV, r11_reg_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, S390X_GP_REG_RSAVE_AREA_START + (11 - 2) * 8, - R15_HARD_REG, MIR_NON_HARD_REG, 1)); /* restore r11 */ - gen_mov (gen_ctx, anchor, MIR_MOV, r14_reg_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, S390X_GP_REG_RSAVE_AREA_START + (14 - 2) * 8, - R15_HARD_REG, MIR_NON_HARD_REG, 1)); /* restore r14 */ + _MIR_new_var_mem_op (ctx, MIR_T_I64, S390X_GP_REG_RSAVE_AREA_START + (11 - 2) * 8, + R15_HARD_REG, MIR_NON_VAR, 1)); /* restore r11 */ + if (!func->jret_p) + gen_mov (gen_ctx, anchor, MIR_MOV, r14_reg_op, + _MIR_new_var_mem_op (ctx, MIR_T_I64, S390X_GP_REG_RSAVE_AREA_START + (14 - 2) * 8, + R15_HARD_REG, MIR_NON_VAR, 1)); /* restore r14 */ } struct pattern { @@ -1091,7 +1066,7 @@ struct pattern { sD - displacement ([20..31]) used as shift SD - displacement (low part [20..31], high part [32..39]) used as shift T - switch table displacement - Q - stack header + param_area + Q - stack header + param_area + block param area */ const char *replacement; }; @@ -1200,6 +1175,20 @@ static const struct pattern patterns[] = { {MIR_SUB, "r 0 Ms2", "e3:19 r0 m"}, /* sgf r0,m */ // ??? changing sub imm to add imm + {MIR_ADDOS, "r 0 r", "1a* r0 R2"}, /* ar r0,r1 */ + {MIR_ADDOS, "r 0 m2", "5a r0 m"}, /* a r0,m */ + {MIR_ADDO, "r 0 r", "b908 r0 R2"}, /* agr r0,r1 */ + {MIR_ADDO, "r 0 M2", "e3:5a r0 m"}, /* ay r0,m */ + {MIR_ADDO, "r 0 M3", "e3:08 r0 m"}, /* ag r0,m */ + {MIR_ADDO, "r 0 Ms2", "e3:18 r0 m"}, /* agf r0,m */ + + {MIR_SUBOS, "r 0 r", "1b* r0 R2"}, /* sr r0,r1 */ + {MIR_SUBOS, "r 0 m2", "5b r0 m"}, /* s r0,m */ + {MIR_SUBO, "r 0 r", "b909 r0 R2"}, /* sgr r0,r1 */ + {MIR_SUBO, "r 0 M2", "e3:5b r0 m"}, /* sy r0,m */ + {MIR_SUBO, "r 0 M3", "e3:09 r0 m"}, /* sg r0,m */ + {MIR_SUBO, "r 0 Ms2", "e3:19 r0 m"}, /* sgf r0,m */ + {MIR_FSUB, "r 0 r", "b30b r0 R2"}, /* sebr r0,r1*/ {MIR_DSUB, "r 0 r", "b31b r0 R2"}, /* sdbr r0,r1*/ {MIR_FSUB, "r 0 mf", "ed:0b r0 m"}, /* seb r,m*/ @@ -1334,6 +1323,9 @@ static const struct pattern patterns[] = { {MIR_JMP, "L", SBRCL (15)}, /* bcril m15, l */ + {MIR_LADDR, "r L", "c0:0* r0 L"}, /* lalr r,offset */ + {MIR_JMPI, "r", "07* ma15 R0"}, /* br r */ + {MIR_BT, "L r", "b902 r1 R1" BRCL (6)}, /* ltgr r0,r0; bcril m8,l */ {MIR_BF, "L r", "b902 r1 R1" BRCL (8)}, /* ltgr r1,r1; bcril m6,l */ {MIR_BTS, "L r", "12* r1 R1" BRCL (6)}, /* ltr r0,r0; bcril m8,l */ @@ -1369,11 +1361,14 @@ static const struct pattern patterns[] = { BCMPU (MIR_UBLT, MIR_UBLTS, 4) BCMPU (MIR_UBGT, MIR_UBGTS, 2) BCMPU (MIR_UBLE, MIR_UBLES, 12) BCMPU (MIR_UBGE, MIR_UBGES, 10) - {MIR_NEG, "r r", "b903 r0 R1"}, /* lcgr r0,r1 */ - {MIR_NEGS, "r r", "13* r0 R1"}, /* lcr r0,r1 */ - {MIR_FNEG, "r r", "b303 r0 R1"}, /* lcebr r0,r1 */ - {MIR_DNEG, "r r", "b313 r0 R1"}, /* lcdbr r0,r1 */ - // ldneg is a builtin + {MIR_BO, "L", SBRCL (1)}, /* jo l */ + {MIR_BNO, "L", SBRCL (14)}, /* jno l */ + + {MIR_NEG, "r r", "b903 r0 R1"}, /* lcgr r0,r1 */ + {MIR_NEGS, "r r", "13* r0 R1"}, /* lcr r0,r1 */ + {MIR_FNEG, "r r", "b303 r0 R1"}, /* lcebr r0,r1 */ + {MIR_DNEG, "r r", "b313 r0 R1"}, /* lcdbr r0,r1 */ + // ldneg is a builtin {MIR_LSH, "r r r", "eb:0d r0 R1 s2"}, /* sllg r0,r2,b3 */ {MIR_LSH, "r r D", "eb:0d r0 R1 mD"}, /* sllg r0,r2,d */ @@ -1434,7 +1429,10 @@ static const struct pattern patterns[] = { {MIR_CALL, "X r $", "0d* h14 R1"}, /* basr h14,r0 */ {MIR_RET, "$", "07* ma15 H14"}, /* bcr m15,14 */ -/* sgr h15,r0; lg h0,(h15,r0); stg h0,0(h15); lay r0,160+param_area_size(h15): */ + {MIR_JCALL, "X r $", "07* ma15 R1"}, /* br r */ + {MIR_JRET, "r $", "07* ma15 R0"}, /* br r */ + +/* sgr h15,r0; lg h0,(h15,r0); stg h0,0(h15); lay r0,160+param_area_size+blkparamsize(h15): */ #define ALLOCA_END "; b909 h15 R0; e3:04 h0 hs15 x0; e3:24 h0 hs15; e3:71 r0 Q hs15" /* la r0,7(r1);nill r0,0xfff8; ... : */ @@ -1453,7 +1451,7 @@ static const struct pattern patterns[] = { static void target_get_early_clobbered_hard_regs (MIR_insn_t insn, MIR_reg_t *hr1, MIR_reg_t *hr2) { MIR_insn_code_t code = insn->code; - *hr1 = *hr2 = MIR_NON_HARD_REG; + *hr1 = *hr2 = MIR_NON_VAR; if (code == MIR_DIV || code == MIR_DIVS || code == MIR_UDIV || code == MIR_UDIVS || code == MIR_MOD || code == MIR_MODS || code == MIR_UMOD || code == MIR_UMODS) { *hr1 = R0_HARD_REG; @@ -1508,8 +1506,8 @@ static int nth_uint16_p (uint64_t u, int n) { return !(u & ~(((uint64_t) 0xffff) static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_insn_t insn) { MIR_context_t ctx = gen_ctx->ctx; - int nop, n; - size_t nops = MIR_insn_nops (ctx, insn); + int n; + size_t nop, nops = MIR_insn_nops (ctx, insn); const char *p; char ch, start_ch; MIR_op_mode_t mode; @@ -1525,10 +1523,10 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in switch (start_ch = *p) { case 'X': break; case 'r': - if (op.mode != MIR_OP_HARD_REG) return FALSE; + if (op.mode != MIR_OP_VAR) return FALSE; break; case 'h': - if (op.mode != MIR_OP_HARD_REG) return FALSE; + if (op.mode != MIR_OP_VAR) return FALSE; ch = *++p; gen_assert ('0' <= ch && ch <= '9'); hr = ch - '0'; @@ -1538,14 +1536,14 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in else --p; gen_assert (hr <= MAX_HARD_REG); - if (op.u.hard_reg != hr) return FALSE; + if (op.u.var != hr) return FALSE; break; case 'm': case 'M': { MIR_type_t type, type2, type3 = MIR_T_BOUND; int l_p = FALSE, u_p = TRUE, s_p = TRUE, index_p = TRUE; - if (op.mode != MIR_OP_HARD_REG_MEM) return FALSE; + if (op.mode != MIR_OP_VAR_MEM) return FALSE; ch = *++p; switch (ch) { case 'f': @@ -1568,7 +1566,7 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in u_p = ch == 'u'; s_p = ch == 's'; ch = *++p; - /* Fall through: */ + /* fall through */ default: gen_assert ('0' <= ch && ch <= '3'); if (ch == '0') { @@ -1592,18 +1590,16 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in #endif } } - if (op.u.hard_reg_mem.type != type && op.u.hard_reg_mem.type != type2 - && op.u.hard_reg_mem.type != type3) + if (op.u.var_mem.type != type && op.u.var_mem.type != type2 && op.u.var_mem.type != type3) return FALSE; - if ((!index_p && op.u.hard_reg_mem.base != MIR_NON_HARD_REG - && op.u.hard_reg_mem.index != MIR_NON_HARD_REG) - || (op.u.hard_reg_mem.index != MIR_NON_HARD_REG && op.u.hard_reg_mem.scale != 1) - || op.u.hard_reg_mem.base == R0_HARD_REG || op.u.hard_reg_mem.index == R0_HARD_REG - || !((start_ch == 'm' && uint12_p (op.u.hard_reg_mem.disp)) - || (start_ch != 'm' && int20_p (op.u.hard_reg_mem.disp))) + if ((!index_p && op.u.var_mem.base != MIR_NON_VAR && op.u.var_mem.index != MIR_NON_VAR) + || (op.u.var_mem.index != MIR_NON_VAR && op.u.var_mem.scale != 1) + || op.u.var_mem.base == R0_HARD_REG || op.u.var_mem.index == R0_HARD_REG + || !((start_ch == 'm' && uint12_p (op.u.var_mem.disp)) + || (start_ch != 'm' && int20_p (op.u.var_mem.disp))) || (l_p - && !((start_ch == 'm' && uint12_p (op.u.hard_reg_mem.disp + 8)) - || (start_ch != 'm' && int20_p (op.u.hard_reg_mem.disp + 8))))) + && !((start_ch == 'm' && uint12_p (op.u.var_mem.disp + 8)) + || (start_ch != 'm' && int20_p (op.u.var_mem.disp + 8))))) return FALSE; break; } @@ -1655,16 +1651,16 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in case '8': case '9': n = start_ch - '0'; - gen_assert (n < nop); + gen_assert (n < (int) nop); original = insn->ops[n]; mode = op.mode; if (mode == MIR_OP_UINT) mode = MIR_OP_INT; if (original.mode != mode && (original.mode != MIR_OP_UINT || mode != MIR_OP_INT)) return FALSE; - gen_assert (mode == MIR_OP_HARD_REG || mode == MIR_OP_INT || mode == MIR_OP_FLOAT - || mode == MIR_OP_DOUBLE || mode == MIR_OP_LDOUBLE || mode == MIR_OP_HARD_REG_MEM + gen_assert (mode == MIR_OP_VAR || mode == MIR_OP_INT || mode == MIR_OP_FLOAT + || mode == MIR_OP_DOUBLE || mode == MIR_OP_LDOUBLE || mode == MIR_OP_VAR_MEM || mode == MIR_OP_LABEL); - if (mode == MIR_OP_HARD_REG && op.u.hard_reg != original.u.hard_reg) + if (mode == MIR_OP_VAR && op.u.var != original.u.var) return FALSE; else if (mode == MIR_OP_INT && op.u.i != original.u.i) return FALSE; @@ -1676,11 +1672,12 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in return FALSE; else if (mode == MIR_OP_LABEL && op.u.label != original.u.label) return FALSE; - else if (mode == MIR_OP_HARD_REG_MEM && op.u.hard_reg_mem.type != original.u.hard_reg_mem.type - && op.u.hard_reg_mem.scale != original.u.hard_reg_mem.scale - && op.u.hard_reg_mem.base != original.u.hard_reg_mem.base - && op.u.hard_reg_mem.index != original.u.hard_reg_mem.index - && op.u.hard_reg_mem.disp != original.u.hard_reg_mem.disp) + else if (mode == MIR_OP_VAR_MEM + && (op.u.var_mem.type != original.u.var_mem.type + || op.u.var_mem.scale != original.u.var_mem.scale + || op.u.var_mem.base != original.u.var_mem.base + || op.u.var_mem.index != original.u.var_mem.index + || op.u.var_mem.disp != original.u.var_mem.disp)) return FALSE; break; default: gen_assert (FALSE); @@ -1810,7 +1807,8 @@ static void check_and_set_mask (uint64_t *binsn_mask, uint64_t mask, int start_b *binsn_mask |= place_field (mask, start_bit, len); } -static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacement) { +static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacement, + void **jump_addrs) { MIR_context_t ctx = gen_ctx->ctx; size_t nops = MIR_insn_nops (ctx, insn); size_t offset; @@ -1896,10 +1894,10 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen case 's': case 'n': ch2 = *++p; - gen_assert ('0' <= ch2 && ch2 <= '2' && ch2 - '0' < nops); + gen_assert ('0' <= ch2 && ch2 <= '2' && ch2 - '0' < (int) nops); op = insn->ops[ch2 - '0']; - gen_assert (op.mode == MIR_OP_HARD_REG); - reg = op.u.hard_reg; + gen_assert (op.mode == MIR_OP_VAR); + reg = op.u.var; gen_assert (ch != 'n' || reg >= F0_HARD_REG); if (start_ch == 'n') reg += 2; gen_assert (reg <= F15_HARD_REG); @@ -1949,27 +1947,27 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen } } else { if (ch2 != 'n') p--; - if (insn->ops[0].mode == MIR_OP_HARD_REG_MEM) { + if (insn->ops[0].mode == MIR_OP_VAR_MEM) { op = insn->ops[0]; - } else if (nops >= 2 && insn->ops[1].mode == MIR_OP_HARD_REG_MEM) { + } else if (nops >= 2 && insn->ops[1].mode == MIR_OP_VAR_MEM) { op = insn->ops[1]; - } else if (nops >= 3 && insn->ops[2].mode == MIR_OP_HARD_REG_MEM) { + } else if (nops >= 3 && insn->ops[2].mode == MIR_OP_VAR_MEM) { op = insn->ops[2]; } else { gen_assert (FALSE); } gen_assert (rs < 0 && rx < 0); - if (ch2 == 'n') op.u.hard_reg_mem.disp += 8; - gen_assert (op.u.hard_reg_mem.index == MIR_NON_HARD_REG || op.u.hard_reg_mem.scale == 1); - if (op.u.hard_reg_mem.base == MIR_NON_HARD_REG) { - if (op.u.hard_reg_mem.index != MIR_NON_HARD_REG) rs = op.u.hard_reg_mem.index; + if (ch2 == 'n') op.u.var_mem.disp += 8; + gen_assert (op.u.var_mem.index == MIR_NON_VAR || op.u.var_mem.scale == 1); + if (op.u.var_mem.base == MIR_NON_VAR) { + if (op.u.var_mem.index != MIR_NON_VAR) rs = op.u.var_mem.index; } else { - rs = op.u.hard_reg_mem.base; - if (op.u.hard_reg_mem.index != MIR_NON_HARD_REG) rx = op.u.hard_reg_mem.index; + rs = op.u.var_mem.base; + if (op.u.var_mem.index != MIR_NON_VAR) rx = op.u.var_mem.index; } gen_assert (d < 0 && dh < 0); - d = op.u.hard_reg_mem.disp & 0xfff; - dh = (op.u.hard_reg_mem.disp >> 12) & 0xff; + d = op.u.var_mem.disp & 0xfff; + dh = (op.u.var_mem.disp >> 12) & 0xff; if (dh == 0) dh = -1; } break; @@ -1998,13 +1996,13 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen IMM = u & 0xffff; break; case 'I': { - uint64_t imm; + uint64_t imm_val; ch2 = *++p; gen_assert (ch2 == 'a'); gen_assert (const_ref_num < 0); - imm = get_imm (gen_ctx, insn); - const_ref_num = setup_imm_addr (gen_ctx, imm); + imm_val = get_imm (gen_ctx, insn); + const_ref_num = setup_imm_addr (gen_ctx, imm_val); break; } case 'S': @@ -2023,11 +2021,14 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen label_off /= 2; break; case 'L': - op = insn->ops[insn->code != MIR_CALL ? 0 : 1]; + op = insn->ops[insn->code != MIR_CALL && insn->code != MIR_LADDR ? 0 : 1]; gen_assert (op.mode == MIR_OP_LABEL); lr.abs_addr_p = FALSE; lr.label_val_disp = 0; - lr.label = op.u.label; + if (jump_addrs == NULL) + lr.u.label = op.u.label; + else + lr.u.jump_addr = jump_addrs[0]; label_ref_num = VARR_LENGTH (label_ref_t, label_refs); VARR_PUSH (label_ref_t, label_refs, lr); break; @@ -2036,7 +2037,7 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen switch_table_addr_p = TRUE; break; case 'Q': { - int64_t size = S390X_STACK_HEADER_SIZE + param_save_area_size; + int64_t size = S390X_STACK_HEADER_SIZE + param_save_area_size + blk_ld_value_save_area_size; gen_assert (d < 0 && dh < 0 && int20_p (size)); d = (size) &0xfff; dh = ((size) >> 12) & 0xff; @@ -2132,21 +2133,15 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen set_insn_field (&binsn, label_off, 16, 16); check_and_set_mask (&binsn_mask, 0xffff, 16, 16); } - if (const_ref_num >= 0) - VARR_ADDR (const_ref_t, const_refs) - [const_ref_num].insn_pc - = VARR_LENGTH (uint8_t, result_code); - if (label_ref_num >= 0) - VARR_ADDR (label_ref_t, label_refs) - [label_ref_num].label_val_disp - = VARR_LENGTH (uint8_t, result_code); + if (const_ref_num >= 0) VARR_ADDR (const_ref_t, const_refs) + [const_ref_num].insn_pc = VARR_LENGTH (uint8_t, result_code); + if (label_ref_num >= 0) VARR_ADDR (label_ref_t, label_refs) + [label_ref_num].label_val_disp = VARR_LENGTH (uint8_t, result_code); if (switch_table_addr_p) switch_table_addr_insn_start = VARR_LENGTH (uint8_t, result_code); VARR_PUSH_ARR (uint8_t, result_code, (uint8_t *) &binsn, len); /* output the machine insn */ - if (const_ref_num >= 0) - VARR_ADDR (const_ref_t, const_refs) - [const_ref_num].next_insn_pc - = VARR_LENGTH (uint8_t, result_code); + if (const_ref_num >= 0) VARR_ADDR (const_ref_t, const_refs) + [const_ref_num].next_insn_pc = VARR_LENGTH (uint8_t, result_code); if (*p == 0) break; } @@ -2161,21 +2156,53 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen gen_assert (insn->ops[i].mode == MIR_OP_LABEL); lr.abs_addr_p = TRUE; lr.label_val_disp = VARR_LENGTH (uint8_t, result_code); - lr.label = insn->ops[i].u.label; + if (jump_addrs == NULL) + lr.u.label = insn->ops[i].u.label; + else + lr.u.jump_addr = jump_addrs[i - 1]; VARR_PUSH (label_ref_t, label_refs, lr); /* reserve space for absolute label address: */ VARR_PUSH_ARR (uint8_t, result_code, (uint8_t *) &zero64, sizeof (zero64)); } } +static int target_memory_ok_p (gen_ctx_t gen_ctx MIR_UNUSED, MIR_op_t *op_ref) { + if (op_ref->mode != MIR_OP_VAR_MEM) return FALSE; + if (((op_ref->u.var_mem.type != MIR_T_U64 && op_ref->u.var_mem.type != MIR_T_U64) + || op_ref->u.var_mem.base == MIR_NON_VAR || op_ref->u.var_mem.index == MIR_NON_VAR) + && (op_ref->u.var_mem.index == MIR_NON_VAR || op_ref->u.var_mem.scale == 1) + && int20_p (op_ref->u.var_mem.disp) + && (op_ref->u.var_mem.type != MIR_T_LD || int20_p (op_ref->u.var_mem.disp + 8))) + return TRUE; + return FALSE; +} + static int target_insn_ok_p (gen_ctx_t gen_ctx, MIR_insn_t insn) { return find_insn_pattern_replacement (gen_ctx, insn) != NULL; } +static void add_consts (gen_ctx_t gen_ctx) { + while (VARR_LENGTH (uint8_t, result_code) % 16 != 0) /* Align the pool */ + VARR_PUSH (uint8_t, result_code, 0); + for (size_t i = 0; i < VARR_LENGTH (const_ref_t, const_refs); i++) { /* Add pool constants */ + const_ref_t cr = VARR_GET (const_ref_t, const_refs, i); + int64_t offset = VARR_LENGTH (uint8_t, result_code) - cr.insn_pc; + uint64_t zero64 = 0; + + gen_assert (offset > 0 && offset % 2 == 0); + offset /= 2; + gen_assert ((offset >> 31) == 0); + set_int32 (VARR_ADDR (uint8_t, result_code) + cr.insn_pc + 2 /* start disp in LALR */, offset); + put_arr (gen_ctx, &VARR_ADDR (uint64_t, const_pool)[cr.const_num], 8); + put_arr (gen_ctx, &zero64, sizeof (zero64)); /* keep 16 bytes align */ + } +} + +static void target_split_insns (gen_ctx_t gen_ctx MIR_UNUSED) {} + static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { MIR_context_t ctx = gen_ctx->ctx; size_t i; - uint64_t zero64 = 0; MIR_insn_t insn, old_insn; const char *replacement; @@ -2189,6 +2216,7 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { insn = DLIST_NEXT (MIR_insn_t, insn)) { MIR_insn_code_t code = insn->code; + /* ???split/change */ if ((code == MIR_RSH || code == MIR_LSH || code == MIR_URSH || code == MIR_RSHS || code == MIR_LSHS || code == MIR_URSHS) && (insn->ops[2].mode == MIR_OP_INT || insn->ops[2].mode == MIR_OP_UINT)) { @@ -2219,7 +2247,7 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { } if (insn->code == MIR_LABEL) { set_label_disp (gen_ctx, insn, VARR_LENGTH (uint8_t, result_code)); - } else { + } else if (insn->code != MIR_USE) { replacement = find_insn_pattern_replacement (gen_ctx, insn); if (replacement == NULL) { fprintf (stderr, "fatal failure in matching insn:"); @@ -2227,7 +2255,7 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { exit (1); } else { gen_assert (replacement != NULL); - out_insn (gen_ctx, insn, replacement); + out_insn (gen_ctx, insn, replacement, NULL); } } } @@ -2237,10 +2265,10 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { if (lr.abs_addr_p) { set_int64 (&VARR_ADDR (uint8_t, result_code)[lr.label_val_disp], - (int64_t) get_label_disp (gen_ctx, lr.label)); + (int64_t) get_label_disp (gen_ctx, lr.u.label)); VARR_PUSH (uint64_t, abs_address_locs, lr.label_val_disp); } else { /* 32-bit relative address */ - int64_t offset = (int64_t) get_label_disp (gen_ctx, lr.label) - (int64_t) lr.label_val_disp; + int64_t offset = (int64_t) get_label_disp (gen_ctx, lr.u.label) - (int64_t) lr.label_val_disp; gen_assert (offset % 2 == 0); offset /= 2; gen_assert (((offset < 0 ? -offset : offset) & ~(int64_t) 0x7fffffff) == 0); @@ -2248,26 +2276,93 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { |= (offset & 0xffffffff); } } - while (VARR_LENGTH (uint8_t, result_code) % 16 != 0) /* Align the pool */ - VARR_PUSH (uint8_t, result_code, 0); - for (i = 0; i < VARR_LENGTH (const_ref_t, const_refs); i++) { /* Add pool constants */ - const_ref_t cr = VARR_GET (const_ref_t, const_refs, i); - int64_t offset = VARR_LENGTH (uint8_t, result_code) - cr.insn_pc; + add_consts (gen_ctx); + *len = VARR_LENGTH (uint8_t, result_code); + return VARR_ADDR (uint8_t, result_code); +} - gen_assert (offset > 0 && offset % 2 == 0); - offset /= 2; - gen_assert ((offset >> 31) == 0); - set_int32 (VARR_ADDR (uint8_t, result_code) + cr.insn_pc + 2 /* start disp in LALR */, offset); - put_arr (gen_ctx, &VARR_ADDR (uint64_t, const_pool)[cr.const_num], 8); - put_arr (gen_ctx, &zero64, sizeof (zero64)); /* keep 16 bytes align */ +static void target_rebase (gen_ctx_t gen_ctx, uint8_t *base) { + MIR_code_reloc_t reloc; + + VARR_TRUNC (MIR_code_reloc_t, relocs, 0); + for (size_t i = 0; i < VARR_LENGTH (uint64_t, abs_address_locs); i++) { + reloc.offset = VARR_GET (uint64_t, abs_address_locs, i); + reloc.value = base + get_int64 (base + reloc.offset); + VARR_PUSH (MIR_code_reloc_t, relocs, reloc); } + _MIR_update_code_arr (gen_ctx->ctx, base, VARR_LENGTH (MIR_code_reloc_t, relocs), + VARR_ADDR (MIR_code_reloc_t, relocs)); + gen_setup_lrefs (gen_ctx, base); +} + +static void target_change_to_direct_calls (MIR_context_t ctx MIR_UNUSED) {} + +struct target_bb_version { + uint8_t *base; + label_ref_t branch_ref; /* label cand used for jump to this bb version */ +}; + +static void target_init_bb_version_data (target_bb_version_t data) { + data->base = NULL; /* we don't know origin branch */ +} + +static void target_bb_translate_start (gen_ctx_t gen_ctx) { + VARR_TRUNC (uint8_t, result_code, 0); + VARR_TRUNC (const_ref_t, const_refs, 0); + VARR_TRUNC (label_ref_t, label_refs, 0); + VARR_TRUNC (uint64_t, abs_address_locs, 0); +} + +static void target_bb_insn_translate (gen_ctx_t gen_ctx, MIR_insn_t insn, void **jump_addrs) { + const char *replacement; + if (insn->code == MIR_LABEL) return; + replacement = find_insn_pattern_replacement (gen_ctx, insn); + gen_assert (replacement != NULL); + out_insn (gen_ctx, insn, replacement, jump_addrs); +} + +static void target_output_jump (gen_ctx_t gen_ctx, void **jump_addrs) { + out_insn (gen_ctx, temp_jump, temp_jump_replacement, jump_addrs); +} + +static uint8_t *target_bb_translate_finish (gen_ctx_t gen_ctx, size_t *len) { + add_consts (gen_ctx); *len = VARR_LENGTH (uint8_t, result_code); return VARR_ADDR (uint8_t, result_code); } -static void target_rebase (gen_ctx_t gen_ctx, uint8_t *base) { +static void setup_rel (gen_ctx_t gen_ctx, label_ref_t *lr, uint8_t *base, void *addr) { + MIR_context_t ctx = gen_ctx->ctx; + int64_t offset = (int64_t) addr - (int64_t) (base + lr->label_val_disp); + int32_t rel32 = offset; + + gen_assert ((offset & 0x1) == 0); + offset >>= 1; + gen_assert (((offset < 0 ? -offset : offset) & ~(int64_t) 0x7fffffff) == 0); + /* check max 32-bit offset with possible branch conversion (see offset): */ + if (lr->abs_addr_p || ((offset < 0 ? -offset : offset) & ~(int64_t) 0x7fffffff) != 0) { + fprintf (stderr, "too big offset (%lld) in setup_rel", (long long) offset); + exit (1); + } + /* ??? thread safe: */ + uint32_t *insn_ptr = (uint32_t *) (base + lr->label_val_disp); + rel32 = offset & 0xffffffff; + _MIR_change_code (ctx, (uint8_t *) insn_ptr + 2, (uint8_t *) &rel32, 4); +} + +static void target_bb_rebase (gen_ctx_t gen_ctx, uint8_t *base) { + MIR_context_t ctx = gen_ctx->ctx; MIR_code_reloc_t reloc; + /* Setting up relative labels */ + for (size_t i = 0; i < VARR_LENGTH (label_ref_t, label_refs); i++) { + label_ref_t lr = VARR_GET (label_ref_t, label_refs, i); + if (lr.abs_addr_p) { + _MIR_change_code (ctx, (uint8_t *) base + lr.label_val_disp, (uint8_t *) &lr.u.jump_addr, 8); + } else { + setup_rel (gen_ctx, &lr, base, lr.u.jump_addr); + } + } VARR_TRUNC (MIR_code_reloc_t, relocs, 0); for (size_t i = 0; i < VARR_LENGTH (uint64_t, abs_address_locs); i++) { reloc.offset = VARR_GET (uint64_t, abs_address_locs, i); @@ -2278,7 +2373,38 @@ static void target_rebase (gen_ctx_t gen_ctx, uint8_t *base) { VARR_ADDR (MIR_code_reloc_t, relocs)); } +static void target_setup_succ_bb_version_data (gen_ctx_t gen_ctx, uint8_t *base) { + if (VARR_LENGTH (label_ref_t, label_refs) + != VARR_LENGTH (target_bb_version_t, target_succ_bb_versions)) + /* We can have more one possible branch from original insn + (e.g. SWITCH, FBNE). If it is so, we will make jumps only + through BB thunk. */ + return; + for (size_t i = 0; i < VARR_LENGTH (target_bb_version_t, target_succ_bb_versions); i++) { + target_bb_version_t data = VARR_GET (target_bb_version_t, target_succ_bb_versions, i); + if (data == NULL) continue; + data->branch_ref = VARR_GET (label_ref_t, label_refs, i); + data->base = base; + } +} + +static void target_redirect_bb_origin_branch (gen_ctx_t gen_ctx, target_bb_version_t data, + void *addr) { + MIR_context_t ctx = gen_ctx->ctx; + + if (data->base == NULL) return; + if (data->branch_ref.abs_addr_p) { + _MIR_change_code (ctx, (uint8_t *) data->base + data->branch_ref.label_val_disp, + (uint8_t *) &addr, 8); + } else { + setup_rel (gen_ctx, &data->branch_ref, data->base, addr); + } + data->base = NULL; +} + static void target_init (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; + gen_ctx->target_ctx = gen_malloc (gen_ctx, sizeof (struct target_ctx)); VARR_CREATE (uint8_t, result_code, 0); VARR_CREATE (uint64_t, const_pool, 0); @@ -2288,10 +2414,13 @@ static void target_init (gen_ctx_t gen_ctx) { VARR_CREATE (MIR_code_reloc_t, relocs, 0); VARR_CREATE (uint64_t, ld_addr_regs, 0); patterns_init (gen_ctx); + temp_jump = MIR_new_insn (ctx, MIR_JMP, MIR_new_label_op (ctx, NULL)); + temp_jump_replacement = find_insn_pattern_replacement (gen_ctx, temp_jump); } static void target_finish (gen_ctx_t gen_ctx) { patterns_finish (gen_ctx); + _MIR_free_insn (gen_ctx->ctx, temp_jump); VARR_DESTROY (uint8_t, result_code); VARR_DESTROY (uint64_t, const_pool); VARR_DESTROY (const_ref_t, const_refs); diff --git a/mir/mir-gen-stub.c b/mir/mir-gen-stub.c index d4c82510..57f53e59 100644 --- a/mir/mir-gen-stub.c +++ b/mir/mir-gen-stub.c @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . Stub for MIR generator machine dependent file. It contains definitions used by MIR generator. You can use this file for @@ -90,6 +90,7 @@ static void target_get_early_clobbered_hard_regs (MIR_insn_t insn, MIR_reg_t *hr static int target_insn_ok_p (MIR_context_t ctx, MIR_insn_t insn) { return FALSE; } static uint8_t *target_translate (MIR_context_t ctx, size_t *len) { return NULL; } static void target_rebase (MIR_context_t ctx, uint8_t *base) {} +static void target_change_to_direct_calls (MIR_context_t ctx) {} static void target_init (MIR_context_t ctx) { fprintf (stderr, "Your generator target dependent file is just a stub!\n"); diff --git a/mir/mir-gen-x86_64.c b/mir/mir-gen-x86_64.c index d70e8e62..84c7a11c 100644 --- a/mir/mir-gen-x86_64.c +++ b/mir/mir-gen-x86_64.c @@ -1,59 +1,18 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ #include -#define HREG_EL(h) h##_HARD_REG -#define REP_SEP , -enum { - REP8 (HREG_EL, AX, CX, DX, BX, SP, BP, SI, DI), - REP8 (HREG_EL, R8, R9, R10, R11, R12, R13, R14, R15), - REP8 (HREG_EL, XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7), - REP8 (HREG_EL, XMM8, XMM9, XMM10, XMM11, XMM12, XMM13, XMM14, XMM15), - REP2 (HREG_EL, ST0, ST1), -}; -#undef REP_SEP +#include "mir-x86_64.h" -static const MIR_reg_t MAX_HARD_REG = ST1_HARD_REG; static const MIR_reg_t FP_HARD_REG = BP_HARD_REG; -static int target_locs_num (MIR_reg_t loc, MIR_type_t type) { - return loc > MAX_HARD_REG && type == MIR_T_LD ? 2 : 1; -} - -static inline MIR_reg_t target_nth_loc (MIR_reg_t loc, MIR_type_t type, int n) { return loc + n; } - -/* Hard regs not used in machinized code, preferably call used ones. */ -const MIR_reg_t TEMP_INT_HARD_REG1 = R10_HARD_REG, TEMP_INT_HARD_REG2 = R11_HARD_REG; -#ifndef _WIN32 -const MIR_reg_t TEMP_FLOAT_HARD_REG1 = XMM8_HARD_REG, TEMP_FLOAT_HARD_REG2 = XMM9_HARD_REG; -const MIR_reg_t TEMP_DOUBLE_HARD_REG1 = XMM8_HARD_REG, TEMP_DOUBLE_HARD_REG2 = XMM9_HARD_REG; -#else -const MIR_reg_t TEMP_FLOAT_HARD_REG1 = XMM4_HARD_REG, TEMP_FLOAT_HARD_REG2 = XMM5_HARD_REG; -const MIR_reg_t TEMP_DOUBLE_HARD_REG1 = XMM4_HARD_REG, TEMP_DOUBLE_HARD_REG2 = XMM5_HARD_REG; -#endif -const MIR_reg_t TEMP_LDOUBLE_HARD_REG1 = MIR_NON_HARD_REG; -const MIR_reg_t TEMP_LDOUBLE_HARD_REG2 = MIR_NON_HARD_REG; - -static inline int target_hard_reg_type_ok_p (MIR_reg_t hard_reg, MIR_type_t type) { - assert (hard_reg <= MAX_HARD_REG); - /* For LD we need x87 stack regs and it is too complicated so no - hard register allocation for LD: */ - if (type == MIR_T_LD) return FALSE; - return MIR_int_type_p (type) ? hard_reg < XMM0_HARD_REG : hard_reg >= XMM0_HARD_REG; -} - -static inline int target_fixed_hard_reg_p (MIR_reg_t hard_reg) { - assert (hard_reg <= MAX_HARD_REG); - return (hard_reg == BP_HARD_REG || hard_reg == SP_HARD_REG || hard_reg == TEMP_INT_HARD_REG1 - || hard_reg == TEMP_INT_HARD_REG2 || hard_reg == TEMP_FLOAT_HARD_REG1 - || hard_reg == TEMP_FLOAT_HARD_REG2 || hard_reg == TEMP_DOUBLE_HARD_REG1 - || hard_reg == TEMP_DOUBLE_HARD_REG2 || hard_reg == ST0_HARD_REG - || hard_reg == ST1_HARD_REG); +static inline MIR_reg_t target_nth_loc (MIR_reg_t loc, MIR_type_t type MIR_UNUSED, int n) { + return loc + n; } -static inline int target_call_used_hard_reg_p (MIR_reg_t hard_reg, MIR_type_t type) { +static inline int target_call_used_hard_reg_p (MIR_reg_t hard_reg, MIR_type_t type MIR_UNUSED) { assert (hard_reg <= MAX_HARD_REG); #ifndef _WIN32 return !(hard_reg == BX_HARD_REG || (hard_reg >= R12_HARD_REG && hard_reg <= R15_HARD_REG)); @@ -67,29 +26,34 @@ static inline int target_call_used_hard_reg_p (MIR_reg_t hard_reg, MIR_type_t ty /* Stack layout (sp refers to the last reserved stack slot address) from higher address to lower address memory: - | ... | prev func stack frame (start address should be aligned to 16 bytes) - |---------------| - | return pc | value of sp before prologue = start sp hard reg - |---------------| - | old bp | bp for previous func stack frame; new bp refers for here - |---------------| - | reg save | 176 bytes - | area | optional area for vararg func reg save area - |---------------| - | slots assigned| can be absent for small functions (known only after RA) - | to pseudos | - |---------------| - | saved regs | callee saved regs used in the func (known only after RA) - |---------------| - | alloca areas | optional - |---------------| - | slots for | dynamically allocated/deallocated by caller - | passing args | - |---------------| - | spill space | WIN32 only, 32 bytes spill space for register args - |---------------| + FP is required: FP omitted: + +| | prev func stack frame | | +| ... | (start addr aligned to 16 bytes) | ... | +|---------------| |---------------| +| return pc | sp before call = start sp hard reg | | +| | absent for jcall/jret func | return pc | +|---------------| | | +| old bp | new bp refers here | | +|---------------| |---------------| +| reg save | 176 bytes optional area for | reg save | +| area | vararg func reg save area | area | +|---------------| |---------------| +| slots assigned| can be absent for small functions | saved regs | +| to pseudos | (known only after RA) | | +|---------------| | | +| saved regs | callee saved regs used in the func | | +|---------------| (known only after RA) |---------------| +| alloca areas | optional | | +|---------------| | slot assigned | +| slots for | dynamically reserved/freed | to pseudos | +| passing args | by caller | | +|---------------| |---------------| +| spill space | WIN32 only: 32 bytes spill space | spill space | +|---------------| for reg args (allocated at call) |---------------| size of slots and saved regs is multiple of 16 bytes + will be fp ommited is defined after machinize */ @@ -101,21 +65,14 @@ static const int reg_save_area_size = 0; static const int spill_space_size = 32; #endif -static MIR_disp_t target_get_stack_slot_offset (gen_ctx_t gen_ctx, MIR_type_t type, - MIR_reg_t slot) { - /* slot is 0, 1, ... */ - return -((MIR_disp_t) (slot + (type == MIR_T_LD ? 2 : 1)) * 8 - + (curr_func_item->u.func->vararg_p ? reg_save_area_size : 0)); -} - static const MIR_insn_code_t target_io_dup_op_insn_codes[] = { /* see possible patterns */ - MIR_FADD, MIR_DADD, MIR_LDADD, MIR_SUB, MIR_SUBS, MIR_FSUB, MIR_DSUB, - MIR_LDSUB, MIR_MUL, MIR_MULS, MIR_FMUL, MIR_DMUL, MIR_LDMUL, MIR_DIV, - MIR_DIVS, MIR_UDIV, MIR_FDIV, MIR_DDIV, MIR_LDDIV, MIR_MOD, MIR_MODS, - MIR_UMOD, MIR_UMODS, MIR_AND, MIR_ANDS, MIR_OR, MIR_ORS, MIR_XOR, - MIR_XORS, MIR_LSH, MIR_LSHS, MIR_RSH, MIR_RSHS, MIR_URSH, MIR_URSHS, - MIR_NEG, MIR_NEGS, MIR_FNEG, MIR_DNEG, MIR_LDNEG, MIR_INSN_BOUND, + MIR_ADD, MIR_ADDS, MIR_FADD, MIR_DADD, MIR_LDADD, MIR_SUB, MIR_SUBS, MIR_FSUB, + MIR_DSUB, MIR_LDSUB, MIR_MUL, MIR_MULS, MIR_FMUL, MIR_DMUL, MIR_LDMUL, MIR_FDIV, + MIR_DDIV, MIR_LDDIV, MIR_AND, MIR_ANDS, MIR_OR, MIR_ORS, MIR_XOR, MIR_XORS, + MIR_LSH, MIR_LSHS, MIR_RSH, MIR_RSHS, MIR_URSH, MIR_URSHS, MIR_NEG, MIR_NEGS, + MIR_FNEG, MIR_DNEG, MIR_LDNEG, MIR_ADDO, MIR_ADDOS, MIR_SUBO, MIR_SUBOS, MIR_MULO, + MIR_MULOS, MIR_UMULO, MIR_UMULOS, MIR_INSN_BOUND, }; static MIR_insn_code_t get_ext_code (MIR_type_t type) { @@ -142,8 +99,8 @@ static MIR_reg_t get_fp_arg_reg (size_t fp_arg_num) { case 6: case 7: #endif - return XMM0_HARD_REG + fp_arg_num; - default: return MIR_NON_HARD_REG; + return (MIR_reg_t) (XMM0_HARD_REG + fp_arg_num); + default: return MIR_NON_VAR; } } @@ -164,7 +121,7 @@ static MIR_reg_t get_int_arg_reg (size_t int_arg_num) { #endif case 4: return R8_HARD_REG; case 5: return R9_HARD_REG; - default: return MIR_NON_HARD_REG; + default: return MIR_NON_VAR; } } @@ -185,7 +142,7 @@ static MIR_reg_t get_arg_reg (MIR_type_t arg_type, size_t *int_arg_num, size_t * MIR_reg_t arg_reg; if (arg_type == MIR_T_LD) { - arg_reg = MIR_NON_HARD_REG; + arg_reg = MIR_NON_VAR; *mov_code = MIR_LDMOV; } else if (arg_type == MIR_T_F || arg_type == MIR_T_D) { arg_reg = get_fp_arg_reg (*fp_arg_num); @@ -210,12 +167,15 @@ static void gen_mov (gen_ctx_t gen_ctx, MIR_insn_t anchor, MIR_insn_code_t code, gen_add_insn_before (gen_ctx, anchor, MIR_new_insn (gen_ctx->ctx, code, dst_op, src_op)); } +static void prohibit_omitting_fp (gen_ctx_t gen_ctx); + static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { MIR_context_t ctx = gen_ctx->ctx; MIR_func_t func = curr_func_item->u.func; MIR_proto_t proto = call_insn->ops[0].u.ref->u.proto; size_t size, nargs, nops = MIR_insn_nops (ctx, call_insn), start = proto->nres + 2; - size_t int_arg_num = 0, fp_arg_num = 0, xmm_args = 0, arg_stack_size = spill_space_size; + size_t int_arg_num = 0, fp_arg_num = 0, xmm_args = 0; + size_t init_arg_stack_size = spill_space_size, arg_stack_size = init_arg_stack_size; #ifdef _WIN32 size_t block_offset = spill_space_size; #endif @@ -239,8 +199,8 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { nargs = VARR_LENGTH (MIR_var_t, proto->args); arg_vars = VARR_ADDR (MIR_var_t, proto->args); } - if (call_insn->ops[1].mode != MIR_OP_REG && call_insn->ops[1].mode != MIR_OP_HARD_REG) { - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + if (call_insn->ops[1].mode != MIR_OP_VAR && call_insn->ops[1].mode != MIR_OP_REF) { + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); new_insn = MIR_new_insn (ctx, MIR_MOV, temp_op, call_insn->ops[1]); call_insn->ops[1] = temp_op; gen_add_insn_before (gen_ctx, call_insn, new_insn); @@ -250,14 +210,12 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { #endif for (size_t i = start; i < nops; i++) { arg_op = call_insn->ops[i]; - gen_assert ( - arg_op.mode == MIR_OP_REG || arg_op.mode == MIR_OP_HARD_REG - || (arg_op.mode == MIR_OP_MEM && MIR_all_blk_type_p (arg_op.u.mem.type)) - || (arg_op.mode == MIR_OP_HARD_REG_MEM && MIR_all_blk_type_p (arg_op.u.hard_reg_mem.type))); + gen_assert (arg_op.mode == MIR_OP_VAR + || (arg_op.mode == MIR_OP_VAR_MEM && MIR_all_blk_type_p (arg_op.u.var_mem.type))); if (i - start < nargs) { type = arg_vars[i - start].type; - } else if (arg_op.mode == MIR_OP_MEM || arg_op.mode == MIR_OP_HARD_REG_MEM) { - type = arg_op.mode == MIR_OP_MEM ? arg_op.u.mem.type : arg_op.u.hard_reg_mem.type; + } else if (arg_op.mode == MIR_OP_VAR_MEM) { + type = arg_op.u.var_mem.type; assert (MIR_all_blk_type_p (type)); } else { mode = call_insn->ops[i].value_mode; // ??? smaller ints @@ -271,43 +229,45 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { if (xmm_args < 8 && (type == MIR_T_F || type == MIR_T_D)) xmm_args++; ext_insn = NULL; if ((ext_code = get_ext_code (type)) != MIR_INVALID_INSN) { /* extend arg if necessary */ - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); ext_insn = MIR_new_insn (ctx, ext_code, temp_op, arg_op); call_insn->ops[i] = arg_op = temp_op; } size = 0; if (MIR_blk_type_p (type)) { - gen_assert (arg_op.mode == MIR_OP_MEM); - size = (arg_op.u.mem.disp + 7) / 8 * 8; + gen_assert (arg_op.mode == MIR_OP_VAR_MEM); + size = (arg_op.u.var_mem.disp + 7) / 8 * 8; gen_assert (prev_call_insn != NULL); /* call_insn should not be 1st after simplification */ } #ifndef _WIN32 - if ((type == MIR_T_BLK + 1 && get_int_arg_reg (int_arg_num) != MIR_NON_HARD_REG - && (size <= 8 || get_int_arg_reg (int_arg_num + 1) != MIR_NON_HARD_REG)) - || (type == MIR_T_BLK + 2 && get_fp_arg_reg (fp_arg_num) != MIR_NON_HARD_REG - && (size <= 8 || get_fp_arg_reg (fp_arg_num + 1) != MIR_NON_HARD_REG))) { + if ((type == MIR_T_BLK + 1 && get_int_arg_reg (int_arg_num) != MIR_NON_VAR + && (size <= 8 || get_int_arg_reg (int_arg_num + 1) != MIR_NON_VAR)) + || (type == MIR_T_BLK + 2 && get_fp_arg_reg (fp_arg_num) != MIR_NON_VAR + && (size <= 8 || get_fp_arg_reg (fp_arg_num + 1) != MIR_NON_VAR))) { /* all is passed in gprs or fprs */ MIR_type_t mov_type = type == MIR_T_BLK + 1 ? MIR_T_I64 : MIR_T_D; MIR_insn_code_t mov_code; MIR_reg_t reg2, reg1 = get_arg_reg (mov_type, &int_arg_num, &fp_arg_num, &mov_code); assert (size <= 16); - new_insn = MIR_new_insn (ctx, mov_code, _MIR_new_hard_reg_op (ctx, reg1), - MIR_new_mem_op (ctx, mov_type, 0, arg_op.u.mem.base, 0, 1)); + new_insn = MIR_new_insn (ctx, mov_code, _MIR_new_var_op (ctx, reg1), + _MIR_new_var_mem_op (ctx, mov_type, 0, arg_op.u.var_mem.base, + MIR_NON_VAR, 1)); gen_add_insn_before (gen_ctx, call_insn, new_insn); setup_call_hard_reg_args (gen_ctx, call_insn, reg1); - call_insn->ops[i].u.mem.base = 0; /* not used anymore */ + call_insn->ops[i].u.var_mem.base = MIR_NON_VAR; /* not used anymore */ if (size > 8) { reg2 = get_arg_reg (mov_type, &int_arg_num, &fp_arg_num, &mov_code); - new_insn = MIR_new_insn (ctx, mov_code, _MIR_new_hard_reg_op (ctx, reg2), - MIR_new_mem_op (ctx, mov_type, 8, arg_op.u.mem.base, 0, 1)); + new_insn = MIR_new_insn (ctx, mov_code, _MIR_new_var_op (ctx, reg2), + _MIR_new_var_mem_op (ctx, mov_type, 8, arg_op.u.var_mem.base, + MIR_NON_VAR, 1)); gen_add_insn_before (gen_ctx, call_insn, new_insn); setup_call_hard_reg_args (gen_ctx, call_insn, reg2); } continue; } else if ((type == MIR_T_BLK + 3 || type == MIR_T_BLK + 4) - && get_int_arg_reg (int_arg_num) != MIR_NON_HARD_REG - && get_fp_arg_reg (fp_arg_num) != MIR_NON_HARD_REG) { + && get_int_arg_reg (int_arg_num) != MIR_NON_VAR + && get_fp_arg_reg (fp_arg_num) != MIR_NON_VAR) { /* gpr and then fpr or fpr and then gpr */ MIR_type_t mov_type1 = type == MIR_T_BLK + 3 ? MIR_T_I64 : MIR_T_D; MIR_type_t mov_type2 = type == MIR_T_BLK + 3 ? MIR_T_D : MIR_T_I64; @@ -316,13 +276,15 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { MIR_reg_t reg2 = get_arg_reg (mov_type2, &int_arg_num, &fp_arg_num, &mov_code2); assert (size > 8 && size <= 16); - new_insn = MIR_new_insn (ctx, mov_code1, _MIR_new_hard_reg_op (ctx, reg1), - MIR_new_mem_op (ctx, mov_type1, 0, arg_op.u.mem.base, 0, 1)); + new_insn = MIR_new_insn (ctx, mov_code1, _MIR_new_var_op (ctx, reg1), + _MIR_new_var_mem_op (ctx, mov_type1, 0, arg_op.u.var_mem.base, + MIR_NON_VAR, 1)); setup_call_hard_reg_args (gen_ctx, call_insn, reg1); - call_insn->ops[i].u.mem.base = 0; /* not used anymore */ + call_insn->ops[i].u.var_mem.base = MIR_NON_VAR; /* not used anymore */ gen_add_insn_before (gen_ctx, call_insn, new_insn); - new_insn = MIR_new_insn (ctx, mov_code2, _MIR_new_hard_reg_op (ctx, reg2), - MIR_new_mem_op (ctx, mov_type2, 8, arg_op.u.mem.base, 0, 1)); + new_insn = MIR_new_insn (ctx, mov_code2, _MIR_new_var_op (ctx, reg2), + _MIR_new_var_mem_op (ctx, mov_type2, 8, arg_op.u.var_mem.base, + MIR_NON_VAR, 1)); gen_add_insn_before (gen_ctx, call_insn, new_insn); setup_call_hard_reg_args (gen_ctx, call_insn, reg2); continue; @@ -337,17 +299,18 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { by_val_p = size <= 8; #endif if (by_val_p) { - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - mem_op = MIR_new_mem_op (ctx, MIR_T_I64, 0, arg_op.u.mem.base, 0, 1); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + mem_op = _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, arg_op.u.var_mem.base, MIR_NON_VAR, 1); load_insn = MIR_new_insn (ctx, MIR_MOV, temp_op, mem_op); gen_add_insn_after (gen_ctx, prev_call_insn, load_insn); arg_op = temp_op; } else if (size > 0 && size <= 2 * 8) { /* upto 2 moves */ disp = 0; first_p = TRUE; - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + temp_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); while (size != 0) { - mem_op = MIR_new_mem_op (ctx, MIR_T_I64, disp, arg_op.u.mem.base, 0, 1); + mem_op + = _MIR_new_var_mem_op (ctx, MIR_T_I64, disp, arg_op.u.var_mem.base, MIR_NON_VAR, 1); load_insn = MIR_new_insn (ctx, MIR_MOV, temp_op, mem_op); gen_add_insn_after (gen_ctx, prev_call_insn, load_insn); disp += 8; @@ -359,21 +322,20 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { dest_disp = arg_stack_size; arg_stack_size += 8; #endif - mem_op = _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, dest_disp, SP_HARD_REG, - MIR_NON_HARD_REG, 1); + mem_op = _MIR_new_var_mem_op (ctx, MIR_T_I64, dest_disp, SP_HARD_REG, MIR_NON_VAR, 1); new_insn = MIR_new_insn (ctx, MIR_MOV, mem_op, temp_op); size -= 8; gen_add_insn_after (gen_ctx, load_insn, new_insn); if (first_p) { call_insn->ops[i] - = _MIR_new_hard_reg_mem_op (ctx, type, dest_disp, SP_HARD_REG, MIR_NON_HARD_REG, 1); + = _MIR_new_var_mem_op (ctx, type, dest_disp, SP_HARD_REG, MIR_NON_VAR, 1); first_p = FALSE; } } #ifdef _WIN32 arg_op - = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, curr_func_item->u.func)); - new_insn = MIR_new_insn (ctx, MIR_ADD, arg_op, _MIR_new_hard_reg_op (ctx, SP_HARD_REG), + = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, curr_func_item->u.func)); + new_insn = MIR_new_insn (ctx, MIR_ADD, arg_op, _MIR_new_var_op (ctx, SP_HARD_REG), MIR_new_int_op (ctx, start_dest_disp)); gen_add_insn_before (gen_ctx, call_insn, new_insn); #endif @@ -386,19 +348,20 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { MIR_item_t memcpy_import_item = _MIR_builtin_func (ctx, curr_func_item->module, "mir.arg_memcpy", memcpy); freg_op - = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, curr_func_item->u.func)); + = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, curr_func_item->u.func)); dest_reg = gen_new_temp_reg (gen_ctx, MIR_T_I64, curr_func_item->u.func); - dest_reg_op = MIR_new_reg_op (ctx, dest_reg); + dest_reg_op = _MIR_new_var_op (ctx, dest_reg); ops[0] = MIR_new_ref_op (ctx, memcpy_proto_item); ops[1] = freg_op; - ops[2] = _MIR_new_hard_reg_op (ctx, get_int_arg_reg (0)); - ops[3] = _MIR_new_hard_reg_op (ctx, get_int_arg_reg (1)); - ops[4] = _MIR_new_hard_reg_op (ctx, get_int_arg_reg (2)); + ops[2] = _MIR_new_var_op (ctx, get_int_arg_reg (0)); + ops[3] = _MIR_new_var_op (ctx, get_int_arg_reg (1)); + ops[4] = _MIR_new_var_op (ctx, get_int_arg_reg (2)); new_insn = MIR_new_insn_arr (ctx, MIR_CALL, 5, ops); gen_add_insn_after (gen_ctx, prev_call_insn, new_insn); new_insn = MIR_new_insn (ctx, MIR_MOV, ops[4], MIR_new_int_op (ctx, size)); gen_add_insn_after (gen_ctx, prev_call_insn, new_insn); - new_insn = MIR_new_insn (ctx, MIR_MOV, ops[3], MIR_new_reg_op (ctx, arg_op.u.mem.base)); + new_insn + = MIR_new_insn (ctx, MIR_MOV, ops[3], _MIR_new_var_op (ctx, arg_op.u.var_mem.base)); gen_add_insn_after (gen_ctx, prev_call_insn, new_insn); new_insn = MIR_new_insn (ctx, MIR_MOV, ops[2], dest_reg_op); gen_add_insn_after (gen_ctx, prev_call_insn, new_insn); @@ -409,25 +372,25 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { start_dest_disp = arg_stack_size; arg_stack_size += size; #endif - new_insn = MIR_new_insn (ctx, MIR_ADD, dest_reg_op, _MIR_new_hard_reg_op (ctx, SP_HARD_REG), + new_insn = MIR_new_insn (ctx, MIR_ADD, dest_reg_op, _MIR_new_var_op (ctx, SP_HARD_REG), MIR_new_int_op (ctx, start_dest_disp)); gen_add_insn_after (gen_ctx, prev_call_insn, new_insn); new_insn = MIR_new_insn (ctx, MIR_MOV, ops[1], MIR_new_ref_op (ctx, memcpy_import_item)); gen_add_insn_after (gen_ctx, prev_call_insn, new_insn); - call_insn->ops[i] = MIR_new_mem_op (ctx, MIR_T_BLK, arg_op.u.mem.disp, dest_reg, 0, 1); + call_insn->ops[i] + = _MIR_new_var_mem_op (ctx, MIR_T_BLK, arg_op.u.var_mem.disp, dest_reg, MIR_NON_VAR, 1); #ifdef _WIN32 arg_op = dest_reg_op; #endif } #ifdef _WIN32 if ((arg_reg = get_arg_reg (MIR_T_P, &int_arg_num, &fp_arg_num, &new_insn_code)) - != MIR_NON_HARD_REG) { - new_arg_op = _MIR_new_hard_reg_op (ctx, arg_reg); + != MIR_NON_VAR) { + new_arg_op = _MIR_new_var_op (ctx, arg_reg); new_insn = MIR_new_insn (ctx, MIR_MOV, new_arg_op, arg_op); call_insn->ops[i] = new_arg_op; } else { - mem_op = _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, arg_stack_size, SP_HARD_REG, - MIR_NON_HARD_REG, 1); + mem_op = _MIR_new_var_mem_op (ctx, MIR_T_I64, arg_stack_size, SP_HARD_REG, MIR_NON_VAR, 1); new_insn = MIR_new_insn (ctx, MIR_MOV, mem_op, arg_op); call_insn->ops[i] = mem_op; arg_stack_size += 8; @@ -435,23 +398,18 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { gen_add_insn_before (gen_ctx, call_insn, new_insn); #endif } else if ((arg_reg = get_arg_reg (type, &int_arg_num, &fp_arg_num, &new_insn_code)) - != MIR_NON_HARD_REG) { + != MIR_NON_VAR) { /* put arguments to argument hard regs */ if (ext_insn != NULL) gen_add_insn_before (gen_ctx, call_insn, ext_insn); if (type != MIR_T_RBLK) { - new_arg_op = _MIR_new_hard_reg_op (ctx, arg_reg); + new_arg_op = _MIR_new_var_op (ctx, arg_reg); new_insn = MIR_new_insn (ctx, new_insn_code, new_arg_op, arg_op); - } else if (arg_op.mode == MIR_OP_MEM) { - new_insn = MIR_new_insn (ctx, new_insn_code, _MIR_new_hard_reg_op (ctx, arg_reg), - MIR_new_reg_op (ctx, arg_op.u.mem.base)); - new_arg_op = _MIR_new_hard_reg_mem_op (ctx, MIR_T_RBLK, arg_op.u.mem.disp, arg_reg, - MIR_NON_HARD_REG, 1); } else { - assert (arg_op.mode == MIR_OP_HARD_REG_MEM); - new_insn = MIR_new_insn (ctx, new_insn_code, _MIR_new_hard_reg_op (ctx, arg_reg), - _MIR_new_hard_reg_op (ctx, arg_op.u.hard_reg_mem.base)); - new_arg_op = _MIR_new_hard_reg_mem_op (ctx, MIR_T_RBLK, arg_op.u.hard_reg_mem.disp, arg_reg, - MIR_NON_HARD_REG, 1); + assert (arg_op.mode == MIR_OP_VAR_MEM); + new_insn = MIR_new_insn (ctx, new_insn_code, _MIR_new_var_op (ctx, arg_reg), + _MIR_new_var_op (ctx, arg_op.u.var_mem.base)); + new_arg_op + = _MIR_new_var_mem_op (ctx, MIR_T_RBLK, arg_op.u.var_mem.disp, arg_reg, MIR_NON_VAR, 1); } gen_add_insn_before (gen_ctx, call_insn, new_insn); call_insn->ops[i] = new_arg_op; @@ -462,27 +420,25 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { arg_reg = get_int_arg_reg (int_arg_num - 1); setup_call_hard_reg_args (gen_ctx, call_insn, arg_reg); /* mir does not support moving fp to int regs directly, spill and load them instead */ - mem_op = _MIR_new_hard_reg_mem_op (ctx, MIR_T_D, 8, SP_HARD_REG, MIR_NON_HARD_REG, 1); + mem_op = _MIR_new_var_mem_op (ctx, MIR_T_D, 8, SP_HARD_REG, MIR_NON_VAR, 1); new_insn = MIR_new_insn (ctx, MIR_DMOV, mem_op, arg_op); gen_add_insn_before (gen_ctx, call_insn, new_insn); - mem_op = _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 8, SP_HARD_REG, MIR_NON_HARD_REG, 1); - new_insn = MIR_new_insn (ctx, MIR_MOV, _MIR_new_hard_reg_op (ctx, arg_reg), mem_op); + mem_op = _MIR_new_var_mem_op (ctx, MIR_T_I64, 8, SP_HARD_REG, MIR_NON_VAR, 1); + new_insn = MIR_new_insn (ctx, MIR_MOV, _MIR_new_var_op (ctx, arg_reg), mem_op); gen_add_insn_before (gen_ctx, call_insn, new_insn); } #endif } else { /* put arguments on the stack */ if (type == MIR_T_RBLK) { - assert (arg_op.mode == MIR_OP_MEM || arg_op.mode == MIR_OP_HARD_REG_MEM); - arg_op = arg_op.mode == MIR_OP_MEM ? MIR_new_reg_op (ctx, arg_op.u.mem.base) - : _MIR_new_hard_reg_op (ctx, arg_op.u.hard_reg_mem.base); + assert (arg_op.mode == MIR_OP_VAR_MEM); + arg_op = _MIR_new_var_op (ctx, arg_op.u.var_mem.base); } mem_type = type == MIR_T_F || type == MIR_T_D || type == MIR_T_LD ? type : MIR_T_I64; new_insn_code = (type == MIR_T_F ? MIR_FMOV : type == MIR_T_D ? MIR_DMOV : type == MIR_T_LD ? MIR_LDMOV : MIR_MOV); - mem_op = _MIR_new_hard_reg_mem_op (ctx, mem_type, arg_stack_size, SP_HARD_REG, - MIR_NON_HARD_REG, 1); + mem_op = _MIR_new_var_mem_op (ctx, mem_type, arg_stack_size, SP_HARD_REG, MIR_NON_VAR, 1); new_insn = MIR_new_insn (ctx, new_insn_code, mem_op, arg_op); gen_assert (prev_call_insn != NULL); /* call_insn should not be 1st after simplification */ MIR_insert_insn_after (ctx, curr_func_item, prev_call_insn, new_insn); @@ -501,7 +457,7 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { #ifndef _WIN32 if (proto->vararg_p) { setup_call_hard_reg_args (gen_ctx, call_insn, AX_HARD_REG); - new_insn = MIR_new_insn (ctx, MIR_MOV, _MIR_new_hard_reg_op (ctx, AX_HARD_REG), + new_insn = MIR_new_insn (ctx, MIR_MOV, _MIR_new_var_op (ctx, AX_HARD_REG), MIR_new_int_op (ctx, xmm_args)); gen_add_insn_before (gen_ctx, call_insn, new_insn); } @@ -513,26 +469,22 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { n_iregs = n_xregs = n_fregs = 0; for (size_t i = 0; i < proto->nres; i++) { ret_reg_op = call_insn->ops[i + 2]; - gen_assert (ret_reg_op.mode == MIR_OP_REG || ret_reg_op.mode == MIR_OP_HARD_REG); + gen_assert (ret_reg_op.mode == MIR_OP_VAR); if (proto->res_types[i] == MIR_T_F && n_xregs < 2) { - new_insn - = MIR_new_insn (ctx, MIR_FMOV, ret_reg_op, - _MIR_new_hard_reg_op (ctx, n_xregs == 0 ? XMM0_HARD_REG : XMM1_HARD_REG)); + new_insn = MIR_new_insn (ctx, MIR_FMOV, ret_reg_op, + _MIR_new_var_op (ctx, n_xregs == 0 ? XMM0_HARD_REG : XMM1_HARD_REG)); n_xregs++; } else if (proto->res_types[i] == MIR_T_D && n_xregs < 2) { - new_insn - = MIR_new_insn (ctx, MIR_DMOV, ret_reg_op, - _MIR_new_hard_reg_op (ctx, n_xregs == 0 ? XMM0_HARD_REG : XMM1_HARD_REG)); + new_insn = MIR_new_insn (ctx, MIR_DMOV, ret_reg_op, + _MIR_new_var_op (ctx, n_xregs == 0 ? XMM0_HARD_REG : XMM1_HARD_REG)); n_xregs++; } else if (proto->res_types[i] == MIR_T_LD && n_fregs < 2) { - new_insn - = MIR_new_insn (ctx, MIR_LDMOV, ret_reg_op, - _MIR_new_hard_reg_op (ctx, n_fregs == 0 ? ST0_HARD_REG : ST1_HARD_REG)); + new_insn = MIR_new_insn (ctx, MIR_LDMOV, ret_reg_op, + _MIR_new_var_op (ctx, n_fregs == 0 ? ST0_HARD_REG : ST1_HARD_REG)); n_fregs++; } else if (n_iregs < 2) { - new_insn - = MIR_new_insn (ctx, MIR_MOV, ret_reg_op, - _MIR_new_hard_reg_op (ctx, n_iregs == 0 ? AX_HARD_REG : DX_HARD_REG)); + new_insn = MIR_new_insn (ctx, MIR_MOV, ret_reg_op, + _MIR_new_var_op (ctx, n_iregs == 0 ? AX_HARD_REG : DX_HARD_REG)); n_iregs++; } else { (*MIR_get_error_func (ctx)) (MIR_ret_error, @@ -552,25 +504,26 @@ static void machinize_call (gen_ctx_t gen_ctx, MIR_insn_t call_insn) { #endif if (arg_stack_size != 0) { /* allocate/deallocate stack for args passed on stack */ arg_stack_size = (arg_stack_size + 15) / 16 * 16; /* make it of several 16 bytes */ - new_insn = MIR_new_insn (ctx, MIR_SUB, _MIR_new_hard_reg_op (ctx, SP_HARD_REG), - _MIR_new_hard_reg_op (ctx, SP_HARD_REG), - MIR_new_int_op (ctx, arg_stack_size)); + new_insn + = MIR_new_insn (ctx, MIR_SUB, _MIR_new_var_op (ctx, SP_HARD_REG), + _MIR_new_var_op (ctx, SP_HARD_REG), MIR_new_int_op (ctx, arg_stack_size)); MIR_insert_insn_after (ctx, curr_func_item, prev_call_insn, new_insn); next_insn = DLIST_NEXT (MIR_insn_t, new_insn); create_new_bb_insns (gen_ctx, prev_call_insn, next_insn, call_insn); - new_insn = MIR_new_insn (ctx, MIR_ADD, _MIR_new_hard_reg_op (ctx, SP_HARD_REG), - _MIR_new_hard_reg_op (ctx, SP_HARD_REG), - MIR_new_int_op (ctx, arg_stack_size)); + new_insn + = MIR_new_insn (ctx, MIR_ADD, _MIR_new_var_op (ctx, SP_HARD_REG), + _MIR_new_var_op (ctx, SP_HARD_REG), MIR_new_int_op (ctx, arg_stack_size)); MIR_insert_insn_after (ctx, curr_func_item, call_insn, new_insn); next_insn = DLIST_NEXT (MIR_insn_t, new_insn); create_new_bb_insns (gen_ctx, call_insn, next_insn, call_insn); } + if (arg_stack_size != 0) prohibit_omitting_fp (gen_ctx); } -static float mir_ui2f (uint64_t i) { return i; } -static double mir_ui2d (uint64_t i) { return i; } -static long double mir_ui2ld (uint64_t i) { return i; } -static int64_t mir_ld2i (long double ld) { return ld; } +static float mir_ui2f (uint64_t i) { return (float) i; } +static double mir_ui2d (uint64_t i) { return (double) i; } +static long double mir_ui2ld (uint64_t i) { return (long double) i; } +static int64_t mir_ld2i (long double ld) { return (int64_t) ld; } static const char *UI2F = "mir.ui2f"; static const char *UI2D = "mir.ui2d"; static const char *UI2LD = "mir.ui2ld"; @@ -633,10 +586,6 @@ static void get_builtin (gen_ctx_t gen_ctx, MIR_insn_code_t code, MIR_item_t *pr } } -DEF_VARR (int); -DEF_VARR (uint8_t); -DEF_VARR (uint64_t); - struct insn_pattern_info { int start, num; }; @@ -646,6 +595,8 @@ typedef struct insn_pattern_info insn_pattern_info_t; DEF_VARR (insn_pattern_info_t); struct const_ref { + int call_p; /* flag that constant from call insn */ + MIR_item_t func_item; /* non-null for constant representing reference to func item */ size_t pc; /* where rel32 address should be in code */ size_t next_insn_disp; /* displacement of the next insn */ size_t const_num; @@ -655,22 +606,33 @@ typedef struct const_ref const_ref_t; DEF_VARR (const_ref_t); struct label_ref { - int abs_addr_p; + char abs_addr_p, short_p; /* 8 or 32-bit target */ size_t label_val_disp, next_insn_disp; - MIR_label_t label; + union { + MIR_label_t label; + void *jump_addr; /* absolute addr for BBV */ + } u; }; typedef struct label_ref label_ref_t; DEF_VARR (label_ref_t); -DEF_VARR (MIR_code_reloc_t); - #define MOVDQA_CODE 0 +struct call_ref { + MIR_item_t ref_func_item; /* func where the ref is located and referenced func */ + uint8_t *call_addr; /* addr of rex call disp32(rip) or call *disp32(rip) */ +}; + +typedef struct call_ref call_ref_t; +DEF_VARR (call_ref_t); + struct target_ctx { - unsigned char alloca_p, block_arg_func_p, leaf_p; + unsigned char alloca_p, block_arg_func_p, leaf_p, keep_fp_p; int start_sp_from_bp_offset; - VARR (int) * pattern_indexes; + MIR_insn_t temp_jump; + int temp_jump_pat_ind; + VARR (int) * pattern_indexes, *insn_pattern_indexes; VARR (insn_pattern_info_t) * insn_pattern_info; VARR (uint8_t) * result_code; VARR (uint64_t) * const_pool; @@ -678,13 +640,18 @@ struct target_ctx { VARR (label_ref_t) * label_refs; VARR (uint64_t) * abs_address_locs; VARR (MIR_code_reloc_t) * relocs; + VARR (call_ref_t) * call_refs; }; #define alloca_p gen_ctx->target_ctx->alloca_p #define block_arg_func_p gen_ctx->target_ctx->block_arg_func_p #define leaf_p gen_ctx->target_ctx->leaf_p +#define keep_fp_p gen_ctx->target_ctx->keep_fp_p #define start_sp_from_bp_offset gen_ctx->target_ctx->start_sp_from_bp_offset +#define temp_jump gen_ctx->target_ctx->temp_jump +#define temp_jump_pat_ind gen_ctx->target_ctx->temp_jump_pat_ind #define pattern_indexes gen_ctx->target_ctx->pattern_indexes +#define insn_pattern_indexes gen_ctx->target_ctx->insn_pattern_indexes #define insn_pattern_info gen_ctx->target_ctx->insn_pattern_info #define result_code gen_ctx->target_ctx->result_code #define const_pool gen_ctx->target_ctx->const_pool @@ -693,21 +660,30 @@ struct target_ctx { #define abs_address_locs gen_ctx->target_ctx->abs_address_locs #define relocs gen_ctx->target_ctx->relocs -static void prepend_insn (gen_ctx_t gen_ctx, MIR_insn_t new_insn) { - MIR_prepend_insn (gen_ctx->ctx, curr_func_item, new_insn); - create_new_bb_insns (gen_ctx, NULL, DLIST_NEXT (MIR_insn_t, new_insn), NULL); +static void prohibit_omitting_fp (gen_ctx_t gen_ctx) { keep_fp_p = TRUE; } + +static MIR_disp_t target_get_stack_slot_offset (gen_ctx_t gen_ctx, MIR_type_t type, + MIR_reg_t slot) { + /* slot is 0, 1, ... */ + if (keep_fp_p) + return -((MIR_disp_t) (slot + (type == MIR_T_LD ? 2 : 1)) * 8 + + (curr_func_item->u.func->vararg_p ? reg_save_area_size : 0)); + return (MIR_disp_t) slot * 8; } -static int target_valid_mem_offset_p (gen_ctx_t gen_ctx, MIR_type_t type, MIR_disp_t offset) { +static MIR_reg_t target_get_stack_slot_base_reg (gen_ctx_t gen_ctx) { + return keep_fp_p ? FP_HARD_REG : SP_HARD_REG; +} + +static int target_valid_mem_offset_p (gen_ctx_t gen_ctx MIR_UNUSED, MIR_type_t type MIR_UNUSED, + MIR_disp_t offset MIR_UNUSED) { return TRUE; } -#define SWAP(v1, v2, t) \ - do { \ - t = v1; \ - v1 = v2; \ - v2 = t; \ - } while (0) +static void prepend_insn (gen_ctx_t gen_ctx, MIR_insn_t new_insn) { + MIR_prepend_insn (gen_ctx->ctx, curr_func_item, new_insn); + create_new_bb_insns (gen_ctx, NULL, DLIST_NEXT (MIR_insn_t, new_insn), NULL); +} static void target_machinize (gen_ctx_t gen_ctx) { MIR_context_t ctx = gen_ctx->ctx; @@ -723,16 +699,17 @@ static void target_machinize (gen_ctx_t gen_ctx) { func = curr_func_item->u.func; block_arg_func_p = FALSE; start_sp_from_bp_offset = 8; + keep_fp_p = func->vararg_p; for (i = 0; i < func->nargs; i++) { /* Argument extensions is already done in simplify */ /* Prologue: generate arg_var = hard_reg|stack mem|stack addr ... */ type = VARR_GET (MIR_var_t, func->vars, i).type; blk_size = MIR_blk_type_p (type) ? (VARR_GET (MIR_var_t, func->vars, i).size + 7) / 8 * 8 : 0; #ifndef _WIN32 - if ((type == MIR_T_BLK + 1 && get_int_arg_reg (int_arg_num) != MIR_NON_HARD_REG - && (blk_size <= 8 || get_int_arg_reg (int_arg_num + 1) != MIR_NON_HARD_REG)) - || (type == MIR_T_BLK + 2 && get_fp_arg_reg (fp_arg_num) != MIR_NON_HARD_REG - && (blk_size <= 8 || get_fp_arg_reg (fp_arg_num + 1) != MIR_NON_HARD_REG))) { + if ((type == MIR_T_BLK + 1 && get_int_arg_reg (int_arg_num) != MIR_NON_VAR + && (blk_size <= 8 || get_int_arg_reg (int_arg_num + 1) != MIR_NON_VAR)) + || (type == MIR_T_BLK + 2 && get_fp_arg_reg (fp_arg_num) != MIR_NON_VAR + && (blk_size <= 8 || get_fp_arg_reg (fp_arg_num + 1) != MIR_NON_VAR))) { /* all is passed in gprs or fprs */ MIR_type_t mov_type = type == MIR_T_BLK + 1 ? MIR_T_I64 : MIR_T_D; MIR_insn_code_t mov_code1, mov_code2; @@ -741,20 +718,24 @@ static void target_machinize (gen_ctx_t gen_ctx) { assert (blk_size <= 16); if (blk_size > 8) { reg2 = get_arg_reg (mov_type, &int_arg_num, &fp_arg_num, &mov_code2); - new_insn = MIR_new_insn (ctx, mov_code1, MIR_new_mem_op (ctx, mov_type, 8, i + 1, 0, 1), - _MIR_new_hard_reg_op (ctx, reg2)); + new_insn = MIR_new_insn (ctx, mov_code1, + _MIR_new_var_mem_op (ctx, mov_type, 8, i + MAX_HARD_REG + 1, + MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, reg2)); prepend_insn (gen_ctx, new_insn); } - new_insn = MIR_new_insn (ctx, mov_code1, MIR_new_mem_op (ctx, mov_type, 0, i + 1, 0, 1), - _MIR_new_hard_reg_op (ctx, reg1)); + new_insn = MIR_new_insn (ctx, mov_code1, + _MIR_new_var_mem_op (ctx, mov_type, 0, i + MAX_HARD_REG + 1, + MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, reg1)); prepend_insn (gen_ctx, new_insn); - new_insn = MIR_new_insn (ctx, MIR_ALLOCA, MIR_new_reg_op (ctx, i + 1), + new_insn = MIR_new_insn (ctx, MIR_ALLOCA, _MIR_new_var_op (ctx, i + MAX_HARD_REG + 1), MIR_new_int_op (ctx, blk_size)); prepend_insn (gen_ctx, new_insn); continue; } else if ((type == MIR_T_BLK + 3 || type == MIR_T_BLK + 4) - && get_int_arg_reg (int_arg_num) != MIR_NON_HARD_REG - && get_fp_arg_reg (fp_arg_num) != MIR_NON_HARD_REG) { + && get_int_arg_reg (int_arg_num) != MIR_NON_VAR + && get_fp_arg_reg (fp_arg_num) != MIR_NON_VAR) { /* gpr and then fpr or fpr and then gpr */ MIR_type_t mov_type1 = type == MIR_T_BLK + 3 ? MIR_T_I64 : MIR_T_D; MIR_type_t mov_type2 = type == MIR_T_BLK + 3 ? MIR_T_D : MIR_T_I64; @@ -763,13 +744,17 @@ static void target_machinize (gen_ctx_t gen_ctx) { MIR_reg_t reg2 = get_arg_reg (mov_type2, &int_arg_num, &fp_arg_num, &mov_code2); assert (blk_size > 8 && blk_size <= 16); - new_insn = MIR_new_insn (ctx, mov_code2, MIR_new_mem_op (ctx, mov_type2, 8, i + 1, 0, 1), - _MIR_new_hard_reg_op (ctx, reg2)); + new_insn = MIR_new_insn (ctx, mov_code2, + _MIR_new_var_mem_op (ctx, mov_type2, 8, i + MAX_HARD_REG + 1, + MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, reg2)); prepend_insn (gen_ctx, new_insn); - new_insn = MIR_new_insn (ctx, mov_code1, MIR_new_mem_op (ctx, mov_type1, 0, i + 1, 0, 1), - _MIR_new_hard_reg_op (ctx, reg1)); + new_insn = MIR_new_insn (ctx, mov_code1, + _MIR_new_var_mem_op (ctx, mov_type1, 0, i + MAX_HARD_REG + 1, + MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, reg1)); prepend_insn (gen_ctx, new_insn); - new_insn = MIR_new_insn (ctx, MIR_ALLOCA, MIR_new_reg_op (ctx, i + 1), + new_insn = MIR_new_insn (ctx, MIR_ALLOCA, _MIR_new_var_op (ctx, i + MAX_HARD_REG + 1), MIR_new_int_op (ctx, blk_size)); prepend_insn (gen_ctx, new_insn); continue; @@ -783,53 +768,57 @@ static void target_machinize (gen_ctx_t gen_ctx) { } #endif if (blk_p) { - block_arg_func_p = TRUE; + keep_fp_p = block_arg_func_p = TRUE; #ifdef _WIN32 assert (blk_size <= 8); if ((arg_reg = get_arg_reg (MIR_T_I64, &int_arg_num, &fp_arg_num, &new_insn_code)) - == MIR_NON_HARD_REG) { - new_insn = MIR_new_insn (ctx, MIR_ADD, MIR_new_reg_op (ctx, i + 1), - _MIR_new_hard_reg_op (ctx, FP_HARD_REG), - MIR_new_int_op (ctx, mem_size + 8 /* ret */ - + start_sp_from_bp_offset)); + == MIR_NON_VAR) { + new_insn + = MIR_new_insn (ctx, MIR_ADD, _MIR_new_var_op (ctx, (MIR_reg_t) (i + MAX_HARD_REG + 1)), + _MIR_new_var_op (ctx, FP_HARD_REG), + MIR_new_int_op (ctx, mem_size + 8 /* ret */ + + start_sp_from_bp_offset)); mem_size += 8; } else { /* put reg into spill space and use its address: prepend in reverse order: */ - int disp = (mem_size + 8 /* ret */ + start_sp_from_bp_offset - spill_space_size - + 8 * get_int_arg_reg_num (arg_reg)); + int disp = (int) (mem_size + 8 /* ret */ + start_sp_from_bp_offset - spill_space_size + + 8 * get_int_arg_reg_num (arg_reg)); new_insn - = MIR_new_insn (ctx, MIR_ADD, MIR_new_reg_op (ctx, i + 1), - _MIR_new_hard_reg_op (ctx, FP_HARD_REG), MIR_new_int_op (ctx, disp)); + = MIR_new_insn (ctx, MIR_ADD, _MIR_new_var_op (ctx, (MIR_reg_t) (i + MAX_HARD_REG + 1)), + _MIR_new_var_op (ctx, FP_HARD_REG), MIR_new_int_op (ctx, disp)); prepend_insn (gen_ctx, new_insn); - arg_reg_op = _MIR_new_hard_reg_op (ctx, arg_reg); - mem_op = _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, disp, FP_HARD_REG, MIR_NON_HARD_REG, 1); + arg_reg_op = _MIR_new_var_op (ctx, arg_reg); + mem_op = _MIR_new_var_mem_op (ctx, MIR_T_I64, disp, FP_HARD_REG, MIR_NON_VAR, 1); new_insn = MIR_new_insn (ctx, MIR_MOV, mem_op, arg_reg_op); } #else - new_insn = MIR_new_insn (ctx, MIR_ADD, MIR_new_reg_op (ctx, i + 1), - _MIR_new_hard_reg_op (ctx, FP_HARD_REG), + new_insn = MIR_new_insn (ctx, MIR_ADD, _MIR_new_var_op (ctx, i + MAX_HARD_REG + 1), + _MIR_new_var_op (ctx, FP_HARD_REG), MIR_new_int_op (ctx, mem_size + 8 /* ret addr */ + start_sp_from_bp_offset)); mem_size += blk_size; #endif prepend_insn (gen_ctx, new_insn); } else if ((arg_reg = get_arg_reg (type, &int_arg_num, &fp_arg_num, &new_insn_code)) - != MIR_NON_HARD_REG) { - arg_reg_op = _MIR_new_hard_reg_op (ctx, arg_reg); - new_insn = MIR_new_insn (ctx, new_insn_code, MIR_new_reg_op (ctx, i + 1), arg_reg_op); + != MIR_NON_VAR) { + arg_reg_op = _MIR_new_var_op (ctx, arg_reg); + new_insn + = MIR_new_insn (ctx, new_insn_code, + _MIR_new_var_op (ctx, (MIR_reg_t) (i + MAX_HARD_REG + 1)), arg_reg_op); prepend_insn (gen_ctx, new_insn); } else { /* arg is on the stack */ - block_arg_func_p = TRUE; + keep_fp_p = block_arg_func_p = TRUE; mem_type = type == MIR_T_F || type == MIR_T_D || type == MIR_T_LD ? type : MIR_T_I64; new_insn_code = (type == MIR_T_F ? MIR_FMOV : type == MIR_T_D ? MIR_DMOV : type == MIR_T_LD ? MIR_LDMOV : MIR_MOV); - mem_op = _MIR_new_hard_reg_mem_op (ctx, mem_type, - mem_size + 8 /* ret */ - + start_sp_from_bp_offset, - FP_HARD_REG, MIR_NON_HARD_REG, 1); - new_insn = MIR_new_insn (ctx, new_insn_code, MIR_new_reg_op (ctx, i + 1), mem_op); + mem_op = _MIR_new_var_mem_op (ctx, mem_type, + mem_size + 8 /* ret */ + + start_sp_from_bp_offset, + FP_HARD_REG, MIR_NON_VAR, 1); + new_insn = MIR_new_insn (ctx, new_insn_code, + _MIR_new_var_op (ctx, (MIR_reg_t) (i + MAX_HARD_REG + 1)), mem_op); prepend_insn (gen_ctx, new_insn); mem_size += type == MIR_T_LD ? 16 : 8; } @@ -849,8 +838,9 @@ static void target_machinize (gen_ctx_t gen_ctx) { MIR_op_t freg_op, res_reg_op = insn->ops[0], op_reg_op = insn->ops[1], ops[4]; get_builtin (gen_ctx, code, &proto_item, &func_import_item); - assert (res_reg_op.mode == MIR_OP_REG && op_reg_op.mode == MIR_OP_REG); - freg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, curr_func_item->u.func)); + assert (res_reg_op.mode == MIR_OP_VAR && op_reg_op.mode == MIR_OP_VAR); + freg_op + = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, curr_func_item->u.func)); next_insn = new_insn = MIR_new_insn (ctx, MIR_MOV, freg_op, MIR_new_ref_op (ctx, func_import_item)); gen_add_insn_before (gen_ctx, insn, new_insn); @@ -865,16 +855,16 @@ static void target_machinize (gen_ctx_t gen_ctx) { } case MIR_VA_START: { MIR_op_t treg_op - = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, curr_func_item->u.func)); + = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, curr_func_item->u.func)); MIR_op_t va_op = insn->ops[0]; MIR_reg_t va_reg; #ifndef _WIN32 int gp_offset = 0, fp_offset = 48, mem_offset = 0; MIR_var_t var; - assert (func->vararg_p && (va_op.mode == MIR_OP_REG || va_op.mode == MIR_OP_HARD_REG)); - for (uint32_t i = 0; i < func->nargs; i++) { - var = VARR_GET (MIR_var_t, func->vars, i); + assert (func->vararg_p && va_op.mode == MIR_OP_VAR); + for (uint32_t narg = 0; narg < func->nargs; narg++) { + var = VARR_GET (MIR_var_t, func->vars, narg); if (var.type == MIR_T_F || var.type == MIR_T_D) { fp_offset += 16; if (gp_offset >= 176) mem_offset += 8; @@ -887,33 +877,38 @@ static void target_machinize (gen_ctx_t gen_ctx) { if (gp_offset >= 48) mem_offset += 8; } } - va_reg = va_op.mode == MIR_OP_REG ? va_op.u.reg : va_op.u.hard_reg; + va_reg = va_op.u.var; /* Insns can be not simplified as soon as they match a machine insn. */ /* mem32[va_reg] = gp_offset; mem32[va_reg] = fp_offset */ - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_U32, 0, va_reg, 0, 1), + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_U32, 0, va_reg, MIR_NON_VAR, 1), MIR_new_int_op (ctx, gp_offset)); next_insn = DLIST_PREV (MIR_insn_t, insn); - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_U32, 4, va_reg, 0, 1), + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_U32, 4, va_reg, MIR_NON_VAR, 1), MIR_new_int_op (ctx, fp_offset)); /* overflow_arg_area_reg: treg = start sp + 8 + mem_offset; mem64[va_reg + 8] = treg */ new_insn - = MIR_new_insn (ctx, MIR_ADD, treg_op, _MIR_new_hard_reg_op (ctx, FP_HARD_REG), + = MIR_new_insn (ctx, MIR_ADD, treg_op, _MIR_new_var_op (ctx, FP_HARD_REG), MIR_new_int_op (ctx, 8 /*ret*/ + mem_offset + start_sp_from_bp_offset)); gen_add_insn_before (gen_ctx, insn, new_insn); - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 8, va_reg, 0, 1), treg_op); + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 8, va_reg, MIR_NON_VAR, 1), treg_op); /* reg_save_area: treg = start sp - reg_save_area_size; mem64[va_reg + 16] = treg */ - new_insn = MIR_new_insn (ctx, MIR_ADD, treg_op, _MIR_new_hard_reg_op (ctx, FP_HARD_REG), + new_insn = MIR_new_insn (ctx, MIR_ADD, treg_op, _MIR_new_var_op (ctx, FP_HARD_REG), MIR_new_int_op (ctx, -reg_save_area_size)); gen_add_insn_before (gen_ctx, insn, new_insn); - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 16, va_reg, 0, 1), treg_op); + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 16, va_reg, MIR_NON_VAR, 1), treg_op); #else /* init va_list */ mem_size = 8 /*ret*/ + start_sp_from_bp_offset + func->nargs * 8; - new_insn = MIR_new_insn (ctx, MIR_ADD, treg_op, _MIR_new_hard_reg_op (ctx, FP_HARD_REG), + new_insn = MIR_new_insn (ctx, MIR_ADD, treg_op, _MIR_new_var_op (ctx, FP_HARD_REG), MIR_new_int_op (ctx, mem_size)); gen_add_insn_before (gen_ctx, insn, new_insn); - va_reg = va_op.mode == MIR_OP_REG ? va_op.u.reg : va_op.u.hard_reg; - gen_mov (gen_ctx, insn, MIR_MOV, MIR_new_mem_op (ctx, MIR_T_I64, 0, va_reg, 0, 1), treg_op); + va_reg = va_op.u.var; + gen_mov (gen_ctx, insn, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, va_reg, MIR_NON_VAR, 1), treg_op); #endif gen_delete_insn (gen_ctx, insn); break; @@ -929,16 +924,16 @@ static void target_machinize (gen_ctx_t gen_ctx) { MIR_op_t res_reg_op = insn->ops[0], va_reg_op = insn->ops[1], op3 = insn->ops[2]; get_builtin (gen_ctx, code, &proto_item, &func_import_item); - assert (res_reg_op.mode == MIR_OP_REG && va_reg_op.mode == MIR_OP_REG - && op3.mode == (code == MIR_VA_ARG ? MIR_OP_MEM : MIR_OP_REG)); - func_reg_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); - reg_op3 = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + assert (res_reg_op.mode == MIR_OP_VAR && va_reg_op.mode == MIR_OP_VAR + && op3.mode == (code == MIR_VA_ARG ? MIR_OP_VAR_MEM : MIR_OP_VAR)); + func_reg_op = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + reg_op3 = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); next_insn = new_insn = MIR_new_insn (ctx, MIR_MOV, func_reg_op, MIR_new_ref_op (ctx, func_import_item)); gen_add_insn_before (gen_ctx, insn, new_insn); if (code == MIR_VA_ARG) { - new_insn - = MIR_new_insn (ctx, MIR_MOV, reg_op3, MIR_new_int_op (ctx, (int64_t) op3.u.mem.type)); + new_insn = MIR_new_insn (ctx, MIR_MOV, reg_op3, + MIR_new_int_op (ctx, (int64_t) op3.u.var_mem.type)); op3 = reg_op3; gen_add_insn_before (gen_ctx, insn, new_insn); } @@ -953,7 +948,7 @@ static void target_machinize (gen_ctx_t gen_ctx) { gen_delete_insn (gen_ctx, insn); break; } - case MIR_ALLOCA: alloca_p = TRUE; break; + case MIR_ALLOCA: keep_fp_p = alloca_p = TRUE; break; case MIR_RET: { /* In simplify we already transformed code for one return insn and added extension in return (if any). */ @@ -965,9 +960,8 @@ static void target_machinize (gen_ctx_t gen_ctx) { "Windows x86-64 doesn't support multiple return values"); #endif assert (curr_func_item->u.func->nres == MIR_insn_nops (ctx, insn)); - for (size_t i = 0; i < curr_func_item->u.func->nres; i++) { - assert (insn->ops[i].mode == MIR_OP_REG); - res_type = curr_func_item->u.func->res_types[i]; + for (size_t nres = 0; nres < curr_func_item->u.func->nres; nres++) { + res_type = curr_func_item->u.func->res_types[nres]; if ((res_type == MIR_T_F || res_type == MIR_T_D) && n_xregs < 2) { new_insn_code = res_type == MIR_T_F ? MIR_FMOV : MIR_DMOV; ret_reg = n_xregs++ == 0 ? XMM0_HARD_REG : XMM1_HARD_REG; @@ -982,10 +976,10 @@ static void target_machinize (gen_ctx_t gen_ctx) { (*MIR_get_error_func (ctx)) (MIR_ret_error, "x86-64 can not handle this combination of return values"); } - ret_reg_op = _MIR_new_hard_reg_op (ctx, ret_reg); - new_insn = MIR_new_insn (ctx, new_insn_code, ret_reg_op, insn->ops[i]); + ret_reg_op = _MIR_new_var_op (ctx, ret_reg); + new_insn = MIR_new_insn (ctx, new_insn_code, ret_reg_op, insn->ops[nres]); gen_add_insn_before (gen_ctx, insn, new_insn); - insn->ops[i] = ret_reg_op; + insn->ops[nres] = ret_reg_op; } break; } @@ -996,19 +990,31 @@ static void target_machinize (gen_ctx_t gen_ctx) { case MIR_RSHS: case MIR_URSHS: { /* We can access only cl as shift register: */ - MIR_op_t creg_op = _MIR_new_hard_reg_op (ctx, CX_HARD_REG); + MIR_op_t creg_op = _MIR_new_var_op (ctx, CX_HARD_REG); new_insn = MIR_new_insn (ctx, MIR_MOV, creg_op, insn->ops[2]); gen_add_insn_before (gen_ctx, insn, new_insn); insn->ops[2] = creg_op; break; } + case MIR_UMULO: + case MIR_UMULOS: { + /* We can use only ax as zero and the 1st operand: */ + MIR_op_t areg_op = _MIR_new_var_op (ctx, AX_HARD_REG); + + new_insn = MIR_new_insn (ctx, MIR_MOV, areg_op, insn->ops[1]); + gen_add_insn_before (gen_ctx, insn, new_insn); + new_insn = MIR_new_insn (ctx, MIR_MOV, insn->ops[0], areg_op); + gen_add_insn_after (gen_ctx, insn, new_insn); + insn->ops[0] = insn->ops[1] = areg_op; + break; + } case MIR_DIV: case MIR_UDIV: case MIR_DIVS: case MIR_UDIVS: { /* Divide uses ax/dx as operands: */ - MIR_op_t areg_op = _MIR_new_hard_reg_op (ctx, AX_HARD_REG); + MIR_op_t areg_op = _MIR_new_var_op (ctx, AX_HARD_REG); new_insn = MIR_new_insn (ctx, MIR_MOV, areg_op, insn->ops[1]); gen_add_insn_before (gen_ctx, insn, new_insn); @@ -1022,8 +1028,8 @@ static void target_machinize (gen_ctx_t gen_ctx) { case MIR_MODS: case MIR_UMODS: { /* Divide uses ax/dx as operands: */ - MIR_op_t areg_op = _MIR_new_hard_reg_op (ctx, AX_HARD_REG); - MIR_op_t dreg_op = _MIR_new_hard_reg_op (ctx, DX_HARD_REG); + MIR_op_t areg_op = _MIR_new_var_op (ctx, AX_HARD_REG); + MIR_op_t dreg_op = _MIR_new_var_op (ctx, DX_HARD_REG); new_insn = MIR_new_insn (ctx, MIR_MOV, areg_op, insn->ops[1]); gen_add_insn_before (gen_ctx, insn, new_insn); @@ -1065,12 +1071,8 @@ static void target_machinize (gen_ctx_t gen_ctx) { case MIR_DLE: case MIR_DGT: case MIR_DGE: { - /* We can access only 4 regs in setxx -- use ax as the result: */ - MIR_op_t areg_op = _MIR_new_hard_reg_op (ctx, AX_HARD_REG); - - new_insn = MIR_new_insn (ctx, MIR_MOV, insn->ops[0], areg_op); + new_insn = MIR_new_insn (ctx, MIR_UEXT8, insn->ops[0], insn->ops[0]); gen_add_insn_after (gen_ctx, insn, new_insn); - insn->ops[0] = areg_op; /* Following conditional branches are changed to correctly process unordered numbers: */ switch (code) { case MIR_FLT: @@ -1140,16 +1142,16 @@ static void isave (gen_ctx_t gen_ctx, MIR_insn_t anchor, int disp, MIR_reg_t har MIR_context_t ctx = gen_ctx->ctx; gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, disp, SP_HARD_REG, MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (ctx, hard_reg)); + _MIR_new_var_mem_op (ctx, MIR_T_I64, disp, SP_HARD_REG, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, hard_reg)); } static void dsave (gen_ctx_t gen_ctx, MIR_insn_t anchor, int disp, MIR_reg_t hard_reg) { MIR_context_t ctx = gen_ctx->ctx; gen_mov (gen_ctx, anchor, MIR_DMOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_D, disp, SP_HARD_REG, MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (ctx, hard_reg)); + _MIR_new_var_mem_op (ctx, MIR_T_D, disp, SP_HARD_REG, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, hard_reg)); } static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_regs, @@ -1161,53 +1163,59 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg #ifdef MIR_NO_RED_ZONE_ABI MIR_op_t temp_reg_op; #endif + MIR_reg_t base_reg; int64_t bp_saved_reg_offset, offset; size_t i, service_area_size, saved_hard_regs_size, stack_slots_size, block_size; assert (curr_func_item->item_type == MIR_func_item); func = curr_func_item->u.func; for (i = saved_hard_regs_size = 0; i <= R15_HARD_REG; i++) - if (!target_call_used_hard_reg_p (i, MIR_T_UNDEF) && bitmap_bit_p (used_hard_regs, i)) + if (!target_call_used_hard_reg_p ((MIR_reg_t) i, MIR_T_UNDEF) + && bitmap_bit_p (used_hard_regs, i)) saved_hard_regs_size += 8; #ifdef _WIN32 for (; i <= XMM15_HARD_REG; i++) - if (!target_call_used_hard_reg_p (i, MIR_T_UNDEF) && bitmap_bit_p (used_hard_regs, i)) + if (!target_call_used_hard_reg_p ((MIR_reg_t) i, MIR_T_UNDEF) + && bitmap_bit_p (used_hard_regs, i)) saved_hard_regs_size += 16; #endif if (leaf_p && !alloca_p && !block_arg_func_p && saved_hard_regs_size == 0 && !func->vararg_p && stack_slots_num == 0) return; anchor = DLIST_HEAD (MIR_insn_t, func->insns); - sp_reg_op = _MIR_new_hard_reg_op (ctx, SP_HARD_REG); - fp_reg_op = _MIR_new_hard_reg_op (ctx, FP_HARD_REG); + sp_reg_op = _MIR_new_var_op (ctx, SP_HARD_REG); + if (keep_fp_p) { + fp_reg_op = _MIR_new_var_op (ctx, FP_HARD_REG); #ifdef MIR_NO_RED_ZONE_ABI - temp_reg_op = _MIR_new_hard_reg_op (ctx, TEMP_INT_HARD_REG1); + temp_reg_op = _MIR_new_var_op (ctx, TEMP_INT_HARD_REG1); #endif - /* Prologue: */ - /* Use add for matching LEA: */ + /* Prologue: */ #ifdef MIR_NO_RED_ZONE_ABI - new_insn = MIR_new_insn (ctx, MIR_ADD, temp_reg_op, sp_reg_op, MIR_new_int_op (ctx, -8)); - gen_add_insn_before (gen_ctx, anchor, new_insn); /* temp = sp - 8 */ + new_insn = MIR_new_insn (ctx, MIR_ADD, temp_reg_op, sp_reg_op, MIR_new_int_op (ctx, -8)); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* temp = sp - 8 */ #else - new_insn - = MIR_new_insn (ctx, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, -8, SP_HARD_REG, MIR_NON_HARD_REG, 1), - fp_reg_op); - gen_add_insn_before (gen_ctx, anchor, new_insn); /* -8(sp) = bp */ - /* Use add for matching LEA: */ - new_insn = MIR_new_insn (ctx, MIR_ADD, fp_reg_op, sp_reg_op, MIR_new_int_op (ctx, -8)); - gen_add_insn_before (gen_ctx, anchor, new_insn); /* bp = sp - 8 */ + new_insn = MIR_new_insn (ctx, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, -8, SP_HARD_REG, MIR_NON_VAR, 1), + fp_reg_op); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* -8(sp) = bp */ + /* Use add for matching LEA: */ + new_insn = MIR_new_insn (ctx, MIR_ADD, fp_reg_op, sp_reg_op, MIR_new_int_op (ctx, -8)); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* bp = sp - 8 */ #endif + } #ifdef _WIN32 if (func->vararg_p) { /* filling spill space */ + assert (keep_fp_p); for (i = 0, offset = 16 /* ret & bp */; i < 4; i++, offset += 8) gen_mov (gen_ctx, anchor, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, offset, FP_HARD_REG, MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (ctx, get_int_arg_reg (i))); + _MIR_new_var_mem_op (ctx, MIR_T_I64, offset, FP_HARD_REG, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, get_int_arg_reg (i))); } #endif - service_area_size = func->vararg_p ? reg_save_area_size + 8 : 8; + service_area_size = func->vararg_p ? reg_save_area_size : 0; + if (!func->jret_p) service_area_size += 8; /* return address */ stack_slots_size = stack_slots_num * 8; + if (!keep_fp_p) stack_slots_size = (stack_slots_size + 15) / 16 * 16; /* stack slots, and saved regs as multiple of 16 bytes: */ block_size = (stack_slots_size + saved_hard_regs_size + 15) / 16 * 16; new_insn = MIR_new_insn (ctx, MIR_SUB, sp_reg_op, sp_reg_op, @@ -1215,14 +1223,16 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg gen_add_insn_before (gen_ctx, anchor, new_insn); /* sp -= block size + service_area_size */ bp_saved_reg_offset = block_size; #ifdef MIR_NO_RED_ZONE_ABI - new_insn - = MIR_new_insn (ctx, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, block_size + service_area_size - 8, - SP_HARD_REG, MIR_NON_HARD_REG, 1), - fp_reg_op); - gen_add_insn_before (gen_ctx, anchor, new_insn); /* -8(old sp) = bp */ - new_insn = MIR_new_insn (ctx, MIR_MOV, fp_reg_op, temp_reg_op); - gen_add_insn_before (gen_ctx, anchor, new_insn); /* bp = temp */ + if (keep_fp_p) { + new_insn + = MIR_new_insn (ctx, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, block_size + service_area_size - 8, + SP_HARD_REG, MIR_NON_VAR, 1), + fp_reg_op); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* -8(old sp) = bp */ + new_insn = MIR_new_insn (ctx, MIR_MOV, fp_reg_op, temp_reg_op); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* bp = temp */ + } #endif #ifndef _WIN32 if (func->vararg_p) { @@ -1245,73 +1255,84 @@ static void target_make_prolog_epilog (gen_ctx_t gen_ctx, bitmap_t used_hard_reg } #endif /* Saving callee saved hard registers: */ - offset = -bp_saved_reg_offset; + offset = keep_fp_p ? -bp_saved_reg_offset : (int64_t) stack_slots_size; + base_reg = keep_fp_p ? FP_HARD_REG : SP_HARD_REG; #ifdef _WIN32 for (i = XMM0_HARD_REG; i <= XMM15_HARD_REG; i++) - if (!target_call_used_hard_reg_p (i, MIR_T_UNDEF) && bitmap_bit_p (used_hard_regs, i)) { + if (!target_call_used_hard_reg_p ((MIR_reg_t) i, MIR_T_UNDEF) + && bitmap_bit_p (used_hard_regs, i)) { new_insn = _MIR_new_unspec_insn (ctx, 3, MIR_new_int_op (ctx, MOVDQA_CODE), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_D, offset, FP_HARD_REG, - MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (ctx, i)); - gen_add_insn_before (gen_ctx, anchor, new_insn); /* disp(sp) = saved hard reg */ + _MIR_new_var_mem_op (ctx, MIR_T_D, offset, base_reg, + MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, (MIR_reg_t) i)); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* disp(bp|sp) = saved hard reg */ offset += 16; } #endif for (i = 0; i <= R15_HARD_REG; i++) - if (!target_call_used_hard_reg_p (i, MIR_T_UNDEF) && bitmap_bit_p (used_hard_regs, i)) { - new_insn = MIR_new_insn (ctx, MIR_MOV, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, offset, FP_HARD_REG, - MIR_NON_HARD_REG, 1), - _MIR_new_hard_reg_op (ctx, i)); - gen_add_insn_before (gen_ctx, anchor, new_insn); /* disp(sp) = saved hard reg */ + if (!target_call_used_hard_reg_p ((MIR_reg_t) i, MIR_T_UNDEF) + && bitmap_bit_p (used_hard_regs, i)) { + new_insn + = MIR_new_insn (ctx, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, offset, base_reg, MIR_NON_VAR, 1), + _MIR_new_var_op (ctx, (MIR_reg_t) i)); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* disp(bp|sp) = saved hard reg */ offset += 8; } /* Epilogue: */ - anchor = DLIST_TAIL (MIR_insn_t, func->insns); - /* It might be infinite loop after CCP with dead code elimination: */ - if (anchor->code == MIR_JMP) return; + for (anchor = DLIST_TAIL (MIR_insn_t, func->insns); anchor != NULL; + anchor = DLIST_PREV (MIR_insn_t, anchor)) + if (anchor->code == MIR_RET || anchor->code == MIR_JRET) break; + if (anchor == NULL) return; /* Restoring hard registers: */ - offset = -bp_saved_reg_offset; + offset = keep_fp_p ? -bp_saved_reg_offset : (int64_t) stack_slots_size; #ifdef _WIN32 for (i = XMM0_HARD_REG; i <= XMM15_HARD_REG; i++) - if (!target_call_used_hard_reg_p (i, MIR_T_UNDEF) && bitmap_bit_p (used_hard_regs, i)) { + if (!target_call_used_hard_reg_p ((MIR_reg_t) i, MIR_T_UNDEF) + && bitmap_bit_p (used_hard_regs, i)) { new_insn = _MIR_new_unspec_insn (ctx, 3, MIR_new_int_op (ctx, MOVDQA_CODE), - _MIR_new_hard_reg_op (ctx, i), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_D, offset, FP_HARD_REG, - MIR_NON_HARD_REG, 1)); - gen_add_insn_before (gen_ctx, anchor, new_insn); /* hard reg = disp(sp) */ + _MIR_new_var_op (ctx, (MIR_reg_t) i), + _MIR_new_var_mem_op (ctx, MIR_T_D, offset, base_reg, + MIR_NON_VAR, 1)); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* hard reg = disp(bp|sp) */ offset += 16; } #endif for (i = 0; i <= R15_HARD_REG; i++) - if (!target_call_used_hard_reg_p (i, MIR_T_UNDEF) && bitmap_bit_p (used_hard_regs, i)) { - new_insn = MIR_new_insn (ctx, MIR_MOV, _MIR_new_hard_reg_op (ctx, i), - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, offset, FP_HARD_REG, - MIR_NON_HARD_REG, 1)); - gen_add_insn_before (gen_ctx, anchor, new_insn); /* hard reg = disp(sp) */ + if (!target_call_used_hard_reg_p ((MIR_reg_t) i, MIR_T_UNDEF) + && bitmap_bit_p (used_hard_regs, i)) { + new_insn + = MIR_new_insn (ctx, MIR_MOV, _MIR_new_var_op (ctx, (MIR_reg_t) i), + _MIR_new_var_mem_op (ctx, MIR_T_I64, offset, base_reg, MIR_NON_VAR, 1)); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* hard reg = disp(bp|sp) */ offset += 8; } + if (!keep_fp_p) { + new_insn = MIR_new_insn (ctx, MIR_ADD, sp_reg_op, sp_reg_op, + MIR_new_int_op (ctx, block_size + service_area_size)); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* sp += block size + service_area_size */ + } else { #ifdef MIR_NO_RED_ZONE_ABI - new_insn = MIR_new_insn (ctx, MIR_MOV, temp_reg_op, fp_reg_op); - gen_add_insn_before (gen_ctx, anchor, new_insn); /* temp = bp */ - new_insn = MIR_new_insn (ctx, MIR_MOV, fp_reg_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 0, TEMP_INT_HARD_REG1, - MIR_NON_HARD_REG, 1)); - gen_add_insn_before (gen_ctx, anchor, new_insn); /* bp = 0(bp) */ - new_insn = MIR_new_insn (ctx, MIR_ADD, sp_reg_op, temp_reg_op, MIR_new_int_op (ctx, 8)); - gen_add_insn_before (gen_ctx, anchor, new_insn); /* sp = temp + 8 */ + new_insn = MIR_new_insn (ctx, MIR_MOV, temp_reg_op, fp_reg_op); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* temp = bp */ + new_insn + = MIR_new_insn (ctx, MIR_MOV, fp_reg_op, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, TEMP_INT_HARD_REG1, MIR_NON_VAR, 1)); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* bp = 0(bp) */ + new_insn = MIR_new_insn (ctx, MIR_ADD, sp_reg_op, temp_reg_op, MIR_new_int_op (ctx, 8)); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* sp = temp + 8 */ #else - new_insn = MIR_new_insn (ctx, MIR_ADD, sp_reg_op, fp_reg_op, MIR_new_int_op (ctx, 8)); - gen_add_insn_before (gen_ctx, anchor, new_insn); /* sp = bp + 8 */ - new_insn = MIR_new_insn (ctx, MIR_MOV, fp_reg_op, - _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, -8, SP_HARD_REG, - MIR_NON_HARD_REG, 1)); - gen_add_insn_before (gen_ctx, anchor, new_insn); /* bp = -8(sp) */ + new_insn = MIR_new_insn (ctx, MIR_ADD, sp_reg_op, fp_reg_op, MIR_new_int_op (ctx, 8)); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* sp = bp + 8 */ + new_insn = MIR_new_insn (ctx, MIR_MOV, fp_reg_op, + _MIR_new_var_mem_op (ctx, MIR_T_I64, -8, SP_HARD_REG, MIR_NON_VAR, 1)); + gen_add_insn_before (gen_ctx, anchor, new_insn); /* bp = -8(sp) */ #endif + } } struct pattern { - MIR_insn_code_t code; + const MIR_insn_code_t code; /* Pattern elements: blank - ignore X - match everything @@ -1320,8 +1341,7 @@ struct pattern { t - ax, cx, dx, or bx register h[0-31] - hard register with given number z - operand is zero - i[0-3] - immediate of size 8,16,32,64-bits - p[0-3] - reference + i[0-3] - immediate (including refs) of size 8,16,32,64-bits s - immediate 1, 2, 4, or 8 (scale) c - immediate integer m[0-3] - int (signed or unsigned) type memory of size 8,16,32,64-bits @@ -1330,7 +1350,8 @@ struct pattern { mf - memory of float md - memory of double mld - memory of long double - l - label which can be present by 32-bit + L - label which can be present by 32-bit + l - label which can be present by 8-bit [0-9] - an operand matching n-th operand (n should be less than given operand number) Remember we have no float or (long) double immediate at this stage. They are represented by @@ -1345,6 +1366,7 @@ struct pattern { [0-9A-F]+ pairs of hexidecimal digits opcode r[0-2] = n-th operand in ModRM:reg R[0-2] = n-th operand in ModRM:rm with mod == 3 + S[0-2] = n-th operand in ModRM:rm with mod == 3, 8-bit registers m[0-2] = n-th operand is mem mt = temp memory in red zone (-16(sp)) mT = switch table memory (h11,r,8) @@ -1354,20 +1376,25 @@ struct pattern { i[0-2] - n-th operand in byte immediate (should be imm of type i8) I[0-2] - n-th operand in 4 byte immediate (should be imm of type i32) J[0-2] - n-th operand in 8 byte immediate - P[0-2] - n-th operand in 8 byte address - T - absolute 8-byte switch table address - l[0-2] - n-th operand-label in 32-bit + P[0-2] - n-th operand is 64-bit call address in memory pool + T - relative switch table address + q - mod==0, rm==5 (ip-relative addressing) + L[0-2] - n-th operand-label in 32-bit + l[0-2] - n-th operand-label in 8-bit /[0-7] - opmod with given value (reg of MOD-RM) +[0-2] - lower 3-bit part of opcode used for n-th reg operand + +h - lower 3-bit part of opcode used for 0-15 hard reg operand c - address of 32-bit or 64-bit constant in memory pool (we keep always 64-bit in memory pool. x86_64 is LE) h - hardware register with given number in reg of ModRM:reg; one bit of 8-15 in REX.R H - hardware register with given number in rm of MOD-RM with and mod=3 - (register); one bit of 8-15 in REX.B v - 8-bit immediate with given hex value V - - 32-bit immediate with given hex value + (register); one bit of 8-15 in REX.B + v - 8-bit immediate with given hex value + V - 32-bit immediate with given hex value */ const char *replacement; + int max_insn_size; }; // make imm always second operand (simplify for cmp and commutative op) @@ -1375,54 +1402,56 @@ struct pattern { // but not for FP (NAN) (simplify) // for FP cmp first operand should be always reg (machinize) -#define IOP0(ICODE, SUFF, RRM_CODE, MR_CODE, RMI8_CODE, RMI32_CODE) \ - {ICODE##SUFF, "r 0 r", "X " RRM_CODE " r0 R2"}, /* op r0,r2*/ \ - {ICODE##SUFF, "r 0 m3", "X " RRM_CODE " r0 m2"}, /* op r0,m2*/ \ - {ICODE##SUFF, "m3 0 r", "X " MR_CODE " r2 m0"}, /* op m0,r2*/ \ - {ICODE##SUFF, "r 0 i0", "X " RMI8_CODE " R0 i2"}, /* op r0,i2*/ \ - {ICODE##SUFF, "m3 0 i0", "X " RMI8_CODE " m0 i2"}, /* op m0,i2*/ \ - {ICODE##SUFF, "r 0 i2", "X " RMI32_CODE " R0 I2"}, /* op r0,i2*/ \ - {ICODE##SUFF, "m3 0 i2", "X " RMI32_CODE " m0 I2"}, /* op m0,i2*/ - -#define IOP0S(ICODE, SUFF, RRM_CODE, MR_CODE, RMI8_CODE, RMI32_CODE) \ - {ICODE##SUFF, "r 0 r", "Y " RRM_CODE " r0 R2"}, /* op r0,r2*/ \ - {ICODE##SUFF, "r 0 m2", "Y " RRM_CODE " r0 m2"}, /* op r0,m2*/ \ - {ICODE##SUFF, "m2 0 r", "Y " MR_CODE " r2 m0"}, /* op m0,r2*/ \ - {ICODE##SUFF, "r 0 i0", "Y " RMI8_CODE " R0 i2"}, /* op r0,i2*/ \ - {ICODE##SUFF, "m2 0 i0", "Y " RMI8_CODE " m0 i2"}, /* op m0,i2*/ \ - {ICODE##SUFF, "r 0 i2", "Y " RMI32_CODE " R0 I2"}, /* op r0,i2*/ \ - {ICODE##SUFF, "m2 0 i2", "Y " RMI32_CODE " m0 I2"}, /* op m0,i2*/ +#define IOP0(ICODE, SUFF, RRM_CODE, MR_CODE, RMI8_CODE, RMI32_CODE) \ + {ICODE##SUFF, "r 0 r", "X " RRM_CODE " r0 R2", 0}, /* op r0,r2*/ \ + {ICODE##SUFF, "r 0 m3", "X " RRM_CODE " r0 m2", 0}, /* op r0,m2*/ \ + {ICODE##SUFF, "m3 0 r", "X " MR_CODE " r2 m0", 0}, /* op m0,r2*/ \ + {ICODE##SUFF, "r 0 i0", "X " RMI8_CODE " R0 i2", 0}, /* op r0,i2*/ \ + {ICODE##SUFF, "m3 0 i0", "X " RMI8_CODE " m0 i2", 0}, /* op m0,i2*/ \ + {ICODE##SUFF, "r 0 i2", "X " RMI32_CODE " R0 I2", 0}, /* op r0,i2*/ \ + {ICODE##SUFF, "m3 0 i2", "X " RMI32_CODE " m0 I2", 0}, /* op m0,i2*/ + +#define IOP0S(ICODE, SUFF, RRM_CODE, MR_CODE, RMI8_CODE, RMI32_CODE) \ + {ICODE##SUFF, "r 0 r", "Y " RRM_CODE " r0 R2", 0}, /* op r0,r2*/ \ + {ICODE##SUFF, "r 0 m2", "Y " RRM_CODE " r0 m2", 0}, /* op r0,m2*/ \ + {ICODE##SUFF, "m2 0 r", "Y " MR_CODE " r2 m0", 0}, /* op m0,r2*/ \ + {ICODE##SUFF, "r 0 i0", "Y " RMI8_CODE " R0 i2", 0}, /* op r0,i2*/ \ + {ICODE##SUFF, "m2 0 i0", "Y " RMI8_CODE " m0 i2", 0}, /* op m0,i2*/ \ + {ICODE##SUFF, "r 0 i2", "Y " RMI32_CODE " R0 I2", 0}, /* op r0,i2*/ \ + {ICODE##SUFF, "m2 0 i2", "Y " RMI32_CODE " m0 I2", 0}, /* op m0,i2*/ #define IOP(ICODE, RRM_CODE, MR_CODE, RMI8_CODE, RMI32_CODE) \ IOP0 (ICODE, , RRM_CODE, MR_CODE, RMI8_CODE, RMI32_CODE) \ IOP0S (ICODE, S, RRM_CODE, MR_CODE, RMI8_CODE, RMI32_CODE) -#define FOP(ICODE, OP_CODE) {ICODE, "r 0 r", OP_CODE " r0 R2"}, {ICODE, "r 0 mf", OP_CODE " r0 m2"}, +#define FOP(ICODE, OP_CODE) \ + {ICODE, "r 0 r", OP_CODE " r0 R2", 0}, {ICODE, "r 0 mf", OP_CODE " r0 m2", 0}, -#define DOP(ICODE, OP_CODE) {ICODE, "r 0 r", OP_CODE " r0 R2"}, {ICODE, "r 0 md", OP_CODE " r0 m2"}, +#define DOP(ICODE, OP_CODE) \ + {ICODE, "r 0 r", OP_CODE " r0 R2", 0}, {ICODE, "r 0 md", OP_CODE " r0 m2", 0}, #define LDOP(ICODE, OP_CODE) \ /* fld m1;fld m2;op;fstp m0: */ \ - {ICODE, "mld mld mld", "DB /5 m1; DB /5 m2; " OP_CODE "; DB /7 m0"}, + {ICODE, "mld mld mld", "DB /5 m1; DB /5 m2; " OP_CODE "; DB /7 m0", 0}, -#define SHOP0(ICODE, SUFF, PREF, CL_OP_CODE, I8_OP_CODE) \ - {ICODE##SUFF, "r 0 h1", #PREF " " CL_OP_CODE " R0"}, /* sh r0,cl */ \ - {ICODE##SUFF, "m3 0 h1", #PREF " " CL_OP_CODE " m0"}, /* sh m0,cl */ \ - {ICODE##SUFF, "r 0 i0", #PREF " " I8_OP_CODE " R0 i2"}, /* sh r0,i2 */ \ - {ICODE##SUFF, "m3 0 i0", #PREF " " I8_OP_CODE " m0 i2"}, /* sh m0,i2 */ +#define SHOP0(ICODE, SUFF, PREF, CL_OP_CODE, I8_OP_CODE) \ + {ICODE##SUFF, "r 0 h1", #PREF " " CL_OP_CODE " R0", 0}, /* sh r0,cl */ \ + {ICODE##SUFF, "m3 0 h1", #PREF " " CL_OP_CODE " m0", 0}, /* sh m0,cl */ \ + {ICODE##SUFF, "r 0 i0", #PREF " " I8_OP_CODE " R0 i2", 0}, /* sh r0,i2 */ \ + {ICODE##SUFF, "m3 0 i0", #PREF " " I8_OP_CODE " m0 i2", 0}, /* sh m0,i2 */ #define SHOP(ICODE, CL_OP_CODE, I8_OP_CODE) \ SHOP0 (ICODE, , X, CL_OP_CODE, I8_OP_CODE) \ SHOP0 (ICODE, S, Y, CL_OP_CODE, I8_OP_CODE) -/* cmp ...; setx r0; movzbl r0,r0: */ -#define CMP0(ICODE, SUFF, PREF, SETX) \ - {ICODE##SUFF, "t r r", #PREF " 3B r1 R2;" SETX " R0;X 0F B6 r0 R0"}, /* cmp r1,r2;...*/ \ - {ICODE##SUFF, "t r m3", #PREF " 3B r1 m2;" SETX " R0;X 0F B6 r0 R0"}, /* cmp r1,m2;...*/ \ - {ICODE##SUFF, "t r i0", #PREF " 83 /7 R1 i2;" SETX " R0;X 0F B6 r0 R0"}, /* cmp r1,i2;...*/ \ - {ICODE##SUFF, "t r i2", #PREF " 81 /7 R1 I2;" SETX " R0;X 0F B6 r0 R0"}, /* cmp r1,i2;...*/ \ - {ICODE##SUFF, "t m3 i0", #PREF " 83 /7 m1 i2;" SETX " R0;X 0F B6 r0 R0"}, /* cmp m1,i2;...*/ \ - {ICODE##SUFF, "t m3 i2", #PREF " 81 /7 m1 I2;" SETX " R0;X 0F B6 r0 R0"}, /* cmp m1,i2;...*/ +/* cmp ...; setx r0: */ +#define CMP0(ICODE, SUFF, PREF, SETX) \ + {ICODE##SUFF, "r r r", #PREF " 3B r1 R2; Y " SETX " S0", 0}, /* cmp r1,r2;...*/ \ + {ICODE##SUFF, "r r m3", #PREF " 3B r1 m2; Y " SETX " S0", 0}, /* cmp r1,m2;...*/ \ + {ICODE##SUFF, "r r i0", #PREF " 83 /7 R1 i2; Y " SETX " S0", 0}, /* cmp r1,i2;...*/ \ + {ICODE##SUFF, "r r i2", #PREF " 81 /7 R1 I2; Y " SETX " S0", 0}, /* cmp r1,i2;...*/ \ + {ICODE##SUFF, "r m3 i0", #PREF " 83 /7 m1 i2; Y " SETX " S0", 0}, /* cmp m1,i2;...*/ \ + {ICODE##SUFF, "r m3 i2", #PREF " 81 /7 m1 I2; Y " SETX " S0", 0}, /* cmp m1,i2;...*/ #define CMP(ICODE, SET_OPCODE) \ CMP0 (ICODE, , X, SET_OPCODE) \ @@ -1431,210 +1460,265 @@ struct pattern { #define FEQ(ICODE, V, SET_OPCODE) \ /*xor %eax,%eax;ucomiss r1,{r,m2};mov V,%edx;set[n]p r0;cmovne %rdx,%rax; mov %rax,r0: */ \ {ICODE, "r r r", \ - "33 h0 H0; 0F 2E r1 R2; BA " V "; " SET_OPCODE " H0; X 0F 45 h0 H2; X 8B r0 H0"}, \ + "33 h0 H0; 0F 2E r1 R2; BA " V "; " SET_OPCODE " H0; X 0F 45 h0 H2; X 8B r0 H0", 0}, \ {ICODE, "r r md", \ - "33 h0 H0; 0F 2E r1 m2; BA " V "; " SET_OPCODE " H0; X 0F 45 h0 H2; X 8B r0 H0"}, + "33 h0 H0; 0F 2E r1 m2; BA " V "; " SET_OPCODE " H0; X 0F 45 h0 H2; X 8B r0 H0", 0}, #define DEQ(ICODE, V, SET_OPCODE) \ /*xor %eax,%eax;ucomisd r1,{r,m2};mov V,%edx;set[n]p r0;cmovne %rdx,%rax; mov %rax,r0: */ \ {ICODE, "r r r", \ - "33 h0 H0; 66 Y 0F 2E r1 R2; BA " V "; " SET_OPCODE " H0; X 0F 45 h0 H2; X 8B r0 H0"}, \ + "33 h0 H0; 66 Y 0F 2E r1 R2; BA " V "; " SET_OPCODE " H0; X 0F 45 h0 H2; X 8B r0 H0", 0}, \ {ICODE, "r r md", \ - "33 h0 H0; 66 Y 0F 2E r1 m2; BA " V "; " SET_OPCODE " H0; X 0F 45 h0 H2; X 8B r0 H0"}, + "33 h0 H0; 66 Y 0F 2E r1 m2; BA " V "; " SET_OPCODE " H0; X 0F 45 h0 H2; X 8B r0 H0", 0}, #define LDEQ(ICODE, V, SET_OPCODE) \ /*fld m2;fld m1;xor %eax,%eax;fucomip st,st(1);fstp %st;mov V,%edx; \ set[n]p r0;cmovne %rdx,%rax;mov %rax,r0: */ \ {ICODE, "r mld mld", \ "DB /5 m2; DB /5 m1; 33 h0 H0; DF E9; DD D8; BA " V "; " SET_OPCODE \ - " H0; X 0F 45 h0 H2; X 8B r0 H0"}, - -#define FCMP(ICODE, SET_OPCODE) \ - /*xor %eax,%eax;ucomiss r1,r2;setx az; mov %rax,r0: */ \ - {ICODE, "r r r", "33 h0 H0; Y 0F 2E r1 R2; " SET_OPCODE " H0;X 8B r0 H0"}, \ - {ICODE, "r r mf", "33 h0 H0; Y 0F 2E r1 m2; " SET_OPCODE " H0;X 8B r0 H0"}, - -#define DCMP(ICODE, SET_OPCODE) \ - /*xor %eax,%eax;ucomisd r1,r2;setx az; mov %rax,r0: */ \ - {ICODE, "r r r", "33 h0 H0; 66 Y 0F 2E r1 R2; " SET_OPCODE " H0;X 8B r0 H0"}, \ - {ICODE, "r r md", "33 h0 H0; 66 Y 0F 2E r1 m2; " SET_OPCODE " H0;X 8B r0 H0"}, - -#define LDCMP(ICODE, SET_OPCODE) \ - /*fld m2;fld m1;xor %eax,%eax;fcomip st,st(1);fstp %st;setx az; mov %rax,r0: */ \ - {ICODE, "r mld mld", "DB /5 m2; DB /5 m1; 33 h0 H0; DF F1; DD D8; " SET_OPCODE " H0;X 8B r0 H0"}, - -#define BR0(ICODE, SUFF, PREF, LONG_JMP_OPCODE) \ - {ICODE##SUFF, "l r", #PREF " 83 /7 R1 v0;" LONG_JMP_OPCODE " l0"}, /*cmp r0,$0;jxx rel32*/ \ - {ICODE##SUFF, "l m3", #PREF " 83 /7 m1 v0;" LONG_JMP_OPCODE " l0"}, /*cmp m0,$0;jxx rel8*/ - -#define BR(ICODE, LONG_JMP_OPCODE) \ - BR0 (ICODE, , X, LONG_JMP_OPCODE) \ - BR0 (ICODE, S, Y, LONG_JMP_OPCODE) - -#define BCMP0(ICODE, SUFF, PREF, LONG_JMP_OPCODE) \ - {ICODE##SUFF, "l r r", #PREF " 3B r1 R2;" LONG_JMP_OPCODE " l0"}, /*cmp r0,r1;jxx rel32*/ \ - {ICODE##SUFF, "l r m3", #PREF " 3B r1 m2;" LONG_JMP_OPCODE " l0"}, /*cmp r0,m1;jxx rel8*/ \ - {ICODE##SUFF, "l r i0", #PREF " 83 /7 R1 i2;" LONG_JMP_OPCODE " l0"}, /*cmp r0,i1;jxx rel32*/ \ - {ICODE##SUFF, "l r i2", #PREF " 81 /7 R1 I2;" LONG_JMP_OPCODE " l0"}, /*cmp r0,i1;jxx rel32*/ \ - {ICODE##SUFF, "l m3 i0", #PREF " 83 /7 m1 i2;" LONG_JMP_OPCODE " l0"}, /*cmp m0,i1;jxx rel32*/ \ - {ICODE##SUFF, "l m3 i2", #PREF " 81 /7 m1 I2;" LONG_JMP_OPCODE " l0"}, /*cmp m0,i1;jxx rel32*/ + " H0; X 0F 45 h0 H2; X 8B r0 H0", \ + 0}, + +#define FCMP(ICODE, SET_OPCODE) \ + /*xor %eax,%eax;ucomiss r1,r2;setx az; mov %rax,r0: */ \ + {ICODE, "r r r", "33 h0 H0; Y 0F 2E r1 R2; " SET_OPCODE " H0;X 8B r0 H0", 0}, \ + {ICODE, "r r mf", "33 h0 H0; Y 0F 2E r1 m2; " SET_OPCODE " H0;X 8B r0 H0", 0}, + +#define DCMP(ICODE, SET_OPCODE) \ + /*xor %eax,%eax;ucomisd r1,r2;setx az; mov %rax,r0: */ \ + {ICODE, "r r r", "33 h0 H0; 66 Y 0F 2E r1 R2; " SET_OPCODE " H0;X 8B r0 H0", 0}, \ + {ICODE, "r r md", "33 h0 H0; 66 Y 0F 2E r1 m2; " SET_OPCODE " H0;X 8B r0 H0", 0}, + +#define LDCMP(ICODE, SET_OPCODE) \ + /*fld m2;fld m1;xor %eax,%eax;fcomip st,st(1);fstp %st;setx az; mov %rax,r0: */ \ + {ICODE, "r mld mld", "DB /5 m2; DB /5 m1; 33 h0 H0; DF F1; DD D8; " SET_OPCODE " H0;X 8B r0 H0", \ + 0}, + +#define BRS0(ICODE, SUFF, PREF, SHORT_JMP_OPCODE) \ + {ICODE##SUFF, "l r", #PREF " 85 r1 R1;" SHORT_JMP_OPCODE " l0", 0}, /*test r0,r0;jxx rel8*/ \ + {ICODE##SUFF, "l m3", #PREF " 83 /7 m1 v0;" SHORT_JMP_OPCODE " l0", 0}, /*cmp m0,$0;jxx rel8*/ + +#define BRS1(ICODE, SUFF, PREF, SHORT_JMP_OPCODE) \ + {ICODE##SUFF, "l m0", #PREF " 80 /7 m1 v0;" SHORT_JMP_OPCODE " l0", 0}, /*cmpb m0,$0;jxx rel8*/ \ + {ICODE##SUFF, "l m1", "66 " #PREF " 83 /7 m1 v0;" SHORT_JMP_OPCODE " l0", 0}, /*cmpw ...*/ \ + {ICODE##SUFF, "l m2", #PREF " 83 /7 m1 v0;" SHORT_JMP_OPCODE " l0", 0}, /*cmpl ...*/ + +#define BRS(ICODE, SHORT_JMP_OPCODE) \ + BRS0 (ICODE, , X, SHORT_JMP_OPCODE) \ + BRS0 (ICODE, S, Y, SHORT_JMP_OPCODE) \ + BRS1 (ICODE, , Y, SHORT_JMP_OPCODE) \ + BRS1 (ICODE, S, Y, SHORT_JMP_OPCODE) + +#define BR0(ICODE, SUFF, PREF, LONG_JMP_OPCODE) \ + {ICODE##SUFF, "L r", #PREF " 85 r1 R1;" LONG_JMP_OPCODE " L0", 0}, /*test r0,r0;jxx rel32*/ \ + {ICODE##SUFF, "L m3", #PREF " 83 /7 m1 v0;" LONG_JMP_OPCODE " L0", 0}, /*cmp m0,$0;jxx rel32*/ + +#define BR1(ICODE, SUFF, PREF, LONG_JMP_OPCODE) \ + {ICODE##SUFF, "L m0", #PREF " 80 /7 m1 v0;" LONG_JMP_OPCODE " L0", 0}, /*cmpb m0,$0;jxx rel32*/ \ + {ICODE##SUFF, "L m1", "66 " #PREF " 83 /7 m1 v0;" LONG_JMP_OPCODE " L0", 0}, /*cmpw ...*/ \ + {ICODE##SUFF, "L m2", #PREF " 83 /7 m1 v0;" LONG_JMP_OPCODE " L0", 0}, /*cmpl ...*/ + +#define BR(ICODE, LONG_JMP_OPCODE) \ + BR0 (ICODE, , X, LONG_JMP_OPCODE) \ + BR0 (ICODE, S, Y, LONG_JMP_OPCODE) \ + BR1 (ICODE, , Y, LONG_JMP_OPCODE) \ + BR1 (ICODE, S, Y, LONG_JMP_OPCODE) + +#define BCMPS0(ICODE, SUFF, PREF, SHORT_JMP_OPCODE) \ + {ICODE##SUFF, "l r r", #PREF " 3B r1 R2;" SHORT_JMP_OPCODE " l0", 0}, /*cmp r0,r1;jxx rel8*/ \ + {ICODE##SUFF, "l r m3", #PREF " 3B r1 m2;" SHORT_JMP_OPCODE " l0", 0}, /*cmp r0,m1;...*/ \ + {ICODE##SUFF, "l r i0", #PREF " 83 /7 R1 i2;" SHORT_JMP_OPCODE " l0", 0}, /*cmp r0,i1;...*/ \ + {ICODE##SUFF, "l r i2", #PREF " 81 /7 R1 I2;" SHORT_JMP_OPCODE " l0", 0}, /*cmp r0,i1;...*/ \ + {ICODE##SUFF, "l m3 i0", #PREF " 83 /7 m1 i2;" SHORT_JMP_OPCODE " l0", 0}, /*cmp m0,i1;...*/ \ + {ICODE##SUFF, "l m3 i2", #PREF " 81 /7 m1 I2;" SHORT_JMP_OPCODE " l0", 0}, /*cmp m0,i1;...*/ + +#define BCMPS(ICODE, SHORT_JMP_OPCODE) \ + BCMPS0 (ICODE, , X, SHORT_JMP_OPCODE) \ + BCMPS0 (ICODE, S, Y, SHORT_JMP_OPCODE) + +#define BCMP0(ICODE, SUFF, PREF, LONG_JMP_OPCODE) \ + {ICODE##SUFF, "L r r", #PREF " 3B r1 R2;" LONG_JMP_OPCODE " L0", 0}, /*cmp r0,r1;jxx rel32*/ \ + {ICODE##SUFF, "L r m3", #PREF " 3B r1 m2;" LONG_JMP_OPCODE " L0", 0}, /*cmp r0,m1;...*/ \ + {ICODE##SUFF, "L r i0", #PREF " 83 /7 R1 i2;" LONG_JMP_OPCODE " L0", 0}, /*cmp r0,i1;...*/ \ + {ICODE##SUFF, "L r i2", #PREF " 81 /7 R1 I2;" LONG_JMP_OPCODE " L0", 0}, /*cmp r0,i1;...*/ \ + {ICODE##SUFF, "L m3 i0", #PREF " 83 /7 m1 i2;" LONG_JMP_OPCODE " L0", 0}, /*cmp m0,i1;...*/ \ + {ICODE##SUFF, "L m3 i2", #PREF " 81 /7 m1 I2;" LONG_JMP_OPCODE " L0", 0}, /*cmp m0,i1;...*/ #define BCMP(ICODE, LONG_JMP_OPCODE) \ BCMP0 (ICODE, , X, LONG_JMP_OPCODE) \ BCMP0 (ICODE, S, Y, LONG_JMP_OPCODE) -#define FBCMP(ICODE, LONG_JMP_OPCODE) \ - {ICODE, "l r r", "Y 0F 2E r1 R2;" LONG_JMP_OPCODE " l0"}, /* ucomiss r0,r1;jxx rel32*/ +#define FBCMPS(ICODE, SHORT_JMP_OPCODE) \ + {ICODE, "l r r", "Y 0F 2E r1 R2;" SHORT_JMP_OPCODE " l0", 0}, /* ucomiss r0,r1;jxx rel8*/ +#define DBCMPS(ICODE, SHORT_JMP_OPCODE) \ + {ICODE, "l r r", "66 Y 0F 2E r1 R2;" SHORT_JMP_OPCODE " l0", 0}, /* ucomisd r0,r1;jxx rel8*/ +#define LDBCMPS(ICODE, SHORT_JMP_OPCODE) \ + /* fld m2;fld m1; fcomip st,st(1); fstp st; jxx rel8*/ \ + {ICODE, "l mld mld", "DB /5 m2; DB /5 m1; DF F1; DD D8; " SHORT_JMP_OPCODE " l0", 0}, +#define FBCMP(ICODE, LONG_JMP_OPCODE) \ + {ICODE, "L r r", "Y 0F 2E r1 R2;" LONG_JMP_OPCODE " L0", 0}, /* ucomiss r0,r1;jxx rel32*/ #define DBCMP(ICODE, LONG_JMP_OPCODE) \ - {ICODE, "l r r", "66 Y 0F 2E r1 R2;" LONG_JMP_OPCODE " l0"}, /* ucomisd r0,r1;jxx rel32*/ - + {ICODE, "L r r", "66 Y 0F 2E r1 R2;" LONG_JMP_OPCODE " L0", 0}, /* ucomisd r0,r1;jxx rel32*/ #define LDBCMP(ICODE, LONG_JMP_OPCODE) \ /* fld m2;fld m1; fcomip st,st(1); fstp st; jxx rel32*/ \ - {ICODE, "l mld mld", "DB /5 m2; DB /5 m1; DF F1; DD D8; " LONG_JMP_OPCODE " l0"}, - -static const struct pattern patterns[] = { - {MIR_MOV, "r z", "Y 33 r0 R0"}, /* xor r0,r0 -- 32 bit xor */ - {MIR_MOV, "r r", "X 8B r0 R1"}, /* mov r0,r1 */ - {MIR_MOV, "r m3", "X 8B r0 m1"}, /* mov r0,m1 */ - {MIR_MOV, "m3 r", "X 89 r1 m0"}, /* mov m0,r1 */ - {MIR_MOV, "r i2", "X C7 /0 R0 I1"}, /* mov r0,i32 */ - {MIR_MOV, "m3 i2", "X C7 /0 m0 I1"}, /* mov m0,i32 */ - {MIR_MOV, "r i3", "X B8 +0 J1"}, /* mov r0,i64 */ - {MIR_MOV, "r p", "X B8 +0 P1"}, /* mov r0,a64 */ - - {MIR_MOV, "m0 r", "Z 88 r1 m0"}, /* mov m0, r1 */ - {MIR_MOV, "m1 r", "66 Y 89 r1 m0"}, /* mov m0, r1 */ - {MIR_MOV, "m2 r", "Y 89 r1 m0"}, /* mov m0, r1 */ - - {MIR_MOV, "r ms0", "X 0F BE r0 m1"}, /* movsx r0, m1 */ - {MIR_MOV, "r ms1", "X 0F BF r0 m1"}, /* movsx r0, m1 */ - {MIR_MOV, "r ms2", "X 63 r0 m1"}, /* movsx r0, m1 */ - - {MIR_MOV, "r mu0", "X 0F B6 r0 m1"}, /* movzx r0, m1 */ - {MIR_MOV, "r mu1", "X 0F B7 r0 m1"}, /* movzx r0, m1 */ - {MIR_MOV, "r mu2", "Y 8B r0 m1"}, /* mov r0, m1 */ - - {MIR_MOV, "m0 i0", "Y C6 /0 m0 i1"}, /* mov m0,i8 */ - {MIR_MOV, "m2 i2", "Y C7 /0 m0 I1"}, /* mov m0,i32 */ - - {MIR_FMOV, "r r", "Y 0F 28 r0 R1"}, /* movaps r0,r1 */ - {MIR_FMOV, "r mf", "F3 Y 0F 10 r0 m1"}, /* movss r0,m32 */ - {MIR_FMOV, "mf r", "F3 Y 0F 11 r1 m0"}, /* movss r0,m32 */ - - {MIR_DMOV, "r r", "66 Y 0F 28 r0 R1"}, /* movapd r0,r1 */ - {MIR_DMOV, "r md", "F2 Y 0F 10 r0 m1"}, /* movsd r0,m64 */ - {MIR_DMOV, "md r", "F2 Y 0F 11 r1 m0"}, /* movsd m64,r0 */ - - {MIR_LDMOV, "mld h32", "DB /7 m0"}, /*only for ret and calls in given order: fstp m0 */ - {MIR_LDMOV, "h32 mld", "DB /5 m1"}, /*only for ret and calls in given order: fld m1 */ - {MIR_LDMOV, "mld h33", "D9 C9; DB /7 m0"}, /*only for ret and calls: fxch;fstp m0 */ - {MIR_LDMOV, "h33 mld", "DB /5 m1; D9 C9"}, /*only for ret and calls: fld m1; fxch */ - {MIR_LDMOV, "mld mld", "DB /5 m1; DB /7 m0"}, /* fld m1; fstp m0 */ + {ICODE, "L mld mld", "DB /5 m2; DB /5 m1; DF F1; DD D8; " LONG_JMP_OPCODE " L0", 0}, + +static struct pattern patterns[] = { + {MIR_MOV, "r z", "Y 33 r0 R0", 0}, /* xor r0,r0 -- 32 bit xor */ + {MIR_MOV, "r r", "X 8B r0 R1", 0}, /* mov r0,r1 */ + {MIR_MOV, "r m3", "X 8B r0 m1", 0}, /* mov r0,m1 */ + {MIR_MOV, "m3 r", "X 89 r1 m0", 0}, /* mov m0,r1 */ + {MIR_MOV, "r i2", "X C7 /0 R0 I1", 0}, /* mov r0,i32 */ + {MIR_MOV, "m3 i2", "X C7 /0 m0 I1", 0}, /* mov m0,i32 */ + {MIR_MOV, "r i3", "X B8 +0 J1", 0}, /* mov r0,i64 */ + + {MIR_MOV, "m0 r", "Z 88 r1 m0", 0}, /* mov m0, r1 */ + {MIR_MOV, "m1 r", "66 Y 89 r1 m0", 0}, /* mov m0, r1 */ + {MIR_MOV, "m2 r", "Y 89 r1 m0", 0}, /* mov m0, r1 */ + + {MIR_MOV, "r ms0", "X 0F BE r0 m1", 0}, /* movsx r0, m1 */ + {MIR_MOV, "r ms1", "X 0F BF r0 m1", 0}, /* movsx r0, m1 */ + {MIR_MOV, "r ms2", "X 63 r0 m1", 0}, /* movsx r0, m1 */ + + {MIR_MOV, "r mu0", "X 0F B6 r0 m1", 0}, /* movzx r0, m1 */ + {MIR_MOV, "r mu1", "X 0F B7 r0 m1", 0}, /* movzx r0, m1 */ + {MIR_MOV, "r mu2", "Y 8B r0 m1", 0}, /* mov r0, m1 */ + + {MIR_MOV, "m0 i0", "Y C6 /0 m0 i1", 0}, /* mov m0,i8 */ + {MIR_MOV, "m2 i2", "Y C7 /0 m0 I1", 0}, /* mov m0,i32 */ + + {MIR_FMOV, "r r", "Y 0F 28 r0 R1", 0}, /* movaps r0,r1 */ + {MIR_FMOV, "r mf", "F3 Y 0F 10 r0 m1", 0}, /* movss r0,m32 */ + {MIR_FMOV, "mf r", "F3 Y 0F 11 r1 m0", 0}, /* movss r0,m32 */ + + {MIR_DMOV, "r r", "66 Y 0F 28 r0 R1", 0}, /* movapd r0,r1 */ + {MIR_DMOV, "r md", "F2 Y 0F 10 r0 m1", 0}, /* movsd r0,m64 */ + {MIR_DMOV, "md r", "F2 Y 0F 11 r1 m0", 0}, /* movsd m64,r0 */ + + {MIR_LDMOV, "mld h32", "DB /7 m0", 0}, /*only for ret and calls in given order: fstp m0 */ + {MIR_LDMOV, "h32 mld", "DB /5 m1", 0}, /*only for ret and calls in given order: fld m1 */ + {MIR_LDMOV, "mld h33", "D9 C9; DB /7 m0", 0}, /*only for ret and calls: fxch;fstp m0 */ + {MIR_LDMOV, "h33 mld", "DB /5 m1; D9 C9", 0}, /*only for ret and calls: fld m1; fxch */ + {MIR_LDMOV, "mld mld", "DB /5 m1; DB /7 m0", 0}, /* fld m1; fstp m0 */ #define STR(c) #c #define STR_VAL(c) STR (c) - {MIR_UNSPEC, "c" STR_VAL (MOVDQA_CODE) " r r", "66 Y 0F 6F r1 R2"}, /* movdqa r0,r1 */ - {MIR_UNSPEC, "c" STR_VAL (MOVDQA_CODE) " r md", "66 Y 0F 6F r1 m2"}, /* movdqa r0,m128 */ - {MIR_UNSPEC, "c" STR_VAL (MOVDQA_CODE) " md r", "66 Y 0F 7F r2 m1"}, /* movdqa m128,r0 */ - - {MIR_EXT8, "r r", "X 0F BE r0 R1"}, /* movsx r0,r1 */ - {MIR_EXT8, "r m0", "X 0F BE r0 m1"}, /* movsx r0,m1 */ - {MIR_EXT16, "r r", "X 0F BF r0 R1"}, /* movsx r0,r1 */ - {MIR_EXT16, "r m1", "X 0F BF r0 m1"}, /* movsx r0,m1 */ - {MIR_EXT32, "r r", "X 63 r0 R1"}, /* movsx r0,r1 */ - {MIR_EXT32, "r m2", "X 63 r0 m1"}, /* movsx r0,m1 */ - {MIR_UEXT8, "r r", "X 0F B6 r0 R1"}, /* movzx r0,r1 */ - {MIR_UEXT8, "r m0", "X 0F B6 r0 m1"}, /* movzx r0,m1 */ - {MIR_UEXT16, "r r", "X 0F B7 r0 R1"}, /* movzx r0,r1 */ - {MIR_UEXT16, "r m1", "X 0F B7 r0 m1"}, /* movzx r0,m1 */ - {MIR_UEXT32, "r r", "Y 8B r0 R1"}, /* mov r0,r1 */ - {MIR_UEXT32, "r m2", "Y 8B r0 m1"}, /* mov r0,m1 */ - - {MIR_I2F, "r r", "F3 X 0F 2A r0 R1"}, /* cvtsi2ss r0,r1 */ - {MIR_I2F, "r mf", "F3 X 0F 2A r0 m1"}, /* cvtsi2ss r0,m1 */ - {MIR_I2D, "r r", "F2 X 0F 2A r0 R1"}, /* cvtsi2sd r0,r1 */ - {MIR_I2D, "r md", "F2 X 0F 2A r0 m1"}, /* cvtsi2sd r0,m1 */ - {MIR_I2LD, "mld r", "X 89 r1 mt; DF /5 mt; DB /7 m0"}, /*mov -16(sp),r1;fild -16(sp);fstp m0 */ - - {MIR_F2I, "r r", "F3 X 0F 2C r0 R1"}, /* cvttss2si r0,r1 */ - {MIR_F2I, "r mf", "F3 X 0F 2C r0 m1"}, /* cvttss2si r0,m1 */ - {MIR_D2I, "r r", "F2 X 0F 2C r0 R1"}, /* cvttsd2si r0,r1 */ - {MIR_D2I, "r md", "F2 X 0F 2C r0 m1"}, /* cvttsd2si r0,m1 */ - - {MIR_F2D, "r r", "F3 Y 0F 5A r0 R1"}, /* cvtss2sd r0,r1 */ - {MIR_F2D, "r mf", "F3 Y 0F 5A r0 m1"}, /* cvtss2sd r0,m1 */ - /* fld m1;fstpl -16(sp);movsd r0,-16(sp): */ - {MIR_LD2D, "r mld", "DB /5 m1; DD /3 mt; F2 Y 0F 10 r0 mt"}, - - {MIR_D2F, "r r", "F2 Y 0F 5A r0 R1"}, /* cvtsd2ss r0,r1 */ - {MIR_D2F, "r md", "F2 Y 0F 5A r0 m1"}, /* cvtsd2ss r0,m1 */ + {MIR_UNSPEC, "c" STR_VAL (MOVDQA_CODE) " r r", "66 Y 0F 6F r1 R2", 0}, /* movdqa r0,r1 */ + {MIR_UNSPEC, "c" STR_VAL (MOVDQA_CODE) " r md", "66 Y 0F 6F r1 m2", 0}, /* movdqa r0,m128 */ + {MIR_UNSPEC, "c" STR_VAL (MOVDQA_CODE) " md r", "66 Y 0F 7F r2 m1", 0}, /* movdqa m128,r0 */ + + {MIR_EXT8, "r r", "X 0F BE r0 R1", 0}, /* movsx r0,r1 */ + {MIR_EXT8, "r m0", "X 0F BE r0 m1", 0}, /* movsx r0,m1 */ + {MIR_EXT16, "r r", "X 0F BF r0 R1", 0}, /* movsx r0,r1 */ + {MIR_EXT16, "r m1", "X 0F BF r0 m1", 0}, /* movsx r0,m1 */ + {MIR_EXT32, "r r", "X 63 r0 R1", 0}, /* movsx r0,r1 */ + {MIR_EXT32, "r m2", "X 63 r0 m1", 0}, /* movsx r0,m1 */ + {MIR_UEXT8, "r r", "Y 0F B6 r0 S1", 0}, /* movzx r0,r1 */ + {MIR_UEXT8, "r m0", "X 0F B6 r0 m1", 0}, /* movzx r0,m1 */ + {MIR_UEXT16, "r r", "X 0F B7 r0 R1", 0}, /* movzx r0,r1 */ + {MIR_UEXT16, "r m1", "X 0F B7 r0 m1", 0}, /* movzx r0,m1 */ + {MIR_UEXT32, "r r", "Y 8B r0 R1", 0}, /* mov r0,r1 */ + {MIR_UEXT32, "r m2", "Y 8B r0 m1", 0}, /* mov r0,m1 */ + + {MIR_I2F, "r r", "F3 X 0F 2A r0 R1", 0}, /* cvtsi2ss r0,r1 */ + {MIR_I2F, "r mf", "F3 X 0F 2A r0 m1", 0}, /* cvtsi2ss r0,m1 */ + {MIR_I2D, "r r", "F2 X 0F 2A r0 R1", 0}, /* cvtsi2sd r0,r1 */ + {MIR_I2D, "r md", "F2 X 0F 2A r0 m1", 0}, /* cvtsi2sd r0,m1 */ + {MIR_I2LD, "mld r", "X 89 r1 mt; DF /5 mt; DB /7 m0", 0}, /*mov -16(sp),r1;fild -16(sp);fstp m0 */ + + {MIR_F2I, "r r", "F3 X 0F 2C r0 R1", 0}, /* cvttss2si r0,r1 */ + {MIR_F2I, "r mf", "F3 X 0F 2C r0 m1", 0}, /* cvttss2si r0,m1 */ + {MIR_D2I, "r r", "F2 X 0F 2C r0 R1", 0}, /* cvttsd2si r0,r1 */ + {MIR_D2I, "r md", "F2 X 0F 2C r0 m1", 0}, /* cvttsd2si r0,m1 */ + + {MIR_F2D, "r r", "F3 Y 0F 5A r0 R1", 0}, /* cvtss2sd r0,r1 */ + {MIR_F2D, "r mf", "F3 Y 0F 5A r0 m1", 0}, /* cvtss2sd r0,m1 */ + /* fld m1;fstpl -16(sp);movsd r0,-16(sp): */ + {MIR_LD2D, "r mld", "DB /5 m1; DD /3 mt; F2 Y 0F 10 r0 mt", 0}, + + {MIR_D2F, "r r", "F2 Y 0F 5A r0 R1", 0}, /* cvtsd2ss r0,r1 */ + {MIR_D2F, "r md", "F2 Y 0F 5A r0 m1", 0}, /* cvtsd2ss r0,m1 */ /* fld m1;fstps -16(sp);movss r0, -16(sp): */ - {MIR_LD2F, "r mld", "DB /5 m1; D9 /3 mt; F3 Y 0F 10 r0 mt"}, + {MIR_LD2F, "r mld", "DB /5 m1; D9 /3 mt; F3 Y 0F 10 r0 mt", 0}, /* movss -16(sp), r1; flds -16(sp); fstp m0: */ - {MIR_F2LD, "mld r", "F3 Y 0F 11 r1 mt; D9 /0 mt; DB /7 m0"}, - {MIR_F2LD, "mld mf", "D9 /0 m1; DB /7 m0"}, /* flds m1; fstp m0 */ + {MIR_F2LD, "mld r", "F3 Y 0F 11 r1 mt; D9 /0 mt; DB /7 m0", 0}, + {MIR_F2LD, "mld mf", "D9 /0 m1; DB /7 m0", 0}, /* flds m1; fstp m0 */ /* movsd -16(sp), r1; fldl -16(sp); fstp m0: */ - {MIR_D2LD, "mld r", "F2 Y 0F 11 r1 mt; DD /0 mt; DB /7 m0"}, - {MIR_D2LD, "mld md", "DD /0 m1; DB /7 m0"}, /* fldl m1; fstp m0 */ + {MIR_D2LD, "mld r", "F2 Y 0F 11 r1 mt; DD /0 mt; DB /7 m0", 0}, + {MIR_D2LD, "mld md", "DD /0 m1; DB /7 m0", 0}, /* fldl m1; fstp m0 */ /* lea r0, 15(r1); and r0, r0, -16; sub sp, r0; mov r0, sp: */ - {MIR_ALLOCA, "r r", "Y 8D r0 adF; X 81 /4 R0 VFFFFFFF0; X 2B h04 R0; X 8B r0 H04"}, - {MIR_ALLOCA, "r i2", "X 81 /5 H04 I1; X 8B r0 H04"}, /* sub sp, i2; mov r0, sp */ + {MIR_ALLOCA, "r r", "Y 8D r0 adF; X 81 /4 R0 VFFFFFFF0; X 2B h04 R0; X 8B r0 H04", 0}, + {MIR_ALLOCA, "r i2", "X 81 /5 H04 I1; X 8B r0 H04", 0}, /* sub sp, i2; mov r0, sp */ - {MIR_BSTART, "r", "X 8B r0 H4"}, /* r0 = sp */ - {MIR_BEND, "r", "X 8B h4 R0"}, /* sp = r0 */ + {MIR_BSTART, "r", "X 8B r0 H4", 0}, /* r0 = sp */ + {MIR_BEND, "r", "X 8B h4 R0", 0}, /* sp = r0 */ - {MIR_NEG, "r 0", "X F7 /3 R1"}, /* neg r0 */ - {MIR_NEG, "m3 0", "X F7 /3 m1"}, /* neg m0 */ - {MIR_NEGS, "r 0", "Y F7 /3 R1"}, /* neg r0 */ - {MIR_NEGS, "m2 0", "Y F7 /3 m1"}, /* neg m0 */ + {MIR_NEG, "r 0", "X F7 /3 R1", 0}, /* neg r0 */ + {MIR_NEG, "m3 0", "X F7 /3 m1", 0}, /* neg m0 */ + {MIR_NEGS, "r 0", "Y F7 /3 R1", 0}, /* neg r0 */ + {MIR_NEGS, "m2 0", "Y F7 /3 m1", 0}, /* neg m0 */ - {MIR_FNEG, "r 0", "Y 0F 57 r0 c0000000080000000"}, /* xorps r0,80000000 */ - {MIR_DNEG, "r 0", "66 Y 0F 57 r0 c8000000000000000"}, /* xorpd r0,0x8000000000000000 */ - {MIR_LDNEG, "mld mld", "DB /5 m1; D9 E0; DB /7 m0"}, /* fld m1; fchs; fstp m0 */ + {MIR_FNEG, "r 0", "Y 0F 57 r0 c0000000080000000", 0}, /* xorps r0,80000000 */ + {MIR_DNEG, "r 0", "66 Y 0F 57 r0 c8000000000000000", 0}, /* xorpd r0,0x8000000000000000 */ + {MIR_LDNEG, "mld mld", "DB /5 m1; D9 E0; DB /7 m0", 0}, /* fld m1; fchs; fstp m0 */ IOP (MIR_ADD, "03", "01", "83 /0", "81 /0") /* x86_64 int additions */ - {MIR_ADD, "r r r", "X 8D r0 ap"}, /* lea r0,(r1,r2)*/ - {MIR_ADD, "r r i2", "X 8D r0 ap"}, /* lea r0,i2(r1)*/ - {MIR_ADDS, "r r r", "Y 8D r0 ap"}, /* lea r0,(r1,r2)*/ - {MIR_ADDS, "r r i2", "Y 8D r0 ap"}, /* lea r0,i2(r1)*/ + {MIR_ADD, "r r r", "X 8D r0 ap", 0}, /* lea r0,(r1,r2)*/ + {MIR_ADD, "r r i2", "X 8D r0 ap", 0}, /* lea r0,i2(r1)*/ + {MIR_ADDS, "r r r", "Y 8D r0 ap", 0}, /* lea r0,(r1,r2)*/ + {MIR_ADDS, "r r i2", "Y 8D r0 ap", 0}, /* lea r0,i2(r1)*/ IOP (MIR_SUB, "2B", "29", "83 /5", "81 /5") /* x86_64 int subtractions */ - {MIR_MUL, "r 0 r", "X 0F AF r0 R2"}, /* imul r0,r1*/ - {MIR_MUL, "r 0 m3", "X 0F AF r0 m2"}, /* imul r0,m1*/ - {MIR_MUL, "r r i2", "X 69 r0 R1 I2"}, /* imul r0,r1,i32*/ - {MIR_MUL, "r m3 i2", "X 69 r0 m1 I2"}, /* imul r0,m1,i32*/ - {MIR_MUL, "r r s", "X 8D r0 ap"}, /* lea r0,(,r1,s2)*/ - {MIR_MULS, "r 0 r", "Y 0F AF r0 R2"}, /* imul r0,r1*/ - {MIR_MULS, "r 0 m2", "Y 0F AF r0 m2"}, /* imul r0,m1*/ - {MIR_MULS, "r r i2", "Y 69 r0 R1 I2"}, /* imul r0,r1,i32*/ - {MIR_MULS, "r m2 i2", "Y 69 r0 m1 I2"}, /* imul r0,m1,i32*/ - {MIR_MULS, "r r s", "Y 8D r0 ap"}, /* lea r0,(,r1,s2)*/ - - {MIR_DIV, "h0 h0 r", "X 99; X F7 /7 R2"}, /* cqo; idiv r2*/ - {MIR_DIV, "h0 h0 m3", "X 99; X F7 /7 m2"}, /* cqo; idiv m2*/ - {MIR_DIVS, "h0 h0 r", "99; Y F7 /7 R2"}, /* cdq; idiv r2*/ - {MIR_DIVS, "h0 h0 m2", "99; Y F7 /7 m2"}, /* cdq; idiv m2*/ - - {MIR_UDIV, "h0 h0 r", "31 D2; X F7 /6 R2"}, /* xorl edx,edx; div r2*/ - {MIR_UDIV, "h0 h0 m3", "31 D2; X F7 /6 m2"}, /* xorl edx,edx; div m2*/ - {MIR_UDIVS, "h0 h0 r", "31 D2; Y F7 /6 R2"}, /* xorl edx,edx; div r2*/ - {MIR_UDIVS, "h0 h0 m2", "31 D2; Y F7 /6 m2"}, /* xorl edx,edx; div m2*/ - - {MIR_MOD, "h2 h0 r", "X 99; X F7 /7 R2"}, /* cqo; idiv r2*/ - {MIR_MOD, "h2 h0 m3", "X 99; X F7 /7 m2"}, /* cqo; idiv m2*/ - {MIR_MODS, "h2 h0 r", "99; Y F7 /7 R2"}, /* cdq; idiv r2*/ - {MIR_MODS, "h2 h0 m2", "99; Y F7 /7 m2"}, /* cdq; idiv m2*/ - - {MIR_UMOD, "h2 h0 r", "31 D2; X F7 /6 R2"}, /* xorl edx,edx; div r2*/ - {MIR_UMOD, "h2 h0 m3", "31 D2; X F7 /6 m2"}, /* xorl edx,edx; div m2*/ - {MIR_UMODS, "h2 h0 r", "31 D2; Y F7 /6 R2"}, /* xorl edx,edx; div r2*/ - {MIR_UMODS, "h2 h0 m2", "31 D2; Y F7 /6 m2"}, /* xorl edx,edx; div m2*/ + IOP (MIR_ADDO, "03", "01", "83 /0", "81 /0") /* x86_64 int additions with ovfl flag */ + IOP (MIR_SUBO, "2B", "29", "83 /5", "81 /5") /* x86_64 int subtractions with ovfl flag */ + +#define IMULL(ICODE, ICODES) \ + {ICODE, "r 0 r", "X 0F AF r0 R2", 0}, /* imul r0,r1*/ \ + {ICODE, "r 0 m3", "X 0F AF r0 m2", 0}, /* imul r0,m1*/ \ + {ICODE, "r r i2", "X 69 r0 R1 I2", 0}, /* imul r0,r1,i32*/ \ + {ICODE, "r m3 i2", "X 69 r0 m1 I2", 0}, /* imul r0,m1,i32*/ \ + {ICODES, "r 0 r", "Y 0F AF r0 R2", 0}, /* imul r0,r1*/ \ + {ICODES, "r 0 m2", "Y 0F AF r0 m2", 0}, /* imul r0,m1*/ \ + {ICODES, "r r i2", "Y 69 r0 R1 I2", 0}, /* imul r0,r1,i32*/ \ + {ICODES, "r m2 i2", "Y 69 r0 m1 I2", 0}, /* imul r0,m1,i32*/ + + IMULL (MIR_MUL, MIR_MULS) + + {MIR_MUL, "r r s", "X 8D r0 ap", 0}, /* lea r0,(,r1,s2)*/ + {MIR_MULS, "r r s", "Y 8D r0 ap", 0}, /* lea r0,(,r1,s2)*/ + + IMULL (MIR_MULO, MIR_MULOS) + + {MIR_UMULO, "h0 0 r", "X F7 /4 R2", 0}, /* mul rax,r1*/ + {MIR_UMULO, "h0 0 m3", "X F7 /4 m2", 0}, /* mul rax,m1*/ + {MIR_UMULOS, "h0 0 r", "Y F7 /4 R2", 0}, /* mul rax,r1*/ + {MIR_UMULOS, "h0 0 m2", "Y F7 /4 m2", 0}, /* mul rax,m1*/ + + {MIR_DIV, "h0 h0 r", "X 99; X F7 /7 R2", 0}, /* cqo; idiv r2*/ + {MIR_DIV, "h0 h0 m3", "X 99; X F7 /7 m2", 0}, /* cqo; idiv m2*/ + {MIR_DIVS, "h0 h0 r", "99; Y F7 /7 R2", 0}, /* cdq; idiv r2*/ + {MIR_DIVS, "h0 h0 m2", "99; Y F7 /7 m2", 0}, /* cdq; idiv m2*/ + + {MIR_UDIV, "h0 h0 r", "31 D2; X F7 /6 R2", 0}, /* xorl edx,edx; div r2*/ + {MIR_UDIV, "h0 h0 m3", "31 D2; X F7 /6 m2", 0}, /* xorl edx,edx; div m2*/ + {MIR_UDIVS, "h0 h0 r", "31 D2; Y F7 /6 R2", 0}, /* xorl edx,edx; div r2*/ + {MIR_UDIVS, "h0 h0 m2", "31 D2; Y F7 /6 m2", 0}, /* xorl edx,edx; div m2*/ + + {MIR_MOD, "h2 h0 r", "X 99; X F7 /7 R2", 0}, /* cqo; idiv r2*/ + {MIR_MOD, "h2 h0 m3", "X 99; X F7 /7 m2", 0}, /* cqo; idiv m2*/ + {MIR_MODS, "h2 h0 r", "99; Y F7 /7 R2", 0}, /* cdq; idiv r2*/ + {MIR_MODS, "h2 h0 m2", "99; Y F7 /7 m2", 0}, /* cdq; idiv m2*/ + + {MIR_UMOD, "h2 h0 r", "31 D2; X F7 /6 R2", 0}, /* xorl edx,edx; div r2*/ + {MIR_UMOD, "h2 h0 m3", "31 D2; X F7 /6 m2", 0}, /* xorl edx,edx; div m2*/ + {MIR_UMODS, "h2 h0 r", "31 D2; Y F7 /6 R2", 0}, /* xorl edx,edx; div r2*/ + {MIR_UMODS, "h2 h0 m2", "31 D2; Y F7 /6 m2", 0}, /* xorl edx,edx; div m2*/ IOP (MIR_AND, "23", "21", "83 /4", "81 /4") /*ands*/ IOP (MIR_OR, "0B", "09", "83 /1", "81 /1") IOP (MIR_XOR, "33", "31", "83 /6", "81 /6") /*(x)ors*/ @@ -1663,13 +1747,35 @@ static const struct pattern patterns[] = { FCMP (MIR_FGT, "0F 97") DCMP (MIR_DGT, "0F 97") LDCMP (MIR_LDGT, "0F 97") /*6*/ FCMP (MIR_FGE, "0F 93") DCMP (MIR_DGE, "0F 93") LDCMP (MIR_LDGE, "0F 93") /*7*/ - {MIR_JMP, "l", "E9 l0"}, /* 32-bit offset jmp */ + {MIR_JMP, "L", "E9 L0", 0}, /* 32-bit offset jmp */ + {MIR_JMP, "l", "EB l0", 0}, /* 8-bit offset jmp */ + + {MIR_LADDR, "r L", "X 8D r0 q L1", 0}, /* ip-relative addressing */ + {MIR_JMPI, "r", "Y FF /4 R0", 0}, /* jmp *r */ + {MIR_JMPI, "m3", "Y FF /4 m0", 0}, /* jmp *m0 */ - /* movq TableAddress,r11; mov (r11,r,8),r11; jmp *r11; TableContent */ - {MIR_SWITCH, "r $", "49 BB T; X 8B hB mT; 41 FF E3"}, + /* lea table_offset(rip),r11; jmp *(r11,r,8); TableContent */ + {MIR_SWITCH, "r $", "X 8D hB T; Y FF /4 mT", 0}, + BRS (MIR_BT, "75") BRS (MIR_BF, "74") /* short branches */ BR (MIR_BT, "0F 85") BR (MIR_BF, "0F 84") /* branches */ + {MIR_BO, "l", "70 l0", 0}, /* 8-bit offset jmp on signed overflow */ + {MIR_UBO, "l", "72 l0", 0}, /* 8-bit offset jmp on unsigned overflow */ + {MIR_BNO, "l", "71 l0", 0}, /* 8-bit offset jmp on signed non-overflow */ + {MIR_UBNO, "l", "73 l0", 0}, /* 8-bit offset jmp on unsigned non-overflow */ + + {MIR_BO, "L", "0F 80 L0", 0}, /* 32-bit offset jmp on signed overflow */ + {MIR_UBO, "L", "0F 82 L0", 0}, /* 32-bit offset jmp on unsigned overflow */ + {MIR_BNO, "L", "0F 81 L0", 0}, /* 32-bit offset jmp on signed non-overflow */ + {MIR_UBNO, "L", "0F 83 L0", 0}, /* 32-bit offset jmp on unsigned non-overflow */ + + BCMPS (MIR_BEQ, "74") BCMPS (MIR_BNE, "75") /* 1. int compare and branch */ + BCMPS (MIR_BLT, "7C") BCMPS (MIR_UBLT, "72") /* 2. int compare and branch */ + BCMPS (MIR_BLE, "7E") BCMPS (MIR_UBLE, "76") /* 3. int compare and branch */ + BCMPS (MIR_BGT, "7F") BCMPS (MIR_UBGT, "77") /* 4. int compare and branch */ + BCMPS (MIR_BGE, "7D") BCMPS (MIR_UBGE, "73") /* 5. int compare and branch */ + BCMP (MIR_BEQ, "0F 84") BCMP (MIR_BNE, "0F 85") /* 1. int compare and branch */ BCMP (MIR_BLT, "0F 8C") BCMP (MIR_UBLT, "0F 82") /* 2. int compare and branch */ BCMP (MIR_BLE, "0F 8E") BCMP (MIR_UBLE, "0F 86") /* 3. int compare and branch */ @@ -1682,31 +1788,46 @@ static const struct pattern patterns[] = { DBCMP (MIR_DBLE, "0F 86") LDBCMP (MIR_LDBLE, "0F 86") /* 3. fp cmp and branch */ #endif - FBCMP (MIR_FBGT, "0F 87") DBCMP (MIR_DBGT, "0F 87") /* 4. fp cmp and branch */ - LDBCMP (MIR_LDBGT, "0F 87") FBCMP (MIR_FBGE, "0F 83") /* 5. fp cmp and branch */ - DBCMP (MIR_DBGE, "0F 83") LDBCMP (MIR_LDBGE, "0F 83") /* 6. fp cmp and branch */ + FBCMPS (MIR_FBGT, "77") DBCMPS (MIR_DBGT, "77") /* fp cmp and short branch */ + LDBCMPS (MIR_LDBGT, "77") FBCMPS (MIR_FBGE, "73") /* fp cmp and short branch */ + DBCMPS (MIR_DBGE, "73") LDBCMPS (MIR_LDBGE, "73") /* fp cmp and short branch */ - {MIR_FBEQ, "l r r", "Y 0F 2E r1 R2; 7A v6; 0F 84 l0"}, /* ucomiss r0,r1;jp L;je rel32 L: */ - {MIR_DBEQ, "l r r", "66 Y 0F 2E r1 R2; 7A v6; 0F 84 l0"}, /* ucomisd r0,r1;jp L;je rel32 L: */ - /* fld m2;fld m1;fucomip st,st1;fstp st;jp L;je rel32 L: */ - {MIR_LDBEQ, "l mld mld", "DB /5 m2; DB /5 m1; DF E9; DD D8; 7A v6; 0F 84 l0"}, + FBCMP (MIR_FBGT, "0F 87") DBCMP (MIR_DBGT, "0F 87") /* fp cmp and branch */ + LDBCMP (MIR_LDBGT, "0F 87") FBCMP (MIR_FBGE, "0F 83") /* fp cmp and branch */ + DBCMP (MIR_DBGE, "0F 83") LDBCMP (MIR_LDBGE, "0F 83") /* fp cmp and branch */ + + /* we don't have short branch patterns for NE as the label will be in two branches: */ + {MIR_FBEQ, "l r r", "Y 0F 2E r1 R2; 7A v2; 74 l0", 0}, /* ucomiss r0,r1;jp l;je rel32 l: */ + {MIR_DBEQ, "l r r", "66 Y 0F 2E r1 R2; 7A v2; 74 l0", 0}, /* ucomisd r0,r1;jp l;je rel32 l: */ + /* fld m2;fld m1;fucomip st,st1;fstp st;jp l;je rel32 l: */ + {MIR_LDBEQ, "l mld mld", "DB /5 m2; DB /5 m1; DF E9; DD D8; 7A v2; 74 l0", 0}, - {MIR_FBNE, "l r r", "Y 0F 2E r1 R2; 0F 8A l0; 0F 85 l0"}, /* ucomiss r0,r1;jp rel32;jne rel32*/ - {MIR_DBNE, "l r r", "66 Y 0F 2E r1 R2; 0F 8A l0; 0F 85 l0"}, /* ucomisd r0,r1;jp rel32;jne rel32*/ + {MIR_FBEQ, "L r r", "Y 0F 2E r1 R2; 7A v6; 0F 84 L0", 0}, /* ucomiss r0,r1;jp L;je rel32 L: */ + {MIR_DBEQ, "L r r", "66 Y 0F 2E r1 R2; 7A v6; 0F 84 L0", 0}, /* ucomisd r0,r1;jp L;je rel32 L: */ + /* fld m2;fld m1;fucomip st,st1;fstp st;jp L;je rel32 L: */ + {MIR_LDBEQ, "L mld mld", "DB /5 m2; DB /5 m1; DF E9; DD D8; 7A v6; 0F 84 L0", 0}, + {MIR_FBNE, "L r r", "Y 0F 2E r1 R2; 0F 8A L0; 0F 85 L0", 0}, /* ucomiss r0,r1;jp rel32;jne rel32*/ + {MIR_DBNE, "L r r", "66 Y 0F 2E r1 R2; 0F 8A L0; 0F 85 L0", + 0}, /* ucomisd r0,r1;jp rel32;jne rel32*/ /* fld m2;fld m1;fucomip st,st1;fstp st;jp rel32;jne rel32 */ - {MIR_LDBNE, "l mld mld", "DB /5 m2; DB /5 m1; DF E9; DD D8; 0F 8A l0; 0F 85 l0"}, + {MIR_LDBNE, "L mld mld", "DB /5 m2; DB /5 m1; DF E9; DD D8; 0F 8A L0; 0F 85 L0", 0}, - {MIR_CALL, "X r $", "Y FF /2 R1"}, /* call *r1 */ + {MIR_CALL, "X i3 $", "FF /2 P1", 0}, /* call *rel32(rip) */ + {MIR_CALL, "X r $", "Y FF /2 R1", 0}, /* call *r1 */ + {MIR_RET, "$", "C3", 0}, /* ret ax, dx, xmm0, xmm1, st0, st1 */ - {MIR_RET, "$", "C3"}, /* ret ax, dx, xmm0, xmm1, st0, st1 */ + {MIR_JCALL, "X i3 $", "FF /4 P1", 0}, /* jmp *rel32(rip) */ + {MIR_JCALL, "X r $", "Y FF /4 R1", 0}, /* jmp *r */ + {MIR_JRET, "r $", "Y FF /4 R0", 0}, /* jmp *r */ }; static void target_get_early_clobbered_hard_regs (MIR_insn_t insn, MIR_reg_t *hr1, MIR_reg_t *hr2) { MIR_insn_code_t code = insn->code; - *hr1 = *hr2 = MIR_NON_HARD_REG; + *hr1 = *hr2 = MIR_NON_VAR; if (code == MIR_DIV || code == MIR_UDIV || code == MIR_DIVS || code == MIR_UDIVS - || code == MIR_MOD || code == MIR_UMOD || code == MIR_MODS || code == MIR_UMODS) { + || code == MIR_MOD || code == MIR_UMOD || code == MIR_MODS || code == MIR_UMODS + || code == MIR_UMULO || code == MIR_UMULOS) { *hr1 = DX_HARD_REG; } else if (code == MIR_FEQ || code == MIR_FNE || code == MIR_DEQ || code == MIR_DNE || code == MIR_LDEQ || code == MIR_LDNE) { @@ -1751,6 +1872,8 @@ static int pattern_index_cmp (const void *a1, const void *a2) { return c1 != c2 ? c1 - c2 : (long) i1 - (long) i2; } +static int get_max_insn_size (gen_ctx_t gen_ctx, const char *replacement); + static void patterns_init (gen_ctx_t gen_ctx) { int i, ind, n = sizeof (patterns) / sizeof (struct pattern); MIR_insn_code_t prev_code, code; @@ -1758,7 +1881,14 @@ static void patterns_init (gen_ctx_t gen_ctx) { insn_pattern_info_t pinfo = {0, 0}; VARR_CREATE (int, pattern_indexes, 0); - for (i = 0; i < n; i++) VARR_PUSH (int, pattern_indexes, i); + for (i = 0; i < n; i++) { + patterns[i].max_insn_size = get_max_insn_size (gen_ctx, patterns[i].replacement); +#if 0 + fprintf (stderr, "size of \"%s\" = %d\n", patterns[i].replacement, + patterns[i].max_insn_size); +#endif + VARR_PUSH (int, pattern_indexes, i); + } qsort (VARR_ADDR (int, pattern_indexes), n, sizeof (int), pattern_index_cmp); VARR_CREATE (insn_pattern_info_t, insn_pattern_info, 0); for (i = 0; i < MIR_INSN_BOUND; i++) VARR_PUSH (insn_pattern_info_t, insn_pattern_info, pinfo); @@ -1775,14 +1905,20 @@ static void patterns_init (gen_ctx_t gen_ctx) { info_addr[prev_code].num = n - info_addr[prev_code].start; } -static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_insn_t insn) { +static int64_t int_value (gen_ctx_t gen_ctx, const MIR_op_t *op) { + gen_assert (op->mode == MIR_OP_REF || op->mode == MIR_OP_INT || op->mode == MIR_OP_UINT); + return (op->mode != MIR_OP_REF ? op->u.i : (int64_t) get_ref_value (gen_ctx, op)); +} + +static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_insn_t insn, + int try_short_jump_p) { MIR_context_t ctx = gen_ctx->ctx; - int nop, n; - size_t nops = MIR_insn_nops (ctx, insn); + size_t nop, nops = MIR_insn_nops (ctx, insn); const char *p; char ch, start_ch; MIR_op_mode_t mode; - MIR_op_t op, original; + MIR_op_t original; + const MIR_op_t *op_ref; MIR_reg_t hr; for (nop = 0, p = pat->pattern; *p != 0; p++, nop++) { @@ -1790,19 +1926,20 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in if (*p == '$') return TRUE; if (MIR_call_code_p (insn->code) && nop >= nops) return FALSE; gen_assert (nop < nops); - op = insn->ops[nop]; + op_ref = &insn->ops[nop]; switch (start_ch = *p) { case 'X': break; case 'r': - if (op.mode != MIR_OP_HARD_REG) return FALSE; + if (op_ref->mode != MIR_OP_VAR) return FALSE; break; case 't': - if (op.mode != MIR_OP_HARD_REG - || !(AX_HARD_REG <= op.u.hard_reg && op.u.hard_reg <= BX_HARD_REG)) + if (op_ref->mode != MIR_OP_VAR + || !(AX_HARD_REG == op_ref->u.var || op_ref->u.var == CX_HARD_REG + || op_ref->u.var == DX_HARD_REG || op_ref->u.var == BX_HARD_REG)) return FALSE; break; case 'h': - if (op.mode != MIR_OP_HARD_REG) return FALSE; + if (op_ref->mode != MIR_OP_VAR) return FALSE; ch = *++p; gen_assert ('0' <= ch && ch <= '9'); hr = ch - '0'; @@ -1811,39 +1948,39 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in hr = hr * 10 + ch - '0'; else --p; - if (op.u.hard_reg != hr) return FALSE; + if (op_ref->u.var != hr) return FALSE; break; case 'z': - if ((op.mode != MIR_OP_INT && op.mode != MIR_OP_UINT) || op.u.i != 0) return FALSE; + if ((op_ref->mode != MIR_OP_INT && op_ref->mode != MIR_OP_UINT) || op_ref->u.i != 0) + return FALSE; break; - case 'i': - if (op.mode != MIR_OP_INT && op.mode != MIR_OP_UINT) return FALSE; + case 'i': { + if (op_ref->mode != MIR_OP_INT && op_ref->mode != MIR_OP_UINT && op_ref->mode != MIR_OP_REF) + return FALSE; ch = *++p; gen_assert ('0' <= ch && ch <= '3'); - if ((ch == '0' && !int8_p (op.u.i)) || (ch == '1' && !int16_p (op.u.i)) - || (ch == '2' && !int32_p (op.u.i))) + int64_t n = int_value (gen_ctx, op_ref); + if ((ch == '0' && !int8_p (n)) || (ch == '1' && !int16_p (n)) || (ch == '2' && !int32_p (n))) return FALSE; break; - case 'p': - if (op.mode != MIR_OP_REF) return FALSE; - break; + } case 's': - if ((op.mode != MIR_OP_INT && op.mode != MIR_OP_UINT) - || (op.u.i != 1 && op.u.i != 2 && op.u.i != 4 && op.u.i != 8)) + if ((op_ref->mode != MIR_OP_INT && op_ref->mode != MIR_OP_UINT) + || (op_ref->u.i != 1 && op_ref->u.i != 2 && op_ref->u.i != 4 && op_ref->u.i != 8)) return FALSE; break; case 'c': { - uint64_t n; + uint64_t dec_val; p++; - n = read_dec (&p); - if ((op.mode != MIR_OP_INT && op.mode != MIR_OP_UINT) || op.u.u != n) return FALSE; + dec_val = read_dec (&p); + if ((op_ref->mode != MIR_OP_INT && op_ref->mode != MIR_OP_UINT) || op_ref->u.u != dec_val) + return FALSE; break; } case 'm': { MIR_type_t type, type2, type3 = MIR_T_BOUND; int u_p, s_p; - if (op.mode != MIR_OP_HARD_REG_MEM) return FALSE; u_p = s_p = TRUE; ch = *++p; switch (ch) { @@ -1866,7 +2003,7 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in u_p = ch == 'u'; s_p = ch == 's'; ch = *++p; - /* Fall through: */ + /* falls through */ default: gen_assert ('0' <= ch && ch <= '3'); if (ch == '0') { @@ -1889,19 +2026,29 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in #endif } } - if (op.u.hard_reg_mem.type != type && op.u.hard_reg_mem.type != type2 - && op.u.hard_reg_mem.type != type3) + /* LD pseudos always get memory: */ + if (type == MIR_T_LD && op_ref->mode == MIR_OP_VAR && op_ref->u.var > MAX_HARD_REG) break; + if (op_ref->mode != MIR_OP_VAR_MEM) return FALSE; + if (op_ref->u.var_mem.type != type && op_ref->u.var_mem.type != type2 + && op_ref->u.var_mem.type != type3) return FALSE; - if (op.u.hard_reg_mem.index != MIR_NON_HARD_REG && op.u.hard_reg_mem.scale != 1 - && op.u.hard_reg_mem.scale != 2 && op.u.hard_reg_mem.scale != 4 - && op.u.hard_reg_mem.scale != 8) + if (op_ref->u.var_mem.index != MIR_NON_VAR && op_ref->u.var_mem.scale != 1 + && op_ref->u.var_mem.scale != 2 && op_ref->u.var_mem.scale != 4 + && op_ref->u.var_mem.scale != 8) return FALSE; - if (!int32_p (op.u.hard_reg_mem.disp)) return FALSE; + if (!int32_p (op_ref->u.var_mem.disp)) return FALSE; break; } + case 'L': break; case 'l': - if (op.mode != MIR_OP_LABEL) return FALSE; - break; + if (op_ref->mode != MIR_OP_LABEL) return FALSE; + if (!try_short_jump_p) return FALSE; /* we are in size estimation mode */ + int64_t disp = ((int64_t) get_label_disp (gen_ctx, op_ref->u.label) + - (int64_t) VARR_LENGTH (uint8_t, result_code)); + /* short->long (+1 for long jump prefix +3 for offset), minimal jump is 2 bytes: */ + disp = disp < 0 ? disp - (pat->max_insn_size + 4) : disp - 2; + if (-128 <= disp && disp < 128) break; + return FALSE; case '0': case '1': case '2': @@ -1911,36 +2058,39 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in case '6': case '7': case '8': - case '9': - n = start_ch - '0'; + case '9': { + size_t n = start_ch - '0'; gen_assert (n < nop); original = insn->ops[n]; - mode = op.mode; + mode = op_ref->mode; if (mode == MIR_OP_UINT) mode = MIR_OP_INT; if (original.mode != mode && (original.mode != MIR_OP_UINT || mode != MIR_OP_INT)) return FALSE; - gen_assert (mode == MIR_OP_HARD_REG || mode == MIR_OP_INT || mode == MIR_OP_FLOAT - || mode == MIR_OP_DOUBLE || mode == MIR_OP_LDOUBLE || mode == MIR_OP_HARD_REG_MEM + gen_assert (mode == MIR_OP_VAR || mode == MIR_OP_INT || mode == MIR_OP_FLOAT + || mode == MIR_OP_DOUBLE || mode == MIR_OP_LDOUBLE || mode == MIR_OP_VAR_MEM || mode == MIR_OP_LABEL); - if (mode == MIR_OP_HARD_REG && op.u.hard_reg != original.u.hard_reg) + if (mode == MIR_OP_VAR && op_ref->u.var != original.u.var) return FALSE; - else if (mode == MIR_OP_INT && op.u.i != original.u.i) + else if (mode == MIR_OP_INT && op_ref->u.i != original.u.i) return FALSE; - else if (mode == MIR_OP_FLOAT && op.u.f != original.u.f) + else if (mode == MIR_OP_FLOAT && op_ref->u.f != original.u.f) return FALSE; - else if (mode == MIR_OP_DOUBLE && op.u.d != original.u.d) + else if (mode == MIR_OP_DOUBLE && op_ref->u.d != original.u.d) return FALSE; - else if (mode == MIR_OP_LDOUBLE && op.u.ld != original.u.ld) + else if (mode == MIR_OP_LDOUBLE && op_ref->u.ld != original.u.ld) return FALSE; - else if (mode == MIR_OP_LABEL && op.u.label != original.u.label) + else if (mode == MIR_OP_LABEL && op_ref->u.label != original.u.label) return FALSE; - else if (mode == MIR_OP_HARD_REG_MEM && op.u.hard_reg_mem.type != original.u.hard_reg_mem.type - && op.u.hard_reg_mem.scale != original.u.hard_reg_mem.scale - && op.u.hard_reg_mem.base != original.u.hard_reg_mem.base - && op.u.hard_reg_mem.index != original.u.hard_reg_mem.index - && op.u.hard_reg_mem.disp != original.u.hard_reg_mem.disp) + else if (mode == MIR_OP_VAR_MEM + && (op_ref->u.var_mem.type != original.u.var_mem.type + || op_ref->u.var_mem.disp != original.u.var_mem.disp + || op_ref->u.var_mem.base != original.u.var_mem.base + || op_ref->u.var_mem.index != original.u.var_mem.index + || (op_ref->u.var_mem.index != MIR_NON_VAR + && op_ref->u.var_mem.scale != original.u.var_mem.scale))) return FALSE; break; + } default: gen_assert (FALSE); } } @@ -1948,16 +2098,20 @@ static int pattern_match_p (gen_ctx_t gen_ctx, const struct pattern *pat, MIR_in return TRUE; } -static const char *find_insn_pattern_replacement (gen_ctx_t gen_ctx, MIR_insn_t insn) { - int i; +static int find_insn_pattern (gen_ctx_t gen_ctx, MIR_insn_t insn, int *size) { + int i, ind; const struct pattern *pat; insn_pattern_info_t info = VARR_GET (insn_pattern_info_t, insn_pattern_info, insn->code); for (i = 0; i < info.num; i++) { - pat = &patterns[VARR_GET (int, pattern_indexes, info.start + i)]; - if (pattern_match_p (gen_ctx, pat, insn)) return pat->replacement; + ind = VARR_GET (int, pattern_indexes, info.start + i); + pat = &patterns[ind]; + if (pattern_match_p (gen_ctx, pat, insn, size == NULL)) { + if (size != NULL) *size = patterns[ind].max_insn_size; + return ind; + } } - return NULL; + return -1; } static void patterns_finish (gen_ctx_t gen_ctx) { @@ -1993,6 +2147,19 @@ static void setup_r (int *rex, int *r, int v) { *r = v; } +static void setup_rm_byte (int *rex, int *high, int *r, int v) { + gen_assert ((rex == NULL || *rex < 0) && *r < 0 && v >= 0 && v <= MAX_HARD_REG); + if (v >= 16) v -= 16; + if (v >= 4) { + if (rex != NULL) *rex = 1; + } + if (v >= 8) { + if (high != NULL) *high = 1; + v -= 8; + } + *r = v; +} + static void setup_reg (int *rex_reg, int *reg, int v) { setup_r (rex_reg, reg, v); } static void setup_rm (int *rex_b, int *rm, int v) { setup_r (rex_b, rm, v); } @@ -2024,12 +2191,12 @@ static void setup_mem (MIR_mem_t mem, int *mod, int *rm, int *scale, int *base, MIR_disp_t disp = mem.disp; gen_assert (*disp8 < 0 && *disp32 < 0 && mem.index != SP_HARD_REG); - if (mem.index == MIR_NON_HARD_REG && mem.base == MIR_NON_HARD_REG) { /* SIB: disp only */ + if (mem.index == MIR_NON_VAR && mem.base == MIR_NON_VAR) { /* SIB: disp only */ setup_rm (NULL, rm, 4); *disp32 = (uint32_t) disp; setup_base (NULL, base, BP_HARD_REG); setup_index (NULL, index, SP_HARD_REG); - } else if (mem.index == MIR_NON_HARD_REG && mem.base != SP_HARD_REG && mem.base != R12_HARD_REG) { + } else if (mem.index == MIR_NON_VAR && mem.base != SP_HARD_REG && mem.base != R12_HARD_REG) { setup_rm (rex_b, rm, mem.base); if (disp == 0 && mem.base != BP_HARD_REG && mem.base != R13_HARD_REG) { setup_mod (mod, 0); @@ -2040,7 +2207,7 @@ static void setup_mem (MIR_mem_t mem, int *mod, int *rm, int *scale, int *base, setup_mod (mod, 2); *disp32 = (uint32_t) disp; } - } else if (mem.index == MIR_NON_HARD_REG) { /* SIB: only base = sp or r12 */ + } else if (mem.index == MIR_NON_VAR) { /* SIB: only base = sp or r12 */ setup_rm (NULL, rm, 4); setup_index (NULL, index, SP_HARD_REG); setup_base (rex_b, base, mem.base); @@ -2053,7 +2220,7 @@ static void setup_mem (MIR_mem_t mem, int *mod, int *rm, int *scale, int *base, setup_mod (mod, 2); *disp32 = (uint32_t) disp; } - } else if (mem.base == MIR_NON_HARD_REG) { /* SIB: index with scale only */ + } else if (mem.base == MIR_NON_VAR) { /* SIB: index with scale only */ setup_rm (NULL, rm, 4); setup_index (rex_x, index, mem.index); setup_base (NULL, base, BP_HARD_REG); @@ -2112,25 +2279,191 @@ static size_t add_to_const_pool (struct gen_ctx *gen_ctx, uint64_t v) { return len; } -static int setup_imm_addr (struct gen_ctx *gen_ctx, uint64_t v, int *mod, int *rm, - int64_t *disp32) { +static int setup_imm_addr (struct gen_ctx *gen_ctx, uint64_t v, int *mod, int *rm, int64_t *disp32, + int call_p, MIR_item_t func_item) { const_ref_t cr; size_t n; n = add_to_const_pool (gen_ctx, v); setup_rip_rel_addr (0, mod, rm, disp32); + cr.call_p = call_p; + cr.func_item = func_item; cr.pc = 0; cr.next_insn_disp = 0; cr.const_num = n; VARR_PUSH (const_ref_t, const_refs, cr); - return VARR_LENGTH (const_ref_t, const_refs) - 1; + return (int) VARR_LENGTH (const_ref_t, const_refs) - 1; +} + +static int get_max_insn_size (gen_ctx_t gen_ctx MIR_UNUSED, const char *replacement) { + const char *p, *insn_str; + int size = 0; + + for (insn_str = replacement;; insn_str = p + 1) { + char ch, start_ch; + int opcode0_p = FALSE, opcode1_p = FALSE, opcode2_p = FALSE; + int rex_p = FALSE, modrm_p = FALSE, addr_p = FALSE, prefix_p = FALSE; + int disp8_p = FALSE, imm8_p = FALSE, disp32_p = FALSE, imm32_p = FALSE, imm64_p = FALSE; + int switch_table_addr_p = FALSE; + + for (p = insn_str; (ch = *p) != '\0' && ch != ';'; p++) { + if (hex_value (ch = *p) >= 0) { + hex_value (ch = *++p); + if (!opcode0_p) + opcode0_p = TRUE; + else if (!opcode1_p) + opcode1_p = TRUE; + else { + gen_assert (!opcode2_p); + opcode2_p = TRUE; + } + p++; + } + if ((ch = *p) == 0 || ch == ';') break; + switch ((start_ch = ch = *p)) { + case ' ': + case '\t': break; + case 'X': + case 'Y': + case 'Z': + if (opcode0_p) { + gen_assert (!opcode1_p); + prefix_p = opcode0_p; + opcode0_p = FALSE; + } + rex_p = TRUE; + break; + case 'r': + case 'R': + case 'S': + ch = *++p; + gen_assert ('0' <= ch && ch <= '2'); + modrm_p = TRUE; + break; + case 'm': + ch = *++p; + modrm_p = TRUE; + addr_p = TRUE; + if (ch == 't') { /* -16(%rsp) */ + disp8_p = TRUE; + } else if (ch == 'T') { + disp8_p = TRUE; + } else { + gen_assert ('0' <= ch && ch <= '2'); + disp32_p = TRUE; + } + break; + case 'a': + ch = *++p; + addr_p = TRUE; + if (ch == 'p') { + disp32_p = TRUE; + } else if (ch == 'd') { + ++p; + uint64_t disp = read_hex (&p); + if (int8_p (disp)) + disp8_p = TRUE; + else + disp32_p = TRUE; + } else { + gen_assert (ch == 'm'); + } + break; + case 'i': + case 'I': + case 'J': + ch = *++p; + gen_assert ('0' <= ch && ch <= '7'); + if (start_ch == 'i') { + imm8_p = TRUE; + } else if (start_ch == 'I') { + imm32_p = TRUE; + } else { + imm64_p = TRUE; + } + break; + case 'T': switch_table_addr_p = modrm_p = TRUE; break; + case 'q': modrm_p = TRUE; break; + case 'l': disp8_p = TRUE; goto label_rest; + case 'L': + disp32_p = TRUE; + label_rest: + ch = *++p; + gen_assert ('0' <= ch && ch <= '2'); + break; + case 'P': + ch = *++p; + gen_assert ('0' <= ch && ch <= '7'); + modrm_p = TRUE; + disp32_p = TRUE; + break; + case '/': + ch = *++p; + gen_assert ('0' <= ch && ch <= '7'); + modrm_p = TRUE; + break; + case '+': + ch = *++p; + if (ch == 'h') { + ch = *++p; + } else { + gen_assert ('0' <= ch && ch <= '2'); + } + opcode0_p = TRUE; + break; + case 'c': // ??? + ++p; + read_hex (&p); + gen_assert (!disp32_p); + disp32_p = TRUE; + break; + case 'h': + ++p; + read_hex (&p); + modrm_p = TRUE; + break; + case 'H': + ++p; + read_hex (&p); + modrm_p = TRUE; + break; + case 'v': + case 'V': + ++p; + read_hex (&p); + if (start_ch == 'v') { + imm8_p = TRUE; + } else { + imm32_p = TRUE; + } + break; + default: gen_assert (FALSE); + } + } + if (prefix_p) size++; + if (rex_p) size++; + if (opcode0_p) size++; + if (opcode1_p) size++; + if (opcode2_p) size++; + if (modrm_p) size++; + if (addr_p) size++; + if (disp8_p) size++; + if (disp32_p) size += 4; + if (imm8_p) size++; + if (imm32_p) size += 4; + if (imm64_p) size += 8; + if (switch_table_addr_p) size += 4; + if (ch == '\0') break; + } + return size; } -static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacement) { +static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacement, + void **jump_addrs) { MIR_context_t ctx = gen_ctx->ctx; const char *p, *insn_str; label_ref_t lr; - int switch_table_addr_start = -1; + int switch_table_addr_start_offset = -1; if (insn->code == MIR_ALLOCA && (insn->ops[1].mode == MIR_OP_INT || insn->ops[1].mode == MIR_OP_UINT)) @@ -2146,7 +2479,7 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen int64_t disp32 = -1, imm32 = -1; int imm64_p = FALSE; uint64_t imm64 = 0, v; - MIR_op_t op; + const MIR_op_t *op_ref; int const_ref_num = -1, label_ref_num = -1, switch_table_addr_p = FALSE; for (p = insn_str; (ch = *p) != '\0' && ch != ';'; p++) { @@ -2194,14 +2527,18 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen break; case 'r': case 'R': + case 'S': ch = *++p; gen_assert ('0' <= ch && ch <= '2'); - op = insn->ops[ch - '0']; - gen_assert (op.mode == MIR_OP_HARD_REG); + op_ref = &insn->ops[ch - '0']; + gen_assert (op_ref->mode == MIR_OP_VAR); if (start_ch == 'r') - setup_reg (&rex_r, ®, op.u.hard_reg); - else { - setup_rm (&rex_b, &rm, op.u.hard_reg); + setup_reg (&rex_r, ®, op_ref->u.var); + else if (start_ch == 'R') { + setup_rm (&rex_b, &rm, op_ref->u.var); + setup_mod (&mod, 3); + } else if (start_ch == 'S') { + setup_rm_byte (&rex_0, &rex_b, &rm, op_ref->u.var); setup_mod (&mod, 3); } break; @@ -2216,16 +2553,16 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen } else if (ch == 'T') { MIR_op_t mem; - op = insn->ops[0]; - gen_assert (op.mode == MIR_OP_HARD_REG); - mem = _MIR_new_hard_reg_mem_op (ctx, MIR_T_I64, 0, R11_HARD_REG, op.u.hard_reg, 8); - setup_mem (mem.u.hard_reg_mem, &mod, &rm, &scale, &base, &rex_b, &index, &rex_x, &disp8, + op_ref = &insn->ops[0]; + gen_assert (op_ref->mode == MIR_OP_VAR); + mem = _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, R11_HARD_REG, op_ref->u.var, 8); + setup_mem (mem.u.var_mem, &mod, &rm, &scale, &base, &rex_b, &index, &rex_x, &disp8, &disp32); } else { gen_assert ('0' <= ch && ch <= '2'); - op = insn->ops[ch - '0']; - gen_assert (op.mode == MIR_OP_HARD_REG_MEM); - setup_mem (op.u.hard_reg_mem, &mod, &rm, &scale, &base, &rex_b, &index, &rex_x, &disp8, + op_ref = &insn->ops[ch - '0']; + gen_assert (op_ref->mode == MIR_OP_VAR_MEM); + setup_mem (op_ref->u.var_mem, &mod, &rm, &scale, &base, &rex_b, &index, &rex_x, &disp8, &disp32); } break; @@ -2234,36 +2571,37 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen MIR_op_t op2; ch = *++p; - op = insn->ops[1]; - gen_assert (op.mode == MIR_OP_HARD_REG); + op_ref = &insn->ops[1]; + gen_assert (op_ref->mode == MIR_OP_VAR); mem.type = MIR_T_I8; if (ch == 'p') { op2 = insn->ops[2]; - mem.base = op.u.hard_reg; + mem.base = op_ref->u.var; mem.scale = 1; - if (op2.mode == MIR_OP_HARD_REG) { - mem.index = op2.u.hard_reg; + if (op2.mode == MIR_OP_VAR) { + mem.index = op2.u.var; mem.disp = 0; } else { - gen_assert (op2.mode == MIR_OP_INT || op2.mode == MIR_OP_UINT); - mem.index = MIR_NON_HARD_REG; - mem.disp = op2.u.i; + gen_assert (op2.mode == MIR_OP_INT || op2.mode == MIR_OP_UINT + || op2.mode == MIR_OP_REF); + mem.index = MIR_NON_VAR; + mem.disp = int_value (gen_ctx, &op2); } } else if (ch == 'd') { - mem.base = op.u.hard_reg; - mem.index = MIR_NON_HARD_REG; + mem.base = op_ref->u.var; + mem.index = MIR_NON_VAR; mem.scale = 1; ++p; mem.disp = read_hex (&p); } else { gen_assert (ch == 'm'); op2 = insn->ops[2]; - mem.index = op.u.hard_reg; - mem.base = MIR_NON_HARD_REG; + mem.index = op_ref->u.var; + mem.base = MIR_NON_VAR; mem.disp = 0; gen_assert ((op2.mode == MIR_OP_INT || op2.mode == MIR_OP_UINT) && (op2.u.i == 1 || op2.u.i == 2 || op2.u.i == 4 || op2.u.i == 8)); - mem.scale = op2.u.i; + mem.scale = (MIR_scale_t) op2.u.i; } setup_mem (mem, &mod, &rm, &scale, &base, &rex_b, &index, &rex_x, &disp8, &disp32); break; @@ -2273,79 +2611,107 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen case 'J': ch = *++p; gen_assert ('0' <= ch && ch <= '7'); - op = insn->ops[ch - '0']; - gen_assert (op.mode == MIR_OP_INT || op.mode == MIR_OP_UINT); + op_ref = &insn->ops[ch - '0']; + gen_assert (op_ref->mode == MIR_OP_INT || op_ref->mode == MIR_OP_UINT + || op_ref->mode == MIR_OP_REF); + int64_t n = int_value (gen_ctx, op_ref); if (start_ch == 'i') { - gen_assert (int8_p (op.u.i)); - imm8 = (uint8_t) op.u.i; + gen_assert (int8_p (n)); + imm8 = (uint8_t) n; } else if (start_ch == 'I') { - gen_assert (int32_p (op.u.i)); - imm32 = (uint32_t) op.u.i; + gen_assert (int32_p (n)); + imm32 = (uint32_t) n; } else { imm64_p = TRUE; - imm64 = (uint64_t) op.u.i; + imm64 = (uint64_t) n; } break; - case 'P': - ch = *++p; - gen_assert ('0' <= ch && ch <= '7'); - op = insn->ops[ch - '0']; - gen_assert (op.mode == MIR_OP_REF); - imm64_p = TRUE; - if (op.u.ref->item_type == MIR_data_item && op.u.ref->u.data->name != NULL - && _MIR_reserved_ref_name_p (ctx, op.u.ref->u.data->name)) - imm64 = (uint64_t) op.u.ref->u.data->u.els; - else - imm64 = (uint64_t) op.u.ref->addr; - break; case 'T': { - gen_assert (!switch_table_addr_p && switch_table_addr_start < 0); + gen_assert (!switch_table_addr_p && switch_table_addr_start_offset < 0); switch_table_addr_p = TRUE; + mod = 0; + rm = 5; break; } - case 'l': { + case 'q': + mod = 0; + rm = 5; + break; + case 'l': + gen_assert (disp32 < 0 && disp8 < 0); + lr.short_p = TRUE; + disp8 = 0; + goto label_rest; + case 'L': + gen_assert (disp32 < 0 && disp8 < 0); + lr.short_p = FALSE; + disp32 = 0; /* To reserve the space */ + label_rest: ch = *++p; gen_assert ('0' <= ch && ch <= '2'); - op = insn->ops[ch - '0']; - gen_assert (op.mode == MIR_OP_LABEL); + op_ref = &insn->ops[ch - '0']; + gen_assert (op_ref->mode == MIR_OP_LABEL); lr.abs_addr_p = FALSE; lr.label_val_disp = lr.next_insn_disp = 0; - lr.label = op.u.label; - gen_assert (label_ref_num < 0 && disp32 < 0); - disp32 = 0; /* To reserve the space */ - label_ref_num = VARR_LENGTH (label_ref_t, label_refs); + if (jump_addrs == NULL) + lr.u.label = op_ref->u.label; + else + lr.u.jump_addr = jump_addrs[0]; + gen_assert (label_ref_num < 0); + label_ref_num = (int) VARR_LENGTH (label_ref_t, label_refs); VARR_PUSH (label_ref_t, label_refs, lr); break; - } + case 'P': + ch = *++p; + gen_assert ('0' <= ch && ch <= '7'); + op_ref = &insn->ops[ch - '0']; + gen_assert (op_ref->mode == MIR_OP_INT || op_ref->mode == MIR_OP_UINT + || op_ref->mode == MIR_OP_REF); + v = (uint64_t) int_value (gen_ctx, op_ref); + gen_assert (const_ref_num < 0 && disp32 < 0); + MIR_item_t func_item + = op_ref->mode != MIR_OP_REF || op_ref->u.ref->item_type != MIR_func_item ? NULL + : op_ref->u.ref; + const_ref_num = setup_imm_addr (gen_ctx, v, &mod, &rm, &disp32, TRUE, func_item); + break; case '/': ch = *++p; gen_assert ('0' <= ch && ch <= '7'); setup_reg (NULL, ®, ch - '0'); break; - case '+': + case '+': { + int hreg; ch = *++p; - gen_assert ('0' <= ch && ch <= '2'); - op = insn->ops[ch - '0']; - gen_assert (op.mode == MIR_OP_HARD_REG); - setup_reg (&rex_b, &lb, op.u.hard_reg); + if (ch == 'h') { + ch = *++p; + hreg = hex_value (*p++); + gen_assert (hreg >= 0); + } else { + gen_assert ('0' <= ch && ch <= '2'); + op_ref = &insn->ops[ch - '0']; + gen_assert (op_ref->mode == MIR_OP_VAR); + hreg = op_ref->u.var; + } + setup_reg (&rex_b, &lb, hreg); break; + } case 'c': ++p; v = read_hex (&p); gen_assert (const_ref_num < 0 && disp32 < 0); - const_ref_num = setup_imm_addr (gen_ctx, v, &mod, &rm, &disp32); + const_ref_num = setup_imm_addr (gen_ctx, v, &mod, &rm, &disp32, FALSE, NULL); break; case 'h': ++p; v = read_hex (&p); gen_assert (v <= 31); - setup_reg (&rex_r, ®, v); + setup_reg (&rex_r, ®, (int) v); break; case 'H': ++p; v = read_hex (&p); gen_assert (v <= 31); - setup_rm (&rex_b, &rm, v); + setup_rm (&rex_b, &rm, (int) v); setup_mod (&mod, 3); break; case 'v': @@ -2354,7 +2720,7 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen v = read_hex (&p); if (start_ch == 'v') { gen_assert (uint8_p (v)); - imm8 = v; + imm8 = (int) v; } else { gen_assert (uint32_p (v)); imm32 = v; @@ -2406,8 +2772,8 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen if (imm64_p) put_uint64 (gen_ctx, imm64, 8); if (switch_table_addr_p) { - switch_table_addr_start = VARR_LENGTH (uint8_t, result_code); - put_uint64 (gen_ctx, 0, 8); + switch_table_addr_start_offset = (int) VARR_LENGTH (uint8_t, result_code); + put_uint64 (gen_ctx, 0, 4); } if (label_ref_num >= 0) VARR_ADDR (label_ref_t, label_refs) @@ -2417,16 +2783,20 @@ static void out_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, const char *replacemen [const_ref_num].next_insn_disp = VARR_LENGTH (uint8_t, result_code); if (ch == '\0') break; } - if (switch_table_addr_start < 0) return; - gen_assert (insn->code == MIR_SWITCH); - VARR_PUSH (uint64_t, abs_address_locs, switch_table_addr_start); - set_int64 (&VARR_ADDR (uint8_t, result_code)[switch_table_addr_start], - (int64_t) VARR_LENGTH (uint8_t, result_code), 8); + if (switch_table_addr_start_offset < 0) return; + while (VARR_LENGTH (uint8_t, result_code) % 8 != 0) put_byte (gen_ctx, 0); /* align the table */ + gen_assert (insn->code == MIR_SWITCH + && (int) VARR_LENGTH (uint8_t, result_code) > switch_table_addr_start_offset); + set_int64 (&VARR_ADDR (uint8_t, result_code)[switch_table_addr_start_offset], + (int64_t) VARR_LENGTH (uint8_t, result_code) - switch_table_addr_start_offset - 4, 4); for (size_t i = 1; i < insn->nops; i++) { gen_assert (insn->ops[i].mode == MIR_OP_LABEL); lr.abs_addr_p = TRUE; lr.label_val_disp = VARR_LENGTH (uint8_t, result_code); - lr.label = insn->ops[i].u.label; + if (jump_addrs == NULL) + lr.u.label = insn->ops[i].u.label; + else + lr.u.jump_addr = jump_addrs[i - 1]; VARR_PUSH (label_ref_t, label_refs, lr); put_uint64 (gen_ctx, 0, 8); } @@ -2437,54 +2807,255 @@ static uint8_t MIR_UNUSED get_short_jump_opcode (uint8_t *long_jump_opcode) { return long_jump_opcode[1] - 0x10; } -static int target_insn_ok_p (gen_ctx_t gen_ctx, MIR_insn_t insn) { - return find_insn_pattern_replacement (gen_ctx, insn) != NULL; +static int target_memory_ok_p (gen_ctx_t gen_ctx MIR_UNUSED, MIR_op_t *op_ref) { + if (op_ref->mode != MIR_OP_VAR_MEM) return FALSE; + if (op_ref->u.var_mem.index != MIR_NON_VAR && op_ref->u.var_mem.scale != 1 + && op_ref->u.var_mem.scale != 2 && op_ref->u.var_mem.scale != 4 + && op_ref->u.var_mem.scale != 8) + return FALSE; + return int32_p (op_ref->u.var_mem.disp); } -static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { - MIR_context_t ctx = gen_ctx->ctx; - size_t i; - MIR_insn_t insn; - const char *replacement; +static int target_insn_ok_p (gen_ctx_t gen_ctx, MIR_insn_t insn) { + return find_insn_pattern (gen_ctx, insn, NULL) >= 0; +} - gen_assert (curr_func_item->item_type == MIR_func_item); +static void translate_init (gen_ctx_t gen_ctx) { VARR_TRUNC (uint8_t, result_code, 0); VARR_TRUNC (uint64_t, const_pool, 0); VARR_TRUNC (const_ref_t, const_refs, 0); VARR_TRUNC (label_ref_t, label_refs, 0); VARR_TRUNC (uint64_t, abs_address_locs, 0); +} + +static uint8_t *translate_finish (gen_ctx_t gen_ctx, size_t *len) { + /* Setting up labels */ + for (size_t i = 0; i < VARR_LENGTH (label_ref_t, label_refs); i++) { + label_ref_t lr = VARR_GET (label_ref_t, label_refs, i); + + if (lr.abs_addr_p) { + set_int64 (&VARR_ADDR (uint8_t, result_code)[lr.label_val_disp], + (int64_t) get_label_disp (gen_ctx, lr.u.label), 8); + VARR_PUSH (uint64_t, abs_address_locs, lr.label_val_disp); + } else if (lr.short_p) { + int64_t disp = (int64_t) get_label_disp (gen_ctx, lr.u.label) - (int64_t) lr.next_insn_disp; + gen_assert (-128 <= disp && disp < 128); + set_int64 (&VARR_ADDR (uint8_t, result_code)[lr.label_val_disp], disp, 1); + } else { + set_int64 (&VARR_ADDR (uint8_t, result_code)[lr.label_val_disp], + (int64_t) get_label_disp (gen_ctx, lr.u.label) - (int64_t) lr.next_insn_disp, 4); + } + } + while (VARR_LENGTH (uint8_t, result_code) % 16 != 0) /* Align the pool */ + VARR_PUSH (uint8_t, result_code, 0); + for (size_t i = 0; i < VARR_LENGTH (const_ref_t, const_refs); i++) { /* Add pool constants */ + const_ref_t cr = VARR_GET (const_ref_t, const_refs, i); + + set_int64 (VARR_ADDR (uint8_t, result_code) + cr.pc, + VARR_LENGTH (uint8_t, result_code) - cr.next_insn_disp, 4); + put_uint64 (gen_ctx, VARR_GET (uint64_t, const_pool, cr.const_num), 8); + put_uint64 (gen_ctx, 0, 8); /* keep 16 bytes align */ + } + *len = VARR_LENGTH (uint8_t, result_code); + return VARR_ADDR (uint8_t, result_code); +} + +static void target_split_insns (gen_ctx_t gen_ctx MIR_UNUSED) {} + +#define LOOP_ALIGN 8 + +static const char *nop_pats[] = { + "", + "\x90" /* 1:nop */, + "\x66\x90", /* 2: xchg ax,ax */ + "\x0f\x1f\xc0" /* 3:nopl eax */, + "\x48\x0f\x1f\xc0" /* 4:nop rax */, + "\x0f\x1f\x44\x00\x00" /* 5: nopl 0x0(%rax,%rax,1) */, + "\x66\x0f\x1f\x44\x00\x00" /* 6: nopw 0x0(%rax,%rax,1) */, + "\x0f\x1f\x80\x00\x00\x00\x00" /* 7: nopl 0x0(%rax) */, +}; + +static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_insn_t insn; + int ind, max_insn_size; + size_t curr_size, n; + + gen_assert (curr_func_item->item_type == MIR_func_item); + translate_init (gen_ctx); + curr_size = 0; + VARR_TRUNC (int, insn_pattern_indexes, 0); for (insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); insn != NULL; insn = DLIST_NEXT (MIR_insn_t, insn)) { if (insn->code == MIR_LABEL) { - set_label_disp (gen_ctx, insn, VARR_LENGTH (uint8_t, result_code)); - } else { - replacement = find_insn_pattern_replacement (gen_ctx, insn); - if (replacement == NULL) { - fprintf (stderr, "%d: fatal failure in matching insn:", gen_ctx->gen_num); + if (gen_nested_loop_label_p (gen_ctx, insn)) curr_size += LOOP_ALIGN; + set_label_disp (gen_ctx, insn, curr_size); /* estimation */ + } else if (insn->code != MIR_USE) { + ind = find_insn_pattern (gen_ctx, insn, &max_insn_size); + if (ind < 0) { + fprintf (stderr, "Fatal failure in matching insn:"); MIR_output_insn (ctx, stderr, insn, curr_func_item->u.func, TRUE); exit (1); - } else { - gen_assert (replacement != NULL); - out_insn (gen_ctx, insn, replacement); } + curr_size += max_insn_size; + if (insn->code == MIR_SWITCH) curr_size += (insn->nops - 1) * 8; /* label addresses */ + VARR_PUSH (int, insn_pattern_indexes, ind); } } - /* Setting up labels */ - for (i = 0; i < VARR_LENGTH (label_ref_t, label_refs); i++) { - label_ref_t lr = VARR_GET (label_ref_t, label_refs, i); + for (n = 0, insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); insn != NULL; + insn = DLIST_NEXT (MIR_insn_t, insn)) { + if (insn->code == MIR_LABEL) { + if (gen_nested_loop_label_p (gen_ctx, insn)) { + int padn = LOOP_ALIGN - (int) (VARR_LENGTH (uint8_t, result_code) % LOOP_ALIGN); + if (padn == LOOP_ALIGN) padn = 0; + gen_assert ((size_t) padn < sizeof (nop_pats) / sizeof (char *)); + if (padn != 0) VARR_PUSH_ARR (uint8_t, result_code, (uint8_t *) nop_pats[padn], padn); + } + set_label_disp (gen_ctx, insn, VARR_LENGTH (uint8_t, result_code)); + } else if (insn->code != MIR_USE) { + ind = VARR_GET (int, insn_pattern_indexes, n++); + if (MIR_branch_code_p (insn->code)) /* possible replacement change */ + ind = find_insn_pattern (gen_ctx, insn, NULL); + gen_assert (ind >= 0); +#ifndef NDEBUG + size_t len_before = VARR_LENGTH (uint8_t, result_code); +#endif + out_insn (gen_ctx, insn, patterns[ind].replacement, NULL); +#ifndef NDEBUG + size_t insn_len = VARR_LENGTH (uint8_t, result_code) - len_before; + if (insn_len > (size_t) patterns[ind].max_insn_size && insn->code != MIR_SWITCH) { + fprintf (stderr, "\"%s\" max size(%d) < real size(%d)\n", patterns[ind].replacement, + patterns[ind].max_insn_size, (int) insn_len); + gen_assert (FALSE); + } +#endif + } + } + return translate_finish (gen_ctx, len); +} - if (!lr.abs_addr_p) { - set_int64 (&VARR_ADDR (uint8_t, result_code)[lr.label_val_disp], - (int64_t) get_label_disp (gen_ctx, lr.label) - (int64_t) lr.next_insn_disp, 4); - } else { - set_int64 (&VARR_ADDR (uint8_t, result_code)[lr.label_val_disp], - (int64_t) get_label_disp (gen_ctx, lr.label), 8); - VARR_PUSH (uint64_t, abs_address_locs, lr.label_val_disp); +static void store_call_ref (gen_ctx_t gen_ctx, MIR_item_t ref_func_item, uint8_t *call_addr) { + call_ref_t call_ref; + + if (MIR_get_func_redef_permission_p (gen_ctx->ctx)) return; + call_ref.ref_func_item = ref_func_item; + call_ref.call_addr = call_addr; + VARR_PUSH (call_ref_t, gen_ctx->target_ctx->call_refs, call_ref); +} + +static void change_calls (gen_ctx_t gen_ctx, uint8_t *base) { + MIR_context_t ctx = gen_ctx->ctx; + /* changing calls to rel32 calls: */ + for (size_t i = 0; i < VARR_LENGTH (const_ref_t, const_refs); i++) { + const_ref_t cr = VARR_GET (const_ref_t, const_refs, i); + if (!cr.call_p) continue; + gen_assert (base[cr.pc - 2] == 0xff); + gen_assert (base[cr.pc - 1] == 0x15 || base[cr.pc - 1] == 0x25); + if (cr.func_item != NULL) store_call_ref (gen_ctx, cr.func_item, (uint8_t *) base + cr.pc - 2); + uint64_t v = VARR_GET (uint64_t, const_pool, cr.const_num); + int64_t off = (int64_t) v - (int64_t) (base + cr.next_insn_disp); + if (!int32_p (off)) continue; + uint8_t rel_insn[] = {0x40, 0xe8, 0, 0, 0, 0}; /* rex call rel32 */ + if (base[cr.pc - 1] == 0x25) rel_insn[1] = 0xe9; /* rex jmp rel32 */ + set_int64 (rel_insn + 2, off, 4); + _MIR_change_code (ctx, (uint8_t *) base + cr.pc - 2, (uint8_t *) rel_insn, 6); + } +} + +static void target_rebase (gen_ctx_t gen_ctx, uint8_t *base) { + MIR_code_reloc_t reloc; + + VARR_TRUNC (MIR_code_reloc_t, relocs, 0); + for (size_t i = 0; i < VARR_LENGTH (uint64_t, abs_address_locs); i++) { + reloc.offset = VARR_GET (uint64_t, abs_address_locs, i); + reloc.value = base + get_int64 (base + reloc.offset, 8); + VARR_PUSH (MIR_code_reloc_t, relocs, reloc); + } + _MIR_update_code_arr (gen_ctx->ctx, base, VARR_LENGTH (MIR_code_reloc_t, relocs), + VARR_ADDR (MIR_code_reloc_t, relocs)); + change_calls (gen_ctx, base); + gen_setup_lrefs (gen_ctx, base); +} + +static void target_change_to_direct_calls (MIR_context_t ctx) { + gen_ctx_t gen_ctx = *gen_ctx_loc (ctx); + size_t len = VARR_LENGTH (call_ref_t, gen_ctx->target_ctx->call_refs); + if (len == 0) return; + call_ref_t *call_refs_addr = VARR_ADDR (call_ref_t, gen_ctx->target_ctx->call_refs); + for (size_t i = 0; i < len; i++) { + MIR_item_t ref_func_item = call_refs_addr[i].ref_func_item; + MIR_func_t ref_func = ref_func_item->u.func; + uint8_t *addr_loc, *addr_before, *addr = ref_func->machine_code; + uint8_t *call_addr = call_refs_addr[i].call_addr; + int32_t off = *(int32_t *) (call_addr + 2); + int call32_p = FALSE; + if (call_addr[0] == 0xff) { /* call *rel32(rip) */ + addr_loc = call_addr + 6 + off; + addr_before = (uint8_t *) *(uint64_t *) addr_loc; + if (addr_before == addr) continue; + _MIR_change_code (ctx, addr_loc, (uint8_t *) &addr, sizeof (uint64_t)); + } else { /* rex call rel32(rip) */ + gen_assert (call_addr[0] == 0x40); + addr_loc = call_addr; + addr_before = call_addr + 6 + off; + int64_t new_off = addr - (call_addr + 6); + if (addr_before == addr || !int32_p (new_off)) { + DEBUG (2, { + fprintf (stderr, + "Failing to make direct 32-bit call of func %s at 0x%llx (addr: before=0x%llx, " + "after=0x%llx)\n", + ref_func->name, (unsigned long long) addr_loc, (unsigned long long) addr_before, + (unsigned long long) addr); + }); + continue; + } + off = (int32_t) new_off; + _MIR_change_code (ctx, addr_loc + 2, (uint8_t *) &off, sizeof (uint32_t)); + call32_p = TRUE; } + DEBUG (2, { + fprintf (stderr, + "Making direct %s-bit call of func %s at 0x%llx (addr: before=0x%llx, " + "after=0x%llx)\n", + (call32_p ? "32" : "64"), ref_func->name, (unsigned long long) addr_loc, + (unsigned long long) addr_before, (unsigned long long) addr); + }); } + VARR_TRUNC (call_ref_t, gen_ctx->target_ctx->call_refs, 0); +} + +struct target_bb_version { + uint8_t *base; + label_ref_t branch_ref; /* label cand used for jump to this bb version */ +}; + +static void target_init_bb_version_data (target_bb_version_t data) { + data->base = NULL; /* we don't know origin branch */ +} + +static void target_bb_translate_start (gen_ctx_t gen_ctx) { + VARR_TRUNC (uint8_t, result_code, 0); + VARR_TRUNC (uint64_t, const_pool, 0); + VARR_TRUNC (const_ref_t, const_refs, 0); + VARR_TRUNC (label_ref_t, label_refs, 0); + VARR_TRUNC (uint64_t, abs_address_locs, 0); +} + +static void target_bb_insn_translate (gen_ctx_t gen_ctx, MIR_insn_t insn, void **jump_addrs) { + if (insn->code == MIR_LABEL) return; + int ind = find_insn_pattern (gen_ctx, insn, &ind); /* &ind for no short jumps */ + gen_assert (ind >= 0); + out_insn (gen_ctx, insn, patterns[ind].replacement, jump_addrs); +} + +static void target_output_jump (gen_ctx_t gen_ctx, void **jump_addrs) { + out_insn (gen_ctx, temp_jump, patterns[temp_jump_pat_ind].replacement, jump_addrs); +} + +static uint8_t *target_bb_translate_finish (gen_ctx_t gen_ctx, size_t *len) { while (VARR_LENGTH (uint8_t, result_code) % 16 != 0) /* Align the pool */ VARR_PUSH (uint8_t, result_code, 0); - for (i = 0; i < VARR_LENGTH (const_ref_t, const_refs); i++) { /* Add pool constants */ + for (size_t i = 0; i < VARR_LENGTH (const_ref_t, const_refs); i++) { /* Add pool constants */ const_ref_t cr = VARR_GET (const_ref_t, const_refs, i); set_int64 (VARR_ADDR (uint8_t, result_code) + cr.pc, @@ -2496,9 +3067,32 @@ static uint8_t *target_translate (gen_ctx_t gen_ctx, size_t *len) { return VARR_ADDR (uint8_t, result_code); } -static void target_rebase (gen_ctx_t gen_ctx, uint8_t *base) { +static void setup_rel32 (gen_ctx_t gen_ctx, label_ref_t *lr, uint8_t *base, void *addr) { + MIR_context_t ctx = gen_ctx->ctx; + int64_t offset = (int64_t) addr - (int64_t) (base + lr->next_insn_disp); + int32_t rel32 = (int32_t) offset; + + if (lr->abs_addr_p || !(INT32_MIN <= offset && offset <= INT32_MAX)) { + fprintf (stderr, "too big offset (%lld) in setup_rel32", (long long) offset); + exit (1); + } + _MIR_change_code (ctx, (uint8_t *) base + lr->label_val_disp, (uint8_t *) &rel32, 4); +} + +static void target_bb_rebase (gen_ctx_t gen_ctx, uint8_t *base) { + MIR_context_t ctx = gen_ctx->ctx; MIR_code_reloc_t reloc; + /* Setting up relative labels */ + for (size_t i = 0; i < VARR_LENGTH (label_ref_t, label_refs); i++) { + label_ref_t lr = VARR_GET (label_ref_t, label_refs, i); + if (lr.abs_addr_p) { + _MIR_change_code (ctx, (uint8_t *) base + lr.label_val_disp, (uint8_t *) &lr.u.jump_addr, 8); + } else { + setup_rel32 (gen_ctx, &lr, base, lr.u.jump_addr); + } + } + change_calls (gen_ctx, base); VARR_TRUNC (MIR_code_reloc_t, relocs, 0); for (size_t i = 0; i < VARR_LENGTH (uint64_t, abs_address_locs); i++) { reloc.offset = VARR_GET (uint64_t, abs_address_locs, i); @@ -2509,28 +3103,66 @@ static void target_rebase (gen_ctx_t gen_ctx, uint8_t *base) { VARR_ADDR (MIR_code_reloc_t, relocs)); } +static void target_setup_succ_bb_version_data (gen_ctx_t gen_ctx, uint8_t *base) { + if (VARR_LENGTH (label_ref_t, label_refs) + != VARR_LENGTH (target_bb_version_t, target_succ_bb_versions)) + /* We can have more one possible branch from original insn + (e.g. SWITCH, FBNE). If it is so, we will make jumps only + through BB thunk. */ + return; + for (size_t i = 0; i < VARR_LENGTH (target_bb_version_t, target_succ_bb_versions); i++) { + target_bb_version_t data = VARR_GET (target_bb_version_t, target_succ_bb_versions, i); + if (data == NULL) continue; + data->branch_ref = VARR_GET (label_ref_t, label_refs, i); + data->base = base; + } +} + +static void target_redirect_bb_origin_branch (gen_ctx_t gen_ctx, target_bb_version_t data, + void *addr) { + MIR_context_t ctx = gen_ctx->ctx; + + if (data->base == NULL) return; + if (data->branch_ref.abs_addr_p) { + _MIR_change_code (ctx, (uint8_t *) data->base + data->branch_ref.label_val_disp, + (uint8_t *) &addr, 8); + } else { + setup_rel32 (gen_ctx, &data->branch_ref, data->base, addr); + } + data->base = NULL; +} + static void target_init (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; + gen_ctx->target_ctx = gen_malloc (gen_ctx, sizeof (struct target_ctx)); VARR_CREATE (uint8_t, result_code, 0); + VARR_CREATE (int, insn_pattern_indexes, 0); VARR_CREATE (uint64_t, const_pool, 0); VARR_CREATE (const_ref_t, const_refs, 0); VARR_CREATE (label_ref_t, label_refs, 0); VARR_CREATE (uint64_t, abs_address_locs, 0); VARR_CREATE (MIR_code_reloc_t, relocs, 0); + VARR_CREATE (call_ref_t, gen_ctx->target_ctx->call_refs, 0); MIR_type_t res = MIR_T_D; - MIR_var_t args[] = {{MIR_T_D, "src"}}; + MIR_var_t args[] = {{MIR_T_D, "src", 0}}; _MIR_register_unspec_insn (gen_ctx->ctx, MOVDQA_CODE, "movdqa", 1, &res, 1, FALSE, args); patterns_init (gen_ctx); + temp_jump = MIR_new_insn (ctx, MIR_JMP, MIR_new_label_op (ctx, NULL)); + temp_jump_pat_ind = find_insn_pattern (gen_ctx, temp_jump, NULL); } static void target_finish (gen_ctx_t gen_ctx) { patterns_finish (gen_ctx); + _MIR_free_insn (gen_ctx->ctx, temp_jump); VARR_DESTROY (uint8_t, result_code); + VARR_DESTROY (int, insn_pattern_indexes); VARR_DESTROY (uint64_t, const_pool); VARR_DESTROY (const_ref_t, const_refs); VARR_DESTROY (label_ref_t, label_refs); VARR_DESTROY (uint64_t, abs_address_locs); VARR_DESTROY (MIR_code_reloc_t, relocs); + VARR_DESTROY (call_ref_t, gen_ctx->target_ctx->call_refs); free (gen_ctx->target_ctx); gen_ctx->target_ctx = NULL; } diff --git a/mir/mir-gen.c b/mir/mir-gen.c index 022bacdd..3dd75934 100644 --- a/mir/mir-gen.c +++ b/mir/mir-gen.c @@ -1,62 +1,92 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ /* Optimization pipeline: - ------------- ------------- - ---------- ----------- ----------- | Copy | | Global | - MIR -->| Simplify |-->| Build CFG |-->| Build SSA |-->| Propagation |-->| Value | - ---------- ----------- ----------- | | | Numbering | - ------------- ------------- + --------------- ------------ + ---------- ----------- ----------- | Address | | Block | + MIR -->| Simplify |-->| Build CFG |-->| Build SSA |-->| Transformation|-->| Cloning | + ---------- ----------- ----------- --------------- ------------ | - ------------- V - ------- --------- -------- | Sparse | ------------- - | Build | | Finding | ----------- | Out of | | Conditional | | Dead Code | - | Live |<--| Loops |<--| Machinize |<--| SSA |<--| Constant |<--| Elimination | - | Info | --------- ----------- -------- | Propagation | ------------- - ------- ------------- - | - V - -------- ---------- - | Build | -------- --------- --------- ------------- | Generate | - | Live |-->| Assign |-->| Rewrite |-->| Combine |-->| Dead Code |-->| Machine |--> Machine - | Ranges | -------- --------- --------- | Elimination | | Insns | Insns - -------- ------------- ---------- + V + ------------ ------------ ------------ --------------------------- + |Dead Code | |Dead Store | | Copy | | Global Value Numbering, | + |Elimination |<--- |Elimination |<---| Propagation|<---| Constant Propagation, | + ------------ ------------ ------------ | Redundat Load Elimination | + | --------------------------- + V + ----------- -------- ------- ------ ---- ------- + | Loop | |Register| | SSA | |Out of| |Jump| --------- | Build | + | Invariant |-->|Pressure|-->|Combine|-->| SSA |-->|Opts|-->|Machinize|-->| Live | + | Motion | | Relief | ------- ------ ---- --------- | Info | + ----------- -------- ------- + | + V + -------- ---------- --------- + |Generate| ----- ------- |Register | -------- |Build | + Machine <---|Machine |<--| DCE |<--|Combine|<--|Allocator |<--|Coalesce|<--|Register | + Insns | Insns | ----- ------- ---------- -------- |Conflicts| + -------- --------- Simplify: Lowering MIR (in mir.c). Always. - Build CGF: Building Control Flow Graph (basic blocks and CFG edges). Only for -O1 and above. + Build CGF: Building Control Flow Graph (basic blocks and CFG edges). Always. Build SSA: Building Single Static Assignment Form by adding phi nodes and SSA edges - Copy Propagation: SSA copy propagation keeping conventional SSA form and removing redundant - extension insns - Global Value Numbering: Removing redundant insns through GVN. Only for -O2 and above. - Dead code elimination: Removing insns with unused outputs. Only for -O2 and above. - Sparse Conditional Constant Propagation: Constant propagation and removing death paths of CFG. - Only for -O2 and above. - Out of SSA: Removing phi nodes and SSA edges (we keep conventional SSA all the time) + (for -O2 and above). + Address Transformation: Optional pass to remove or change ADDR insns (for -O2 and above). + Block Cloning: Cloning insns and BBs to improve hot path optimization opportunities + (for -O2 and above). + Global Value Numbering: Removing redundant insns through GVN. This includes constant + propagation and redundant load eliminations (for -O2 and above). + Copy Propagation: SSA copy propagation and removing redundant extension insns + (for -O2 and above). + Dead store elimination: Removing redundant stores (for -O2 and above). + Dead code elimination: Removing insns with unused outputs (for -O2 and above). + Loop invariant motion (LICM): Moving invarinat insns out of loop (for -O2 and above). + Pressure relief: Moving insns to decrease register pressure (for -O2 and above). + SSA combine: Combining addresses and cmp and branch pairs (for -O2 and above). + Out of SSA: Making conventional SSA and removing phi nodes and SSA edges (for -O2 and above). + Jump optimizations: Different optimizations on jumps and branches (for -O2 and above). Machinize: Machine-dependent code (e.g. in mir-gen-x86_64.c) transforming MIR for calls ABI, 2-op insns, etc. Always. - Finding Loops: Building loop tree which is used in subsequent register allocation. - Only for -O1 and above. - Building Live Info: Calculating live in and live out for the basic blocks. - Build Live Ranges: Calculating program point ranges for registers. Only for -O1 and above. - Assign: Fast RA for -O0 or Priority-based linear scan RA for -O1 and above. - Rewrite: Transform MIR according to the assign using reserved hard regs. - Combine (code selection): Merging data-depended insns into one. Only for -O1 and above. - Dead code elimination: Removing insns with unused outputs. Only for -O1 and above. - Generate machine insns: Machine-dependent code (e.g. in - mir-gen-x86_64.c) creating machine insns. Always. - - -O0 is 2 times faster than -O1 but generates much slower code. + Building Live Info: Calculating live in and live out for the basic blocks. Always. + Build Register Conflicts: Build conflict matrix for registers involved in moves. + It is used for register coalescing + Coalesce: Aggressive register coalescing + Register Allocator (RA): Priority-based linear scan RA (always) with live range splitting + (for -O2 and above). + Combine: Code selection by merging data-depended insns into one (for -O1 and above). + Dead code elimination (DCE): Removing insns with unused outputs (for -O1 and above). + Generate machine insns: Machine-dependent code (e.g. in mir-gen-x86_64.c) creating + machine insns. Always. + + -O0 and -O1 are 2-3 times faster than -O2 but generate considerably slower code. Terminology: - reg - MIR (pseudo-)register (their numbers are in MIR_OP_REG and MIR_OP_MEM) - hard reg - MIR hard register (their numbers are in MIR_OP_HARD_REG and MIR_OP_HARD_REG_MEM) - breg (based reg) - function pseudo registers whose numbers start with zero - var - pseudo and hard register (var numbers for pseudo-registers - are based reg numbers + MAX_HARD_REG + 1) + reg - MIR (pseudo-)register (their numbers are in MIR_OP_VAR and MIR_OP_VAR_MEM > MAX_HARD_REG) + hard reg - MIR hard register (their numbers are in MIR_OP_VAR and MIR_OP_VAR_MEM + and less or equal MAX_HARD_REG) + var - pseudo and hard register (MIR_NON_VAR means no var) loc - hard register and stack locations (stack slot numbers start with MAX_HARD_REG + 1). - We use conventional SSA to make out-of-ssa fast and simple. + Memory aliasing rules: + + * Memory has aliases and they are used for recognizing aliased memory + + * Memory has nloc attribute. Memory with the same nloc always refer for the same memory + although memory with different nloc still may refer for the same memory. Memory with + the same nloc has the same alias attributes + + * Memory found aliased with alias attributes can be recognized as non-aliased one by + using alloca flags described below + + * Memory can have flags 'must alloca' and 'may alloca'. 'Must alloca' always goes + with 'may alloca'. 'Must alloca' means that we guarantee memory can be allocated + only alloca in the func. 'May alloca' means that it is not excluded that memory is + allocated by alloca + + * Memory with 'must alloca' flag can have disp attribute. We can define that + 'must alloca' memory refers the same memory using disp attribute + */ #include @@ -84,6 +114,7 @@ static void varr_error (const char *message) { util_error (NULL, message); } /* Functions used by target dependent code: */ static void *gen_malloc (gen_ctx_t gen_ctx, size_t size); static MIR_reg_t gen_new_temp_reg (gen_ctx_t gen_ctx, MIR_type_t type, MIR_func_t func); +static int gen_nested_loop_label_p (gen_ctx_t gen_ctx, MIR_insn_t insn); static void set_label_disp (gen_ctx_t gen_ctx, MIR_insn_t insn, size_t disp); static size_t get_label_disp (gen_ctx_t gen_ctx, MIR_insn_t insn); static void create_new_bb_insns (gen_ctx_t gen_ctx, MIR_insn_t before, MIR_insn_t after, @@ -92,6 +123,16 @@ static void gen_delete_insn (gen_ctx_t gen_ctx, MIR_insn_t insn); static void gen_add_insn_before (gen_ctx_t gen_ctx, MIR_insn_t before, MIR_insn_t insn); static void gen_add_insn_after (gen_ctx_t gen_ctx, MIR_insn_t after, MIR_insn_t insn); static void setup_call_hard_reg_args (gen_ctx_t gen_ctx, MIR_insn_t call_insn, MIR_reg_t hard_reg); +static uint64_t get_ref_value (gen_ctx_t gen_ctx, const MIR_op_t *ref_op); +static void gen_setup_lrefs (gen_ctx_t gen_ctx, uint8_t *func_code); +static int64_t gen_int_log2 (int64_t i); + +#define SWAP(v1, v2, temp) \ + do { \ + temp = v1; \ + v1 = v2; \ + v2 = temp; \ + } while (0) #ifndef MIR_GEN_CALL_TRACE #define MIR_GEN_CALL_TRACE 0 @@ -112,11 +153,10 @@ struct target_ctx; struct data_flow_ctx; struct ssa_ctx; struct gvn_ctx; -struct ccp_ctx; struct lr_ctx; +struct coalesce_ctx; struct ra_ctx; -struct selection_ctx; -struct fg_ctx; +struct combine_ctx; typedef struct loop_node *loop_node_t; DEF_VARR (loop_node_t); @@ -130,18 +170,37 @@ struct dead_var { }; DEF_DLIST (dead_var_t, dead_var_link); -struct all_gen_ctx; - typedef struct bb_insn *bb_insn_t; DEF_VARR (bb_insn_t); +typedef struct target_bb_version *target_bb_version_t; +DEF_VARR (target_bb_version_t); + +typedef void *void_ptr_t; +DEF_VARR (void_ptr_t); + +typedef struct { + unsigned char alloca_flag; + unsigned char disp_def_p; /* can be true only for MUST_ALLOCA */ + MIR_type_t type; /* memory type */ + MIR_alias_t alias, nonalias; /* memory aliases */ + MIR_insn_t def_insn; /* base def insn: its value + disp form address */ + int64_t disp; /* defined only when disp_def_p, otherwise disp is unknown */ +} mem_attr_t; + +DEF_VARR (mem_attr_t); + +typedef struct spot_attr { + uint32_t spot, prop; + MIR_op_t *mem_ref; /* ref for memory if the spot is memory, NULL otherwise */ +} spot_attr_t; + +DEF_VARR (spot_attr_t); + +DEF_VARR (MIR_op_t); +DEF_VARR (MIR_insn_t); + struct gen_ctx { - struct all_gen_ctx *all_gen_ctx; - int gen_num; /* always 1 for non-parallel generation */ -#if MIR_PARALLEL_GEN - pthread_t gen_thread; - int busy_p; -#endif MIR_context_t ctx; unsigned optimize_level; /* 0:fast gen; 1:RA+combiner; 2: +GVN/CCP (default); >=3: everything */ MIR_item_t curr_func_item; @@ -149,72 +208,103 @@ struct gen_ctx { FILE *debug_file; int debug_level; #endif - bitmap_t insn_to_consider, temp_bitmap, temp_bitmap2; - bitmap_t call_used_hard_regs[MIR_T_BOUND], func_used_hard_regs; + VARR (void_ptr_t) * to_free; + int addr_insn_p; /* true if we have address insns in the input func */ + bitmap_t tied_regs; /* regs tied to hard reg */ + bitmap_t addr_regs; /* regs in addr insns as 2nd op */ + bitmap_t insn_to_consider, temp_bitmap, temp_bitmap2, temp_bitmap3; + bitmap_t call_used_hard_regs[MIR_T_BOUND]; + bitmap_t func_used_hard_regs; /* before prolog: used hard regs except global var hard regs */ func_cfg_t curr_cfg; uint32_t curr_bb_index, curr_loop_node_index; DLIST (dead_var_t) free_dead_vars; + unsigned long long overall_bbs_num, overall_gen_bbs_num; struct target_ctx *target_ctx; struct data_flow_ctx *data_flow_ctx; struct ssa_ctx *ssa_ctx; struct gvn_ctx *gvn_ctx; - struct ccp_ctx *ccp_ctx; struct lr_ctx *lr_ctx; + struct coalesce_ctx *coalesce_ctx; struct ra_ctx *ra_ctx; - struct selection_ctx *selection_ctx; - struct fg_ctx *fg_ctx; - VARR (bb_insn_t) * dead_bb_insns; + struct combine_ctx *combine_ctx; + VARR (MIR_op_t) * temp_ops; + VARR (MIR_insn_t) * temp_insns, *temp_insns2; + VARR (bb_insn_t) * temp_bb_insns, *temp_bb_insns2; VARR (loop_node_t) * loop_nodes, *queue_nodes, *loop_entries; /* used in building loop tree */ + /* true when alloca memory escapes by assigning alloca address to memory: */ + unsigned char full_escape_p; + VARR (mem_attr_t) * mem_attrs; /* nloc (> 0) => mem attributes */ int max_int_hard_regs, max_fp_hard_regs; /* Slots num for variables. Some variable can take several slots and can be aligned. */ size_t func_stack_slots_num; + VARR (target_bb_version_t) * target_succ_bb_versions; + VARR (void_ptr_t) * succ_bb_addrs; + void *bb_wrapper; /* to jump to lazy basic block generation */ + VARR (spot_attr_t) * spot2attr; /* map: spot number -> spot_attr */ + VARR (spot_attr_t) * spot_attrs; /* spot attrs wit only non-zero properies */ }; #define optimize_level gen_ctx->optimize_level #define curr_func_item gen_ctx->curr_func_item #define debug_file gen_ctx->debug_file #define debug_level gen_ctx->debug_level +#define to_free gen_ctx->to_free +#define addr_insn_p gen_ctx->addr_insn_p +#define tied_regs gen_ctx->tied_regs +#define addr_regs gen_ctx->addr_regs #define insn_to_consider gen_ctx->insn_to_consider #define temp_bitmap gen_ctx->temp_bitmap #define temp_bitmap2 gen_ctx->temp_bitmap2 +#define temp_bitmap3 gen_ctx->temp_bitmap3 #define call_used_hard_regs gen_ctx->call_used_hard_regs #define func_used_hard_regs gen_ctx->func_used_hard_regs #define curr_cfg gen_ctx->curr_cfg #define curr_bb_index gen_ctx->curr_bb_index #define curr_loop_node_index gen_ctx->curr_loop_node_index +#define full_escape_p gen_ctx->full_escape_p +#define mem_attrs gen_ctx->mem_attrs #define free_dead_vars gen_ctx->free_dead_vars -#define dead_bb_insns gen_ctx->dead_bb_insns +#define overall_bbs_num gen_ctx->overall_bbs_num +#define overall_gen_bbs_num gen_ctx->overall_gen_bbs_num +#define temp_ops gen_ctx->temp_ops +#define temp_insns gen_ctx->temp_insns +#define temp_insns2 gen_ctx->temp_insns2 +#define temp_bb_insns gen_ctx->temp_bb_insns +#define temp_bb_insns2 gen_ctx->temp_bb_insns2 #define loop_nodes gen_ctx->loop_nodes #define queue_nodes gen_ctx->queue_nodes #define loop_entries gen_ctx->loop_entries #define max_int_hard_regs gen_ctx->max_int_hard_regs #define max_fp_hard_regs gen_ctx->max_fp_hard_regs #define func_stack_slots_num gen_ctx->func_stack_slots_num +#define target_succ_bb_versions gen_ctx->target_succ_bb_versions +#define succ_bb_addrs gen_ctx->succ_bb_addrs +#define bb_wrapper gen_ctx->bb_wrapper +#define spot_attrs gen_ctx->spot_attrs +#define spot2attr gen_ctx->spot2attr -DEF_VARR (MIR_item_t); -struct all_gen_ctx { -#if MIR_PARALLEL_GEN - mir_mutex_t queue_mutex; - mir_cond_t generate_signal, done_signal; - size_t funcs_start; - VARR (MIR_item_t) * funcs_to_generate; -#endif - MIR_context_t ctx; - size_t gens_num; /* size of the following array: */ - struct gen_ctx gen_ctx[1]; +#define LOOP_COST_FACTOR 5 + +typedef struct bb_version *bb_version_t; + +struct func_or_bb { + /* full_p is used only when func_p and means generation machine code for full func */ + char func_p, full_p; + union { + MIR_item_t func_item; + bb_version_t bb_version; + } u; }; -#if MIR_PARALLEL_GEN -#define queue_mutex all_gen_ctx->queue_mutex -#define generate_signal all_gen_ctx->generate_signal -#define done_signal all_gen_ctx->done_signal -#define funcs_start all_gen_ctx->funcs_start -#define funcs_to_generate all_gen_ctx->funcs_to_generate -#endif +typedef struct func_or_bb func_or_bb_t; +DEF_VARR (func_or_bb_t); -static inline struct all_gen_ctx **all_gen_ctx_loc (MIR_context_t ctx) { - return (struct all_gen_ctx **) ctx; -} +static inline gen_ctx_t *gen_ctx_loc (MIR_context_t ctx) { return (gen_ctx_t *) ctx; } + +DEF_VARR (int); +DEF_VARR (uint8_t); +DEF_VARR (uint64_t); +DEF_VARR (MIR_code_reloc_t); #if defined(__x86_64__) || defined(_M_AMD64) #include "mir-gen-x86_64.c" @@ -237,6 +327,29 @@ static inline struct all_gen_ctx **all_gen_ctx_loc (MIR_context_t ctx) { #error "undefined or unsupported generation target" #endif +typedef struct bb_stub *bb_stub_t; +DEF_DLIST_LINK (bb_version_t); + +struct bb_version { + bb_stub_t bb_stub; + DLIST_LINK (bb_version_t) bb_version_link; + int call_p; + void *addr; /* bb code address or generator creating and returning address */ + void *machine_code; + struct target_bb_version target_data; /* data container for the target code */ + uint32_t n_attrs; + spot_attr_t attrs[1]; +}; + +/* Definition of double list of bb_version_t type elements */ +DEF_DLIST (bb_version_t, bb_version_link); + +struct bb_stub { + DLIST (bb_version_t) bb_versions; + MIR_item_t func_item; + MIR_insn_t first_insn, last_insn; +}; + static void MIR_NO_RETURN util_error (gen_ctx_t gen_ctx, const char *message) { (*MIR_get_error_func (gen_ctx->ctx)) (MIR_alloc_error, message); } @@ -247,59 +360,14 @@ static void *gen_malloc (gen_ctx_t gen_ctx, size_t size) { return res; } -#define DEFAULT_INIT_BITMAP_BITS_NUM 256 - -static void make_io_dup_op_insns (gen_ctx_t gen_ctx) { - MIR_context_t ctx = gen_ctx->ctx; - MIR_func_t func; - MIR_insn_t insn, next_insn; - MIR_insn_code_t code; - MIR_op_t input, output, temp_op; - MIR_op_mode_t mode; - MIR_type_t type; - size_t i; - int out_p; - - gen_assert (curr_func_item->item_type == MIR_func_item); - func = curr_func_item->u.func; - for (i = 0; target_io_dup_op_insn_codes[i] != MIR_INSN_BOUND; i++) - bitmap_set_bit_p (insn_to_consider, target_io_dup_op_insn_codes[i]); - if (bitmap_empty_p (insn_to_consider)) return; - for (insn = DLIST_HEAD (MIR_insn_t, func->insns); insn != NULL; insn = next_insn) { - next_insn = DLIST_NEXT (MIR_insn_t, insn); - code = insn->code; - if (!bitmap_bit_p (insn_to_consider, code)) continue; - gen_assert (MIR_insn_nops (ctx, insn) >= 2 && !MIR_call_code_p (code) && code != MIR_RET); - mode = MIR_insn_op_mode (ctx, insn, 0, &out_p); - gen_assert (out_p && mode == MIR_insn_op_mode (ctx, insn, 1, &out_p) && !out_p); - output = insn->ops[0]; - input = insn->ops[1]; - gen_assert (input.mode == MIR_OP_REG || input.mode == MIR_OP_HARD_REG - || output.mode == MIR_OP_REG || output.mode == MIR_OP_HARD_REG); - if (input.mode == output.mode - && ((input.mode == MIR_OP_HARD_REG && input.u.hard_reg == output.u.hard_reg) - || (input.mode == MIR_OP_REG && input.u.reg == output.u.reg))) - continue; - if (mode == MIR_OP_FLOAT) { - code = MIR_FMOV; - type = MIR_T_F; - } else if (mode == MIR_OP_DOUBLE) { - code = MIR_DMOV; - type = MIR_T_D; - } else if (mode == MIR_OP_LDOUBLE) { - code = MIR_LDMOV; - type = MIR_T_LD; - } else { - code = MIR_MOV; - type = MIR_T_I64; - } - temp_op = MIR_new_reg_op (ctx, gen_new_temp_reg (gen_ctx, type, func)); - gen_add_insn_before (gen_ctx, insn, MIR_new_insn (ctx, code, temp_op, insn->ops[1])); - gen_add_insn_after (gen_ctx, insn, MIR_new_insn (ctx, code, insn->ops[0], temp_op)); - insn->ops[0] = insn->ops[1] = temp_op; - } +static void *gen_malloc_and_mark_to_free (gen_ctx_t gen_ctx, size_t size) { + void *res = gen_malloc (gen_ctx, size); + VARR_PUSH (void_ptr_t, to_free, res); + return res; } +#define DEFAULT_INIT_BITMAP_BITS_NUM 256 + typedef struct bb *bb_t; DEF_DLIST_LINK (bb_t); @@ -321,8 +389,7 @@ struct edge { bb_t src, dst; DLIST_LINK (in_edge_t) in_link; DLIST_LINK (out_edge_t) out_link; - unsigned char back_edge_p; - unsigned char skipped_p; /* used for CCP */ + unsigned char fall_through_p, back_edge_p, flag1, flag2; }; DEF_DLIST (in_edge_t, in_link); @@ -336,14 +403,17 @@ struct insn_data { /* used only for calls/labels in -O0 mode */ } u; }; +#define MAY_ALLOCA 0x1 +#define MUST_ALLOCA 0x2 struct bb_insn { MIR_insn_t insn; - unsigned char flag; /* used for CCP */ - uint32_t gvn_val; /* used for GVN, it is negative index for non GVN expr insns */ - uint32_t index; + unsigned char gvn_val_const_p; /* true for int value, false otherwise */ + unsigned char alloca_flag; /* true for value may and/or must be from alloca */ + uint32_t index, mem_index; + int64_t gvn_val; /* used for GVN, it is negative index for non GVN expr insns */ DLIST_LINK (bb_insn_t) bb_insn_link; bb_t bb; - DLIST (dead_var_t) dead_vars; + DLIST (dead_var_t) insn_dead_vars; bitmap_t call_hard_reg_args; /* non-null for calls */ size_t label_disp; /* for label */ }; @@ -352,14 +422,15 @@ DEF_DLIST (bb_insn_t, bb_insn_link); struct bb { size_t index, pre, rpost, bfs; /* preorder, reverse post order, breadth first order */ - unsigned int flag; /* used for CCP */ DLIST_LINK (bb_t) bb_link; DLIST (in_edge_t) in_edges; /* The out edges order: optional fall through bb, optional label bb, optional exit bb. There is always at least one edge. */ DLIST (out_edge_t) out_edges; DLIST (bb_insn_t) bb_insns; - size_t freq; + unsigned char call_p; /* used in mem avail calculation, true if there is a call in BB */ + unsigned char flag; /* used in different calculation */ + unsigned char reachable_p; /* reachable if its label is used as value */ bitmap_t in, out, gen, kill; /* var bitmaps for different data flow problems */ bitmap_t dom_in, dom_out; /* additional var bitmaps */ loop_node_t loop_node; @@ -376,6 +447,10 @@ struct loop_node { bb_t bb; /* NULL for internal tree node */ loop_node_t entry; loop_node_t parent; + union { /* used in LICM */ + loop_node_t preheader; /* used for non-bb loop it is loop node of preheader bb */ + loop_node_t preheader_loop; /* used for preheader bb it is the loop node */ + } u; DLIST (loop_node_t) children; DLIST_LINK (loop_node_t) children_link; int max_int_pressure, max_fp_pressure; @@ -385,35 +460,10 @@ DEF_DLIST_CODE (loop_node_t, children_link); DEF_DLIST_LINK (func_cfg_t); -typedef struct mv *mv_t; -typedef mv_t dst_mv_t; -typedef mv_t src_mv_t; - -DEF_DLIST_LINK (mv_t); -DEF_DLIST_LINK (dst_mv_t); -DEF_DLIST_LINK (src_mv_t); - -struct mv { - bb_insn_t bb_insn; - size_t freq; - DLIST_LINK (mv_t) mv_link; - DLIST_LINK (dst_mv_t) dst_link; - DLIST_LINK (src_mv_t) src_link; -}; - -DEF_DLIST (mv_t, mv_link); -DEF_DLIST (dst_mv_t, dst_link); -DEF_DLIST (src_mv_t, src_link); - struct reg_info { long freq; /* The following members are defined and used only in RA */ - long thread_freq; /* thread accumulated freq, defined for first thread breg */ - /* first/next breg of the same thread, MIR_MAX_REG_NUM is end mark */ - MIR_reg_t thread_first, thread_next; - size_t live_length; /* # of program points where breg lives */ - DLIST (dst_mv_t) dst_moves; - DLIST (src_mv_t) src_moves; + size_t live_length; /* # of program points where reg lives */ }; typedef struct reg_info reg_info_t; @@ -428,23 +478,12 @@ typedef struct { } u; } const_t; -#if !MIR_NO_GEN_DEBUG -static void print_const (FILE *f, const_t c) { - if (c.uns_p) - fprintf (f, "%" PRIu64, c.u.u); - else - fprintf (f, "%" PRId64, c.u.i); -} -#endif - struct func_cfg { - MIR_reg_t min_reg, max_reg; - uint32_t non_conflicting_moves; /* # of moves with non-conflicting regs */ + MIR_reg_t max_var; uint32_t curr_bb_insn_index; - VARR (reg_info_t) * breg_info; /* bregs */ - bitmap_t call_crossed_bregs; + VARR (reg_info_t) * reg_info; /* regs */ + bitmap_t call_crossed_regs; DLIST (bb_t) bbs; - DLIST (mv_t) used_moves, free_moves; loop_node_t root_loop_node; }; @@ -475,18 +514,18 @@ static void finish_dead_vars (gen_ctx_t gen_ctx) { static void add_bb_insn_dead_var (gen_ctx_t gen_ctx, bb_insn_t bb_insn, MIR_reg_t var) { dead_var_t dv; - for (dv = DLIST_HEAD (dead_var_t, bb_insn->dead_vars); dv != NULL; + for (dv = DLIST_HEAD (dead_var_t, bb_insn->insn_dead_vars); dv != NULL; dv = DLIST_NEXT (dead_var_t, dv)) if (dv->var == var) return; dv = get_dead_var (gen_ctx); dv->var = var; - DLIST_APPEND (dead_var_t, bb_insn->dead_vars, dv); + DLIST_APPEND (dead_var_t, bb_insn->insn_dead_vars, dv); } static dead_var_t find_bb_insn_dead_var (bb_insn_t bb_insn, MIR_reg_t var) { dead_var_t dv; - for (dv = DLIST_HEAD (dead_var_t, bb_insn->dead_vars); dv != NULL; + for (dv = DLIST_HEAD (dead_var_t, bb_insn->insn_dead_vars); dv != NULL; dv = DLIST_NEXT (dead_var_t, dv)) if (dv->var == var) return dv; return NULL; @@ -495,30 +534,35 @@ static dead_var_t find_bb_insn_dead_var (bb_insn_t bb_insn, MIR_reg_t var) { static void clear_bb_insn_dead_vars (gen_ctx_t gen_ctx, bb_insn_t bb_insn) { dead_var_t dv; - while ((dv = DLIST_HEAD (dead_var_t, bb_insn->dead_vars)) != NULL) { - DLIST_REMOVE (dead_var_t, bb_insn->dead_vars, dv); + while ((dv = DLIST_HEAD (dead_var_t, bb_insn->insn_dead_vars)) != NULL) { + DLIST_REMOVE (dead_var_t, bb_insn->insn_dead_vars, dv); free_dead_var (gen_ctx, dv); } } -static void remove_bb_insn_dead_var (gen_ctx_t gen_ctx, bb_insn_t bb_insn, MIR_reg_t hr) { +static void remove_bb_insn_dead_var (gen_ctx_t gen_ctx, bb_insn_t bb_insn, MIR_reg_t var) { dead_var_t dv, next_dv; - gen_assert (hr <= MAX_HARD_REG); - for (dv = DLIST_HEAD (dead_var_t, bb_insn->dead_vars); dv != NULL; dv = next_dv) { + gen_assert (var != MIR_NON_VAR); + for (dv = DLIST_HEAD (dead_var_t, bb_insn->insn_dead_vars); dv != NULL; dv = next_dv) { next_dv = DLIST_NEXT (dead_var_t, dv); - if (dv->var != hr) continue; - DLIST_REMOVE (dead_var_t, bb_insn->dead_vars, dv); + if (dv->var != var) continue; + DLIST_REMOVE (dead_var_t, bb_insn->insn_dead_vars, dv); free_dead_var (gen_ctx, dv); } } -static void move_bb_insn_dead_vars (bb_insn_t bb_insn, bb_insn_t from_bb_insn) { +static void move_bb_insn_dead_vars (gen_ctx_t gen_ctx, bb_insn_t bb_insn, bb_insn_t from_bb_insn, + int (*filter_p) (gen_ctx_t, bb_insn_t, MIR_reg_t)) { dead_var_t dv; - while ((dv = DLIST_HEAD (dead_var_t, from_bb_insn->dead_vars)) != NULL) { - DLIST_REMOVE (dead_var_t, from_bb_insn->dead_vars, dv); - DLIST_APPEND (dead_var_t, bb_insn->dead_vars, dv); + while ((dv = DLIST_HEAD (dead_var_t, from_bb_insn->insn_dead_vars)) != NULL) { + DLIST_REMOVE (dead_var_t, from_bb_insn->insn_dead_vars, dv); + if (filter_p (gen_ctx, bb_insn, dv->var)) { + DLIST_APPEND (dead_var_t, bb_insn->insn_dead_vars, dv); + } else { + free_dead_var (gen_ctx, dv); + } } } @@ -557,20 +601,26 @@ static bb_insn_t create_bb_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, bb_t bb) { insn->data = bb_insn; bb_insn->bb = bb; bb_insn->insn = insn; - bb_insn->flag = FALSE; + bb_insn->gvn_val_const_p = FALSE; + bb_insn->alloca_flag = insn->code == MIR_ALLOCA ? MAY_ALLOCA | MUST_ALLOCA : 0; bb_insn->call_hard_reg_args = NULL; gen_assert (curr_cfg->curr_bb_insn_index != (uint32_t) ~0ull); bb_insn->index = curr_cfg->curr_bb_insn_index++; + bb_insn->mem_index = 0; bb_insn->gvn_val = bb_insn->index; - DLIST_INIT (dead_var_t, bb_insn->dead_vars); + DLIST_INIT (dead_var_t, bb_insn->insn_dead_vars); if (MIR_call_code_p (insn->code)) bb_insn->call_hard_reg_args = bitmap_create2 (MAX_HARD_REG + 1); + bb_insn->label_disp = 0; return bb_insn; } -static bb_insn_t add_new_bb_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, bb_t bb) { +static bb_insn_t add_new_bb_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, bb_t bb, int append_p) { bb_insn_t bb_insn = create_bb_insn (gen_ctx, insn, bb); - DLIST_APPEND (bb_insn_t, bb->bb_insns, bb_insn); + if (append_p) + DLIST_APPEND (bb_insn_t, bb->bb_insns, bb_insn); + else + DLIST_PREPEND (bb_insn_t, bb->bb_insns, bb_insn); return bb_insn; } @@ -634,30 +684,33 @@ static void gen_add_insn_before (gen_ctx_t gen_ctx, MIR_insn_t before, MIR_insn_ MIR_context_t ctx = gen_ctx->ctx; MIR_insn_t insn_for_bb = before; - gen_assert (!MIR_branch_code_p (insn->code) && insn->code != MIR_LABEL); + gen_assert (!MIR_any_branch_code_p (insn->code) && insn->code != MIR_LABEL); if (before->code == MIR_LABEL) { insn_for_bb = DLIST_PREV (MIR_insn_t, before); - gen_assert (insn_for_bb == NULL || !MIR_branch_code_p (insn_for_bb->code)); + gen_assert (insn_for_bb == NULL || !MIR_any_branch_code_p (insn_for_bb->code)); } MIR_insert_insn_before (ctx, curr_func_item, before, insn); create_new_bb_insns (gen_ctx, DLIST_PREV (MIR_insn_t, insn), before, insn_for_bb); } static void gen_add_insn_after (gen_ctx_t gen_ctx, MIR_insn_t after, MIR_insn_t insn) { + MIR_insn_t insn_for_bb = after; + gen_assert (insn->code != MIR_LABEL); - gen_assert (!MIR_branch_code_p (after->code)); + if (MIR_any_branch_code_p (insn_for_bb->code)) insn_for_bb = DLIST_NEXT (MIR_insn_t, insn_for_bb); + gen_assert (!MIR_any_branch_code_p (insn_for_bb->code)); MIR_insert_insn_after (gen_ctx->ctx, curr_func_item, after, insn); - create_new_bb_insns (gen_ctx, after, DLIST_NEXT (MIR_insn_t, insn), after); + create_new_bb_insns (gen_ctx, after, DLIST_NEXT (MIR_insn_t, insn), insn_for_bb); } static void gen_move_insn_before (gen_ctx_t gen_ctx, MIR_insn_t before, MIR_insn_t insn) { + bb_insn_t bb_insn = insn->data, before_bb_insn = before->data; + DLIST_REMOVE (MIR_insn_t, curr_func_item->u.func->insns, insn); MIR_insert_insn_before (gen_ctx->ctx, curr_func_item, before, insn); - if (optimize_level != 0) { - bb_insn_t bb_insn = insn->data, before_bb_insn = before->data; - DLIST_REMOVE (bb_insn_t, bb_insn->bb->bb_insns, bb_insn); - DLIST_INSERT_BEFORE (bb_insn_t, before_bb_insn->bb->bb_insns, before_bb_insn, bb_insn); - } + DLIST_REMOVE (bb_insn_t, bb_insn->bb->bb_insns, bb_insn); + DLIST_INSERT_BEFORE (bb_insn_t, before_bb_insn->bb->bb_insns, before_bb_insn, bb_insn); + bb_insn->bb = before_bb_insn->bb; } static void MIR_UNUSED setup_call_hard_reg_args (gen_ctx_t gen_ctx, MIR_insn_t call_insn, @@ -674,6 +727,19 @@ static void MIR_UNUSED setup_call_hard_reg_args (gen_ctx_t gen_ctx, MIR_insn_t c bitmap_set_bit_p (insn_data->u.call_hard_reg_args, hard_reg); } +static int MIR_UNUSED gen_nested_loop_label_p (gen_ctx_t gen_ctx, MIR_insn_t insn) { + gen_assert (insn->code == MIR_LABEL); + if (optimize_level <= 1) return FALSE; + bb_t bb = get_insn_bb (gen_ctx, insn); + if (bb->loop_node == NULL) return FALSE; + loop_node_t node, parent = bb->loop_node->parent; + if (parent->entry == NULL || parent->entry->bb != bb) return FALSE; + for (node = DLIST_HEAD (loop_node_t, parent->children); node != NULL; + node = DLIST_NEXT (loop_node_t, node)) + if (node->bb == NULL) return FALSE; /* subloop */ + return TRUE; +} + static void set_label_disp (gen_ctx_t gen_ctx, MIR_insn_t insn, size_t disp) { gen_assert (insn->code == MIR_LABEL); if (optimize_level == 0) @@ -687,6 +753,24 @@ static size_t get_label_disp (gen_ctx_t gen_ctx, MIR_insn_t insn) { : ((bb_insn_t) insn->data)->label_disp); } +static uint64_t get_ref_value (gen_ctx_t gen_ctx, const MIR_op_t *ref_op) { + gen_assert (ref_op->mode == MIR_OP_REF); + if (ref_op->u.ref->item_type == MIR_data_item && ref_op->u.ref->u.data->name != NULL + && _MIR_reserved_ref_name_p (gen_ctx->ctx, ref_op->u.ref->u.data->name)) + return (uint64_t) ref_op->u.ref->u.data->u.els; + return (uint64_t) ref_op->u.ref->addr; +} + +static void gen_setup_lrefs (gen_ctx_t gen_ctx, uint8_t *func_code) { + for (MIR_lref_data_t lref = curr_func_item->u.func->first_lref; lref != NULL; + lref = lref->next) { /* set up lrefs */ + int64_t disp = (int64_t) get_label_disp (gen_ctx, lref->label) + lref->disp; + *(void **) lref->load_addr + = lref->label2 == NULL ? (void *) (func_code + disp) + : (void *) (disp - (int64_t) get_label_disp (gen_ctx, lref->label2)); + } +} + static void setup_used_hard_regs (gen_ctx_t gen_ctx, MIR_type_t type, MIR_reg_t hard_reg) { MIR_reg_t curr_hard_reg; int i, slots_num = target_locs_num (hard_reg, type); @@ -707,11 +791,11 @@ static bb_t create_bb (gen_ctx_t gen_ctx, MIR_insn_t insn) { bb_t bb = gen_malloc (gen_ctx, sizeof (struct bb)); bb->pre = bb->rpost = bb->bfs = 0; - bb->flag = FALSE; bb->loop_node = NULL; DLIST_INIT (bb_insn_t, bb->bb_insns); DLIST_INIT (in_edge_t, bb->in_edges); DLIST_INIT (out_edge_t, bb->out_edges); + bb->call_p = bb->flag = bb->reachable_p = FALSE; bb->in = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); bb->out = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); bb->gen = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); @@ -723,17 +807,28 @@ static bb_t create_bb (gen_ctx_t gen_ctx, MIR_insn_t insn) { if (optimize_level == 0) setup_insn_data (gen_ctx, insn, bb); else - add_new_bb_insn (gen_ctx, insn, bb); + add_new_bb_insn (gen_ctx, insn, bb, TRUE); } return bb; } -static void add_bb (gen_ctx_t gen_ctx, bb_t bb) { +static void add_new_bb (gen_ctx_t gen_ctx, bb_t bb) { DLIST_APPEND (bb_t, curr_cfg->bbs, bb); bb->index = curr_bb_index++; } -static edge_t create_edge (gen_ctx_t gen_ctx, bb_t src, bb_t dst, int append_p) { +static void insert_new_bb_after (gen_ctx_t gen_ctx, bb_t after, bb_t bb) { + DLIST_INSERT_AFTER (bb_t, curr_cfg->bbs, after, bb); + bb->index = curr_bb_index++; +} + +static void insert_new_bb_before (gen_ctx_t gen_ctx, bb_t before, bb_t bb) { + DLIST_INSERT_BEFORE (bb_t, curr_cfg->bbs, before, bb); + bb->index = curr_bb_index++; +} + +static edge_t create_edge (gen_ctx_t gen_ctx, bb_t src, bb_t dst, int fall_through_p, + int append_p) { edge_t e = gen_malloc (gen_ctx, sizeof (struct edge)); e->src = src; @@ -745,7 +840,8 @@ static edge_t create_edge (gen_ctx_t gen_ctx, bb_t src, bb_t dst, int append_p) DLIST_PREPEND (in_edge_t, dst->in_edges, e); DLIST_PREPEND (out_edge_t, src->out_edges, e); } - e->back_edge_p = e->skipped_p = FALSE; + e->fall_through_p = fall_through_p; + e->back_edge_p = e->flag1 = e->flag2 = FALSE; return e; } @@ -755,6 +851,13 @@ static void delete_edge (edge_t e) { free (e); } +static edge_t find_edge (bb_t src, bb_t dst) { + for (edge_t e = DLIST_HEAD (out_edge_t, src->out_edges); e != NULL; + e = DLIST_NEXT (out_edge_t, e)) + if (e->dst == dst) return e; + return NULL; +} + static void delete_bb (gen_ctx_t gen_ctx, bb_t bb) { edge_t e, next_e; @@ -766,6 +869,13 @@ static void delete_bb (gen_ctx_t gen_ctx, bb_t bb) { next_e = DLIST_NEXT (in_edge_t, e); delete_edge (e); } + if (bb->loop_node != NULL) { + if (bb->loop_node->parent->entry == bb->loop_node) bb->loop_node->parent->entry = NULL; + DLIST_REMOVE (loop_node_t, bb->loop_node->parent->children, bb->loop_node); + if (bb->loop_node->u.preheader_loop != NULL) + bb->loop_node->u.preheader_loop->u.preheader = NULL; + free (bb->loop_node); + } DLIST_REMOVE (bb_t, curr_cfg->bbs, bb); bitmap_destroy (bb->in); bitmap_destroy (bb->out); @@ -776,6 +886,151 @@ static void delete_bb (gen_ctx_t gen_ctx, bb_t bb) { free (bb); } +static void print_bb_insn (gen_ctx_t gen_ctx, bb_insn_t bb_insn, int with_notes_p); + +/* Return BB to put insns from edge E. If necessary, split edge by creating new bb, bb enumeration + and new bb bitmaps can be invalid after that. Loop info is undefined for the new bb. */ +static bb_t split_edge_if_necessary (gen_ctx_t gen_ctx, edge_t e) { + MIR_context_t ctx = gen_ctx->ctx; + size_t i; + bb_t new_bb, src = e->src, dst = e->dst; + edge_t e2; + bb_insn_t last_bb_insn = DLIST_TAIL (bb_insn_t, src->bb_insns); + bb_insn_t first_bb_insn = DLIST_HEAD (bb_insn_t, dst->bb_insns); + MIR_insn_t insn, tail_insn, last_insn = last_bb_insn->insn, first_insn = first_bb_insn->insn; + DEBUG (4, { + fprintf (debug_file, " Splitting bb%lu->bb%lu:\n", (unsigned long) src->index, + (unsigned long) dst->index); + }); + if (DLIST_HEAD (out_edge_t, src->out_edges) == DLIST_TAIL (out_edge_t, src->out_edges) + || e->fall_through_p) { /* fall through or src with one dest */ + if (e->fall_through_p) { + insn = MIR_new_insn_arr (ctx, MIR_USE, 0, NULL); /* just nop */ + MIR_insert_insn_after (ctx, curr_func_item, last_insn, insn); + } else if (DLIST_HEAD (in_edge_t, src->in_edges) == DLIST_TAIL (in_edge_t, src->in_edges)) { + return src; + } else { /* jump with one dest only: move jmp to new fall-though block */ + gen_assert (last_insn->code == MIR_JMP || last_insn->code == MIR_RET + || last_insn->code == MIR_JRET); + delete_bb_insn (gen_ctx, last_bb_insn); + insn = last_insn; + } + new_bb = create_bb (gen_ctx, insn); + insert_new_bb_after (gen_ctx, src, new_bb); + DLIST_REMOVE (in_edge_t, dst->in_edges, e); + e->dst = new_bb; + DLIST_APPEND (in_edge_t, new_bb->in_edges, e); + create_edge (gen_ctx, new_bb, dst, e->fall_through_p, TRUE); + e->fall_through_p = TRUE; + DEBUG (4, { + fprintf (debug_file, " creating fall through bb%lu after bb%lu, redirect the edge to it", + (unsigned long) new_bb->index, (unsigned long) src->index); + fprintf (debug_file, ", and create edge bb%lu->bb%lu:\n", (unsigned long) new_bb->index, + (unsigned long) dst->index); + fprintf (debug_file, " new bb insn is "); + print_bb_insn (gen_ctx, (bb_insn_t) insn->data, FALSE); + }); + } else if (DLIST_HEAD (in_edge_t, dst->in_edges) == DLIST_TAIL (in_edge_t, dst->in_edges)) { + gen_assert (first_insn->code == MIR_LABEL); + if (first_bb_insn == DLIST_TAIL (bb_insn_t, dst->bb_insns)) return dst; + /* non-fall through dest with one source only: move dest label to new block */ + delete_bb_insn (gen_ctx, first_bb_insn); + new_bb = create_bb (gen_ctx, first_insn); + insert_new_bb_before (gen_ctx, dst, new_bb); + DLIST_REMOVE (in_edge_t, dst->in_edges, e); + e->dst = new_bb; + DLIST_APPEND (in_edge_t, new_bb->in_edges, e); + create_edge (gen_ctx, new_bb, dst, TRUE, TRUE); + DEBUG (4, { + fprintf (debug_file, " creating bb%lu before bb%lu, redirect the edge to it", + (unsigned long) new_bb->index, (unsigned long) dst->index); + fprintf (debug_file, ", and create fall-through edge bb%lu->bb%lu:\n", + (unsigned long) new_bb->index, (unsigned long) dst->index); + fprintf (debug_file, " new bb insn is "); + print_bb_insn (gen_ctx, (bb_insn_t) first_insn->data, FALSE); + }); + } else { /* critical non-fall through edge: */ + gen_assert (first_insn->code == MIR_LABEL); + for (e2 = DLIST_HEAD (in_edge_t, dst->in_edges); e2 != NULL; e2 = DLIST_NEXT (in_edge_t, e2)) { + if (e2->fall_through_p) break; + gen_assert (DLIST_TAIL (bb_insn_t, e2->src->bb_insns) != NULL + && MIR_any_branch_code_p (DLIST_TAIL (bb_insn_t, e2->src->bb_insns)->insn->code)); + } + if (e2 != NULL) { /* make fall through edge to dst a jump edge */ + gen_assert (e2->dst == dst); + insn = MIR_new_insn (ctx, MIR_JMP, MIR_new_label_op (ctx, first_insn)); + tail_insn = DLIST_TAIL (bb_insn_t, e2->src->bb_insns)->insn; + if (DLIST_NEXT (out_edge_t, e2) == NULL && DLIST_PREV (out_edge_t, e2) == NULL) { + /* e2->src with the only output edge: just put jump at the end of e2->src */ + gen_assert (!MIR_any_branch_code_p (tail_insn->code)); + gen_add_insn_after (gen_ctx, tail_insn, insn); + e2->fall_through_p = FALSE; + DEBUG (4, { + fprintf (debug_file, + " Make edge bb%lu->bb%lu a non-fall through, add new insn at the of bb%lu ", + (unsigned long) e2->src->index, (unsigned long) e2->dst->index, + (unsigned long) e2->src->index); + print_bb_insn (gen_ctx, (bb_insn_t) insn->data, FALSE); + }); + } else { + MIR_insert_insn_after (ctx, curr_func_item, tail_insn, insn); + new_bb = create_bb (gen_ctx, insn); + insert_new_bb_after (gen_ctx, e2->src, new_bb); + DLIST_REMOVE (in_edge_t, e2->dst->in_edges, e2); + e2->dst = new_bb; + DLIST_APPEND (in_edge_t, new_bb->in_edges, e2); + create_edge (gen_ctx, new_bb, dst, FALSE, TRUE); + DEBUG (4, { + fprintf (debug_file, + " creating bb%lu after bb%lu, redirect edge bb%lu->bb%lu to bb%lu", + (unsigned long) new_bb->index, (unsigned long) e2->src->index, + (unsigned long) e2->src->index, (unsigned long) dst->index, + (unsigned long) new_bb->index); + fprintf (debug_file, ", and create jump edge bb%lu->bb%lu:\n", + (unsigned long) new_bb->index, (unsigned long) dst->index); + fprintf (debug_file, " new bb insn is "); + print_bb_insn (gen_ctx, (bb_insn_t) insn->data, FALSE); + }); + } + } + /* add fall through new block before dst */ + insn = MIR_new_label (ctx); + MIR_insert_insn_before (ctx, curr_func_item, first_insn, insn); + new_bb = create_bb (gen_ctx, insn); + insert_new_bb_before (gen_ctx, dst, new_bb); + DLIST_REMOVE (in_edge_t, dst->in_edges, e); + e->dst = new_bb; + DLIST_APPEND (in_edge_t, new_bb->in_edges, e); + create_edge (gen_ctx, new_bb, dst, TRUE, TRUE); + DEBUG (4, { + fprintf (debug_file, " creating bb%lu before bb%lu, redirect the edge to it", + (unsigned long) new_bb->index, (unsigned long) dst->index); + fprintf (debug_file, ", and create fall-through edge bb%lu->bb%lu:\n", + (unsigned long) new_bb->index, (unsigned long) dst->index); + fprintf (debug_file, " new bb insn is "); + print_bb_insn (gen_ctx, (bb_insn_t) insn->data, FALSE); + fprintf (debug_file, " change src bb insn "); + print_bb_insn (gen_ctx, last_bb_insn, FALSE); + }); + /* change label first_insn to label insn in src */ + if (last_insn->code != MIR_SWITCH) { + gen_assert (last_insn->ops[0].mode == MIR_OP_LABEL + && last_insn->ops[0].u.label == first_insn); + last_insn->ops[0] = MIR_new_label_op (ctx, insn); + } else { + for (i = 1; i < last_insn->nops; i++) + if (last_insn->ops[i].u.label == first_insn) break; + gen_assert (i < last_insn->nops); + last_insn->ops[i] = MIR_new_label_op (ctx, insn); + } + DEBUG (4, { + fprintf (debug_file, " to insn "); + print_bb_insn (gen_ctx, last_bb_insn, FALSE); + }); + } + return new_bb; +} + static void DFS (bb_t bb, size_t *pre, size_t *rpost) { edge_t e; @@ -811,6 +1066,7 @@ static loop_node_t create_loop_node (gen_ctx_t gen_ctx, bb_t bb) { if (bb != NULL) bb->loop_node = loop_node; loop_node->parent = NULL; loop_node->entry = NULL; + loop_node->u.preheader = NULL; loop_node->max_int_pressure = loop_node->max_fp_pressure = 0; DLIST_INIT (loop_node_t, loop_node->children); return loop_node; @@ -827,7 +1083,9 @@ static int process_loop (gen_ctx_t gen_ctx, bb_t entry_bb) { for (e = DLIST_HEAD (in_edge_t, entry_bb->in_edges); e != NULL; e = DLIST_NEXT (in_edge_t, e)) if (e->back_edge_p && e->src != entry_bb) { loop_node = top_loop_node (e->src); - if (!bitmap_set_bit_p (temp_bitmap, loop_node->index)) continue; + if (!bitmap_set_bit_p (temp_bitmap, loop_node->index) + || (e->src->pre == 0 && e->src->rpost == 0)) + continue; /* processed or unreachable */ VARR_PUSH (loop_node_t, loop_nodes, loop_node); VARR_PUSH (loop_node_t, queue_nodes, loop_node); } @@ -839,7 +1097,9 @@ static int process_loop (gen_ctx_t gen_ctx, bb_t entry_bb) { for (e = DLIST_HEAD (in_edge_t, queue_bb->in_edges); e != NULL; e = DLIST_NEXT (in_edge_t, e)) if (e->src != entry_bb) { loop_node = top_loop_node (e->src); - if (!bitmap_set_bit_p (temp_bitmap, loop_node->index)) continue; + if (!bitmap_set_bit_p (temp_bitmap, loop_node->index) + || (e->src->pre == 0 && e->src->rpost == 0)) + continue; /* processed or unreachable */ VARR_PUSH (loop_node_t, loop_nodes, loop_node); VARR_PUSH (loop_node_t, queue_nodes, loop_node); } @@ -913,6 +1173,7 @@ static void destroy_loop_tree (gen_ctx_t gen_ctx, loop_node_t root) { loop_node_t node, next; if (root->bb != NULL) { + root->u.preheader_loop = NULL; root->bb->loop_node = NULL; } else { for (node = DLIST_HEAD (loop_node_t, root->children); node != NULL; node = next) { @@ -923,53 +1184,33 @@ static void destroy_loop_tree (gen_ctx_t gen_ctx, loop_node_t root) { free (root); } -static void update_min_max_reg (gen_ctx_t gen_ctx, MIR_reg_t reg) { - if (reg == 0) return; - if (curr_cfg->max_reg == 0 || curr_cfg->min_reg > reg) curr_cfg->min_reg = reg; - if (curr_cfg->max_reg < reg) curr_cfg->max_reg = reg; +static void update_max_var (gen_ctx_t gen_ctx, MIR_reg_t reg) { + if (reg == MIR_NON_VAR) return; + if (curr_cfg->max_var < reg) curr_cfg->max_var = reg; } static MIR_reg_t gen_new_temp_reg (gen_ctx_t gen_ctx, MIR_type_t type, MIR_func_t func) { - MIR_reg_t reg = _MIR_new_temp_reg (gen_ctx->ctx, type, func); + MIR_reg_t reg = _MIR_new_temp_reg (gen_ctx->ctx, type, func) + MAX_HARD_REG; - update_min_max_reg (gen_ctx, reg); + update_max_var (gen_ctx, reg); return reg; } -static MIR_reg_t reg2breg (gen_ctx_t gen_ctx, MIR_reg_t reg) { return reg - curr_cfg->min_reg; } -static MIR_reg_t breg2reg (gen_ctx_t gen_ctx, MIR_reg_t breg) { return breg + curr_cfg->min_reg; } -static MIR_reg_t reg2var (gen_ctx_t gen_ctx, MIR_reg_t reg) { - return reg2breg (gen_ctx, reg) + MAX_HARD_REG + 1; -} -static MIR_reg_t var_is_reg_p (MIR_reg_t var) { return var > MAX_HARD_REG; } -static MIR_reg_t var2reg (gen_ctx_t gen_ctx, MIR_reg_t var) { - gen_assert (var > MAX_HARD_REG); - return breg2reg (gen_ctx, var - MAX_HARD_REG - 1); -} -static MIR_reg_t var2breg (gen_ctx_t gen_ctx, MIR_reg_t var) { - gen_assert (var > MAX_HARD_REG); - return var - MAX_HARD_REG - 1; -} - -static MIR_reg_t get_nregs (gen_ctx_t gen_ctx) { - return curr_cfg->max_reg == 0 ? 0 : curr_cfg->max_reg - curr_cfg->min_reg + 1; -} - -static MIR_reg_t get_nvars (gen_ctx_t gen_ctx) { return get_nregs (gen_ctx) + MAX_HARD_REG + 1; } +static MIR_reg_t get_max_var (gen_ctx_t gen_ctx) { return curr_cfg->max_var; } static int move_code_p (MIR_insn_code_t code) { return code == MIR_MOV || code == MIR_FMOV || code == MIR_DMOV || code == MIR_LDMOV; } static int move_p (MIR_insn_t insn) { - return (move_code_p (insn->code) - && (insn->ops[0].mode == MIR_OP_REG || insn->ops[0].mode == MIR_OP_HARD_REG) - && (insn->ops[1].mode == MIR_OP_REG || insn->ops[1].mode == MIR_OP_HARD_REG)); + return (move_code_p (insn->code) && insn->ops[0].mode == MIR_OP_VAR + && insn->ops[1].mode == MIR_OP_VAR && insn->ops[0].u.var > MAX_HARD_REG + && insn->ops[1].u.var > MAX_HARD_REG); } static int imm_move_p (MIR_insn_t insn) { - return (move_code_p (insn->code) - && (insn->ops[0].mode == MIR_OP_REG || insn->ops[0].mode == MIR_OP_HARD_REG) + return (move_code_p (insn->code) && insn->ops[0].mode == MIR_OP_VAR + && insn->ops[0].u.var > MAX_HARD_REG && (insn->ops[1].mode == MIR_OP_INT || insn->ops[1].mode == MIR_OP_UINT || insn->ops[1].mode == MIR_OP_FLOAT || insn->ops[1].mode == MIR_OP_DOUBLE || insn->ops[1].mode == MIR_OP_LDOUBLE || insn->ops[1].mode == MIR_OP_REF)); @@ -977,54 +1218,40 @@ static int imm_move_p (MIR_insn_t insn) { typedef struct { MIR_insn_t insn; - size_t nops, op_num, op_part_num, passed_mem_num; + size_t nops, op_num, op_part_num; } insn_var_iterator_t; -static inline void insn_var_iterator_init (gen_ctx_t gen_ctx, insn_var_iterator_t *iter, - MIR_insn_t insn) { +static inline void insn_var_iterator_init (insn_var_iterator_t *iter, MIR_insn_t insn) { iter->insn = insn; - iter->nops = MIR_insn_nops (gen_ctx->ctx, insn); + iter->nops = insn->nops; iter->op_num = 0; iter->op_part_num = 0; - iter->passed_mem_num = 0; } -static inline int insn_var_iterator_next (gen_ctx_t gen_ctx, insn_var_iterator_t *iter, - MIR_reg_t *var, int *op_num, int *out_p, int *mem_p, - size_t *passed_mem_num) { - MIR_op_t op; +static inline int input_insn_var_iterator_next (gen_ctx_t gen_ctx, insn_var_iterator_t *iter, + MIR_reg_t *var, int *op_num) { + MIR_op_t *op_ref; + int out_p; while (iter->op_num < iter->nops) { - *op_num = iter->op_num; - MIR_insn_op_mode (gen_ctx->ctx, iter->insn, iter->op_num, out_p); - op = iter->insn->ops[iter->op_num]; - *mem_p = FALSE; - *passed_mem_num = iter->passed_mem_num; + *op_num = (int) iter->op_num; + MIR_insn_op_mode (gen_ctx->ctx, iter->insn, iter->op_num, &out_p); + op_ref = &iter->insn->ops[iter->op_num]; + if (out_p && op_ref->mode != MIR_OP_VAR_MEM) { + iter->op_num++; + continue; + } while (iter->op_part_num < 2) { - if (op.mode == MIR_OP_MEM || op.mode == MIR_OP_HARD_REG_MEM) { - *mem_p = TRUE; - *passed_mem_num = ++iter->passed_mem_num; - *out_p = FALSE; - if (op.mode == MIR_OP_MEM) { - *var = iter->op_part_num == 0 ? op.u.mem.base : op.u.mem.index; - if (*var == 0) { - iter->op_part_num++; - continue; - } - *var = reg2var (gen_ctx, *var); - } else { - *var = iter->op_part_num == 0 ? op.u.hard_reg_mem.base : op.u.hard_reg_mem.index; - if (*var == MIR_NON_HARD_REG) { - iter->op_part_num++; - continue; - } + if (op_ref->mode == MIR_OP_VAR_MEM) { + *var = iter->op_part_num == 0 ? op_ref->u.var_mem.base : op_ref->u.var_mem.index; + if (*var == MIR_NON_VAR) { + iter->op_part_num++; + continue; } } else if (iter->op_part_num > 0) { break; - } else if (op.mode == MIR_OP_REG) { - *var = reg2var (gen_ctx, op.u.reg); - } else if (op.mode == MIR_OP_HARD_REG) { - *var = op.u.hard_reg; + } else if (op_ref->mode == MIR_OP_VAR) { + *var = op_ref->u.var; } else break; iter->op_part_num++; @@ -1036,10 +1263,34 @@ static inline int insn_var_iterator_next (gen_ctx_t gen_ctx, insn_var_iterator_t return FALSE; } -#define FOREACH_INSN_VAR(gen_ctx, iterator, insn, var, op_num, out_p, mem_p, passed_mem_num) \ - for (insn_var_iterator_init (gen_ctx, &iterator, insn); \ - insn_var_iterator_next (gen_ctx, &iterator, &var, &op_num, &out_p, &mem_p, \ - &passed_mem_num);) +static inline int output_insn_var_iterator_next (gen_ctx_t gen_ctx, insn_var_iterator_t *iter, + MIR_reg_t *var, int *op_num) { + MIR_op_t *op_ref; + int out_p; + + while (iter->op_num < iter->nops) { + *op_num = (int) iter->op_num; + MIR_insn_op_mode (gen_ctx->ctx, iter->insn, iter->op_num, &out_p); + op_ref = &iter->insn->ops[iter->op_num]; + if (!out_p || op_ref->mode == MIR_OP_VAR_MEM) { + iter->op_num++; + continue; + } + gen_assert (op_ref->mode == MIR_OP_VAR); + *var = op_ref->u.var; + iter->op_num++; + return TRUE; + } + return FALSE; +} + +#define FOREACH_IN_INSN_VAR(gen_ctx, iterator, insn, var, op_num) \ + for (insn_var_iterator_init (&iterator, insn); \ + input_insn_var_iterator_next (gen_ctx, &iterator, &var, &op_num);) + +#define FOREACH_OUT_INSN_VAR(gen_ctx, iterator, insn, var, op_num) \ + for (insn_var_iterator_init (&iterator, insn); \ + output_insn_var_iterator_next (gen_ctx, &iterator, &var, &op_num);) #if !MIR_NO_GEN_DEBUG static void output_in_edges (gen_ctx_t gen_ctx, bb_t bb) { @@ -1047,8 +1298,8 @@ static void output_in_edges (gen_ctx_t gen_ctx, bb_t bb) { fprintf (debug_file, " in edges:"); for (e = DLIST_HEAD (in_edge_t, bb->in_edges); e != NULL; e = DLIST_NEXT (in_edge_t, e)) { - fprintf (debug_file, " %3lu", (unsigned long) e->src->index); - if (e->skipped_p) fprintf (debug_file, "(CCP skip)"); + fprintf (debug_file, " %3lu%s%s", (unsigned long) e->src->index, e->fall_through_p ? "f" : "", + e->back_edge_p ? "*" : ""); } fprintf (debug_file, "\n"); } @@ -1058,14 +1309,17 @@ static void output_out_edges (gen_ctx_t gen_ctx, bb_t bb) { fprintf (debug_file, " out edges:"); for (e = DLIST_HEAD (out_edge_t, bb->out_edges); e != NULL; e = DLIST_NEXT (out_edge_t, e)) { - fprintf (debug_file, " %3lu", (unsigned long) e->dst->index); - if (e->skipped_p) fprintf (debug_file, "(CCP skip)"); + fprintf (debug_file, " %3lu%s%s", (unsigned long) e->dst->index, e->fall_through_p ? "f" : "", + e->back_edge_p ? "*" : ""); } fprintf (debug_file, "\n"); } -static void output_bitmap (gen_ctx_t gen_ctx, const char *head, bitmap_t bm, int var_p) { +/* When print_name_p, treat as reg bitmap. */ +static void output_bitmap (gen_ctx_t gen_ctx, const char *head, bitmap_t bm, int print_name_p, + MIR_reg_t *reg_map) { MIR_context_t ctx = gen_ctx->ctx; + MIR_reg_t reg; size_t nel; bitmap_iterator_t bi; @@ -1073,16 +1327,39 @@ static void output_bitmap (gen_ctx_t gen_ctx, const char *head, bitmap_t bm, int fprintf (debug_file, "%s", head); FOREACH_BITMAP_BIT (bi, bm, nel) { fprintf (debug_file, " %3lu", (unsigned long) nel); - if (var_p && var_is_reg_p (nel)) + if (print_name_p && (reg_map != NULL || nel > MAX_HARD_REG)) { + reg = (MIR_reg_t) nel; + if (reg_map != NULL) reg = reg_map[nel]; + gen_assert (reg >= MAX_HARD_REG); fprintf (debug_file, "(%s:%s)", - MIR_type_str (ctx, - MIR_reg_type (ctx, var2reg (gen_ctx, nel), curr_func_item->u.func)), - MIR_reg_name (ctx, var2reg (gen_ctx, nel), curr_func_item->u.func)); + MIR_type_str (ctx, MIR_reg_type (ctx, reg - MAX_HARD_REG, curr_func_item->u.func)), + MIR_reg_name (ctx, reg - MAX_HARD_REG, curr_func_item->u.func)); + } } fprintf (debug_file, "\n"); } -static int get_op_reg_index (gen_ctx_t gen_ctx, MIR_op_t op); +static void print_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, int newln_p) { + MIR_context_t ctx = gen_ctx->ctx; + int flag; + MIR_op_t op; + + MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, FALSE); + for (size_t i = 0; i < insn->nops; i++) { + op = insn->ops[i]; + if (op.mode == MIR_OP_VAR_MEM && op.u.var_mem.nloc != 0) { + flag = VARR_GET (mem_attr_t, mem_attrs, op.u.var_mem.nloc).alloca_flag; + fprintf (debug_file, " # m%lu%s", (unsigned long) op.u.var_mem.nloc, + !flag ? "" + : flag & (MAY_ALLOCA | MUST_ALLOCA) ? "au" + : flag & MAY_ALLOCA ? "a" + : "u"); + } + } + if (newln_p) fprintf (debug_file, "\n"); +} + +static void print_op_data (gen_ctx_t gen_ctx, void *op_data, bb_insn_t from); static void print_bb_insn (gen_ctx_t gen_ctx, bb_insn_t bb_insn, int with_notes_p) { MIR_context_t ctx = gen_ctx->ctx; MIR_op_t op; @@ -1090,27 +1367,19 @@ static void print_bb_insn (gen_ctx_t gen_ctx, bb_insn_t bb_insn, int with_notes_ size_t nel; bitmap_iterator_t bi; - MIR_output_insn (ctx, debug_file, bb_insn->insn, curr_func_item->u.func, FALSE); + print_insn (gen_ctx, bb_insn->insn, FALSE); fprintf (debug_file, " # indexes: "); for (size_t i = 0; i < bb_insn->insn->nops; i++) { - op = bb_insn->insn->ops[i]; if (i != 0) fprintf (debug_file, ","); - if (op.data == NULL) - fprintf (debug_file, "_"); - else - fprintf (debug_file, "%d", get_op_reg_index (gen_ctx, op)); + print_op_data (gen_ctx, bb_insn->insn->ops[i].data, bb_insn); } if (with_notes_p) { - for (dead_var_t dv = DLIST_HEAD (dead_var_t, bb_insn->dead_vars); dv != NULL; + for (dead_var_t dv = DLIST_HEAD (dead_var_t, bb_insn->insn_dead_vars); dv != NULL; dv = DLIST_NEXT (dead_var_t, dv)) { - if (var_is_reg_p (dv->var)) { - op.mode = MIR_OP_REG; - op.u.reg = var2reg (gen_ctx, dv->var); - } else { - op.mode = MIR_OP_HARD_REG; - op.u.hard_reg = dv->var; - } - fprintf (debug_file, dv == DLIST_HEAD (dead_var_t, bb_insn->dead_vars) ? " # dead: " : " "); + op.mode = MIR_OP_VAR; + op.u.var = dv->var; + fprintf (debug_file, + dv == DLIST_HEAD (dead_var_t, bb_insn->insn_dead_vars) ? " # dead: " : " "); MIR_output_op (ctx, debug_file, op, curr_func_item->u.func); } if (MIR_call_code_p (bb_insn->insn->code)) { @@ -1189,16 +1458,23 @@ static void print_loop_subtree (gen_ctx_t gen_ctx, loop_node_t root, int level, for (int i = 0; i < 2 * level + 2; i++) fprintf (debug_file, " "); if (root->bb != NULL) { gen_assert (DLIST_HEAD (loop_node_t, root->children) == NULL); - fprintf (debug_file, "BB%-3lu (pressure: int=%d, fp=%d)\n", (unsigned long) root->bb->index, + fprintf (debug_file, "BB%-3lu (pressure: int=%d, fp=%d)", (unsigned long) root->bb->index, root->max_int_pressure, root->max_fp_pressure); + if (root->bb != NULL && root->u.preheader_loop != NULL) + fprintf (debug_file, " (loop of the preheader - loop%lu)", + (unsigned long) root->u.preheader_loop->index); + fprintf (debug_file, "\n"); return; } fprintf (debug_file, "Loop%3lu (pressure: int=%d, fp=%d)", (unsigned long) root->index, root->max_int_pressure, root->max_fp_pressure); - if (curr_cfg->root_loop_node == root) + if (curr_cfg->root_loop_node == root || root->entry == NULL) fprintf (debug_file, ":\n"); - else + else { + if (root->bb == NULL && root->u.preheader != NULL) + fprintf (debug_file, " (preheader - bb%lu)", (unsigned long) root->u.preheader->bb->index); fprintf (debug_file, " (entry - bb%lu):\n", (unsigned long) root->entry->bb->index); + } for (loop_node_t node = DLIST_HEAD (loop_node_t, root->children); node != NULL; node = DLIST_NEXT (loop_node_t, node)) print_loop_subtree (gen_ctx, node, level + 1, bb_p); @@ -1213,133 +1489,386 @@ static void print_loop_tree (gen_ctx_t gen_ctx, int bb_p) { #endif static void rename_op_reg (gen_ctx_t gen_ctx, MIR_op_t *op_ref, MIR_reg_t reg, MIR_reg_t new_reg, - MIR_insn_t insn) { + MIR_insn_t insn, int print_p) { MIR_context_t ctx = gen_ctx->ctx; int change_p = FALSE; - if (op_ref->mode == MIR_OP_REG && op_ref->u.reg == reg) { - op_ref->u.reg = new_reg; + gen_assert (reg > MAX_HARD_REG); + if (op_ref->mode == MIR_OP_VAR && op_ref->u.var == reg) { + op_ref->u.var = new_reg; change_p = TRUE; - } else if (op_ref->mode == MIR_OP_MEM) { - if (op_ref->u.mem.base == reg) { - op_ref->u.mem.base = new_reg; + } else if (op_ref->mode == MIR_OP_VAR_MEM) { + if (op_ref->u.var_mem.base == reg) { + op_ref->u.var_mem.base = new_reg; change_p = TRUE; } - if (op_ref->u.mem.index == reg) { - op_ref->u.mem.index = new_reg; + if (op_ref->u.var_mem.index == reg) { + op_ref->u.var_mem.index = new_reg; change_p = TRUE; } } - if (!change_p) return; /* definition was already changed from another use */ + if (!change_p || !print_p) return; /* definition was already changed from another use */ DEBUG (2, { MIR_func_t func = curr_func_item->u.func; - fprintf (debug_file, " Change %s to %s in insn %-5lu", MIR_reg_name (ctx, reg, func), - MIR_reg_name (ctx, new_reg, func), (long unsigned) ((bb_insn_t) insn->data)->index); + fprintf (debug_file, " Change %s to %s in insn %-5lu", + MIR_reg_name (ctx, reg - MAX_HARD_REG, func), + MIR_reg_name (ctx, new_reg - MAX_HARD_REG, func), + (long unsigned) ((bb_insn_t) insn->data)->index); print_bb_insn (gen_ctx, insn->data, FALSE); }); } -static mv_t get_free_move (gen_ctx_t gen_ctx) { - mv_t mv; +static void update_tied_regs (gen_ctx_t gen_ctx, MIR_reg_t reg) { + gen_assert (reg > MAX_HARD_REG); + if (reg == MIR_NON_VAR + || MIR_reg_hard_reg_name (gen_ctx->ctx, reg - MAX_HARD_REG, curr_func_item->u.func) == NULL) + return; + bitmap_set_bit_p (tied_regs, reg); +} - if ((mv = DLIST_HEAD (mv_t, curr_cfg->free_moves)) != NULL) - DLIST_REMOVE (mv_t, curr_cfg->free_moves, mv); - else - mv = gen_malloc (gen_ctx, sizeof (struct mv)); - DLIST_APPEND (mv_t, curr_cfg->used_moves, mv); - return mv; +static long remove_unreachable_bbs (gen_ctx_t gen_ctx); + +static void MIR_UNUSED new_temp_op (gen_ctx_t gen_ctx, MIR_op_t *temp_op) { + MIR_context_t ctx = gen_ctx->ctx; + *temp_op = MIR_new_reg_op (ctx, _MIR_new_temp_reg (ctx, MIR_T_I64, curr_func_item->u.func)); +} + +static MIR_insn_t MIR_UNUSED find_bo (gen_ctx_t gen_ctx MIR_UNUSED, MIR_insn_t insn) { + for (; insn != NULL; insn = DLIST_NEXT (MIR_insn_t, insn)) + if (insn->code == MIR_BO || insn->code == MIR_BNO || insn->code == MIR_UBO + || insn->code == MIR_UBNO) + return insn; + gen_assert (FALSE); + return NULL; } -static void free_move (gen_ctx_t gen_ctx, mv_t mv) { - DLIST_REMOVE (mv_t, curr_cfg->used_moves, mv); - DLIST_APPEND (mv_t, curr_cfg->free_moves, mv); +static int label_cmp (const void *l1, const void *l2) { + const MIR_insn_t lab1 = *(const MIR_insn_t *) l1, lab2 = *(const MIR_insn_t *) l2; + gen_assert (lab1->code == MIR_LABEL && lab2->code == MIR_LABEL); + return lab1 < lab2 ? -1 : lab1 == lab2 ? 0 : 1; } static void build_func_cfg (gen_ctx_t gen_ctx) { MIR_context_t ctx = gen_ctx->ctx; - MIR_insn_t insn, next_insn; - size_t i, nops; - MIR_op_t *op; - MIR_var_t var; - bb_t bb, prev_bb, entry_bb, exit_bb, label_bb; + MIR_func_t func = curr_func_item->u.func; + MIR_insn_t insn, insn2, next_insn, ret_insn, use_insn; + MIR_insn_t new_insn MIR_UNUSED, new_label MIR_UNUSED, bo_insn MIR_UNUSED; + size_t i, j, nops; + MIR_op_t *op, temp_op1 MIR_UNUSED, temp_op2 MIR_UNUSED, temp_op3 MIR_UNUSED; + MIR_reg_t reg; + MIR_var_t mir_var; + bb_t bb, bb2, prev_bb, entry_bb, exit_bb, label_bb; DLIST_INIT (bb_t, curr_cfg->bbs); - DLIST_INIT (mv_t, curr_cfg->used_moves); - DLIST_INIT (mv_t, curr_cfg->free_moves); curr_cfg->curr_bb_insn_index = 0; - curr_cfg->max_reg = 0; - curr_cfg->min_reg = 0; - curr_cfg->non_conflicting_moves = 0; + curr_cfg->max_var = MAX_HARD_REG; curr_cfg->root_loop_node = NULL; curr_bb_index = 0; - for (i = 0; i < VARR_LENGTH (MIR_var_t, curr_func_item->u.func->vars); i++) { - var = VARR_GET (MIR_var_t, curr_func_item->u.func->vars, i); - update_min_max_reg (gen_ctx, MIR_reg (ctx, var.name, curr_func_item->u.func)); + for (i = 0; i < VARR_LENGTH (MIR_var_t, func->vars); i++) { + mir_var = VARR_GET (MIR_var_t, func->vars, i); + update_max_var (gen_ctx, MIR_reg (ctx, mir_var.name, func) + MAX_HARD_REG); } entry_bb = create_bb (gen_ctx, NULL); - add_bb (gen_ctx, entry_bb); + add_new_bb (gen_ctx, entry_bb); exit_bb = create_bb (gen_ctx, NULL); - add_bb (gen_ctx, exit_bb); - insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); - if (insn == NULL || insn->code == MIR_LABEL || MIR_call_code_p (insn->code)) { - /* To deal with special cases like adding insns before call in - machinize or moving invariant out of loop: */ - MIR_prepend_insn (ctx, curr_func_item, MIR_new_label (ctx)); - insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); - } + add_new_bb (gen_ctx, exit_bb); + /* To deal with special cases like adding insns before call in + machinize or moving invariant out of loop: */ + MIR_prepend_insn (ctx, curr_func_item, MIR_new_label (ctx)); bb = create_bb (gen_ctx, NULL); - add_bb (gen_ctx, bb); - for (; insn != NULL; insn = next_insn) { + add_new_bb (gen_ctx, bb); + bitmap_clear (tied_regs); + bitmap_clear (addr_regs); + addr_insn_p = FALSE; + VARR_TRUNC (MIR_insn_t, temp_insns, 0); + VARR_TRUNC (MIR_insn_t, temp_insns2, 0); + for (ret_insn = NULL, insn = DLIST_HEAD (MIR_insn_t, func->insns); insn != NULL; + insn = next_insn) { next_insn = DLIST_NEXT (MIR_insn_t, insn); + if (MIR_addr_code_p (insn->code)) { + addr_insn_p = TRUE; + bitmap_set_bit_p (addr_regs, insn->ops[1].u.reg + MAX_HARD_REG); + } else if (insn->code == MIR_RET) { + if (ret_insn != NULL) { /* we should have only one ret insn before generator */ + gen_assert (ret_insn == insn); + } else if (func->global_vars != NULL) { + VARR_TRUNC (MIR_op_t, temp_ops, 0); + for (i = 0; i < VARR_LENGTH (MIR_var_t, func->global_vars); i++) { + reg = MIR_reg (ctx, VARR_GET (MIR_var_t, func->global_vars, i).name, func) + MAX_HARD_REG; + VARR_PUSH (MIR_op_t, temp_ops, _MIR_new_var_op (ctx, reg)); + } + use_insn = MIR_new_insn_arr (ctx, MIR_USE, VARR_LENGTH (MIR_op_t, temp_ops), + VARR_ADDR (MIR_op_t, temp_ops)); + MIR_insert_insn_before (ctx, curr_func_item, insn, use_insn); + next_insn = use_insn; + ret_insn = insn; + continue; + } + } else if (MIR_call_code_p (insn->code)) { + bb->call_p = TRUE; + } else { + switch (insn->code) { /* ??? should we copy result change before insn and bo */ +#if defined(TARGET_EXPAND_ADDOS) || defined(TARGET_EXPAND_UADDOS) + case MIR_ADDOS: + case MIR_SUBOS: bo_insn = find_bo (gen_ctx, insn); +#ifndef TARGET_EXPAND_UADDOS + if (bo_insn->code == MIR_UBO || bo_insn->code == MIR_UBNO) break; +#endif +#ifndef TARGET_EXPAND_ADDOS + if (bo_insn->code == MIR_BO || bo_insn->code == MIR_BNO) break; +#endif + insn->code = insn->code == MIR_ADDO ? MIR_ADDS : MIR_SUBS; + new_temp_op (gen_ctx, &temp_op1); + if (bo_insn->code == MIR_UBO || bo_insn->code == MIR_UBNO) { + /* t1=a1;adds r,t1,a2; ublts r,t1,ov_label or t1=a1;subs r,t1,a2; ublts t1,res,ov_label */ + next_insn = new_insn = MIR_new_insn (ctx, MIR_MOV, temp_op1, insn->ops[1]); + MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + insn->ops[1] = temp_op1; + new_insn + = MIR_new_insn (ctx, bo_insn->code == MIR_UBO ? MIR_UBLTS : MIR_UBGES, bo_insn->ops[0], + insn->code == MIR_ADDS ? insn->ops[0] : temp_op1, + insn->code == MIR_ADDS ? temp_op1 : insn->ops[0]); + MIR_insert_insn_before (ctx, curr_func_item, bo_insn, new_insn); + } else { + /* ext32 t1,a1; ext32 t2,a2; (adds|subs) r,a1,a2; (add|sub) t1,t1,t2; ext32 t2,r; + bne t1,t2,ov_lab */ + new_temp_op (gen_ctx, &temp_op2); + next_insn = new_insn = MIR_new_insn (ctx, MIR_EXT32, temp_op1, insn->ops[1]); + MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + new_insn = MIR_new_insn (ctx, MIR_EXT32, temp_op2, insn->ops[2]); + MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + new_insn = MIR_new_insn (ctx, insn->code == MIR_ADDS ? MIR_ADD : MIR_SUB, temp_op1, + temp_op1, temp_op2); + MIR_insert_insn_before (ctx, curr_func_item, bo_insn, new_insn); + new_insn = MIR_new_insn (ctx, MIR_EXT32, temp_op2, insn->ops[0]); + MIR_insert_insn_before (ctx, curr_func_item, bo_insn, new_insn); + new_insn = MIR_new_insn (ctx, bo_insn->code == MIR_BO ? MIR_BNE : MIR_BEQ, + bo_insn->ops[0], temp_op1, temp_op2); + MIR_insert_insn_before (ctx, curr_func_item, bo_insn, new_insn); + } + MIR_remove_insn (gen_ctx->ctx, curr_func_item, bo_insn); + continue; +#endif +#if defined(TARGET_EXPAND_ADDO) || defined(TARGET_EXPAND_UADDO) + case MIR_ADDO: + case MIR_SUBO: bo_insn = find_bo (gen_ctx, insn); +#ifndef TARGET_EXPAND_UADDO + if (bo_insn->code == MIR_UBO || bo_insn->code == MIR_UBNO) break; +#endif +#ifndef TARGET_EXPAND_ADDO + if (bo_insn->code == MIR_BO || bo_insn->code == MIR_BNO) break; +#endif + insn->code = insn->code == MIR_ADDO ? MIR_ADD : MIR_SUB; + if (bo_insn->code == MIR_UBO || bo_insn->code == MIR_UBNO) { + /* t1=a1;add r,t1,a2; ublt r,t1,ov_label or t1=a1;sub r,t1,a2; ublt t1,r,ov_lab */ + next_insn = new_insn = MIR_new_insn (ctx, MIR_MOV, temp_op1, insn->ops[1]); + MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + insn->ops[1] = temp_op1; + new_insn = MIR_new_insn (ctx, bo_insn->code == MIR_UBO ? MIR_UBLT : MIR_UBGE, + bo_insn->ops[0], insn->code == MIR_ADD ? insn->ops[0] : temp_op1, + insn->code == MIR_ADD ? temp_op1 : insn->ops[0]); + MIR_insert_insn_before (ctx, curr_func_item, bo_insn, new_insn); + } else { + /* t1=a1;t2=a2;(add|sub) r,t1,t2;(lt t1,r,t1|lt t1,t1,r1);lt t2,t2,0;bne t2,t1,ov_lab */ + new_temp_op (gen_ctx, &temp_op1); + new_temp_op (gen_ctx, &temp_op2); + new_temp_op (gen_ctx, &temp_op3); + next_insn = new_insn = MIR_new_insn (ctx, MIR_MOV, temp_op1, insn->ops[1]); + MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + new_insn = MIR_new_insn (ctx, MIR_MOV, temp_op2, insn->ops[2]); + MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + if (insn->code == MIR_ADDO) + new_insn = MIR_new_insn (ctx, MIR_LT, temp_op1, insn->ops[0], temp_op1); + else + new_insn = MIR_new_insn (ctx, MIR_LT, temp_op1, temp_op1, insn->ops[0]); + MIR_insert_insn_before (ctx, curr_func_item, bo_insn, new_insn); + new_insn = MIR_new_insn (ctx, MIR_MOV, temp_op3, MIR_new_int_op (ctx, 0)); + MIR_insert_insn_before (ctx, curr_func_item, bo_insn, new_insn); + new_insn = MIR_new_insn (ctx, MIR_LT, temp_op2, temp_op2, temp_op3); + MIR_insert_insn_before (ctx, curr_func_item, bo_insn, new_insn); + new_insn = MIR_new_insn (ctx, bo_insn->code == MIR_BO ? MIR_BNE : MIR_BEQ, + bo_insn->ops[0], temp_op1, temp_op2); + MIR_insert_insn_before (ctx, curr_func_item, bo_insn, new_insn); + } + MIR_remove_insn (gen_ctx->ctx, curr_func_item, bo_insn); + continue; +#endif +#if defined(TARGET_EXPAND_MULOS) || defined(TARGET_EXPAND_UMULOS) + case MIR_MULOS: + case MIR_UMULOS: + /* [u]ext32 t1,a1; [u]ext32 t2,a2;[u]mul t1,t1,t2; [u]ext32 r,t1;..; b(ne|eq) lab,t1,r */ + bo_insn = find_bo (gen_ctx, insn); +#ifndef TARGET_EXPAND_UMULOS + if (bo_insn->code == MIR_UBO || bo_insn->code == MIR_UBNO) break; +#endif +#ifndef TARGET_EXPAND_MULOS + if (bo_insn->code == MIR_BO || bo_insn->code == MIR_BNO) break; +#endif + new_temp_op (gen_ctx, &temp_op1); + new_temp_op (gen_ctx, &temp_op2); + MIR_insn_code_t ext_code = insn->code == MIR_MULOS ? MIR_EXT32 : MIR_UEXT32; + next_insn = new_insn = MIR_new_insn (ctx, ext_code, temp_op1, insn->ops[1]); + MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + new_insn = MIR_new_insn (ctx, ext_code, temp_op2, insn->ops[2]); + MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + new_insn = MIR_new_insn (ctx, ext_code, insn->ops[0], temp_op1); + MIR_insert_insn_after (ctx, curr_func_item, insn, new_insn); + insn->code = MIR_MUL; + insn->ops[0] = temp_op1; + insn->ops[1] = temp_op1; + insn->ops[2] = temp_op2; + new_insn + = MIR_new_insn (ctx, + bo_insn->code == MIR_BO || bo_insn->code == MIR_UBO ? MIR_BNE : MIR_BEQ, + bo_insn->ops[0], new_insn->ops[0], new_insn->ops[1]); + MIR_insert_insn_after (ctx, curr_func_item, bo_insn, new_insn); + MIR_remove_insn (gen_ctx->ctx, curr_func_item, bo_insn); + continue; +#endif +#if defined(TARGET_EXPAND_MULO) || defined(TARGET_EXPAND_UMULO) + case MIR_MULO: + case MIR_UMULO: + /* t1=a1;t2=t2;mul r,t1,t2;...; [u]bno: bf lab,t1;[u]div t1,r,t1;bne lab,t,t2 + [u]bo: bf new_lab,t1;[u]div t1,r,t1;bne lab,t,t2;new_lab: */ + bo_insn = find_bo (gen_ctx, insn); +#ifndef TARGET_EXPAND_UMULO + if (bo_insn->code == MIR_UBO || bo_insn->code == MIR_UBNO) break; +#endif +#ifndef TARGET_EXPAND_MULO + if (bo_insn->code == MIR_BO || bo_insn->code == MIR_BNO) break; +#endif + new_temp_op (gen_ctx, &temp_op1); + new_temp_op (gen_ctx, &temp_op2); + next_insn = new_insn = MIR_new_insn (ctx, MIR_MOV, temp_op1, insn->ops[1]); + MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + new_insn = MIR_new_insn (ctx, MIR_MOV, temp_op2, insn->ops[2]); + MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + insn->code = MIR_MUL; + insn->ops[1] = temp_op1; + insn->ops[2] = temp_op2; + if (bo_insn->code == MIR_BNO || bo_insn->code == MIR_UBNO) { + new_insn = MIR_new_insn (ctx, MIR_BF, bo_insn->ops[0], temp_op1); + } else { + new_label = MIR_new_label (ctx); + new_insn = MIR_new_insn (ctx, MIR_BF, MIR_new_label_op (ctx, new_label), temp_op1); + } + MIR_insert_insn_before (ctx, curr_func_item, bo_insn, new_insn); + new_insn + = MIR_new_insn (ctx, + bo_insn->code == MIR_BO || bo_insn->code == MIR_BNO ? MIR_DIV : MIR_UDIV, + temp_op1, insn->ops[0], temp_op1); + MIR_insert_insn_before (ctx, curr_func_item, bo_insn, new_insn); + new_insn + = MIR_new_insn (ctx, + bo_insn->code == MIR_BO || bo_insn->code == MIR_UBO ? MIR_BNE : MIR_BEQ, + bo_insn->ops[0], temp_op1, temp_op2); + MIR_insert_insn_before (ctx, curr_func_item, bo_insn, new_insn); + if (bo_insn->code == MIR_BO || bo_insn->code == MIR_UBO) + MIR_insert_insn_before (ctx, curr_func_item, bo_insn, new_label); + MIR_remove_insn (gen_ctx->ctx, curr_func_item, bo_insn); + continue; +#endif + default: break; + } + } if (insn->data == NULL) { if (optimize_level != 0) - add_new_bb_insn (gen_ctx, insn, bb); + add_new_bb_insn (gen_ctx, insn, bb, TRUE); else setup_insn_data (gen_ctx, insn, bb); } + if (insn->code == MIR_LADDR) { + gen_assert (insn->ops[1].mode == MIR_OP_LABEL); + VARR_PUSH (MIR_insn_t, temp_insns2, insn->ops[1].u.label); + } else if (insn->code == MIR_JMPI) { + VARR_PUSH (MIR_insn_t, temp_insns, insn); + } nops = MIR_insn_nops (ctx, insn); if (next_insn != NULL - && (MIR_branch_code_p (insn->code) || insn->code == MIR_RET || insn->code == MIR_SWITCH + && (MIR_any_branch_code_p (insn->code) || insn->code == MIR_RET || insn->code == MIR_JRET + || insn->code == MIR_PRBEQ || insn->code == MIR_PRBNE || next_insn->code == MIR_LABEL)) { prev_bb = bb; if (next_insn->code == MIR_LABEL && next_insn->data != NULL) bb = get_insn_bb (gen_ctx, next_insn); else bb = create_bb (gen_ctx, next_insn); - add_bb (gen_ctx, bb); - if (insn->code != MIR_JMP && insn->code != MIR_RET && insn->code != MIR_SWITCH) - create_edge (gen_ctx, prev_bb, bb, TRUE); + add_new_bb (gen_ctx, bb); + if (insn->code != MIR_JMP && insn->code != MIR_JMPI && insn->code != MIR_RET + && insn->code != MIR_JRET && insn->code != MIR_SWITCH) + create_edge (gen_ctx, prev_bb, bb, TRUE, TRUE); /* fall through */ } - for (i = 0; i < nops; i++) - if ((op = &insn->ops[i])->mode == MIR_OP_LABEL) { - if (op->u.label->data == NULL) create_bb (gen_ctx, op->u.label); - label_bb = get_insn_bb (gen_ctx, op->u.label); - create_edge (gen_ctx, get_insn_bb (gen_ctx, insn), label_bb, TRUE); - } else if (op->mode == MIR_OP_REG) { - update_min_max_reg (gen_ctx, op->u.reg); + for (i = 0; i < nops; i++) { /* Transform all ops to var ops */ + op = &insn->ops[i]; + if (op->mode == MIR_OP_REG) { + op->mode = MIR_OP_VAR; + op->u.var = op->u.reg == 0 ? MIR_NON_VAR : op->u.reg + MAX_HARD_REG; } else if (op->mode == MIR_OP_MEM) { - update_min_max_reg (gen_ctx, op->u.mem.base); - update_min_max_reg (gen_ctx, op->u.mem.index); + op->mode = MIR_OP_VAR_MEM; + op->u.var_mem.base = op->u.mem.base == 0 ? MIR_NON_VAR : op->u.mem.base + MAX_HARD_REG; + op->u.var_mem.index = op->u.mem.index == 0 ? MIR_NON_VAR : op->u.mem.index + MAX_HARD_REG; + } + if (op->mode == MIR_OP_LABEL) { + if (op->u.label->data == NULL) create_bb (gen_ctx, op->u.label); + if (insn->code != MIR_LADDR) { + label_bb = get_insn_bb (gen_ctx, op->u.label); + create_edge (gen_ctx, get_insn_bb (gen_ctx, insn), label_bb, FALSE, TRUE); + } + } else if (op->mode == MIR_OP_VAR) { + update_max_var (gen_ctx, op->u.var); + update_tied_regs (gen_ctx, op->u.var); + } else if (op->mode == MIR_OP_VAR_MEM) { + update_max_var (gen_ctx, op->u.var_mem.base); + update_tied_regs (gen_ctx, op->u.var_mem.base); + update_max_var (gen_ctx, op->u.var_mem.index); + update_tied_regs (gen_ctx, op->u.var_mem.index); } + } + } + for (MIR_lref_data_t lref = func->first_lref; lref != NULL; lref = lref->next) { + VARR_PUSH (MIR_insn_t, temp_insns2, lref->label); + if (lref->label2 != NULL) VARR_PUSH (MIR_insn_t, temp_insns2, lref->label2); + } + qsort (VARR_ADDR (MIR_insn_t, temp_insns2), VARR_LENGTH (MIR_insn_t, temp_insns2), + sizeof (MIR_insn_t), label_cmp); + for (i = 0; i < VARR_LENGTH (MIR_insn_t, temp_insns); i++) { + insn = VARR_GET (MIR_insn_t, temp_insns, i); + gen_assert (insn->code == MIR_JMPI); + bb = get_insn_bb (gen_ctx, insn); + MIR_insn_t prev_label = NULL; + for (j = 0; j < VARR_LENGTH (MIR_insn_t, temp_insns2); j++) { + insn2 = VARR_GET (MIR_insn_t, temp_insns2, j); + if (insn2 == prev_label) continue; + gen_assert (insn2->code == MIR_LABEL); + prev_label = insn2; + bb2 = get_insn_bb (gen_ctx, insn2); + create_edge (gen_ctx, bb, bb2, FALSE, TRUE); + } + } + for (i = 0; i < VARR_LENGTH (MIR_insn_t, temp_insns2); i++) { + insn = VARR_GET (MIR_insn_t, temp_insns2, i); + gen_assert (insn->code == MIR_LABEL); + bb = get_insn_bb (gen_ctx, insn); + bb->reachable_p = TRUE; } + if (optimize_level > 0) remove_unreachable_bbs (gen_ctx); /* Add additional edges with entry and exit */ for (bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { if (bb != entry_bb && DLIST_HEAD (in_edge_t, bb->in_edges) == NULL) - create_edge (gen_ctx, entry_bb, bb, TRUE); + create_edge (gen_ctx, entry_bb, bb, FALSE, TRUE); if (bb != exit_bb && DLIST_HEAD (out_edge_t, bb->out_edges) == NULL) - create_edge (gen_ctx, bb, exit_bb, TRUE); + create_edge (gen_ctx, bb, exit_bb, FALSE, TRUE); } enumerate_bbs (gen_ctx); - VARR_CREATE (reg_info_t, curr_cfg->breg_info, 128); - curr_cfg->call_crossed_bregs = bitmap_create2 (curr_cfg->max_reg); + VARR_CREATE (reg_info_t, curr_cfg->reg_info, 128); + curr_cfg->call_crossed_regs = bitmap_create2 (curr_cfg->max_var); } static void destroy_func_cfg (gen_ctx_t gen_ctx) { MIR_insn_t insn; bb_insn_t bb_insn; bb_t bb, next_bb; - mv_t mv, next_mv; gen_assert (curr_func_item->item_type == MIR_func_item && curr_func_item->data != NULL); for (insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); insn != NULL; @@ -1356,22 +1885,14 @@ static void destroy_func_cfg (gen_ctx_t gen_ctx) { next_bb = DLIST_NEXT (bb_t, bb); delete_bb (gen_ctx, bb); } - for (mv = DLIST_HEAD (mv_t, curr_cfg->used_moves); mv != NULL; mv = next_mv) { - next_mv = DLIST_NEXT (mv_t, mv); - free (mv); - } - for (mv = DLIST_HEAD (mv_t, curr_cfg->free_moves); mv != NULL; mv = next_mv) { - next_mv = DLIST_NEXT (mv_t, mv); - free (mv); - } - VARR_DESTROY (reg_info_t, curr_cfg->breg_info); - bitmap_destroy (curr_cfg->call_crossed_bregs); + VARR_DESTROY (reg_info_t, curr_cfg->reg_info); + bitmap_destroy (curr_cfg->call_crossed_regs); free (curr_func_item->data); curr_func_item->data = NULL; } static int rpost_cmp (const void *a1, const void *a2) { - return (*(const struct bb **) a1)->rpost - (*(const struct bb **) a2)->rpost; + return (int) ((*(const struct bb **) a1)->rpost - (*(const struct bb **) a2)->rpost); } static int post_cmp (const void *a1, const void *a2) { return -rpost_cmp (a1, a2); } @@ -1455,38 +1976,163 @@ static void finish_data_flow (gen_ctx_t gen_ctx) { /* New Page */ -/* Building SSA. First we build optimized maximal SSA, then we minimize it - getting minimal SSA for reducible CFGs. There are two SSA representations: - - 1. Def pointers only: - - phi|insn: out:v1, in, in - ^ - | - phi|insn: out, in:v1, ... - - 2. Def-use chains (we don't use mir-lists to use less memory): - - phi|insn: out:v1, in, in - | (op.data) - V - ssa_edge (next_use)---------------> ssa_edge - ^ ^ - | (op.data) | (op.data) - phi|insn: out, in:v1, ... phi|insn: out, in:v1, ... +static MIR_insn_t get_insn_label (gen_ctx_t gen_ctx, MIR_insn_t insn) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_insn_t label; - We use conventional SSA to make out-of-ssa fast and simple. We don't rename all regs for SSA. - All phi insns always use the same reg to guarantee conventional SSA. It also means that - some regs have more one definition but ssa edges represent the correct SSA. The only - optimization in the pipeline which would benefit from full renaming is copy propagation (full - SSA copy propagation would not keep conventional SSA). -*/ + if (insn->code == MIR_LABEL) return insn; + label = MIR_new_label (ctx); + MIR_insert_insn_before (ctx, curr_func_item, insn, label); + add_new_bb_insn (gen_ctx, label, ((bb_insn_t) insn->data)->bb, FALSE); + return label; +} -typedef struct ssa_edge *ssa_edge_t; +/* Clone hot BBs to cold ones (which are after ret insn) to improve + optimization opportunities in hot part. */ +static int clone_bbs (gen_ctx_t gen_ctx) { + const int max_bb_growth_factor = 3; + MIR_context_t ctx = gen_ctx->ctx; + MIR_insn_t dst_insn, last_dst_insn, new_insn, label, next_insn, after; + bb_t bb, dst, new_bb; + edge_t e; + bb_insn_t bb_insn, dst_bb_insn, next_bb_insn; + MIR_func_t func = curr_func_item->u.func; + size_t size, orig_size, len, last_orig_bound; + int res; -struct ssa_edge { - bb_insn_t use, def; - char flag; + gen_assert (optimize_level != 0); + bitmap_clear (temp_bitmap); + for (bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + bitmap_set_bit_p (temp_bitmap, bb->index); + if ((bb_insn = DLIST_TAIL (bb_insn_t, bb->bb_insns)) == NULL) continue; + if (bb_insn->insn->code == MIR_RET || bb_insn->insn->code == MIR_JRET) break; + } + if (bb == NULL) return FALSE; + VARR_TRUNC (bb_t, worklist, 0); + for (bb = DLIST_NEXT (bb_t, bb); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + bb_insn = DLIST_TAIL (bb_insn_t, bb->bb_insns); + gen_assert (bb_insn != NULL); + if (bb_insn->insn->code == MIR_JMP && (e = DLIST_HEAD (out_edge_t, bb->out_edges)) != NULL + && bitmap_bit_p (temp_bitmap, e->dst->index)) + VARR_PUSH (bb_t, worklist, bb); + } + res = FALSE; + last_orig_bound = VARR_LENGTH (bb_t, worklist); + orig_size = size = 0; + while ((len = VARR_LENGTH (bb_t, worklist)) != 0) { + if (last_orig_bound > len) { + last_orig_bound = len; + orig_size = size = DLIST_LENGTH (bb_insn_t, VARR_LAST (bb_t, worklist)->bb_insns); + } + bb = VARR_POP (bb_t, worklist); + e = DLIST_HEAD (out_edge_t, bb->out_edges); + gen_assert (DLIST_NEXT (out_edge_t, e) == NULL); + if (e->back_edge_p) continue; + bb_insn = DLIST_TAIL (bb_insn_t, bb->bb_insns); + gen_assert (bb_insn != NULL && bb_insn->insn->code == MIR_JMP); + dst = e->dst; + dst_bb_insn = DLIST_TAIL (bb_insn_t, dst->bb_insns); + if (dst_bb_insn->insn->code == MIR_RET || dst_bb_insn->insn->code == MIR_JRET + || dst_bb_insn->insn->code == MIR_SWITCH || size > max_bb_growth_factor * orig_size) + continue; + res = TRUE; + DEBUG (2, { + fprintf (debug_file, " Cloning from BB%lu into BB%lu:\n", (unsigned long) dst->index, + (unsigned long) bb->index); + }); + last_dst_insn = DLIST_TAIL (bb_insn_t, dst->bb_insns)->insn; + after = DLIST_PREV (MIR_insn_t, bb_insn->insn); + gen_delete_insn (gen_ctx, bb_insn->insn); + bb_insn = NULL; + for (dst_bb_insn = DLIST_HEAD (bb_insn_t, dst->bb_insns); dst_bb_insn != NULL; + dst_bb_insn = DLIST_NEXT (bb_insn_t, dst_bb_insn)) { + dst_insn = dst_bb_insn->insn; + if (dst_insn->code == MIR_LABEL) continue; + new_insn = MIR_copy_insn (ctx, dst_insn); + /* we can not use gen_add_insn_xxx becuase of some cases (e.g. bb_insn is the last insn): */ + MIR_insert_insn_after (ctx, curr_func_item, after, new_insn); + add_new_bb_insn (gen_ctx, new_insn, bb, TRUE); + after = new_insn; + DEBUG (2, { + fprintf (debug_file, " Adding insn %-5lu", + (unsigned long) ((bb_insn_t) new_insn->data)->index); + MIR_output_insn (ctx, debug_file, new_insn, func, TRUE); + }); + size++; + } + delete_edge (e); + gen_assert (last_dst_insn != NULL); + if (last_dst_insn->code == MIR_JMP) { + label = last_dst_insn->ops[0].u.label; + create_edge (gen_ctx, bb, ((bb_insn_t) label->data)->bb, FALSE, TRUE); + if (bitmap_bit_p (temp_bitmap, ((bb_insn_t) label->data)->index)) + VARR_PUSH (bb_t, worklist, bb); + } else if (!MIR_branch_code_p (last_dst_insn->code)) { + next_insn = DLIST_NEXT (MIR_insn_t, last_dst_insn); + next_bb_insn = next_insn->data; + gen_assert (next_insn->code == MIR_LABEL); + new_insn = MIR_new_insn (ctx, MIR_JMP, MIR_new_label_op (ctx, next_insn)); + MIR_insert_insn_after (ctx, curr_func_item, after, new_insn); + add_new_bb_insn (gen_ctx, new_insn, bb, TRUE); + if (bitmap_bit_p (temp_bitmap, next_bb_insn->index)) VARR_PUSH (bb_t, worklist, bb); + create_edge (gen_ctx, bb, ((bb_insn_t) next_insn->data)->bb, FALSE, TRUE); + } else { + label = get_insn_label (gen_ctx, DLIST_NEXT (MIR_insn_t, last_dst_insn)); /* fallthrough */ + new_insn = MIR_new_insn (ctx, MIR_JMP, MIR_new_label_op (ctx, label)); + MIR_insert_insn_after (ctx, curr_func_item, after, new_insn); + new_bb = create_bb (gen_ctx, new_insn); + insert_new_bb_after (gen_ctx, bb, new_bb); + if (bitmap_bit_p (temp_bitmap, ((bb_insn_t) label->data)->bb->index)) + VARR_PUSH (bb_t, worklist, new_bb); + create_edge (gen_ctx, bb, new_bb, TRUE, TRUE); /* fall through */ + create_edge (gen_ctx, bb, ((bb_insn_t) last_dst_insn->ops[0].u.label->data)->bb, FALSE, + TRUE); /* branch */ + create_edge (gen_ctx, new_bb, ((bb_insn_t) label->data)->bb, FALSE, TRUE); + } + DEBUG (2, { + fprintf (debug_file, " Result BB%lu:\n", (unsigned long) bb->index); + output_in_edges (gen_ctx, bb); + output_out_edges (gen_ctx, bb); + for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; + bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) { + fprintf (debug_file, " %-5lu", (unsigned long) bb_insn->index); + MIR_output_insn (ctx, debug_file, bb_insn->insn, func, TRUE); + } + }); + } + if (res) enumerate_bbs (gen_ctx); + return res; +} + +/* New Page */ + +/* Building SSA. First we build optimized maximal SSA, then we minimize it + getting minimal SSA for reducible CFGs. There are two SSA representations: + + 1. Def pointers only: + + phi|insn: out:v1, in, in + ^ + | + phi|insn: out, in:v1, ... + + 2. Def-use chains (we don't use mir-lists to use less memory): + + phi|insn: out:v1, in, in + | (op.data) + V + ssa_edge (next_use)---------------> ssa_edge + ^ ^ + | (op.data) | (op.data) + phi|insn: out, in:v1, ... phi|insn: out, in:v1, ... + +*/ + +typedef struct ssa_edge *ssa_edge_t; + +struct ssa_edge { + bb_insn_t use, def; + char flag; uint16_t def_op_num; uint32_t use_op_num; ssa_edge_t prev_use, next_use; /* of the same def: we have only head in op.data */ @@ -1499,17 +2145,14 @@ typedef struct def_tab_el { } def_tab_el_t; DEF_HTAB (def_tab_el_t); -DEF_VARR (MIR_op_t); DEF_VARR (ssa_edge_t); -DEF_VARR (char); DEF_VARR (size_t); +DEF_VARR (char); struct ssa_ctx { - int def_use_repr_p; /* flag of def_use_chains */ - /* Insns defining undef and initial arg values. They are not in insn lists. */ + /* Different fake insns: defining undef, initial arg values. They are not in insn lists. */ VARR (bb_insn_t) * arg_bb_insns, *undef_insns; VARR (bb_insn_t) * phis, *deleted_phis; - VARR (MIR_op_t) * temp_ops; HTAB (def_tab_el_t) * def_tab; /* reg,bb -> insn defining reg */ /* used for renaming: */ VARR (ssa_edge_t) * ssa_edges_to_process; @@ -1517,27 +2160,21 @@ struct ssa_ctx { VARR (char) * reg_name; }; -#define def_use_repr_p gen_ctx->ssa_ctx->def_use_repr_p #define arg_bb_insns gen_ctx->ssa_ctx->arg_bb_insns #define undef_insns gen_ctx->ssa_ctx->undef_insns #define phis gen_ctx->ssa_ctx->phis #define deleted_phis gen_ctx->ssa_ctx->deleted_phis -#define temp_ops gen_ctx->ssa_ctx->temp_ops #define def_tab gen_ctx->ssa_ctx->def_tab #define ssa_edges_to_process gen_ctx->ssa_ctx->ssa_edges_to_process #define curr_reg_indexes gen_ctx->ssa_ctx->curr_reg_indexes #define reg_name gen_ctx->ssa_ctx->reg_name -static int get_op_reg_index (gen_ctx_t gen_ctx, MIR_op_t op) { - return def_use_repr_p ? ((ssa_edge_t) op.data)->def->index : ((bb_insn_t) op.data)->index; -} - -static htab_hash_t def_tab_el_hash (def_tab_el_t el, void *arg) { - return mir_hash_finish ( +static htab_hash_t def_tab_el_hash (def_tab_el_t el, void *arg MIR_UNUSED) { + return (htab_hash_t) mir_hash_finish ( mir_hash_step (mir_hash_step (mir_hash_init (0x33), (uint64_t) el.bb), (uint64_t) el.reg)); } -static int def_tab_el_eq (def_tab_el_t el1, def_tab_el_t el2, void *arg) { +static int def_tab_el_eq (def_tab_el_t el1, def_tab_el_t el2, void *arg MIR_UNUSED) { return el1.reg == el2.reg && el1.bb == el2.bb; } @@ -1548,42 +2185,41 @@ static MIR_insn_code_t get_move_code (MIR_type_t type) { : MIR_MOV); } -static bb_insn_t get_start_insn (gen_ctx_t gen_ctx, VARR (bb_insn_t) * start_insns, MIR_reg_t reg) { +static bb_insn_t get_fake_insn (gen_ctx_t gen_ctx, VARR (bb_insn_t) * fake_insns, MIR_reg_t reg) { MIR_context_t ctx = gen_ctx->ctx; MIR_type_t type; MIR_op_t op; MIR_insn_t insn; bb_insn_t bb_insn; - - gen_assert (DLIST_HEAD (bb_t, curr_cfg->bbs)->index == 0); - op = MIR_new_reg_op (ctx, reg); - while (VARR_LENGTH (bb_insn_t, start_insns) <= reg) VARR_PUSH (bb_insn_t, start_insns, NULL); - if ((bb_insn = VARR_GET (bb_insn_t, start_insns, reg)) == NULL) { - type = MIR_reg_type (ctx, reg, curr_func_item->u.func); + bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); /* enter bb */ + + gen_assert (bb->index == 0); /* enter bb */ + op = _MIR_new_var_op (ctx, reg); + while (VARR_LENGTH (bb_insn_t, fake_insns) <= reg) VARR_PUSH (bb_insn_t, fake_insns, NULL); + if ((bb_insn = VARR_GET (bb_insn_t, fake_insns, reg)) == NULL) { + gen_assert (reg > MAX_HARD_REG); + type = MIR_reg_type (ctx, reg - MAX_HARD_REG, curr_func_item->u.func); insn = MIR_new_insn (ctx, get_move_code (type), op, op); - bb_insn = create_bb_insn (gen_ctx, insn, DLIST_HEAD (bb_t, curr_cfg->bbs)); - VARR_SET (bb_insn_t, start_insns, reg, bb_insn); + bb_insn = create_bb_insn (gen_ctx, insn, bb); + VARR_SET (bb_insn_t, fake_insns, reg, bb_insn); } return bb_insn; } -static int start_insn_p (gen_ctx_t gen_ctx, bb_insn_t bb_insn) { - return bb_insn->bb == DLIST_HEAD (bb_t, curr_cfg->bbs); -} +static int fake_insn_p (bb_insn_t bb_insn) { return bb_insn->bb->index == 0; /* enter bb */ } static bb_insn_t redundant_phi_def (gen_ctx_t gen_ctx, bb_insn_t phi, int *def_op_num_ref) { bb_insn_t def, same = NULL; - int op_num; *def_op_num_ref = 0; - for (op_num = 1; op_num < phi->insn->nops; op_num++) { /* check input defs: */ + for (size_t op_num = 1; op_num < phi->insn->nops; op_num++) { /* check input defs: */ def = phi->insn->ops[op_num].data; if (def == same || def == phi) continue; if (same != NULL) return NULL; same = def; } - gen_assert (phi->insn->ops[0].mode == MIR_OP_REG); - if (same == NULL) same = get_start_insn (gen_ctx, undef_insns, phi->insn->ops[0].u.reg); + gen_assert (phi->insn->ops[0].mode == MIR_OP_VAR); + if (same == NULL) same = get_fake_insn (gen_ctx, undef_insns, phi->insn->ops[0].u.var); return same; } @@ -1597,15 +2233,29 @@ static bb_insn_t create_phi (gen_ctx_t gen_ctx, bb_t bb, MIR_op_t op) { while (VARR_LENGTH (MIR_op_t, temp_ops) < len) VARR_PUSH (MIR_op_t, temp_ops, op); phi_insn = MIR_new_insn_arr (ctx, MIR_PHI, len, VARR_ADDR (MIR_op_t, temp_ops)); bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); - if (bb_insn->insn->code == MIR_LABEL) + if (bb_insn->insn->code == MIR_LABEL) { gen_add_insn_after (gen_ctx, bb_insn->insn, phi_insn); - else + } else { gen_add_insn_before (gen_ctx, bb_insn->insn, phi_insn); + } phi_insn->ops[0].data = phi = phi_insn->data; VARR_PUSH (bb_insn_t, phis, phi); return phi; } +static MIR_insn_t get_last_bb_phi_insn (MIR_insn_t phi_insn) { + MIR_insn_t curr_insn, next_insn; + bb_t bb = ((bb_insn_t) phi_insn->data)->bb; + + gen_assert (phi_insn->code == MIR_PHI); + for (curr_insn = phi_insn; + (next_insn = DLIST_NEXT (MIR_insn_t, curr_insn)) != NULL + && ((bb_insn_t) next_insn->data)->bb == bb && next_insn->code == MIR_PHI; + curr_insn = next_insn) + ; + return curr_insn; +} + static bb_insn_t get_def (gen_ctx_t gen_ctx, MIR_reg_t reg, bb_t bb) { MIR_context_t ctx = gen_ctx->ctx; bb_t src; @@ -1618,11 +2268,11 @@ static bb_insn_t get_def (gen_ctx_t gen_ctx, MIR_reg_t reg, bb_t bb) { if (HTAB_DO (def_tab_el_t, def_tab, el, HTAB_FIND, tab_el)) return tab_el.def; if (DLIST_LENGTH (in_edge_t, bb->in_edges) == 1) { if ((src = DLIST_HEAD (in_edge_t, bb->in_edges)->src)->index == 0) { /* start bb: args */ - return get_start_insn (gen_ctx, arg_bb_insns, reg); + return get_fake_insn (gen_ctx, arg_bb_insns, reg); } return get_def (gen_ctx, reg, DLIST_HEAD (in_edge_t, bb->in_edges)->src); } - op = MIR_new_reg_op (ctx, reg); + op = _MIR_new_var_op (ctx, reg); el.def = def = create_phi (gen_ctx, bb, op); HTAB_DO (def_tab_el_t, def_tab, el, HTAB_INSERT, tab_el); return el.def; @@ -1649,8 +2299,7 @@ static void minimize_ssa (gen_ctx_t gen_ctx, size_t insns_num) { MIR_insn_t insn; bb_insn_t phi, def; size_t i, j, saved_bound; - int op_num, change_p, out_p, mem_p; - size_t passed_mem_num; + int op_num, change_p; MIR_reg_t var; insn_var_iterator_t iter; @@ -1682,8 +2331,8 @@ static void minimize_ssa (gen_ctx_t gen_ctx, size_t insns_num) { for (bb_insn_t bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) { insn = bb_insn->insn; - FOREACH_INSN_VAR (gen_ctx, iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (out_p) continue; + FOREACH_IN_INSN_VAR (gen_ctx, iter, insn, var, op_num) { + if (insn->ops[op_num].data == NULL) continue; insn->ops[op_num].data = skip_redundant_phis (insn->ops[op_num].data); } } @@ -1697,27 +2346,55 @@ static void minimize_ssa (gen_ctx_t gen_ctx, size_t insns_num) { } } -static void add_ssa_edge (gen_ctx_t gen_ctx, bb_insn_t def, int def_op_num, bb_insn_t use, - int use_op_num) { +static void print_op_data (gen_ctx_t gen_ctx, void *op_data, bb_insn_t from) { + ssa_edge_t se; + + if (op_data == NULL) { + fprintf (debug_file, "_"); + } else if ((se = op_data)->def != from) { + fprintf (debug_file, "%d", se->def->index); + } else { + for (; se != NULL; se = se->next_use) + fprintf (debug_file, "%s%d", se == op_data ? "(" : ", ", se->use->index); + fprintf (debug_file, ")"); + } +} + +static ssa_edge_t add_ssa_edge_1 (gen_ctx_t gen_ctx, bb_insn_t def, int def_op_num, bb_insn_t use, + int use_op_num, int dup_p MIR_UNUSED) { MIR_op_t *op_ref; ssa_edge_t ssa_edge = gen_malloc (gen_ctx, sizeof (struct ssa_edge)); gen_assert (use_op_num >= 0 && def_op_num >= 0 && def_op_num < (1 << 16)); + gen_assert (def->insn->code != MIR_CALL || def_op_num != 0); ssa_edge->flag = FALSE; ssa_edge->def = def; ssa_edge->def_op_num = def_op_num; ssa_edge->use = use; ssa_edge->use_op_num = use_op_num; - gen_assert (use->insn->ops[use_op_num].data == NULL); + gen_assert (dup_p || use->insn->ops[use_op_num].data == NULL); use->insn->ops[use_op_num].data = ssa_edge; op_ref = &def->insn->ops[def_op_num]; ssa_edge->next_use = op_ref->data; if (ssa_edge->next_use != NULL) ssa_edge->next_use->prev_use = ssa_edge; ssa_edge->prev_use = NULL; op_ref->data = ssa_edge; + return ssa_edge; +} + +static ssa_edge_t add_ssa_edge (gen_ctx_t gen_ctx, bb_insn_t def, int def_op_num, bb_insn_t use, + int use_op_num) { + return add_ssa_edge_1 (gen_ctx, def, def_op_num, use, use_op_num, FALSE); +} + +static ssa_edge_t add_ssa_edge_dup (gen_ctx_t gen_ctx, bb_insn_t def, int def_op_num, bb_insn_t use, + int use_op_num) { + return add_ssa_edge_1 (gen_ctx, def, def_op_num, use, use_op_num, TRUE); } -static void remove_ssa_edge (gen_ctx_t gen_ctx, ssa_edge_t ssa_edge) { +static void free_ssa_edge (ssa_edge_t ssa_edge) { free (ssa_edge); } + +static void remove_ssa_edge (ssa_edge_t ssa_edge) { if (ssa_edge->prev_use != NULL) { ssa_edge->prev_use->next_use = ssa_edge->next_use; } else { @@ -1728,79 +2405,119 @@ static void remove_ssa_edge (gen_ctx_t gen_ctx, ssa_edge_t ssa_edge) { if (ssa_edge->next_use != NULL) ssa_edge->next_use->prev_use = ssa_edge->prev_use; gen_assert (ssa_edge->use->insn->ops[ssa_edge->use_op_num].data == ssa_edge); ssa_edge->use->insn->ops[ssa_edge->use_op_num].data = NULL; - free (ssa_edge); + free_ssa_edge (ssa_edge); +} + +static void remove_insn_ssa_edges (MIR_insn_t insn) { + ssa_edge_t ssa_edge; + for (size_t i = 0; i < insn->nops; i++) { + /* output operand refers to chain of ssa edges -- remove them all: */ + while ((ssa_edge = insn->ops[i].data) != NULL) remove_ssa_edge (ssa_edge); + } } static void change_ssa_edge_list_def (ssa_edge_t list, bb_insn_t new_bb_insn, unsigned new_def_op_num, MIR_reg_t reg, MIR_reg_t new_reg) { + gen_assert (new_reg > MAX_HARD_REG); for (ssa_edge_t se = list; se != NULL; se = se->next_use) { se->def = new_bb_insn; se->def_op_num = new_def_op_num; - if (new_reg != 0) { + if (new_reg != MIR_NON_VAR) { MIR_op_t *op_ref = &se->use->insn->ops[se->use_op_num]; - if (op_ref->mode == MIR_OP_REG) { - op_ref->u.reg = new_reg; + if (op_ref->mode == MIR_OP_VAR) { + if (op_ref->u.var == reg) op_ref->u.var = new_reg; } else { - gen_assert (op_ref->mode == MIR_OP_MEM && op_ref->u.mem.base == reg); - op_ref->u.mem.base = new_reg; + gen_assert (op_ref->mode == MIR_OP_VAR_MEM); + if (op_ref->u.var_mem.base == reg) op_ref->u.var_mem.base = new_reg; + if (op_ref->u.var_mem.index == reg) op_ref->u.var_mem.index = new_reg; } } } } +static void redirect_def (gen_ctx_t gen_ctx, MIR_insn_t insn, MIR_insn_t by, int def_use_ssa_p) { +#ifndef NDEBUG + int out_p, by_out_p; + MIR_insn_op_mode (gen_ctx->ctx, insn, 0, &out_p); + MIR_insn_op_mode (gen_ctx->ctx, by, 0, &by_out_p); + gen_assert (insn->ops[0].mode == MIR_OP_VAR && by->ops[0].mode == MIR_OP_VAR + && (def_use_ssa_p || insn->ops[0].u.var == by->ops[0].u.var) + && !MIR_call_code_p (insn->code) && out_p && by_out_p); +#endif + by->ops[0].data = insn->ops[0].data; + insn->ops[0].data = NULL; /* make redundant insn having no uses */ + change_ssa_edge_list_def (by->ops[0].data, by->data, 0, insn->ops[0].u.var, by->ops[0].u.var); + if (def_use_ssa_p) { + gen_assert (move_p (by) && insn->ops[0].mode == MIR_OP_VAR && by->ops[1].mode == MIR_OP_VAR + && insn->ops[0].u.var == by->ops[1].u.var); + add_ssa_edge (gen_ctx, insn->data, 0, by->data, 1); + } +} + static int get_var_def_op_num (gen_ctx_t gen_ctx, MIR_reg_t var, MIR_insn_t insn) { - int op_num, out_p, mem_p; - size_t passed_mem_num; + int op_num; MIR_reg_t insn_var; insn_var_iterator_t iter; - FOREACH_INSN_VAR (gen_ctx, iter, insn, insn_var, op_num, out_p, mem_p, passed_mem_num) { - if (out_p && var == insn_var) return op_num; + FOREACH_OUT_INSN_VAR (gen_ctx, iter, insn, insn_var, op_num) { + if (var == insn_var) return op_num; } gen_assert (FALSE); return -1; } -static void make_ssa_def_use_repr (gen_ctx_t gen_ctx) { - MIR_insn_t insn; - bb_t bb; - bb_insn_t bb_insn, def; - int op_num, out_p, mem_p; - size_t passed_mem_num; +static void process_insn_inputs_for_ssa_def_use_repr (gen_ctx_t gen_ctx, bb_insn_t bb_insn) { + MIR_insn_t insn = bb_insn->insn; + bb_insn_t def; + int op_num; MIR_reg_t var; insn_var_iterator_t iter; - if (def_use_repr_p) return; - def_use_repr_p = TRUE; - for (bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) + FOREACH_IN_INSN_VAR (gen_ctx, iter, insn, var, op_num) { + if (var <= MAX_HARD_REG) continue; + def = insn->ops[op_num].data; + gen_assert (def != NULL); + insn->ops[op_num].data = NULL; + add_ssa_edge (gen_ctx, def, get_var_def_op_num (gen_ctx, var, def->insn), bb_insn, op_num); + } +} + +static void make_ssa_def_use_repr (gen_ctx_t gen_ctx) { + bb_insn_t bb_insn; + + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; - bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) { - insn = bb_insn->insn; - FOREACH_INSN_VAR (gen_ctx, iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (out_p) continue; - def = insn->ops[op_num].data; - gen_assert (var > MAX_HARD_REG && def != NULL); - insn->ops[op_num].data = NULL; - add_ssa_edge (gen_ctx, def, get_var_def_op_num (gen_ctx, var, def->insn), bb_insn, op_num); - } - } + bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) + process_insn_inputs_for_ssa_def_use_repr (gen_ctx, bb_insn); } -static MIR_reg_t get_new_reg (gen_ctx_t gen_ctx, MIR_reg_t reg, size_t index) { +static void ssa_delete_insn (gen_ctx_t gen_ctx, MIR_insn_t insn) { + remove_insn_ssa_edges (insn); + gen_delete_insn (gen_ctx, insn); +} + +static MIR_reg_t get_new_reg (gen_ctx_t gen_ctx, MIR_reg_t old_reg, int sep, size_t index) { MIR_context_t ctx = gen_ctx->ctx; MIR_func_t func = curr_func_item->u.func; - MIR_type_t type = MIR_reg_type (ctx, reg, func); - const char *name = MIR_reg_name (ctx, reg, func); + MIR_type_t type = MIR_reg_type (ctx, old_reg - MAX_HARD_REG, func); + const char *name = MIR_reg_name (ctx, old_reg - MAX_HARD_REG, func); + const char *hard_reg_name = MIR_reg_hard_reg_name (ctx, old_reg - MAX_HARD_REG, func); char ind_str[30]; MIR_reg_t new_reg; VARR_TRUNC (char, reg_name, 0); VARR_PUSH_ARR (char, reg_name, name, strlen (name)); - VARR_PUSH (char, reg_name, '@'); + VARR_PUSH (char, reg_name, sep); sprintf (ind_str, "%lu", (unsigned long) index); /* ??? should be enough to unique */ VARR_PUSH_ARR (char, reg_name, ind_str, strlen (ind_str) + 1); - new_reg = MIR_new_func_reg (ctx, func, type, VARR_ADDR (char, reg_name)); - update_min_max_reg (gen_ctx, new_reg); + if (hard_reg_name == NULL) { + new_reg = MIR_new_func_reg (ctx, func, type, VARR_ADDR (char, reg_name)) + MAX_HARD_REG; + } else { + new_reg = (MIR_new_global_func_reg (ctx, func, type, VARR_ADDR (char, reg_name), hard_reg_name) + + MAX_HARD_REG); + bitmap_set_bit_p (tied_regs, new_reg); + } + update_max_var (gen_ctx, new_reg); return new_reg; } @@ -1828,53 +2545,49 @@ static int pop_to_rename (gen_ctx_t gen_ctx, ssa_edge_t *ssa_edge) { static void process_insn_to_rename (gen_ctx_t gen_ctx, MIR_insn_t insn, int op_num) { for (ssa_edge_t curr_edge = insn->ops[op_num].data; curr_edge != NULL; curr_edge = curr_edge->next_use) - if (push_to_rename (gen_ctx, curr_edge) && curr_edge->use->insn->code == MIR_PHI) - process_insn_to_rename (gen_ctx, curr_edge->use->insn, 0); - if (insn->code != MIR_PHI) return; - for (size_t i = 1; i < insn->nops; i++) { /* process a def -> the phi use */ - ssa_edge_t ssa_edge = insn->ops[i].data; - bb_insn_t def = ssa_edge->def; + push_to_rename (gen_ctx, curr_edge); +} - /* process the def -> other uses: */ - if (push_to_rename (gen_ctx, ssa_edge)) - process_insn_to_rename (gen_ctx, def->insn, ssa_edge->def_op_num); - } +static MIR_reg_t get_new_ssa_reg (gen_ctx_t gen_ctx, MIR_reg_t reg, int sep, int new_p) { + size_t reg_index; + + while (VARR_LENGTH (size_t, curr_reg_indexes) <= reg) VARR_PUSH (size_t, curr_reg_indexes, 0); + reg_index = VARR_GET (size_t, curr_reg_indexes, reg); + VARR_SET (size_t, curr_reg_indexes, reg, reg_index + 1); + return reg_index == 0 && !new_p ? MIR_NON_VAR : get_new_reg (gen_ctx, reg, sep, reg_index); } static void rename_bb_insn (gen_ctx_t gen_ctx, bb_insn_t bb_insn) { - int op_num, out_p, mem_p; - size_t passed_mem_num, reg_index; + int op_num; MIR_reg_t var, reg, new_reg; MIR_insn_t insn, def_insn, use_insn; ssa_edge_t ssa_edge; insn_var_iterator_t iter; insn = bb_insn->insn; - FOREACH_INSN_VAR (gen_ctx, iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (!out_p || !var_is_reg_p (var)) continue; + FOREACH_OUT_INSN_VAR (gen_ctx, iter, insn, var, op_num) { + if (var <= MAX_HARD_REG) continue; ssa_edge = insn->ops[op_num].data; if (ssa_edge != NULL && ssa_edge->flag) continue; /* already processed */ DEBUG (2, { fprintf (debug_file, " Start def insn %-5lu", (long unsigned) bb_insn->index); print_bb_insn (gen_ctx, bb_insn, FALSE); }); - reg = var2reg (gen_ctx, var); - while (VARR_LENGTH (size_t, curr_reg_indexes) <= reg) VARR_PUSH (size_t, curr_reg_indexes, 0); - reg_index = VARR_GET (size_t, curr_reg_indexes, reg); - VARR_SET (size_t, curr_reg_indexes, reg, reg_index + 1); - new_reg = reg_index == 0 ? 0 : get_new_reg (gen_ctx, reg, reg_index); + reg = var; + new_reg = get_new_ssa_reg (gen_ctx, reg, '@', FALSE); if (ssa_edge == NULL) { /* special case: unused output */ - if (new_reg != 0) rename_op_reg (gen_ctx, &insn->ops[op_num], reg, new_reg, insn); + if (new_reg != MIR_NON_VAR) + rename_op_reg (gen_ctx, &insn->ops[op_num], reg, new_reg, insn, TRUE); continue; } VARR_TRUNC (ssa_edge_t, ssa_edges_to_process, 0); process_insn_to_rename (gen_ctx, insn, op_num); - if (new_reg != 0) { + if (new_reg != MIR_NON_VAR) { while (pop_to_rename (gen_ctx, &ssa_edge)) { def_insn = ssa_edge->def->insn; use_insn = ssa_edge->use->insn; - rename_op_reg (gen_ctx, &def_insn->ops[ssa_edge->def_op_num], reg, new_reg, def_insn); - rename_op_reg (gen_ctx, &use_insn->ops[ssa_edge->use_op_num], reg, new_reg, use_insn); + rename_op_reg (gen_ctx, &def_insn->ops[ssa_edge->def_op_num], reg, new_reg, def_insn, TRUE); + rename_op_reg (gen_ctx, &use_insn->ops[ssa_edge->use_op_num], reg, new_reg, use_insn, TRUE); } } } @@ -1882,8 +2595,7 @@ static void rename_bb_insn (gen_ctx_t gen_ctx, bb_insn_t bb_insn) { static void rename_regs (gen_ctx_t gen_ctx) { bb_insn_t bb_insn; - int op_num, out_p, mem_p; - size_t passed_mem_num; + int op_num; MIR_reg_t var; MIR_insn_t insn; ssa_edge_t ssa_edge; @@ -1893,13 +2605,12 @@ static void rename_regs (gen_ctx_t gen_ctx) { for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) { /* clear all ssa edge flags */ insn = bb_insn->insn; - FOREACH_INSN_VAR (gen_ctx, iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (out_p || !var_is_reg_p (var)) continue; + FOREACH_IN_INSN_VAR (gen_ctx, iter, insn, var, op_num) { + if (var <= MAX_HARD_REG || MIR_addr_code_p (insn->code)) continue; ssa_edge = insn->ops[op_num].data; ssa_edge->flag = FALSE; } } - VARR_TRUNC (size_t, curr_reg_indexes, 0); /* Process arg insns first to have first use of reg in the program with zero index. We need this because machinize for args will use reg with zero index: */ for (size_t i = 0; i < VARR_LENGTH (bb_insn_t, arg_bb_insns); i++) @@ -1911,18 +2622,36 @@ static void rename_regs (gen_ctx_t gen_ctx) { rename_bb_insn (gen_ctx, bb_insn); } -static void build_ssa (gen_ctx_t gen_ctx) { - bb_t bb; - bb_insn_t def, bb_insn, phi; - int op_num, out_p, mem_p; - size_t passed_mem_num, insns_num, i; - MIR_reg_t var; +static void process_bb_insn_for_ssa (gen_ctx_t gen_ctx, bb_insn_t bb_insn) { + bb_t bb = bb_insn->bb; + bb_insn_t def; + int op_num; + MIR_reg_t var, reg; def_tab_el_t el; insn_var_iterator_t iter; + FOREACH_IN_INSN_VAR (gen_ctx, iter, bb_insn->insn, var, op_num) { + gen_assert (var > MAX_HARD_REG); + reg = var; + def = get_def (gen_ctx, reg, bb); + bb_insn->insn->ops[op_num].data = def; + } + FOREACH_OUT_INSN_VAR (gen_ctx, iter, bb_insn->insn, var, op_num) { + reg = var; + el.bb = bb; + el.reg = reg; + el.def = bb_insn; + HTAB_DO (def_tab_el_t, def_tab, el, HTAB_REPLACE, el); + } +} + +static void build_ssa (gen_ctx_t gen_ctx, int rename_p) { + bb_t bb; + bb_insn_t bb_insn, phi; + size_t i, insns_num; + gen_assert (VARR_LENGTH (bb_insn_t, arg_bb_insns) == 0 && VARR_LENGTH (bb_insn_t, undef_insns) == 0); - def_use_repr_p = FALSE; HTAB_CLEAR (def_tab_el_t, def_tab); VARR_TRUNC (bb_t, worklist, 0); for (bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) @@ -1933,72 +2662,136 @@ static void build_ssa (gen_ctx_t gen_ctx) { for (i = 0; i < VARR_LENGTH (bb_t, worklist); i++) { bb = VARR_GET (bb_t, worklist, i); for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; - bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) { + bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) if (bb_insn->insn->code != MIR_PHI) { - FOREACH_INSN_VAR (gen_ctx, iter, bb_insn->insn, var, op_num, out_p, mem_p, passed_mem_num) { - gen_assert (var > MAX_HARD_REG); - if (out_p) continue; - def = get_def (gen_ctx, var - MAX_HARD_REG, bb); - bb_insn->insn->ops[op_num].data = def; - } insns_num++; - FOREACH_INSN_VAR (gen_ctx, iter, bb_insn->insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (!out_p) continue; - el.bb = bb; - el.reg = var - MAX_HARD_REG; - el.def = bb_insn; - HTAB_DO (def_tab_el_t, def_tab, el, HTAB_REPLACE, el); - } + process_bb_insn_for_ssa (gen_ctx, bb_insn); } - } } for (i = 0; i < VARR_LENGTH (bb_insn_t, phis); i++) { phi = VARR_GET (bb_insn_t, phis, i); - add_phi_operands (gen_ctx, phi->insn->ops[0].u.reg, phi); + add_phi_operands (gen_ctx, phi->insn->ops[0].u.var, phi); } /* minimization can not be switched off for def_use representation building as it clears ops[0].data: */ minimize_ssa (gen_ctx, insns_num); make_ssa_def_use_repr (gen_ctx); - rename_regs (gen_ctx); + if (rename_p) { + VARR_TRUNC (size_t, curr_reg_indexes, 0); + rename_regs (gen_ctx); + } +} + +static void make_conventional_ssa (gen_ctx_t gen_ctx) { /* requires life info */ + MIR_context_t ctx = gen_ctx->ctx; + MIR_type_t type; + MIR_reg_t var, dest_var; + MIR_insn_code_t move_code; + MIR_insn_t insn, new_insn; + bb_t bb, prev_bb; + bb_insn_t bb_insn, next_bb_insn, tail, new_bb_insn, after; + edge_t e; + ssa_edge_t se; + + for (bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) + for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; bb_insn = next_bb_insn) { + next_bb_insn = DLIST_NEXT (bb_insn_t, bb_insn); + if ((insn = bb_insn->insn)->code == MIR_LABEL) continue; + if (insn->code != MIR_PHI) break; + gen_assert (insn->ops[0].mode == MIR_OP_VAR && insn->ops[0].u.var > MAX_HARD_REG); + dest_var = var = insn->ops[0].u.var; + type = MIR_reg_type (gen_ctx->ctx, var - MAX_HARD_REG, curr_func_item->u.func); + move_code = get_move_code (type); + dest_var = get_new_ssa_reg (gen_ctx, var, '%', TRUE); + gen_assert (dest_var != MIR_NON_VAR); + e = DLIST_HEAD (in_edge_t, bb->in_edges); + for (size_t i = 1; i < insn->nops; i++) { + se = insn->ops[i].data; + insn->ops[i].data = NULL; + new_insn = MIR_new_insn (ctx, move_code, _MIR_new_var_op (ctx, dest_var), insn->ops[i]); + if ((tail = DLIST_TAIL (bb_insn_t, e->src->bb_insns)) == NULL) { + for (prev_bb = DLIST_PREV (bb_t, e->src), after = NULL; + prev_bb != NULL && (after = DLIST_TAIL (bb_insn_t, prev_bb->bb_insns)) == NULL; + prev_bb = DLIST_PREV (bb_t, prev_bb)) + ; + if (after != NULL) + MIR_insert_insn_after (ctx, curr_func_item, after->insn, new_insn); + else + MIR_prepend_insn (ctx, curr_func_item, new_insn); + new_bb_insn = create_bb_insn (gen_ctx, new_insn, e->src); + DLIST_APPEND (bb_insn_t, e->src->bb_insns, new_bb_insn); + } else if (MIR_any_branch_code_p (tail->insn->code)) { + gen_add_insn_before (gen_ctx, tail->insn, new_insn); + } else { + gen_add_insn_after (gen_ctx, tail->insn, new_insn); + } + new_insn->ops[1].data = se; + se->use = new_insn->data; + se->use_op_num = 1; + add_ssa_edge (gen_ctx, new_insn->data, 0, bb_insn, (int) i); + insn->ops[i].mode = MIR_OP_VAR; + insn->ops[i].u.var = dest_var; + e = DLIST_NEXT (in_edge_t, e); + } + for (se = insn->ops[0].data; se != NULL; se = se->next_use) + if (se->use->bb != bb) break; + if (se == NULL) { /* we should do this only after adding moves at the end of bbs */ + /* r=phi(...), all r uses in the same bb: change new_r = phi(...) and all uses by new_r */ + insn->ops[0].u.var = dest_var; + change_ssa_edge_list_def (insn->ops[0].data, bb_insn, 0, var, dest_var); + } else { + new_insn = MIR_new_insn (ctx, move_code, _MIR_new_var_op (ctx, var), + _MIR_new_var_op (ctx, dest_var)); + gen_add_insn_after (gen_ctx, insn, new_insn); + new_insn->ops[0].data = insn->ops[0].data; + insn->ops[0] = new_insn->ops[1]; + change_ssa_edge_list_def (new_insn->ops[0].data, new_insn->data, 0, MIR_NON_VAR, + MIR_NON_VAR); + add_ssa_edge (gen_ctx, bb_insn, 0, new_insn->data, 1); + } + } +} + +static void free_fake_bb_insns (VARR (bb_insn_t) * bb_insns) { + bb_insn_t bb_insn; + + while (VARR_LENGTH (bb_insn_t, bb_insns) != 0) + if ((bb_insn = VARR_POP (bb_insn_t, bb_insns)) != NULL) { // ??? specialized free funcs + remove_insn_ssa_edges (bb_insn->insn); + free (bb_insn->insn); /* we can not use gen_delete as the insn not in the list */ + free (bb_insn); + } } static void undo_build_ssa (gen_ctx_t gen_ctx) { bb_t bb; bb_insn_t bb_insn, next_bb_insn; - int op_num, out_p, mem_p; - size_t passed_mem_num; + ssa_edge_t se, next_se; + int op_num; MIR_reg_t var; MIR_insn_t insn; insn_var_iterator_t iter; + free_fake_bb_insns (arg_bb_insns); + free_fake_bb_insns (undef_insns); for (bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) { insn = bb_insn->insn; - FOREACH_INSN_VAR (gen_ctx, iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (insn->ops[op_num].data == NULL) continue; - if (!def_use_repr_p) - insn->ops[op_num].data = NULL; - else - remove_ssa_edge (gen_ctx, insn->ops[op_num].data); + FOREACH_OUT_INSN_VAR (gen_ctx, iter, insn, var, op_num) { + /* all sse after ssa combine available only from defs */ + for (se = insn->ops[op_num].data; se != NULL; se = next_se) { + next_se = se->next_use; + free_ssa_edge (se); + } } + for (size_t i = 0; i < insn->nops; i++) insn->ops[i].data = NULL; } for (bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; bb_insn = next_bb_insn) { next_bb_insn = DLIST_NEXT (bb_insn_t, bb_insn); if (bb_insn->insn->code == MIR_PHI) gen_delete_insn (gen_ctx, bb_insn->insn); } - while (VARR_LENGTH (bb_insn_t, arg_bb_insns) != 0) - if ((bb_insn = VARR_POP (bb_insn_t, arg_bb_insns)) != NULL) { // ??? specialized free funcs - free (bb_insn->insn); - free (bb_insn); - } - while (VARR_LENGTH (bb_insn_t, undef_insns) != 0) - if ((bb_insn = VARR_POP (bb_insn_t, undef_insns)) != NULL) { // ??? specialized free funcs - free (bb_insn->insn); - free (bb_insn); - } } static void init_ssa (gen_ctx_t gen_ctx) { @@ -2007,7 +2800,6 @@ static void init_ssa (gen_ctx_t gen_ctx) { VARR_CREATE (bb_insn_t, undef_insns, 0); VARR_CREATE (bb_insn_t, phis, 0); VARR_CREATE (bb_insn_t, deleted_phis, 0); - VARR_CREATE (MIR_op_t, temp_ops, 16); HTAB_CREATE (def_tab_el_t, def_tab, 1024, def_tab_el_hash, def_tab_el_eq, gen_ctx); VARR_CREATE (ssa_edge_t, ssa_edges_to_process, 512); VARR_CREATE (size_t, curr_reg_indexes, 4096); @@ -2019,7 +2811,6 @@ static void finish_ssa (gen_ctx_t gen_ctx) { VARR_DESTROY (bb_insn_t, undef_insns); VARR_DESTROY (bb_insn_t, phis); VARR_DESTROY (bb_insn_t, deleted_phis); - VARR_DESTROY (MIR_op_t, temp_ops); HTAB_DESTROY (def_tab_el_t, def_tab); VARR_DESTROY (ssa_edge_t, ssa_edges_to_process); VARR_DESTROY (size_t, curr_reg_indexes); @@ -2030,114 +2821,581 @@ static void finish_ssa (gen_ctx_t gen_ctx) { /* New Page */ -/* Copy propagation */ +/* If we have addr insns we transforming addressable pseudos to memory if the addr insn can not be + elimnated and memory of addressable pseudos to pseudos otherwise. */ + +/* Add all copies which are uses of bb_insn to temp_bb_insns2. Return TRUE if all bb_insn uses + (skipping moves) are memory address. Collect insns which bb_insn uses are memory in + bb_mem_insns. */ +static int collect_addr_uses (gen_ctx_t gen_ctx, bb_insn_t bb_insn, + VARR (bb_insn_t) * bb_mem_insns) { + int res = TRUE; + + gen_assert (MIR_addr_code_p (bb_insn->insn->code) || move_p (bb_insn->insn)); + for (ssa_edge_t se = bb_insn->insn->ops[0].data; se != NULL; se = se->next_use) { + if (se->use->insn->ops[se->use_op_num].mode == MIR_OP_VAR_MEM) { + gen_assert (move_code_p (se->use->insn->code) && se->use_op_num <= 1); + if (bb_mem_insns != NULL) VARR_PUSH (bb_insn_t, bb_mem_insns, se->use); + continue; + } + if (!move_p (se->use->insn)) { + res = FALSE; + } else if (bitmap_set_bit_p (temp_bitmap2, se->use->index)) { + VARR_PUSH (bb_insn_t, temp_bb_insns2, se->use); + } + } + return res; +} -static int get_ext_params (MIR_insn_code_t code, int *sign_p) { - *sign_p = code == MIR_EXT8 || code == MIR_EXT16 || code == MIR_EXT32; - switch (code) { - case MIR_EXT8: - case MIR_UEXT8: return 8; - case MIR_EXT16: - case MIR_UEXT16: return 16; - case MIR_EXT32: - case MIR_UEXT32: return 32; - default: return 0; +/* Return TRUE if all addr insn (bb_insn) uses (skipping moves) are memory address. + Collect insns which addr uses are memory in bb_mem_insns. */ +static int addr_eliminable_p (gen_ctx_t gen_ctx, bb_insn_t bb_insn, + VARR (bb_insn_t) * bb_mem_insns) { + int res = TRUE; + + bitmap_clear (temp_bitmap2); + VARR_TRUNC (bb_insn_t, temp_bb_insns2, 0); + if (bb_mem_insns != NULL) VARR_TRUNC (bb_insn_t, bb_mem_insns, 0); + if (!collect_addr_uses (gen_ctx, bb_insn, bb_mem_insns)) res = FALSE; + while (VARR_LENGTH (bb_insn_t, temp_bb_insns2) != 0) { + bb_insn_t copy_bb_insn = VARR_POP (bb_insn_t, temp_bb_insns2); + if (!collect_addr_uses (gen_ctx, copy_bb_insn, bb_mem_insns)) res = FALSE; } + return res; } -static void copy_prop (gen_ctx_t gen_ctx) { +// aliasing, loc ??? +static void transform_addrs (gen_ctx_t gen_ctx) { MIR_context_t ctx = gen_ctx->ctx; - MIR_insn_t insn, def_insn; - bb_insn_t bb_insn, next_bb_insn, def; + int op_num, out_p, ssa_rebuild_p = FALSE; + MIR_type_t type; + MIR_insn_code_t move_code; + MIR_reg_t var, reg, addr_reg, new_reg; + MIR_insn_t insn, addr_insn, new_insn; + bb_insn_t bb_insn, next_bb_insn; ssa_edge_t se; - int op_num, out_p, mem_p, w, w2, sign_p, sign2_p; - size_t passed_mem_num; - MIR_reg_t var, reg, new_reg; - insn_var_iterator_t iter; - long deleted_insns_num = 0; + MIR_func_t func = curr_func_item->u.func; - bitmap_clear (temp_bitmap); + gen_assert (addr_insn_p); + bitmap_clear (addr_regs); + VARR_TRUNC (bb_insn_t, temp_bb_insns, 0); for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; - bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) { - if ((insn = bb_insn->insn)->code == MIR_LABEL) continue; - if (insn->code != MIR_PHI) break; - bitmap_set_bit_p (temp_bitmap, insn->ops[0].u.reg); - } + bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) + if (MIR_addr_code_p (bb_insn->insn->code)) { + VARR_PUSH (bb_insn_t, temp_bb_insns, bb_insn); + } else if (move_p (bb_insn->insn)) { + gen_assert (bb_insn->insn->ops[1].data != NULL); + } + if (VARR_LENGTH (bb_insn_t, temp_bb_insns) == 0) + return; /* all addr insns can be unreachable and removed */ + for (size_t i = 0; i < VARR_LENGTH (bb_insn_t, temp_bb_insns); i++) { + bb_insn = VARR_GET (bb_insn_t, temp_bb_insns, i); + insn = bb_insn->insn; + gen_assert (MIR_addr_code_p (insn->code) && insn->ops[0].mode == MIR_OP_VAR + && insn->ops[1].mode == MIR_OP_VAR); + if (!addr_eliminable_p (gen_ctx, bb_insn, NULL)) + bitmap_set_bit_p (addr_regs, insn->ops[1].u.var); + } + addr_insn = NULL; /* to remove warning */ + addr_reg = MIR_NON_VAR; /* to remove warning */ for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; bb_insn = next_bb_insn) { - next_bb_insn = DLIST_NEXT (bb_insn_t, bb_insn); insn = bb_insn->insn; - if (insn->code == MIR_PHI) continue; /* keep conventional SSA */ - FOREACH_INSN_VAR (gen_ctx, iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (out_p || !var_is_reg_p (var)) continue; - for (;;) { - se = insn->ops[op_num].data; - def = se->def; - if (def->bb->index == 0) break; /* arg init or undef insn */ - def_insn = def->insn; - if (se->prev_use != NULL || se->next_use != NULL || !move_p (def_insn) - || bitmap_bit_p (temp_bitmap, def_insn->ops[1].u.reg)) - break; + next_bb_insn = DLIST_NEXT (bb_insn_t, bb_insn); + if (insn->code == MIR_PHI) { + /* we keep conventional SSA -- do nothing when we keep pseudo */ + if (!bitmap_bit_p (addr_regs, insn->ops[0].u.var)) continue; + DEBUG (2, { + fprintf (debug_file, " deleting phi for pseudo transformed into memory "); + print_bb_insn (gen_ctx, insn->data, TRUE); + }); + ssa_delete_insn (gen_ctx, insn); + } else if (insn->code == MIR_USE) { + int change_p = FALSE; + /* we keep conventional SSA -- do nothing */ + for (size_t i = 0; i < insn->nops; i++) { + gen_assert (insn->ops[i].mode == MIR_OP_VAR); + if (!bitmap_bit_p (addr_regs, insn->ops[i].u.var)) continue; + remove_ssa_edge (insn->ops[i].data); + for (size_t j = i; j + 1 < insn->nops; j++) insn->ops[j] = insn->ops[j + 1]; + change_p = TRUE; + i--; + insn->nops--; + } + if (change_p) { DEBUG (2, { - fprintf (debug_file, " Removing copy insn %-5lu", (unsigned long) def->index); - MIR_output_insn (gen_ctx->ctx, debug_file, def_insn, curr_func_item->u.func, TRUE); + fprintf (debug_file, " modifying use to "); + print_bb_insn (gen_ctx, insn->data, TRUE); }); - reg = var2reg (gen_ctx, var); - new_reg = def_insn->ops[1].u.reg; - remove_ssa_edge (gen_ctx, se); - insn->ops[op_num].data = def_insn->ops[1].data; - gen_delete_insn (gen_ctx, def_insn); - deleted_insns_num++; - se = insn->ops[op_num].data; - se->use = bb_insn; - se->use_op_num = op_num; - rename_op_reg (gen_ctx, &insn->ops[op_num], reg, new_reg, insn); } - } - w = get_ext_params (insn->code, &sign_p); - if (w != 0 && insn->ops[1].mode == MIR_OP_REG && var_is_reg_p (insn->ops[1].u.reg)) { - se = insn->ops[1].data; - def_insn = se->def->insn; - w2 = get_ext_params (def_insn->code, &sign2_p); - if (w2 != 0 && sign_p == sign2_p && w2 <= w - && !bitmap_bit_p (temp_bitmap, def_insn->ops[1].u.reg)) { + } else if (!MIR_addr_code_p (insn->code)) { /* change reg to memory */ + MIR_reg_t prev_reg = 0; + for (op_num = 0; op_num < (int) insn->nops; op_num++) { + if (insn->ops[op_num].mode == MIR_OP_VAR) { + var = insn->ops[op_num].u.var; + MIR_insn_op_mode (gen_ctx->ctx, insn, op_num, &out_p); + } else if (insn->ops[op_num].mode == MIR_OP_VAR_MEM) { + var = insn->ops[op_num].u.var_mem.base; + if (var == MIR_NON_VAR) continue; + out_p = FALSE; + } else { + continue; + } + if (var <= MAX_HARD_REG) continue; + reg = var; + if (!bitmap_bit_p (addr_regs, reg)) continue; DEBUG (2, { - fprintf (debug_file, " Change code of insn %lu: before", - (unsigned long) bb_insn->index); - MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, FALSE); + fprintf (debug_file, " "); + print_bb_insn (gen_ctx, bb_insn, TRUE); }); - insn->code = MIR_MOV; + if (reg != prev_reg) { + addr_reg = gen_new_temp_reg (gen_ctx, MIR_T_I64, func); + addr_insn = MIR_new_insn (ctx, MIR_ADDR, _MIR_new_var_op (ctx, addr_reg), + _MIR_new_var_op (ctx, reg)); + gen_add_insn_before (gen_ctx, insn, addr_insn); + prev_reg = reg; + DEBUG (2, { + fprintf (debug_file, " adding before: "); + print_bb_insn (gen_ctx, addr_insn->data, TRUE); + }); + } + type = MIR_reg_type (ctx, reg - MAX_HARD_REG, func); + move_code = get_move_code (type); + new_reg = gen_new_temp_reg (gen_ctx, type, func); + if (out_p) { /* p = ... => addr t2, p (no edge for p); t = ...; mem[t2] = t */ + new_insn = MIR_new_insn (ctx, move_code, + _MIR_new_var_mem_op (ctx, type, 0, addr_reg, MIR_NON_VAR, 0), + _MIR_new_var_op (ctx, new_reg)); + gen_add_insn_after (gen_ctx, insn, new_insn); + gen_assert (insn->ops[op_num].mode == MIR_OP_VAR); + insn->ops[op_num].u.var = new_reg; + while ((se = insn->ops[op_num].data) != NULL) remove_ssa_edge (se); + if (!ssa_rebuild_p) { + add_ssa_edge (gen_ctx, addr_insn->data, 0, new_insn->data, 0); + add_ssa_edge (gen_ctx, bb_insn, op_num, new_insn->data, 1); + } + } else { /* ... = p => addr t2, p (no edge for p); t = mem[t2]; ... = t */ + new_insn = MIR_new_insn (ctx, move_code, _MIR_new_var_op (ctx, new_reg), + _MIR_new_var_mem_op (ctx, type, 0, addr_reg, MIR_NON_VAR, 0)); + gen_add_insn_before (gen_ctx, insn, new_insn); + if (insn->ops[op_num].mode == MIR_OP_VAR) { + insn->ops[op_num].u.var = new_reg; + } else { + gen_assert (insn->ops[op_num].mode == MIR_OP_VAR_MEM + && insn->ops[op_num].u.var_mem.base == reg); + insn->ops[op_num].u.var_mem.base = new_reg; + } + if (insn->ops[op_num].data != NULL) remove_ssa_edge (insn->ops[op_num].data); + if (!ssa_rebuild_p) { + add_ssa_edge (gen_ctx, addr_insn->data, 0, new_insn->data, 1); + add_ssa_edge (gen_ctx, new_insn->data, 0, bb_insn, op_num); + } + } DEBUG (2, { - fprintf (debug_file, " after"); - MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, TRUE); + fprintf (debug_file, " adding %s: ", out_p ? "after" : "before"); + print_bb_insn (gen_ctx, new_insn->data, TRUE); + fprintf (debug_file, " changing to "); + print_bb_insn (gen_ctx, bb_insn, TRUE); }); } + } else if (!bitmap_bit_p (addr_regs, insn->ops[1].u.var)) { + /* addr a, p: change reg mem to reg */ + MIR_UNUSED int res = addr_eliminable_p (gen_ctx, bb_insn, temp_bb_insns); + se = insn->ops[1].data; + gen_assert (res); + while (VARR_LENGTH (bb_insn_t, temp_bb_insns) != 0) { + /* ... = m[a] => ... = p; m[a] = ... => p = ... */ + bb_insn_t use_bb_insn = VARR_POP (bb_insn_t, temp_bb_insns); + MIR_insn_t use_insn = use_bb_insn->insn; + gen_assert (move_code_p (use_insn->code)); + op_num = use_insn->ops[0].mode == MIR_OP_VAR_MEM ? 0 : 1; + ssa_rebuild_p = TRUE; + switch (use_insn->ops[op_num].u.var_mem.type) { + case MIR_T_I8: use_insn->code = MIR_EXT8; break; + case MIR_T_U8: use_insn->code = MIR_UEXT8; break; + case MIR_T_I16: use_insn->code = MIR_EXT16; break; + case MIR_T_U16: use_insn->code = MIR_UEXT16; break; + case MIR_T_I32: use_insn->code = MIR_EXT32; break; + case MIR_T_U32: use_insn->code = MIR_UEXT32; break; + default: break; + } + if (use_insn->ops[op_num].data != NULL) remove_ssa_edge (use_insn->ops[op_num].data); + use_insn->ops[op_num].mode = MIR_OP_VAR; + use_insn->ops[op_num].u.var = insn->ops[1].u.var; + if (!ssa_rebuild_p) add_ssa_edge (gen_ctx, se->def, se->def_op_num, use_bb_insn, op_num); + } + DEBUG (2, { + fprintf (debug_file, " deleting "); + print_bb_insn (gen_ctx, insn->data, TRUE); + }); + ssa_delete_insn (gen_ctx, insn); } } - DEBUG (1, { fprintf (debug_file, "%5ld deleted copy insns\n", deleted_insns_num); }); } /* New Page */ -/* Removing redundant insns through GVN. */ - -typedef struct expr { - MIR_insn_t insn; /* opcode and input operands are the expr keys */ - uint32_t num; /* the expression number (0, 1 ...) */ - MIR_reg_t temp_reg; /* 0 initially and reg used to remove redundant expr */ -} * expr_t; +/* Copy propagation */ -DEF_VARR (expr_t); -DEF_HTAB (expr_t); +static int64_t gen_int_log2 (int64_t i) { + int64_t n; -struct gvn_ctx { - VARR (expr_t) * exprs; /* the expr number -> expression */ + if (i <= 0) return -1; + for (n = 0; (i & 1) == 0; n++, i >>= 1) + ; + return i == 1 ? n : -1; +} + +static int power2_int_op (ssa_edge_t se, MIR_op_t **op_ref) { + MIR_op_t *op; + + *op_ref = NULL; + if (se->def->insn->code != MIR_MOV) return -1; + *op_ref = op = &se->def->insn->ops[1]; + if (op->mode != MIR_OP_INT && op->mode != MIR_OP_UINT) return -1; + return (int) gen_int_log2 (op->u.i); +} + +static MIR_insn_t transform_mul_div (gen_ctx_t gen_ctx, MIR_insn_t insn) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_insn_t new_insns[7]; + MIR_op_t temp[6], *op_ref; + MIR_func_t func = curr_func_item->u.func; + MIR_insn_code_t new_code; + int sh; + ssa_edge_t se; + int n; + + switch (insn->code) { + case MIR_MUL: new_code = MIR_LSH; break; + case MIR_MULS: new_code = MIR_LSHS; break; + case MIR_UDIV: new_code = MIR_URSH; break; + case MIR_UDIVS: new_code = MIR_URSHS; break; + case MIR_DIV: new_code = MIR_RSH; break; + case MIR_DIVS: new_code = MIR_RSHS; break; + default: return insn; + } + sh = power2_int_op (insn->ops[2].data, &op_ref); + if (sh < 0 && (insn->code == MIR_MUL || insn->code == MIR_MULS) + && (sh = power2_int_op (insn->ops[1].data, &op_ref)) >= 0) { + temp[0] = insn->ops[1]; + insn->ops[1] = insn->ops[2]; + insn->ops[2] = temp[0]; + ((ssa_edge_t) insn->ops[1].data)->use_op_num = 1; + ((ssa_edge_t) insn->ops[2].data)->use_op_num = 2; + } + if (sh < 0) return insn; + if (sh == 0) { + new_insns[0] = MIR_new_insn (ctx, MIR_MOV, insn->ops[0], insn->ops[1]); + new_insns[0]->ops[1].data = NULL; + gen_add_insn_before (gen_ctx, insn, new_insns[0]); + redirect_def (gen_ctx, insn, new_insns[0], FALSE); + se = insn->ops[1].data; + add_ssa_edge (gen_ctx, se->def, se->def_op_num, new_insns[0]->data, 1); + n = 1; + } else if (insn->code != MIR_DIV && insn->code != MIR_DIVS) { + temp[0] = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + new_insns[0] = MIR_new_insn (ctx, MIR_MOV, temp[0], MIR_new_int_op (ctx, sh)); + gen_add_insn_before (gen_ctx, insn, new_insns[0]); + new_insns[1] = MIR_new_insn (ctx, new_code, insn->ops[0], insn->ops[1], temp[0]); + new_insns[1]->ops[1].data = NULL; + gen_add_insn_before (gen_ctx, insn, new_insns[1]); + redirect_def (gen_ctx, insn, new_insns[1], FALSE); + se = insn->ops[1].data; + add_ssa_edge (gen_ctx, se->def, se->def_op_num, new_insns[1]->data, 1); + add_ssa_edge (gen_ctx, new_insns[0]->data, 0, new_insns[1]->data, 2); + n = 2; + } else { + for (int i = 0; i < 6; i++) + temp[i] = _MIR_new_var_op (ctx, gen_new_temp_reg (gen_ctx, MIR_T_I64, func)); + if (insn->code == MIR_DIV) { + new_insns[0] = MIR_new_insn (ctx, MIR_MOV, temp[0], MIR_new_int_op (ctx, 63)); + new_insns[1] = MIR_new_insn (ctx, MIR_RSH, temp[1], insn->ops[1], temp[0]); + new_insns[2] = MIR_new_insn (ctx, MIR_MOV, temp[2], MIR_new_int_op (ctx, op_ref->u.i - 1)); + new_insns[3] = MIR_new_insn (ctx, MIR_AND, temp[3], temp[1], temp[2]); + new_insns[4] = MIR_new_insn (ctx, MIR_ADD, temp[4], temp[3], insn->ops[1]); + } else { + new_insns[0] = MIR_new_insn (ctx, MIR_MOV, temp[0], MIR_new_int_op (ctx, 31)); + new_insns[1] = MIR_new_insn (ctx, MIR_RSHS, temp[1], insn->ops[1], temp[0]); + new_insns[2] = MIR_new_insn (ctx, MIR_MOV, temp[2], MIR_new_int_op (ctx, op_ref->u.i - 1)); + new_insns[3] = MIR_new_insn (ctx, MIR_ANDS, temp[3], temp[1], temp[2]); + new_insns[4] = MIR_new_insn (ctx, MIR_ADDS, temp[4], temp[3], insn->ops[1]); + } + new_insns[1]->ops[1].data = NULL; + new_insns[4]->ops[2].data = NULL; + new_insns[5] = MIR_new_insn (ctx, MIR_MOV, temp[5], MIR_new_int_op (ctx, sh)); + new_insns[6] = MIR_new_insn (ctx, new_code, insn->ops[0], temp[4], temp[5]); + for (int i = 0; i < 7; i++) gen_add_insn_before (gen_ctx, insn, new_insns[i]); + add_ssa_edge (gen_ctx, new_insns[0]->data, 0, new_insns[1]->data, 2); + add_ssa_edge (gen_ctx, new_insns[1]->data, 0, new_insns[3]->data, 1); + add_ssa_edge (gen_ctx, new_insns[2]->data, 0, new_insns[3]->data, 2); + add_ssa_edge (gen_ctx, new_insns[3]->data, 0, new_insns[4]->data, 1); + add_ssa_edge (gen_ctx, new_insns[4]->data, 0, new_insns[6]->data, 1); + add_ssa_edge (gen_ctx, new_insns[5]->data, 0, new_insns[6]->data, 2); + se = insn->ops[1].data; + add_ssa_edge (gen_ctx, se->def, se->def_op_num, new_insns[1]->data, 1); + add_ssa_edge (gen_ctx, se->def, se->def_op_num, new_insns[4]->data, 2); + redirect_def (gen_ctx, insn, new_insns[6], FALSE); + n = 7; + } + DEBUG (2, { + for (int i = 0; i < n; i++) { + fprintf (debug_file, i == 0 ? " adding " : " and "); + print_bb_insn (gen_ctx, new_insns[i]->data, TRUE); + } + fprintf (debug_file, " and deleting "); + print_bb_insn (gen_ctx, insn->data, TRUE); + }); + ssa_delete_insn (gen_ctx, insn); + return new_insns[n - 1]; +} + +static int get_ext_params (MIR_insn_code_t code, int *sign_p) { + *sign_p = code == MIR_EXT8 || code == MIR_EXT16 || code == MIR_EXT32; + switch (code) { + case MIR_EXT8: + case MIR_UEXT8: return 8; + case MIR_EXT16: + case MIR_UEXT16: return 16; + case MIR_EXT32: + case MIR_UEXT32: return 32; + default: return 0; + } +} + +static int cmp_res64_p (MIR_insn_code_t cmp_code) { + switch (cmp_code) { +#define REP_SEP : +#define CASE_EL(e) case MIR_##e + REP4 (CASE_EL, EQ, FEQ, DEQ, LDEQ) + : REP4 (CASE_EL, NE, FNE, DNE, LDNE) + : REP5 (CASE_EL, LT, ULT, FLT, DLT, LDLT) + : REP5 (CASE_EL, LE, ULE, FLE, DLE, LDLE) + : REP5 (CASE_EL, GT, UGT, FGT, DGT, LDGT) + : REP5 (CASE_EL, GE, UGE, FGE, DGE, LDGE) : return TRUE; +#undef REP_SEP + default: return FALSE; + } +} + +static void copy_prop (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_func_t func = curr_func_item->u.func; + MIR_insn_t insn, def_insn, new_insn, mov_insn; + MIR_op_t temp_op; + bb_insn_t bb_insn, next_bb_insn, def; + ssa_edge_t se, se2; + int op_num, w, w2, sign_p, sign2_p; + MIR_reg_t var, reg, new_reg, src_reg; + insn_var_iterator_t iter; + + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; bb_insn = next_bb_insn) { + next_bb_insn = DLIST_NEXT (bb_insn_t, bb_insn); + insn = bb_insn->insn; + if (MIR_addr_code_p (insn->code)) { + continue; /* no input reg propagation */ + } + FOREACH_IN_INSN_VAR (gen_ctx, iter, insn, var, op_num) { + if (var <= MAX_HARD_REG) continue; + reg = var; + for (int n = 0; n < 30; n++) { /* unreachable code can create loops in copies */ + se = insn->ops[op_num].data; + def = se->def; + if (def->bb->index == 0) break; /* arg init or undef insn */ + def_insn = def->insn; + if (!move_p (def_insn) || def_insn->ops[0].u.var == def_insn->ops[1].u.var) break; + src_reg = def_insn->ops[1].u.var; + gen_assert (src_reg > MAX_HARD_REG); + if (MIR_reg_hard_reg_name (ctx, def_insn->ops[0].u.var - MAX_HARD_REG, func) + != MIR_reg_hard_reg_name (ctx, src_reg - MAX_HARD_REG, func)) + break; + DEBUG (2, { + fprintf (debug_file, " Propagate from copy insn "); + print_bb_insn (gen_ctx, def, FALSE); + }); + new_reg = def_insn->ops[1].u.var; + gen_assert (reg > MAX_HARD_REG && new_reg > MAX_HARD_REG); + remove_ssa_edge (se); + se = def_insn->ops[1].data; + add_ssa_edge (gen_ctx, se->def, se->def_op_num, bb_insn, op_num); + rename_op_reg (gen_ctx, &insn->ops[op_num], reg, new_reg, insn, TRUE); + reg = new_reg; + } + } + if (move_p (insn) && insn->ops[0].data != NULL && (se = insn->ops[1].data) != NULL + && se->def == DLIST_PREV (bb_insn_t, bb_insn) + && (se = se->def->insn->ops[se->def_op_num].data) != NULL && se->next_use != NULL + && se->next_use->next_use == NULL + && (se->use == DLIST_NEXT (bb_insn_t, bb_insn) + || se->next_use->use == DLIST_NEXT (bb_insn_t, bb_insn))) { + /* a = ...; non-dead insn: b = a; ... = a & only two uses of a => b = ...; ... = b */ + MIR_op_t *def_op_ref = &se->def->insn->ops[se->def_op_num]; + remove_ssa_edge (insn->ops[1].data); + se = def_op_ref->data; + gen_assert (se != NULL && se->next_use == NULL + && se->use == DLIST_NEXT (bb_insn_t, bb_insn)); + def_op_ref->u.var = insn->ops[0].u.var; + MIR_op_t *use_op_ref = &se->use->insn->ops[se->use_op_num]; + gen_assert (use_op_ref->mode == MIR_OP_VAR || use_op_ref->mode == MIR_OP_VAR_MEM); + if (use_op_ref->mode == MIR_OP_VAR) + use_op_ref->u.var = def_op_ref->u.var; + else + use_op_ref->u.var_mem.base = def_op_ref->u.var; + change_ssa_edge_list_def (insn->ops[0].data, se->def, se->def_op_num, MIR_NON_VAR, + MIR_NON_VAR); + se->next_use = insn->ops[0].data; + se->next_use->prev_use = se; + insn->ops[0].data = insn->ops[1].data = NULL; + DEBUG (2, { + fprintf (debug_file, " Remove move %-5lu", (unsigned long) bb_insn->index); + print_bb_insn (gen_ctx, bb_insn, FALSE); + }); + gen_delete_insn (gen_ctx, insn); + continue; + } + insn = transform_mul_div (gen_ctx, insn); + bb_insn = insn->data; + w = get_ext_params (insn->code, &sign_p); + if (w == 0 || insn->ops[1].mode != MIR_OP_VAR) continue; + se = insn->ops[1].data; + def_insn = se->def->insn; + if (cmp_res64_p (def_insn->code)) { + DEBUG (2, { + fprintf (debug_file, " Change code of insn %lu ", (unsigned long) bb_insn->index); + MIR_output_insn (ctx, debug_file, insn, func, FALSE); + fprintf (debug_file, " to move\n"); + }); + insn->code = MIR_MOV; + next_bb_insn = bb_insn; /* process the new move */ + continue; + } + w2 = get_ext_params (def_insn->code, &sign2_p); + if (w2 != 0 && w <= w2) { + /* [u]ext2 b,a; ...[u]ext1 c,b -> [u]ext1 c,a when <= : */ + DEBUG (2, { + fprintf (debug_file, " Change code of insn %lu: before", + (unsigned long) bb_insn->index); + MIR_output_insn (ctx, debug_file, insn, func, FALSE); + }); + insn->ops[1].u.var = def_insn->ops[1].u.var; + remove_ssa_edge (se); + se = def_insn->ops[1].data; + add_ssa_edge (gen_ctx, se->def, se->def_op_num, bb_insn, 1); + DEBUG (2, { + fprintf (debug_file, " after "); + print_bb_insn (gen_ctx, bb_insn, FALSE); + }); + next_bb_insn = bb_insn; /* process ext again */ + continue; + } else if (w2 != 0 && w2 < w && (sign_p || !sign2_p)) { /* exclude ext, uext pair */ + /* [u]ext1 b,a; .. [u]ext c,b -> .. [u]ext1 c,a */ + DEBUG (2, { + fprintf (debug_file, " Change code of insn %lu: before", + (unsigned long) bb_insn->index); + MIR_output_insn (ctx, debug_file, insn, func, FALSE); + }); + insn->code = def_insn->code; + insn->ops[1].u.var = def_insn->ops[1].u.var; + remove_ssa_edge (se); + se = def_insn->ops[1].data; + add_ssa_edge (gen_ctx, se->def, se->def_op_num, bb_insn, 1); + DEBUG (2, { + fprintf (debug_file, " after "); + print_bb_insn (gen_ctx, bb_insn, FALSE); + }); + next_bb_insn = bb_insn; /* process ext again */ + continue; + } + if (!sign_p && (def_insn->code == MIR_AND || def_insn->code == MIR_ANDS)) { + if ((se2 = def_insn->ops[1].data) != NULL && (mov_insn = se2->def->insn)->code == MIR_MOV + && (mov_insn->ops[1].mode == MIR_OP_INT || mov_insn->ops[1].mode == MIR_OP_UINT)) + SWAP (def_insn->ops[1], def_insn->ops[2], temp_op); + if ((se2 = def_insn->ops[2].data) == NULL || (mov_insn = se2->def->insn)->code != MIR_MOV + || (mov_insn->ops[1].mode != MIR_OP_INT && mov_insn->ops[1].mode != MIR_OP_UINT)) + continue; + uint64_t c1 = mov_insn->ops[1].u.u; + uint64_t c2 = w == 8 ? 0xff : w == 16 ? 0xffff : 0xffffffff; + /* and r1,r2,c1; ... uext r, r1 => and r1,r2,c1; ... mov t, c1 & c2; and r, r2, t */ + DEBUG (2, { + fprintf (debug_file, " Change code of insn %lu ", (unsigned long) bb_insn->index); + MIR_output_insn (ctx, debug_file, insn, func, FALSE); + }); + new_reg = gen_new_temp_reg (gen_ctx, MIR_T_I64, func); + mov_insn = MIR_new_insn (ctx, MIR_MOV, _MIR_new_var_op (ctx, new_reg), + MIR_new_int_op (ctx, c1 & c2)); + gen_add_insn_before (gen_ctx, insn, mov_insn); + new_insn = MIR_new_insn (ctx, MIR_AND, insn->ops[0], /* include ssa def list */ + _MIR_new_var_op (ctx, def_insn->ops[1].u.var), + _MIR_new_var_op (ctx, new_reg)); + gen_add_insn_before (gen_ctx, insn, new_insn); + remove_ssa_edge (se); /* r1 */ + add_ssa_edge (gen_ctx, mov_insn->data, 0, new_insn->data, 2); /* t */ + se = def_insn->ops[1].data; + add_ssa_edge (gen_ctx, se->def, se->def_op_num, new_insn->data, 1); /* r2 */ + insn->ops[0].data = NULL; + change_ssa_edge_list_def (new_insn->ops[0].data, new_insn->data, 0, MIR_NON_VAR, + MIR_NON_VAR); /* r */ + ssa_delete_insn (gen_ctx, insn); + DEBUG (2, { + fprintf (debug_file, " on "); + MIR_output_insn (ctx, debug_file, mov_insn, func, FALSE); + fprintf (debug_file, " and "); + MIR_output_insn (ctx, debug_file, new_insn, func, TRUE); + }); + } + } + } +} + +/* New Page */ + +/* Removing redundant insns through GVN. */ + +typedef struct expr { + MIR_insn_t insn; + uint32_t num; /* the expression number (0, 1 ...) */ + MIR_reg_t temp_reg; /* 0 initially and reg used to remove redundant expr */ +} *expr_t; + +DEF_VARR (expr_t); +DEF_HTAB (expr_t); + +typedef struct mem_expr { + MIR_insn_t insn; /* load or store */ + uint32_t mem_num; /* the memory expression number (0, 1 ...) */ + MIR_reg_t temp_reg; /* 0 initially and reg used to remove redundant load/store */ + struct mem_expr *next; +} *mem_expr_t; + +DEF_VARR (mem_expr_t); +DEF_HTAB (mem_expr_t); + +struct insn_nop_pair { + bb_insn_t bb_insn; + size_t nop; +}; +typedef struct insn_nop_pair insn_nop_pair_t; + +DEF_VARR (insn_nop_pair_t); + +struct gvn_ctx { + MIR_insn_t temp_mem_insn; + VARR (expr_t) * exprs; /* the expr number -> expression */ + VARR (mem_expr_t) * mem_exprs; HTAB (expr_t) * expr_tab; /* keys: insn code and input operands */ + /* keys: gvn val of memory address -> list of mem exprs: last added is the first */ + HTAB (mem_expr_t) * mem_expr_tab; + VARR (insn_nop_pair_t) * insn_nop_pairs; }; +#define temp_mem_insn gen_ctx->gvn_ctx->temp_mem_insn #define exprs gen_ctx->gvn_ctx->exprs +#define mem_exprs gen_ctx->gvn_ctx->mem_exprs #define expr_tab gen_ctx->gvn_ctx->expr_tab +#define mem_expr_tab gen_ctx->gvn_ctx->mem_expr_tab +#define insn_nop_pairs gen_ctx->gvn_ctx->insn_nop_pairs static void dom_con_func_0 (bb_t bb) { bitmap_clear (bb->dom_in); } @@ -2168,6 +3426,137 @@ static void calculate_dominators (gen_ctx_t gen_ctx) { solve_dataflow (gen_ctx, TRUE, dom_con_func_0, dom_con_func_n, dom_trans_func); } +#define mem_av_in in +#define mem_av_out out + +static int may_alias_p (MIR_alias_t alias1, MIR_alias_t alias2, MIR_alias_t nonalias1, + MIR_alias_t nonalias2) { + return (alias1 == 0 || alias2 == 0 || alias1 == alias2) + && (nonalias1 == 0 || nonalias2 == 0 || nonalias1 != nonalias2); +} + +static int may_mem_alias_p (const MIR_op_t *mem1, const MIR_op_t *mem2) { + gen_assert (mem1->mode == MIR_OP_VAR_MEM && mem2->mode == MIR_OP_VAR_MEM); + return may_alias_p (mem1->u.var_mem.alias, mem2->u.var_mem.alias, mem1->u.var_mem.nonalias, + mem2->u.var_mem.nonalias); +} + +static void mem_av_con_func_0 (bb_t bb) { bitmap_clear (bb->mem_av_in); } + +static int mem_av_con_func_n (gen_ctx_t gen_ctx, bb_t bb) { + edge_t e, head; + bitmap_t prev_mem_av_in = temp_bitmap; + + bitmap_copy (prev_mem_av_in, bb->mem_av_in); + head = DLIST_HEAD (in_edge_t, bb->in_edges); + bitmap_copy (bb->mem_av_in, head->src->mem_av_out); + for (e = DLIST_NEXT (in_edge_t, head); e != NULL; e = DLIST_NEXT (in_edge_t, e)) + bitmap_and (bb->mem_av_in, bb->mem_av_in, e->src->mem_av_out); /* mem_av_in &= mem_av_out */ + return !bitmap_equal_p (bb->mem_av_in, prev_mem_av_in); +} + +static int mem_av_trans_func (gen_ctx_t gen_ctx, bb_t bb) { + int alias_p; + size_t nel, nel2; + MIR_insn_t insn, mem_insn; + MIR_op_t *mem_ref; + bitmap_iterator_t bi, bi2; + bitmap_t prev_mem_av_out = temp_bitmap; + + bitmap_copy (prev_mem_av_out, bb->mem_av_out); + bitmap_copy (bb->mem_av_out, bb->gen); + if (!bb->call_p) { + FOREACH_BITMAP_BIT (bi, bb->mem_av_in, nel) { + alias_p = FALSE; + insn = VARR_GET (mem_expr_t, mem_exprs, nel)->insn; + mem_ref = insn->ops[0].mode == MIR_OP_VAR_MEM ? &insn->ops[0] : &insn->ops[1]; + FOREACH_BITMAP_BIT (bi2, bb->gen, nel2) { /* consider only stores */ + mem_insn = VARR_GET (mem_expr_t, mem_exprs, nel2)->insn; + if (mem_insn->ops[0].mode == MIR_OP_VAR_MEM + && may_mem_alias_p (mem_ref, &mem_insn->ops[0])) { + alias_p = TRUE; + break; + } + } + if (!alias_p) bitmap_set_bit_p (bb->mem_av_out, nel); + } + } + return !bitmap_equal_p (bb->mem_av_out, prev_mem_av_out); +} + +static void update_mem_availability (gen_ctx_t gen_ctx, bitmap_t mem_av, bb_insn_t mem_bb_insn) { + size_t nel; + bitmap_iterator_t bi; + MIR_insn_t mem_insn; + MIR_op_t *mem_ref = &mem_bb_insn->insn->ops[0]; + int ld_p; + + gen_assert (move_code_p (mem_bb_insn->insn->code)); + if ((ld_p = mem_ref->mode != MIR_OP_VAR_MEM)) mem_ref = &mem_bb_insn->insn->ops[1]; + gen_assert (mem_ref->mode == MIR_OP_VAR_MEM); + FOREACH_BITMAP_BIT (bi, mem_av, nel) { + mem_insn = VARR_GET (mem_expr_t, mem_exprs, nel)->insn; + if (!ld_p + && may_mem_alias_p (&mem_insn->ops[mem_insn->ops[0].mode == MIR_OP_VAR_MEM ? 0 : 1], + mem_ref)) + bitmap_clear_bit_p (mem_av, nel); + } + bitmap_set_bit_p (mem_av, mem_bb_insn->mem_index); +} + +static void calculate_memory_availability (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; + + DEBUG (2, { fprintf (debug_file, "Calculate memory availability:\n"); }); + gen_assert (VARR_LENGTH (mem_expr_t, mem_exprs) == 0); + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + DEBUG (2, { fprintf (debug_file, " BB%lu:\n", (unsigned long) bb->index); }); + bitmap_clear (bb->gen); + for (bb_insn_t bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; + bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) { + MIR_insn_t insn = bb_insn->insn; + mem_expr_t e; + size_t mem_num; + + if (MIR_call_code_p (insn->code)) { /* ??? improving */ + bitmap_clear (bb->gen); + continue; + } + if (!move_code_p (insn->code)) continue; + if (insn->ops[0].mode != MIR_OP_VAR_MEM && insn->ops[1].mode != MIR_OP_VAR_MEM) continue; + mem_num = VARR_LENGTH (mem_expr_t, mem_exprs); + bb_insn->mem_index = (uint32_t) mem_num; + DEBUG (2, { + fprintf (debug_file, " Adding mem insn %-5llu:", (unsigned long long) mem_num); + MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, TRUE); + }); + e = gen_malloc (gen_ctx, sizeof (struct mem_expr)); + e->insn = bb_insn->insn; + e->temp_reg = MIR_NON_VAR; + e->mem_num = (uint32_t) mem_num; + e->next = NULL; + VARR_PUSH (mem_expr_t, mem_exprs, e); + if (insn->ops[0].mode == MIR_OP_VAR_MEM || insn->ops[1].mode == MIR_OP_VAR_MEM) + update_mem_availability (gen_ctx, bb->gen, bb_insn); + } + DEBUG (2, { output_bitmap (gen_ctx, " Mem availabilty gen:", bb->gen, FALSE, NULL); }); + } + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) + bitmap_set_bit_range_p (bb->mem_av_out, 0, VARR_LENGTH (mem_expr_t, mem_exprs)); + solve_dataflow (gen_ctx, TRUE, mem_av_con_func_0, mem_av_con_func_n, mem_av_trans_func); + DEBUG (2, { + fprintf (debug_file, "BB mem availability in/out:\n"); + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + fprintf (debug_file, " BB%lu:\n", (unsigned long) bb->index); + output_bitmap (gen_ctx, " mem av in:", bb->mem_av_in, FALSE, NULL); + output_bitmap (gen_ctx, " mem av out:", bb->mem_av_out, FALSE, NULL); + } + }); +} + +#undef mem_av_in +#undef mem_av_out + static int op_eq (gen_ctx_t gen_ctx, MIR_op_t op1, MIR_op_t op2) { return MIR_op_eq_p (gen_ctx->ctx, op1, op2); } @@ -2178,6 +3567,18 @@ static int multi_out_insn_p (MIR_insn_t insn) { return insn->ops[0].u.ref->u.proto->nres > 1; } +static MIR_type_t canonic_mem_type (MIR_type_t type) { + switch (type) { + case MIR_T_U64: return MIR_T_I64; +#ifdef MIR_PTR32 + case MIR_T_P: return MIR_T_I32; +#else + case MIR_T_P: return MIR_T_I64; +#endif + default: return type; + } +} + static int expr_eq (expr_t e1, expr_t e2, void *arg) { gen_ctx_t gen_ctx = arg; MIR_context_t ctx = gen_ctx->ctx; @@ -2190,17 +3591,22 @@ static int expr_eq (expr_t e1, expr_t e2, void *arg) { nops = MIR_insn_nops (gen_ctx->ctx, insn1); for (i = 0; i < nops; i++) { MIR_insn_op_mode (ctx, insn1, i, &out_p); - if (out_p) continue; - if ((insn1->ops[i].mode != MIR_OP_REG || insn2->ops[i].mode != MIR_OP_REG) + if (out_p && insn1->ops[i].mode != MIR_OP_VAR_MEM) continue; + if ((insn1->ops[i].mode != MIR_OP_VAR || insn2->ops[i].mode != MIR_OP_VAR) + && (insn1->ops[i].mode != MIR_OP_VAR_MEM || insn2->ops[i].mode != MIR_OP_VAR_MEM) && !op_eq (gen_ctx, insn1->ops[i], insn2->ops[i])) return FALSE; ssa_edge1 = insn1->ops[i].data; ssa_edge2 = insn2->ops[i].data; if (ssa_edge1 != NULL && ssa_edge2 != NULL - && (ssa_edge1->def->gvn_val != ssa_edge2->def->gvn_val - /* we can not be sure what defition we use in multi-output insn: */ - || multi_out_insn_p (ssa_edge1->def->insn) - || multi_out_insn_p (ssa_edge2->def->insn))) + && (ssa_edge1->def->gvn_val_const_p != ssa_edge2->def->gvn_val_const_p + || ssa_edge1->def->gvn_val != ssa_edge2->def->gvn_val + /* we can not be sure what definition we use in multi-output insn: */ + || multi_out_insn_p (ssa_edge1->def->insn) || multi_out_insn_p (ssa_edge2->def->insn))) + return FALSE; + if (insn1->ops[i].mode == MIR_OP_VAR_MEM && insn2->ops[i].mode == MIR_OP_VAR_MEM + && canonic_mem_type (insn1->ops[i].u.var_mem.type) + != canonic_mem_type (insn2->ops[i].u.var_mem.type)) return FALSE; } return TRUE; @@ -2212,18 +3618,26 @@ static htab_hash_t expr_hash (expr_t e, void *arg) { size_t i, nops; int out_p; ssa_edge_t ssa_edge; - htab_hash_t h = mir_hash_init (0x42); + htab_hash_t h = (htab_hash_t) mir_hash_init (0x42); - h = mir_hash_step (h, (uint64_t) e->insn->code); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) e->insn->code); nops = MIR_insn_nops (ctx, e->insn); for (i = 0; i < nops; i++) { MIR_insn_op_mode (ctx, e->insn, i, &out_p); - if (out_p) continue; - if (e->insn->ops[i].mode != MIR_OP_REG) h = MIR_op_hash_step (ctx, h, e->insn->ops[i]); - if ((ssa_edge = e->insn->ops[i].data) != NULL) - h = mir_hash_step (h, (uint64_t) ssa_edge->def->gvn_val); + if (out_p && e->insn->ops[i].mode != MIR_OP_VAR_MEM) continue; + if (e->insn->ops[i].mode != MIR_OP_VAR && e->insn->ops[i].mode != MIR_OP_VAR_MEM) + h = MIR_op_hash_step (ctx, h, e->insn->ops[i]); + if ((ssa_edge = e->insn->ops[i].data) != NULL) { + h = (htab_hash_t) mir_hash_step (h, (uint64_t) ssa_edge->def->gvn_val_const_p); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) ssa_edge->def->gvn_val); + if (e->insn->ops[i].mode == MIR_OP_VAR_MEM) { + gen_assert (e->insn->ops[i].u.var_mem.disp == 0); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) canonic_mem_type ( + e->insn->ops[i].u.var_mem.type)); + } + } } - return mir_hash_finish (h); + return (htab_hash_t) mir_hash_finish (h); } static int find_expr (gen_ctx_t gen_ctx, MIR_insn_t insn, expr_t *e) { @@ -2240,46 +3654,127 @@ static void insert_expr (gen_ctx_t gen_ctx, expr_t e) { HTAB_DO (expr_t, expr_tab, e, HTAB_INSERT, e); } -static expr_t add_expr (gen_ctx_t gen_ctx, MIR_insn_t insn) { +static void replace_expr (gen_ctx_t gen_ctx, expr_t e) { + expr_t MIR_UNUSED e2; + + gen_assert (find_expr (gen_ctx, e->insn, &e2)); + HTAB_DO (expr_t, expr_tab, e, HTAB_REPLACE, e); +} + +static expr_t add_expr (gen_ctx_t gen_ctx, MIR_insn_t insn, int replace_p) { expr_t e = gen_malloc (gen_ctx, sizeof (struct expr)); - gen_assert (!MIR_call_code_p (insn->code) && insn->code != MIR_RET); + /* can not be calls, rets, stores */ + gen_assert (!MIR_call_code_p (insn->code) && insn->code != MIR_RET && insn->code != MIR_JRET + && (!move_code_p (insn->code) || insn->ops[0].mode != MIR_OP_VAR_MEM)); e->insn = insn; e->num = ((bb_insn_t) insn->data)->index; - e->temp_reg = 0; + e->temp_reg = MIR_NON_VAR; VARR_PUSH (expr_t, exprs, e); - insert_expr (gen_ctx, e); + if (replace_p) + replace_expr (gen_ctx, e); + else + insert_expr (gen_ctx, e); + return e; +} + +static int mem_expr_eq (mem_expr_t e1, mem_expr_t e2, void *arg MIR_UNUSED) { + MIR_insn_t st1 = e1->insn, st2 = e2->insn; + MIR_op_t *op_ref1 = &st1->ops[0], *op_ref2 = &st2->ops[0]; + ssa_edge_t ssa_edge1, ssa_edge2; + + gen_assert (move_code_p (st1->code) && move_code_p (st2->code)); + if (op_ref1->mode != MIR_OP_VAR_MEM) op_ref1 = &st1->ops[1]; + if (op_ref2->mode != MIR_OP_VAR_MEM) op_ref2 = &st2->ops[1]; + gen_assert (op_ref1->mode == MIR_OP_VAR_MEM && op_ref2->mode == MIR_OP_VAR_MEM); + ssa_edge1 = op_ref1->data; + ssa_edge2 = op_ref2->data; + return (ssa_edge1 != NULL && ssa_edge2 != NULL + && ssa_edge1->def->gvn_val_const_p == ssa_edge2->def->gvn_val_const_p + && ssa_edge1->def->gvn_val == ssa_edge2->def->gvn_val + && canonic_mem_type (op_ref1->u.var_mem.type) + == canonic_mem_type (op_ref2->u.var_mem.type) + && op_ref1->u.var_mem.alias == op_ref2->u.var_mem.alias + && op_ref1->u.var_mem.nonalias == op_ref2->u.var_mem.nonalias); +} + +static htab_hash_t mem_expr_hash (mem_expr_t e, void *arg MIR_UNUSED) { + MIR_insn_t st = e->insn; + MIR_op_t *op_ref; + ssa_edge_t ssa_edge; + htab_hash_t h = (htab_hash_t) mir_hash_init (0x23); + + gen_assert (move_code_p (st->code)); + op_ref = st->ops[0].mode == MIR_OP_VAR_MEM ? &st->ops[0] : &st->ops[1]; + gen_assert (op_ref->mode == MIR_OP_VAR_MEM); + if ((ssa_edge = op_ref->data) != NULL) { + h = (htab_hash_t) mir_hash_step (h, (uint64_t) ssa_edge->def->gvn_val_const_p); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) ssa_edge->def->gvn_val); + } + h = (htab_hash_t) mir_hash_step (h, (uint64_t) canonic_mem_type (op_ref->u.var_mem.type)); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) op_ref->u.var_mem.alias); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) op_ref->u.var_mem.nonalias); + return (htab_hash_t) mir_hash_finish (h); +} + +static mem_expr_t find_mem_expr (gen_ctx_t gen_ctx, MIR_insn_t mem_insn) { + mem_expr_t tab_e, e; + + gen_assert ( + move_code_p (mem_insn->code) + && (mem_insn->ops[0].mode == MIR_OP_VAR_MEM || mem_insn->ops[1].mode == MIR_OP_VAR_MEM)); + e = VARR_GET (mem_expr_t, mem_exprs, ((bb_insn_t) mem_insn->data)->mem_index); + if (HTAB_DO (mem_expr_t, mem_expr_tab, e, HTAB_FIND, tab_e)) return tab_e; + return NULL; +} + +static mem_expr_t add_mem_insn (gen_ctx_t gen_ctx, MIR_insn_t mem_insn) { + bb_insn_t bb_insn = mem_insn->data; + mem_expr_t tab_e, e; + + gen_assert ( + move_code_p (mem_insn->code) + && (mem_insn->ops[0].mode == MIR_OP_VAR_MEM || mem_insn->ops[1].mode == MIR_OP_VAR_MEM)); + e = VARR_GET (mem_expr_t, mem_exprs, bb_insn->mem_index); + e->next = NULL; + if (HTAB_DO (mem_expr_t, mem_expr_tab, e, HTAB_FIND, tab_e)) e->next = tab_e; + HTAB_DO (mem_expr_t, mem_expr_tab, e, HTAB_REPLACE, tab_e); return e; } -static MIR_reg_t get_expr_temp_reg (gen_ctx_t gen_ctx, expr_t e) { +static MIR_type_t mode2type (MIR_op_mode_t mode) { + return (mode == MIR_OP_FLOAT ? MIR_T_F + : mode == MIR_OP_DOUBLE ? MIR_T_D + : mode == MIR_OP_LDOUBLE ? MIR_T_LD + : MIR_T_I64); +} + +static MIR_op_mode_t type2mode (MIR_type_t type) { + return (type == MIR_T_F ? MIR_OP_FLOAT + : type == MIR_T_D ? MIR_OP_DOUBLE + : type == MIR_T_LD ? MIR_OP_LDOUBLE + : MIR_OP_INT); +} + +static MIR_reg_t get_expr_temp_reg (gen_ctx_t gen_ctx, MIR_insn_t insn, MIR_reg_t *temp_reg) { int out_p; MIR_op_mode_t mode; - if (e->temp_reg == 0) { - mode = MIR_insn_op_mode (gen_ctx->ctx, e->insn, 0, &out_p); - e->temp_reg = gen_new_temp_reg (gen_ctx, - mode == MIR_OP_FLOAT ? MIR_T_F - : mode == MIR_OP_DOUBLE ? MIR_T_D - : mode == MIR_OP_LDOUBLE ? MIR_T_LD - : MIR_T_I64, - curr_func_item->u.func); - } - return e->temp_reg; + if (*temp_reg != MIR_NON_VAR) return *temp_reg; + mode = MIR_insn_op_mode (gen_ctx->ctx, insn, 0, &out_p); + *temp_reg = gen_new_temp_reg (gen_ctx, mode2type (mode), curr_func_item->u.func); + return *temp_reg; } -static int gvn_insn_p (MIR_insn_t insn) { - return (!MIR_branch_code_p (insn->code) && insn->code != MIR_RET && insn->code != MIR_SWITCH - && insn->code != MIR_LABEL && !MIR_call_code_p (insn->code) && insn->code != MIR_ALLOCA - && insn->code != MIR_BSTART && insn->code != MIR_BEND && insn->code != MIR_VA_START - && insn->code != MIR_VA_ARG && insn->code != MIR_VA_END - && insn->code != MIR_PHI - /* After simplification we have only mem insn in form: mem = reg or reg = mem. */ - && (!move_code_p (insn->code) - || (insn->ops[0].mode != MIR_OP_MEM && insn->ops[0].mode != MIR_OP_HARD_REG_MEM - && insn->ops[1].mode != MIR_OP_MEM && insn->ops[1].mode != MIR_OP_HARD_REG_MEM))); +static int fixed_place_insn_p (MIR_insn_t insn) { + return (insn->code == MIR_RET || insn->code == MIR_JRET || insn->code == MIR_SWITCH + || insn->code == MIR_LABEL || MIR_call_code_p (insn->code) || insn->code == MIR_ALLOCA + || insn->code == MIR_BSTART || insn->code == MIR_BEND || insn->code == MIR_VA_START + || insn->code == MIR_VA_ARG || insn->code == MIR_VA_END); } +static int gvn_insn_p (MIR_insn_t insn) { return !fixed_place_insn_p (insn); } + #if !MIR_NO_GEN_DEBUG static void print_expr (gen_ctx_t gen_ctx, expr_t e, const char *title) { MIR_context_t ctx = gen_ctx->ctx; @@ -2296,1911 +3791,3902 @@ static void print_expr (gen_ctx_t gen_ctx, expr_t e, const char *title) { } #endif -static int phi_use_p (MIR_insn_t insn) { - for (ssa_edge_t se = insn->ops[0].data; se != NULL; se = se->next_use) - if (se->use->insn->code == MIR_PHI) return TRUE; - return FALSE; +static int add_sub_const_insn_p (gen_ctx_t gen_ctx, MIR_insn_t insn, int64_t *val) { + ssa_edge_t ssa_edge; + bb_insn_t def_bb_insn; + // ??? , minimal gvn->val + if (insn->code != MIR_ADD && insn->code != MIR_SUB && insn->code != MIR_ADDS + && insn->code != MIR_SUBS) + return FALSE; + if ((ssa_edge = insn->ops[2].data) == NULL || !(def_bb_insn = ssa_edge->def)->gvn_val_const_p) + return FALSE; + MIR_func_t func = curr_func_item->u.func; + if (insn->ops[1].mode == MIR_OP_VAR + && MIR_reg_hard_reg_name (gen_ctx->ctx, insn->ops[1].u.var - MAX_HARD_REG, func) != NULL) + return FALSE; + *val = insn->code == MIR_SUB || insn->code == MIR_SUBS ? -def_bb_insn->gvn_val + : def_bb_insn->gvn_val; + return TRUE; } -static void gvn_modify (gen_ctx_t gen_ctx) { +static MIR_insn_t skip_moves (gen_ctx_t gen_ctx, MIR_insn_t insn) { + ssa_edge_t se; + MIR_func_t func = curr_func_item->u.func; + + while (insn->code == MIR_MOV && insn->ops[1].mode == MIR_OP_VAR) { + if ((se = insn->ops[1].data) == NULL + || MIR_reg_hard_reg_name (gen_ctx->ctx, insn->ops[1].u.var - MAX_HARD_REG, func) != NULL) + return insn; + insn = se->def->insn; + } + return insn; +} + +static void print_bb_insn_value (gen_ctx_t gen_ctx, bb_insn_t bb_insn) { MIR_context_t ctx = gen_ctx->ctx; - bb_t bb; - bb_insn_t bb_insn, new_bb_insn, next_bb_insn, expr_bb_insn; - MIR_reg_t temp_reg; - long gvn_insns_num = 0; - for (size_t i = 0; i < VARR_LENGTH (bb_t, worklist); i++) { - bb = VARR_GET (bb_t, worklist, i); - for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; bb_insn = next_bb_insn) { - expr_t e, new_e; - MIR_op_t op; - int add_def_p; - MIR_type_t type; - MIR_insn_code_t move_code; - MIR_insn_t new_insn, def_insn, insn = bb_insn->insn; - ssa_edge_t list; + DEBUG (2, { + fprintf (debug_file, "%s%s=%lld for insn %lu:", + !bb_insn->alloca_flag ? "" + : bb_insn->alloca_flag & (MAY_ALLOCA | MUST_ALLOCA) ? "may/must alloca " + : bb_insn->alloca_flag & MAY_ALLOCA ? "may alloca" + : "must alloca", + bb_insn->gvn_val_const_p ? "const val" : "val", (long long) bb_insn->gvn_val, + (unsigned long) bb_insn->index); + MIR_output_insn (ctx, debug_file, bb_insn->insn, curr_func_item->u.func, TRUE); + }); +} - next_bb_insn = DLIST_NEXT (bb_insn_t, bb_insn); - if (!gvn_insn_p (insn)) continue; - if (!find_expr (gen_ctx, insn, &e)) { - e = add_expr (gen_ctx, insn); - DEBUG (2, { print_expr (gen_ctx, e, "Adding"); }); - } - if (move_p (insn)) - bb_insn->gvn_val = ((ssa_edge_t) insn->ops[1].data)->def->gvn_val; - else - bb_insn->gvn_val = e->num; - DEBUG (2, { - fprintf (debug_file, "Val=%lu for insn %lu:", (unsigned long) bb_insn->gvn_val, - (unsigned long) bb_insn->index); - MIR_output_insn (ctx, debug_file, bb_insn->insn, curr_func_item->u.func, TRUE); - }); - if (e->insn == insn || move_p (insn) - || (imm_move_p (insn) && insn->ops[1].mode != MIR_OP_REF)) - continue; - if (phi_use_p (e->insn)) continue; /* keep conventional SSA */ - expr_bb_insn = e->insn->data; - if (bb->index != expr_bb_insn->bb->index - && !bitmap_bit_p (bb->dom_in, expr_bb_insn->bb->index)) - continue; - add_def_p = e->temp_reg == 0; - temp_reg = get_expr_temp_reg (gen_ctx, e); - op = MIR_new_reg_op (ctx, temp_reg); - type = MIR_reg_type (ctx, temp_reg, curr_func_item->u.func); -#ifndef NDEBUG - int out_p; - MIR_insn_op_mode (ctx, insn, 0, &out_p); /* result here is always 0-th op */ - gen_assert (out_p); -#endif - move_code = get_move_code (type); - if (add_def_p) { - list = e->insn->ops[0].data; - e->insn->ops[0].data = NULL; - new_insn = MIR_new_insn (ctx, move_code, op, e->insn->ops[0]); - gen_add_insn_after (gen_ctx, e->insn, new_insn); - add_ssa_edge (gen_ctx, e->insn->data, 0, new_insn->data, 1); - new_insn->ops[0].data = list; - new_bb_insn = new_insn->data; - change_ssa_edge_list_def (list, new_bb_insn, 0, e->insn->ops[0].u.reg, temp_reg); - if (!find_expr (gen_ctx, new_insn, &new_e)) new_e = add_expr (gen_ctx, new_insn); - new_bb_insn->gvn_val = e->num; - DEBUG (2, { - fprintf (debug_file, " adding insn "); - MIR_output_insn (ctx, debug_file, new_insn, curr_func_item->u.func, FALSE); - fprintf (debug_file, " after def insn "); - MIR_output_insn (ctx, debug_file, e->insn, curr_func_item->u.func, TRUE); - }); - } - list = insn->ops[0].data; - insn->ops[0].data = NULL; /* make redundant insn having no uses */ - new_insn = MIR_new_insn (ctx, move_code, insn->ops[0], op); - gen_add_insn_after (gen_ctx, insn, new_insn); - def_insn = DLIST_NEXT (MIR_insn_t, e->insn); - add_ssa_edge (gen_ctx, def_insn->data, 0, new_insn->data, 1); - new_insn->ops[0].data = list; - new_bb_insn = new_insn->data; - change_ssa_edge_list_def (list, new_bb_insn, 0, 0, 0); - if (!find_expr (gen_ctx, new_insn, &new_e)) new_e = add_expr (gen_ctx, new_insn); - new_bb_insn->gvn_val = e->num; - gvn_insns_num++; - DEBUG (2, { - fprintf (debug_file, " adding insn "); - MIR_output_insn (ctx, debug_file, new_insn, curr_func_item->u.func, FALSE); - fprintf (debug_file, " after use insn "); - MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, TRUE); - }); - } - } - DEBUG (1, { fprintf (debug_file, "%5ld found GVN redundant insns\n", gvn_insns_num); }); -} - -static void gvn (gen_ctx_t gen_ctx) { - calculate_dominators (gen_ctx); - for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) - VARR_PUSH (bb_t, worklist, bb); - qsort (VARR_ADDR (bb_t, worklist), VARR_LENGTH (bb_t, worklist), sizeof (bb_t), rpost_cmp); - gvn_modify (gen_ctx); -} - -static void gvn_clear (gen_ctx_t gen_ctx) { - HTAB_CLEAR (expr_t, expr_tab); - while (VARR_LENGTH (expr_t, exprs) != 0) free (VARR_POP (expr_t, exprs)); -} - -static void init_gvn (gen_ctx_t gen_ctx) { - gen_ctx->gvn_ctx = gen_malloc (gen_ctx, sizeof (struct gvn_ctx)); - VARR_CREATE (expr_t, exprs, 512); - HTAB_CREATE (expr_t, expr_tab, 1024, expr_hash, expr_eq, gen_ctx); -} - -static void finish_gvn (gen_ctx_t gen_ctx) { - VARR_DESTROY (expr_t, exprs); - HTAB_DESTROY (expr_t, expr_tab); - free (gen_ctx->gvn_ctx); - gen_ctx->gvn_ctx = NULL; -} - -/* New Page */ - -/* Sparse Conditional Constant Propagation. Live info should exist. */ - -#define live_in in -#define live_out out - -enum ccp_val_kind { CCP_CONST = 0, CCP_VARYING, CCP_UNKNOWN }; - -struct ccp_val { - enum ccp_val_kind val_kind : 8; - unsigned int flag : 8; - size_t ccp_run; - const_t val; -}; - -typedef struct ccp_val *ccp_val_t; -DEF_VARR (ccp_val_t); - -struct ccp_ctx { - size_t curr_ccp_run; - bitmap_t bb_visited; - VARR (bb_t) * ccp_bbs; - VARR (bb_insn_t) * ccp_insns; - VARR (ccp_val_t) * ccp_vals; -}; - -#define curr_ccp_run gen_ctx->ccp_ctx->curr_ccp_run -#define bb_visited gen_ctx->ccp_ctx->bb_visited -#define ccp_bbs gen_ctx->ccp_ctx->ccp_bbs -#define ccp_insns gen_ctx->ccp_ctx->ccp_insns -#define ccp_vals gen_ctx->ccp_ctx->ccp_vals - -static ccp_val_t get_ccp_val (gen_ctx_t gen_ctx, bb_insn_t bb_insn) { - ccp_val_t ccp_val; - - while (VARR_LENGTH (ccp_val_t, ccp_vals) <= bb_insn->index) VARR_PUSH (ccp_val_t, ccp_vals, NULL); - if ((ccp_val = VARR_GET (ccp_val_t, ccp_vals, bb_insn->index)) == NULL) { - ccp_val = gen_malloc (gen_ctx, sizeof (struct ccp_val)); - VARR_SET (ccp_val_t, ccp_vals, bb_insn->index, ccp_val); - ccp_val->ccp_run = 0; - } - if (ccp_val->ccp_run != curr_ccp_run) { - ccp_val->val_kind = bb_insn->bb == DLIST_HEAD (bb_t, curr_cfg->bbs) ? CCP_VARYING : CCP_UNKNOWN; - ccp_val->flag = FALSE; - ccp_val->ccp_run = curr_ccp_run; - } - return ccp_val; -} - -#undef live_in -#undef live_out - -static void initiate_ccp_info (gen_ctx_t gen_ctx) { - bb_insn_t bb_insn; - ccp_val_t ccp_val; - - for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { - if ((bb_insn = DLIST_TAIL (bb_insn_t, bb->bb_insns)) != NULL - && MIR_branch_code_p (bb_insn->insn->code) && bb_insn->insn->code != MIR_JMP - && bb_insn->insn->code != MIR_SWITCH) { - for (edge_t e = DLIST_HEAD (out_edge_t, bb->out_edges); e != NULL; - e = DLIST_NEXT (out_edge_t, e)) - if (e->dst != DLIST_EL (bb_t, curr_cfg->bbs, 1)) /* ignore exit bb */ - e->skipped_p = TRUE; - } - } - bitmap_clear (bb_visited); - VARR_TRUNC (bb_insn_t, ccp_insns, 0); - VARR_TRUNC (bb_t, ccp_bbs, 0); - while (VARR_LENGTH (ccp_val_t, ccp_vals) != 0) - if ((ccp_val = VARR_POP (ccp_val_t, ccp_vals)) != NULL) free (ccp_val); - VARR_PUSH (bb_t, ccp_bbs, DLIST_HEAD (bb_t, curr_cfg->bbs)); /* entry bb */ -} - -static int var_op_p (MIR_op_t op) { return op.mode == MIR_OP_HARD_REG || op.mode == MIR_OP_REG; } -static int var_insn_op_p (MIR_insn_t insn, size_t nop) { return var_op_p (insn->ops[nop]); } - -static enum ccp_val_kind get_op (gen_ctx_t gen_ctx, MIR_insn_t insn, size_t nop, const_t *val) { - MIR_op_t op; +static int get_gvn_op (MIR_insn_t insn, size_t nop, int64_t *val) { + MIR_op_t *op_ref = &insn->ops[nop]; ssa_edge_t ssa_edge; - ccp_val_t ccp_val; + bb_insn_t def_bb_insn; - if (!var_insn_op_p (insn, nop)) { - if ((op = insn->ops[nop]).mode == MIR_OP_INT) { - val->uns_p = FALSE; - val->u.i = op.u.i; - return CCP_CONST; - } else if (op.mode == MIR_OP_UINT) { - val->uns_p = TRUE; - val->u.u = op.u.u; - return CCP_CONST; - } - return CCP_VARYING; + if ((ssa_edge = op_ref->data) != NULL && (def_bb_insn = ssa_edge->def)->gvn_val_const_p) { + *val = def_bb_insn->gvn_val; + return TRUE; } - ssa_edge = insn->ops[nop].data; - ccp_val = get_ccp_val (gen_ctx, ssa_edge->def); - if (ccp_val->val_kind == CCP_CONST) *val = ccp_val->val; - return ccp_val->val_kind; -} - -static enum ccp_val_kind get_2ops (gen_ctx_t gen_ctx, MIR_insn_t insn, const_t *val1, int out_p) { - if (out_p && !var_insn_op_p (insn, 0)) return CCP_UNKNOWN; - return get_op (gen_ctx, insn, 1, val1); + return FALSE; } -static enum ccp_val_kind get_3ops (gen_ctx_t gen_ctx, MIR_insn_t insn, const_t *val1, const_t *val2, - int out_p) { - enum ccp_val_kind res1, res2; +static int get_gvn_2ops (MIR_insn_t insn, int64_t *val1) { return get_gvn_op (insn, 1, val1); } - if (out_p && !var_insn_op_p (insn, 0)) return CCP_UNKNOWN; - if ((res1 = get_op (gen_ctx, insn, 1, val1)) == CCP_VARYING) return CCP_VARYING; - if ((res2 = get_op (gen_ctx, insn, 2, val2)) == CCP_VARYING) return CCP_VARYING; - return res1 == CCP_UNKNOWN || res2 == CCP_UNKNOWN ? CCP_UNKNOWN : CCP_CONST; +static int get_gvn_3ops (MIR_insn_t insn, int64_t *val1, int64_t *val2) { + if (get_gvn_op (insn, 1, val1) && get_gvn_op (insn, 2, val2)) return TRUE; + return FALSE; } -static enum ccp_val_kind get_2iops (gen_ctx_t gen_ctx, MIR_insn_t insn, int64_t *p, int out_p) { - const_t val; - enum ccp_val_kind res; +static int get_gvn_2iops (MIR_insn_t insn, int64_t *p) { + int64_t val; - if ((res = get_2ops (gen_ctx, insn, &val, out_p))) return res; - *p = val.u.i; - return CCP_CONST; + if (!get_gvn_2ops (insn, &val)) return FALSE; + *p = val; + return TRUE; } -static enum ccp_val_kind get_2isops (gen_ctx_t gen_ctx, MIR_insn_t insn, int32_t *p, int out_p) { - const_t val; - enum ccp_val_kind res; +static int get_gvn_2isops (MIR_insn_t insn, int32_t *p) { + int64_t val; - if ((res = get_2ops (gen_ctx, insn, &val, out_p))) return res; - *p = val.u.i; - return CCP_CONST; + if (!get_gvn_2ops (insn, &val)) return FALSE; + *p = (int32_t) val; + return TRUE; } -static enum ccp_val_kind MIR_UNUSED get_2usops (gen_ctx_t gen_ctx, MIR_insn_t insn, uint32_t *p, - int out_p) { - const_t val; - enum ccp_val_kind res; +static int MIR_UNUSED get_gvn_2usops (MIR_insn_t insn, uint32_t *p) { + int64_t val; - if ((res = get_2ops (gen_ctx, insn, &val, out_p))) return res; - *p = val.u.u; - return CCP_CONST; + if (!get_gvn_2ops (insn, &val)) return FALSE; + *p = (uint32_t) val; + return TRUE; } -static enum ccp_val_kind get_3iops (gen_ctx_t gen_ctx, MIR_insn_t insn, int64_t *p1, int64_t *p2, - int out_p) { - const_t val1, val2; - enum ccp_val_kind res; +static int get_gvn_3iops (MIR_insn_t insn, int64_t *p1, int64_t *p2) { + int64_t val1, val2; - if ((res = get_3ops (gen_ctx, insn, &val1, &val2, out_p))) return res; - *p1 = val1.u.i; - *p2 = val2.u.i; - return CCP_CONST; + if (!get_gvn_3ops (insn, &val1, &val2)) return FALSE; + *p1 = val1; + *p2 = val2; + return TRUE; } -static enum ccp_val_kind get_3isops (gen_ctx_t gen_ctx, MIR_insn_t insn, int32_t *p1, int32_t *p2, - int out_p) { - const_t val1, val2; - enum ccp_val_kind res; +static int get_gvn_3isops (MIR_insn_t insn, int32_t *p1, int32_t *p2) { + int64_t val1, val2; - if ((res = get_3ops (gen_ctx, insn, &val1, &val2, out_p))) return res; - *p1 = val1.u.i; - *p2 = val2.u.i; - return CCP_CONST; + if (!get_gvn_3ops (insn, &val1, &val2)) return FALSE; + *p1 = (int32_t) val1; + *p2 = (int32_t) val2; + return TRUE; } -static enum ccp_val_kind get_3uops (gen_ctx_t gen_ctx, MIR_insn_t insn, uint64_t *p1, uint64_t *p2, - int out_p) { - const_t val1, val2; - enum ccp_val_kind res; +static int get_gvn_3uops (MIR_insn_t insn, uint64_t *p1, uint64_t *p2) { + int64_t val1, val2; - if ((res = get_3ops (gen_ctx, insn, &val1, &val2, out_p))) return res; - *p1 = val1.u.u; - *p2 = val2.u.u; - return CCP_CONST; + if (!get_gvn_3ops (insn, &val1, &val2)) return FALSE; + *p1 = val1; + *p2 = val2; + return TRUE; } -static enum ccp_val_kind get_3usops (gen_ctx_t gen_ctx, MIR_insn_t insn, uint32_t *p1, uint32_t *p2, - int out_p) { - const_t val1, val2; - enum ccp_val_kind res; +static int get_gvn_3usops (MIR_insn_t insn, uint32_t *p1, uint32_t *p2) { + int64_t val1, val2; - if ((res = get_3ops (gen_ctx, insn, &val1, &val2, out_p))) return res; - *p1 = val1.u.u; - *p2 = val2.u.u; - return CCP_CONST; + if (!get_gvn_3ops (insn, &val1, &val2)) return FALSE; + *p1 = (uint32_t) val1; + *p2 = (uint32_t) val2; + return TRUE; } -#define EXT(tp) \ - do { \ - int64_t p; \ - if ((ccp_res = get_2iops (gen_ctx, insn, &p, TRUE)) != CCP_CONST) goto non_const; \ - val.uns_p = FALSE; \ - val.u.i = (tp) p; \ - } while (0) - -#define IOP2(op) \ - do { \ - int64_t p; \ - if ((ccp_res = get_2iops (gen_ctx, insn, &p, TRUE)) != CCP_CONST) goto non_const; \ - val.uns_p = FALSE; \ - val.u.i = op p; \ - } while (0) - -#define IOP2S(op) \ - do { \ - int32_t p; \ - if ((ccp_res = get_2isops (gen_ctx, insn, &p, TRUE)) != CCP_CONST) goto non_const; \ - val.uns_p = FALSE; \ - val.u.i = op p; \ - } while (0) - -#define UOP2S(op) \ - do { \ - uint32_t p; \ - if ((ccp_res = get_2usops (gen_ctx, insn, &p, TRUE)) != CCP_CONST) goto non_const; \ - val.uns_p = FALSE; \ - val.u.u = op p; \ - } while (0) - -#define IOP3(op) \ - do { \ - int64_t p1, p2; \ - if ((ccp_res = get_3iops (gen_ctx, insn, &p1, &p2, TRUE)) != CCP_CONST) goto non_const; \ - val.uns_p = FALSE; \ - val.u.i = p1 op p2; \ - } while (0) - -#define IOP3S(op) \ - do { \ - int32_t p1, p2; \ - if ((ccp_res = get_3isops (gen_ctx, insn, &p1, &p2, TRUE)) != CCP_CONST) goto non_const; \ - val.uns_p = FALSE; \ - val.u.i = p1 op p2; \ +#define GVN_EXT(tp) \ + do { \ + int64_t p; \ + if ((const_p = get_gvn_2iops (insn, &p))) val = (tp) p; \ } while (0) -#define UOP3(op) \ - do { \ - uint64_t p1, p2; \ - if ((ccp_res = get_3uops (gen_ctx, insn, &p1, &p2, TRUE)) != CCP_CONST) goto non_const; \ - val.uns_p = TRUE; \ - val.u.u = p1 op p2; \ +#define GVN_IOP2(op) \ + do { \ + int64_t p; \ + if ((const_p = get_gvn_2iops (insn, &p))) val = op p; \ } while (0) -#define UOP3S(op) \ - do { \ - uint32_t p1, p2; \ - if ((ccp_res = get_3usops (gen_ctx, insn, &p1, &p2, TRUE)) != CCP_CONST) goto non_const; \ - val.uns_p = TRUE; \ - val.u.u = p1 op p2; \ +#define GVN_IOP2S(op) \ + do { \ + int32_t p; \ + if ((const_p = get_gvn_2isops (insn, &p))) val = op p; \ } while (0) -#define IOP30(op) \ - do { \ - if ((ccp_res = get_op (gen_ctx, insn, 2, &val)) != CCP_CONST || val.u.i == 0) goto non_const0; \ - IOP3 (op); \ +#define GVN_IOP3(op) \ + do { \ + int64_t p1, p2; \ + if ((const_p = get_gvn_3iops (insn, &p1, &p2))) val = p1 op p2; \ } while (0) -#define IOP3S0(op) \ - do { \ - if ((ccp_res = get_op (gen_ctx, insn, 2, &val)) != CCP_CONST || val.u.i == 0) goto non_const0; \ - IOP3S (op); \ +#define GVN_IOP3S(op) \ + do { \ + int32_t p1, p2; \ + if ((const_p = get_gvn_3isops (insn, &p1, &p2))) val = p1 op p2; \ } while (0) -#define UOP30(op) \ - do { \ - if ((ccp_res = get_op (gen_ctx, insn, 2, &val)) != CCP_CONST || val.u.u == 0) goto non_const0; \ - UOP3 (op); \ +#define GVN_UOP3(op) \ + do { \ + uint64_t p1, p2; \ + if ((const_p = get_gvn_3uops (insn, &p1, &p2))) val = p1 op p2; \ } while (0) -#define UOP3S0(op) \ - do { \ - if ((ccp_res = get_op (gen_ctx, insn, 2, &val)) != CCP_CONST || val.u.u == 0) goto non_const0; \ - UOP3S (op); \ +#define GVN_UOP3S(op) \ + do { \ + uint32_t p1, p2; \ + if ((const_p = get_gvn_3usops (insn, &p1, &p2))) val = p1 op p2; \ } while (0) -#define ICMP(op) \ - do { \ - int64_t p1, p2; \ - if ((ccp_res = get_3iops (gen_ctx, insn, &p1, &p2, TRUE)) != CCP_CONST) goto non_const; \ - val.uns_p = FALSE; \ - val.u.i = p1 op p2; \ +#define GVN_IOP30(op) \ + do { \ + if (get_gvn_op (insn, 2, &val) && val != 0) GVN_IOP3 (op); \ } while (0) -#define ICMPS(op) \ - do { \ - int32_t p1, p2; \ - if ((ccp_res = get_3isops (gen_ctx, insn, &p1, &p2, TRUE)) != CCP_CONST) goto non_const; \ - val.uns_p = FALSE; \ - val.u.i = p1 op p2; \ +#define GVN_IOP3S0(op) \ + do { \ + if (get_gvn_op (insn, 2, &val) && val != 0) GVN_IOP3S (op); \ } while (0) -#define UCMP(op) \ - do { \ - uint64_t p1, p2; \ - if ((ccp_res = get_3uops (gen_ctx, insn, &p1, &p2, TRUE)) != CCP_CONST) goto non_const; \ - val.uns_p = FALSE; \ - val.u.i = p1 op p2; \ +#define GVN_UOP30(op) \ + do { \ + if (get_gvn_op (insn, 2, &val) && val != 0) GVN_UOP3 (op); \ } while (0) -#define UCMPS(op) \ - do { \ - uint32_t p1, p2; \ - if ((ccp_res = get_3usops (gen_ctx, insn, &p1, &p2, TRUE)) != CCP_CONST) goto non_const; \ - val.uns_p = FALSE; \ - val.u.i = p1 op p2; \ +#define GVN_UOP3S0(op) \ + do { \ + if (get_gvn_op (insn, 2, &val) && val != 0) GVN_UOP3S (op); \ } while (0) -#define BICMP(op) \ - do { \ - int64_t p1, p2; \ - if ((ccp_res = get_3iops (gen_ctx, insn, &p1, &p2, FALSE)) != CCP_CONST) goto non_const; \ - val.uns_p = FALSE; \ - val.u.i = p1 op p2; \ +#define GVN_ICMP(op) \ + do { \ + int64_t p1, p2; \ + if ((const_p = get_gvn_3iops (insn, &p1, &p2))) val = p1 op p2; \ } while (0) -#define BICMPS(op) \ - do { \ - int32_t p1, p2; \ - if ((ccp_res = get_3isops (gen_ctx, insn, &p1, &p2, FALSE)) != CCP_CONST) goto non_const; \ - val.uns_p = FALSE; \ - val.u.i = p1 op p2; \ +#define GVN_ICMPS(op) \ + do { \ + int32_t p1, p2; \ + if ((const_p = get_gvn_3isops (insn, &p1, &p2))) val = p1 op p2; \ } while (0) -#define BUCMP(op) \ - do { \ - uint64_t p1, p2; \ - if ((ccp_res = get_3uops (gen_ctx, insn, &p1, &p2, FALSE)) != CCP_CONST) goto non_const; \ - val.uns_p = FALSE; \ - val.u.i = p1 op p2; \ +#define GVN_UCMP(op) \ + do { \ + uint64_t p1, p2; \ + if ((const_p = get_gvn_3uops (insn, &p1, &p2))) val = p1 op p2; \ } while (0) -#define BUCMPS(op) \ - do { \ - uint32_t p1, p2; \ - if ((ccp_res = get_3usops (gen_ctx, insn, &p1, &p2, FALSE)) != CCP_CONST) goto non_const; \ - val.uns_p = FALSE; \ - val.u.i = p1 op p2; \ +#define GVN_UCMPS(op) \ + do { \ + uint32_t p1, p2; \ + if ((const_p = get_gvn_3usops (insn, &p1, &p2))) val = p1 op p2; \ } while (0) -static int get_ccp_res_op (gen_ctx_t gen_ctx, MIR_insn_t insn, int out_num, MIR_op_t *op) { - MIR_context_t ctx = gen_ctx->ctx; - int out_p; - - gen_assert (!MIR_call_code_p (insn->code)); - if (out_num > 0 || MIR_insn_nops (ctx, insn) < 1) return FALSE; - MIR_insn_op_mode (ctx, insn, 0, &out_p); - if (!out_p) return FALSE; - *op = insn->ops[0]; - return TRUE; -} - -static int ccp_phi_insn_update (gen_ctx_t gen_ctx, bb_insn_t phi) { +static int gvn_phi_val (bb_insn_t phi, int64_t *val) { MIR_insn_t phi_insn = phi->insn; bb_t bb = phi->bb; + bb_insn_t def_bb_insn; edge_t e; - ssa_edge_t ssa_edge; - ccp_val_t res_ccp_val, op_ccp_val; size_t nop; - int change_p = FALSE; + ssa_edge_t se; + int const_p = TRUE, same_p = TRUE; - res_ccp_val = get_ccp_val (gen_ctx, phi); - if (res_ccp_val->val_kind == CCP_VARYING) return FALSE; nop = 1; for (e = DLIST_HEAD (in_edge_t, bb->in_edges); e != NULL; e = DLIST_NEXT (in_edge_t, e), nop++) { /* Update phi value: */ - if (e->skipped_p) continue; gen_assert (nop < phi_insn->nops); - ssa_edge = phi_insn->ops[nop].data; - op_ccp_val = get_ccp_val (gen_ctx, ssa_edge->def); - if (op_ccp_val->val_kind == CCP_UNKNOWN) continue; - if (res_ccp_val->val_kind == CCP_UNKNOWN || op_ccp_val->val_kind == CCP_VARYING) { - change_p = res_ccp_val->val_kind != op_ccp_val->val_kind; - res_ccp_val->val_kind = op_ccp_val->val_kind; - if (op_ccp_val->val_kind == CCP_VARYING) break; - if (op_ccp_val->val_kind == CCP_CONST) res_ccp_val->val = op_ccp_val->val; - } else { - gen_assert (res_ccp_val->val_kind == CCP_CONST && op_ccp_val->val_kind == CCP_CONST); - if (res_ccp_val->val.uns_p != op_ccp_val->val.uns_p - || (res_ccp_val->val.uns_p && res_ccp_val->val.u.u != op_ccp_val->val.u.u) - || (!res_ccp_val->val.uns_p && res_ccp_val->val.u.i != op_ccp_val->val.u.i)) { - res_ccp_val->val_kind = CCP_VARYING; - change_p = TRUE; - break; + if (same_p) { + if ((se = phi_insn->ops[nop].data) == NULL || (def_bb_insn = se->def) == NULL) { + same_p = FALSE; + } else if (nop == 1) { + const_p = def_bb_insn->gvn_val_const_p; + *val = def_bb_insn->gvn_val; + } else if (const_p != def_bb_insn->gvn_val_const_p || *val != def_bb_insn->gvn_val) { + same_p = FALSE; } } + if ((se = phi_insn->ops[nop].data) != NULL) { + phi->alloca_flag = nop == 1 ? se->def->alloca_flag + : ((phi->alloca_flag | se->def->alloca_flag) & MAY_ALLOCA) + | (phi->alloca_flag & se->def->alloca_flag & MUST_ALLOCA); + } } - return change_p; + if (!same_p) *val = phi->index; + return same_p && const_p; } -static int ccp_insn_update (gen_ctx_t gen_ctx, MIR_insn_t insn) { - // ??? should we do CCP for FP (fast-math) too - MIR_op_t op; - int change_p; - enum ccp_val_kind ccp_res; - const_t val; - ccp_val_t ccp_val; - enum ccp_val_kind val_kind; +static void remove_edge_phi_ops (edge_t e) { + size_t i, nop; + edge_t e2; + MIR_insn_t insn; + ssa_edge_t se; - switch (insn->code) { - case MIR_PHI: return ccp_phi_insn_update (gen_ctx, insn->data); - case MIR_MOV: IOP2 (+); break; - case MIR_EXT8: EXT (int8_t); break; - case MIR_EXT16: EXT (int16_t); break; - case MIR_EXT32: EXT (int32_t); break; - case MIR_UEXT8: EXT (uint8_t); break; - case MIR_UEXT16: EXT (uint16_t); break; - case MIR_UEXT32: EXT (uint32_t); break; - - case MIR_NEG: IOP2 (-); break; - case MIR_NEGS: IOP2S (-); break; - - case MIR_ADD: IOP3 (+); break; - case MIR_ADDS: IOP3S (+); break; - - case MIR_SUB: IOP3 (-); break; - case MIR_SUBS: IOP3S (-); break; - - case MIR_MUL: IOP3 (*); break; - case MIR_MULS: IOP3S (*); break; - - case MIR_DIV: IOP30 (/); break; - case MIR_DIVS: IOP3S0 (/); break; - case MIR_UDIV: UOP30 (/); break; - case MIR_UDIVS: UOP3S0 (/); break; - - case MIR_MOD: IOP30 (%); break; - case MIR_MODS: IOP3S0 (%); break; - case MIR_UMOD: UOP30 (%); break; - case MIR_UMODS: UOP3S0 (%); break; - - case MIR_AND: IOP3 (&); break; - case MIR_ANDS: IOP3S (&); break; - case MIR_OR: IOP3 (|); break; - case MIR_ORS: IOP3S (|); break; - case MIR_XOR: IOP3 (^); break; - case MIR_XORS: IOP3S (^); break; - - case MIR_LSH: IOP3 (<<); break; - case MIR_LSHS: IOP3S (<<); break; - case MIR_RSH: IOP3 (>>); break; - case MIR_RSHS: IOP3S (>>); break; - case MIR_URSH: UOP3 (>>); break; - case MIR_URSHS: UOP3S (>>); break; - - case MIR_EQ: ICMP (==); break; - case MIR_EQS: ICMPS (==); break; - case MIR_NE: ICMP (!=); break; - case MIR_NES: ICMPS (!=); break; - - case MIR_LT: ICMP (<); break; - case MIR_LTS: ICMPS (<); break; - case MIR_ULT: UCMP (<); break; - case MIR_ULTS: UCMPS (<); break; - case MIR_LE: ICMP (<=); break; - case MIR_LES: ICMPS (<=); break; - case MIR_ULE: UCMP (<=); break; - case MIR_ULES: UCMPS (<=); break; - case MIR_GT: ICMP (>); break; - case MIR_GTS: ICMPS (>); break; - case MIR_UGT: UCMP (>); break; - case MIR_UGTS: UCMPS (>); break; - case MIR_GE: ICMP (>=); break; - case MIR_GES: ICMPS (>=); break; - case MIR_UGE: UCMP (>=); break; - case MIR_UGES: UCMPS (>=); break; - - default: ccp_res = CCP_VARYING; goto non_const; + for (nop = 1, e2 = DLIST_HEAD (in_edge_t, e->dst->in_edges); e2 != NULL && e2 != e; + e2 = DLIST_NEXT (in_edge_t, e2), nop++) + ; + gen_assert (e2 != NULL); + for (bb_insn_t bb_insn = DLIST_HEAD (bb_insn_t, e->dst->bb_insns); bb_insn != NULL; + bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) { + if ((insn = bb_insn->insn)->code == MIR_LABEL) continue; + if (insn->code != MIR_PHI) break; + if ((se = insn->ops[nop].data) != NULL) remove_ssa_edge (se); + for (i = nop; i + 1 < insn->nops; i++) { + insn->ops[i] = insn->ops[i + 1]; + /* se can be null from some previously removed BB insn: */ + if ((se = insn->ops[i].data) != NULL) { + gen_assert (se->use_op_num == i + 1); + se->use_op_num = (uint32_t) i; + } + } + insn->nops--; } -#ifndef NDEBUG - { - int out_p; +} + +static void MIR_UNUSED remove_dest_phi_ops (bb_t bb) { + for (edge_t e = DLIST_HEAD (out_edge_t, bb->out_edges); e != NULL; e = DLIST_NEXT (out_edge_t, e)) + remove_edge_phi_ops (e); +} + +static void set_alloca_based_flag (bb_insn_t bb_insn, int must_p) { + MIR_insn_t insn = bb_insn->insn; + ssa_edge_t se; - MIR_insn_op_mode (gen_ctx->ctx, insn, 0, &out_p); /* result here is always 0-th op */ - gen_assert (out_p); + gen_assert (insn->nops >= 2); + if (must_p) { + if (((se = insn->ops[1].data) != NULL && (se->def->alloca_flag & MUST_ALLOCA)) + || (insn->nops == 3 && (se = insn->ops[2].data) != NULL + && (se->def->alloca_flag & MUST_ALLOCA))) + bb_insn->alloca_flag |= MUST_ALLOCA; } -#endif - ccp_val = get_ccp_val (gen_ctx, insn->data); - val_kind = ccp_val->val_kind; - gen_assert (val_kind == CCP_UNKNOWN || val_kind == CCP_CONST); - ccp_val->val_kind = CCP_CONST; - ccp_val->val = val; - return val_kind != CCP_CONST; -non_const0: - if (ccp_res == CCP_CONST && val.u.i == 0) ccp_res = CCP_VARYING; -non_const: - if (ccp_res == CCP_UNKNOWN) return FALSE; - gen_assert (ccp_res == CCP_VARYING); - change_p = FALSE; - if (MIR_call_code_p (insn->code)) { - ccp_val = get_ccp_val (gen_ctx, insn->data); - if (ccp_val->val_kind != CCP_VARYING) change_p = TRUE; - ccp_val->val_kind = CCP_VARYING; - } else if (get_ccp_res_op (gen_ctx, insn, 0, &op) - && (op.mode == MIR_OP_HARD_REG || op.mode == MIR_OP_REG)) { - ccp_val = get_ccp_val (gen_ctx, insn->data); - if (ccp_val->val_kind != CCP_VARYING) change_p = TRUE; - ccp_val->val_kind = CCP_VARYING; - gen_assert (!get_ccp_res_op (gen_ctx, insn, 1, &op)); + if (((se = insn->ops[1].data) != NULL && (se->def->alloca_flag & MAY_ALLOCA)) + || (insn->nops == 3 && (se = insn->ops[2].data) != NULL + && (se->def->alloca_flag & MAY_ALLOCA))) + bb_insn->alloca_flag |= MAY_ALLOCA; +} + +static ssa_edge_t skip_move_ssa_edges (ssa_edge_t se, MIR_insn_t *def_insn) { + for (;;) { + gen_assert (se != NULL); + *def_insn = se->def->insn; + if (!move_p (*def_insn)) return se; + se = (*def_insn)->ops[1].data; } - return change_p; } -static enum ccp_val_kind ccp_branch_update (gen_ctx_t gen_ctx, MIR_insn_t insn, int *res) { - enum ccp_val_kind ccp_res; - const_t val; +static MIR_insn_t get_def_disp (ssa_edge_t se, int64_t *disp) { + MIR_insn_t def_insn; - switch (insn->code) { - case MIR_BT: - case MIR_BTS: - case MIR_BF: - case MIR_BFS: - if ((ccp_res = get_op (gen_ctx, insn, 1, &val)) != CCP_CONST) return ccp_res; - if (insn->code == MIR_BTS || insn->code == MIR_BFS) - *res = val.uns_p ? (uint32_t) val.u.u != 0 : (int32_t) val.u.i != 0; - else - *res = val.uns_p ? val.u.u != 0 : val.u.i != 0; - if (insn->code == MIR_BF || insn->code == MIR_BFS) *res = !*res; - return CCP_CONST; - case MIR_BEQ: BICMP (==); break; - case MIR_BEQS: BICMPS (==); break; - case MIR_BNE: BICMP (!=); break; - case MIR_BNES: BICMPS (!=); break; - - case MIR_BLT: BICMP (<); break; - case MIR_BLTS: BICMPS (<); break; - case MIR_UBLT: BUCMP (<); break; - case MIR_UBLTS: BUCMPS (<); break; - case MIR_BLE: BICMP (<=); break; - case MIR_BLES: BICMPS (<=); break; - case MIR_UBLE: BUCMP (<=); break; - case MIR_UBLES: BUCMPS (<=); break; - case MIR_BGT: BICMP (>); break; - case MIR_BGTS: BICMPS (>); break; - case MIR_UBGT: BUCMP (>); break; - case MIR_UBGTS: BUCMPS (>); break; - case MIR_BGE: BICMP (>=); break; - case MIR_BGES: BICMPS (>=); break; - case MIR_UBGE: BUCMP (>=); break; - case MIR_UBGES: BUCMPS (>=); break; - - default: return CCP_VARYING; // ??? should we do CCP for FP BCMP too - } - *res = val.u.i; - return CCP_CONST; -non_const: - return ccp_res; -} - -static void ccp_push_used_insns (gen_ctx_t gen_ctx, ssa_edge_t first_ssa_edge) { - MIR_context_t ctx = gen_ctx->ctx; + *disp = 0; + for (;;) { + se = skip_move_ssa_edges (se, &def_insn); + if ((def_insn->code != MIR_ADD && def_insn->code != MIR_ADDS && def_insn->code != MIR_SUB + && def_insn->code != MIR_SUBS) + || (se = def_insn->ops[2].data) == NULL || !se->def->gvn_val_const_p) + return def_insn; + int add_p = def_insn->code == MIR_ADD || def_insn->code == MIR_ADDS; + *disp += add_p ? se->def->gvn_val : -se->def->gvn_val; + se = def_insn->ops[1].data; /* new base */ + } +} + +static void new_mem_loc (gen_ctx_t gen_ctx, MIR_op_t *mem_op_ref, int flag) { + /* zero loc is fixed: */ + int64_t disp; + mem_attr_t mem_attr; + + if ((mem_op_ref->u.var_mem.nloc = (uint32_t) VARR_LENGTH (mem_attr_t, mem_attrs)) == 0) + mem_op_ref->u.var_mem.nloc = 1; + mem_attr.alloca_flag = flag; + mem_attr.type = mem_op_ref->u.var_mem.type; + mem_attr.alias = mem_op_ref->u.var_mem.alias; + mem_attr.nonalias = mem_op_ref->u.var_mem.nonalias; + mem_attr.disp_def_p = FALSE; + mem_attr.disp = 0; + mem_attr.def_insn = NULL; + if ((flag & MUST_ALLOCA) != 0) { + mem_attr.def_insn = get_def_disp (mem_op_ref->data, &disp); + mem_attr.disp_def_p = TRUE; + mem_attr.disp = disp; + } + if (VARR_LENGTH (mem_attr_t, mem_attrs) == 0) VARR_PUSH (mem_attr_t, mem_attrs, mem_attr); + DEBUG (2, { + fprintf (debug_file, " new m%lu", (unsigned long) mem_op_ref->u.var_mem.nloc); + if (mem_attr.def_insn != NULL) + fprintf (debug_file, "(def_insn=%u)", ((bb_insn_t) mem_attr.def_insn->data)->index); + if (mem_attr.disp_def_p) fprintf (debug_file, "(disp=%lld)", (long long) mem_attr.disp); + if (flag) + fprintf (debug_file, " is %s alloca based", + flag & (MAY_ALLOCA | MUST_ALLOCA) ? "may/must" + : flag & MAY_ALLOCA ? "may" + : "must"); + fprintf (debug_file, "\n"); + }); + VARR_PUSH (mem_attr_t, mem_attrs, mem_attr); +} - for (ssa_edge_t ssa_edge = first_ssa_edge; ssa_edge != NULL; ssa_edge = ssa_edge->next_use) { - bb_insn_t bb_insn = ssa_edge->use; +static void update_mem_loc_alloca_flag (gen_ctx_t gen_ctx, size_t nloc, int flag) { + int old_flag; + mem_attr_t *mem_attr_ref; - if (bb_insn->flag || !bitmap_bit_p (bb_visited, bb_insn->bb->index)) - continue; /* already in ccp_insns or bb is not processed yet */ - VARR_PUSH (bb_insn_t, ccp_insns, bb_insn); - DEBUG (2, { - fprintf (debug_file, " pushing bb%lu insn: ", (unsigned long) bb_insn->bb->index); - MIR_output_insn (ctx, debug_file, bb_insn->insn, curr_func_item->u.func, FALSE); - }); - bb_insn->flag = TRUE; + gen_assert (VARR_LENGTH (mem_attr_t, mem_attrs) > nloc); + mem_attr_ref = &VARR_ADDR (mem_attr_t, mem_attrs)[nloc]; + old_flag = mem_attr_ref->alloca_flag; + mem_attr_ref->alloca_flag = ((old_flag | flag) & MAY_ALLOCA) | (old_flag & flag & MUST_ALLOCA); + DEBUG (2, { + if (flag != old_flag) { + fprintf (debug_file, " m%lu ", (unsigned long) nloc); + if (!flag) + fprintf (debug_file, "is no more alloca based\n"); + else + fprintf (debug_file, "becomes %s alloca based\n", + flag & (MAY_ALLOCA | MUST_ALLOCA) ? "may/must" + : flag & MAY_ALLOCA ? "may" + : "must"); + } + }); +} + +static long remove_bb (gen_ctx_t gen_ctx, bb_t bb) { + MIR_insn_t insn; + bb_insn_t bb_insn, next_bb_insn; + long deleted_insns_num = 0; + + gen_assert (bb->index != 2); + DEBUG (2, { + fprintf (debug_file, " BB%lu is unreachable and removed\n", (unsigned long) bb->index); + }); + for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; bb_insn = next_bb_insn) { + next_bb_insn = DLIST_NEXT (bb_insn_t, bb_insn); + insn = bb_insn->insn; + gen_delete_insn (gen_ctx, insn); + deleted_insns_num++; } + delete_bb (gen_ctx, bb); + return deleted_insns_num; } -static void ccp_process_active_edge (gen_ctx_t gen_ctx, edge_t e) { - if (e->skipped_p && !e->dst->flag) { - DEBUG (2, { - fprintf (debug_file, " Make edge bb%lu->bb%lu active\n", - (unsigned long) e->src->index, (unsigned long) e->dst->index); - }); - e->dst->flag = TRUE; /* just activated edge whose dest is not in ccp_bbs */ - VARR_PUSH (bb_t, ccp_bbs, e->dst); +static void mark_unreachable_bbs (gen_ctx_t gen_ctx) { + bb_t bb = DLIST_EL (bb_t, curr_cfg->bbs, 2); + + if (bb == NULL) return; + gen_assert (bb->index == 2); + bitmap_clear (temp_bitmap); + VARR_TRUNC (bb_t, worklist, 0); + VARR_PUSH (bb_t, worklist, bb); + bitmap_set_bit_p (temp_bitmap, bb->index); + for (bb = DLIST_EL (bb_t, curr_cfg->bbs, 2); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) + if (bb->reachable_p) { + VARR_PUSH (bb_t, worklist, bb); + bitmap_set_bit_p (temp_bitmap, bb->index); + } + while (VARR_LENGTH (bb_t, worklist) != 0) { + bb = VARR_POP (bb_t, worklist); + for (edge_t e = DLIST_HEAD (out_edge_t, bb->out_edges); e != NULL; + e = DLIST_NEXT (out_edge_t, e)) + if (bitmap_set_bit_p (temp_bitmap, e->dst->index)) VARR_PUSH (bb_t, worklist, e->dst); } - e->skipped_p = FALSE; } -static void ccp_make_insn_update (gen_ctx_t gen_ctx, MIR_insn_t insn) { - MIR_op_t op; - ccp_val_t ccp_val; +static long remove_unreachable_bbs (gen_ctx_t gen_ctx) { + long deleted_insns_num = 0; + bb_t next_bb; - if (!ccp_insn_update (gen_ctx, insn)) { - DEBUG (2, { - if (MIR_call_code_p (insn->code)) { - fprintf (debug_file, " -- keep all results varying"); - } else if (get_ccp_res_op (gen_ctx, insn, 0, &op) && var_insn_op_p (insn, 0)) { - ccp_val = get_ccp_val (gen_ctx, insn->data); - if (ccp_val->val_kind == CCP_UNKNOWN) { - fprintf (debug_file, " -- make the result unknown"); - } else if (ccp_val->val_kind == CCP_VARYING) { - fprintf (debug_file, " -- keep the result varying"); - } else { - gen_assert (ccp_val->val_kind == CCP_CONST); - fprintf (debug_file, " -- keep the result a constant "); - print_const (debug_file, ccp_val->val); - } - } - fprintf (debug_file, "\n"); - }); - } else { - ccp_val = NULL; /* to remove an initilized warning */ - if (MIR_call_code_p (insn->code)) { - ccp_val = get_ccp_val (gen_ctx, insn->data); - gen_assert (insn->ops[0].u.ref->item_type == MIR_proto_item); - MIR_proto_t proto = insn->ops[0].u.ref->u.proto; - for (uint32_t i = 0; i < proto->nres; i++) - ccp_push_used_insns (gen_ctx, insn->ops[i + 2].data); - } else if (get_ccp_res_op (gen_ctx, insn, 0, &op) && var_op_p (op)) { - ccp_val = get_ccp_val (gen_ctx, insn->data); - ccp_push_used_insns (gen_ctx, op.data); - } - gen_assert (ccp_val != NULL); - DEBUG (2, { - if (MIR_call_code_p (insn->code)) { - fprintf (debug_file, " -- make all results varying\n"); - } else if (ccp_val->val_kind == CCP_VARYING) { - fprintf (debug_file, " -- make the result varying\n"); - } else { - gen_assert (ccp_val->val_kind == CCP_CONST); - fprintf (debug_file, " -- make the result a constant "); - print_const (debug_file, ccp_val->val); - fprintf (debug_file, "\n"); - } - }); + mark_unreachable_bbs (gen_ctx); + for (bb_t bb = DLIST_EL (bb_t, curr_cfg->bbs, 2); bb != NULL; bb = next_bb) { + next_bb = DLIST_NEXT (bb_t, bb); + if (!bitmap_bit_p (temp_bitmap, bb->index)) deleted_insns_num += remove_bb (gen_ctx, bb); } + return deleted_insns_num; } -static void ccp_process_insn (gen_ctx_t gen_ctx, bb_insn_t bb_insn) { - int res; - enum ccp_val_kind ccp_res; - edge_t e; - bb_t bb = bb_insn->bb; - MIR_insn_t insn = bb_insn->insn; +static void copy_gvn_info (bb_insn_t to, bb_insn_t from) { + to->gvn_val_const_p = from->gvn_val_const_p; + to->gvn_val = from->gvn_val; + to->alloca_flag = from->alloca_flag; +} +static void remove_copy (gen_ctx_t gen_ctx, MIR_insn_t insn) { + ssa_edge_t se, last_se; + bb_insn_t def; + int def_op_num; + + gen_assert (move_p (insn) || (insn->code == MIR_PHI && insn->nops == 2)); + se = insn->ops[1].data; + def = se->def; + def_op_num = se->def_op_num; + remove_ssa_edge (se); + if ((last_se = def->insn->ops[def_op_num].data) != NULL) + while (last_se->next_use != NULL) last_se = last_se->next_use; + change_ssa_edge_list_def (insn->ops[0].data, def, def_op_num, insn->ops[0].u.var, + insn->ops[1].u.var); + if (last_se != NULL) + last_se->next_use = insn->ops[0].data; + else + def->insn->ops[def_op_num].data = insn->ops[0].data; + if (insn->ops[0].data != NULL) ((ssa_edge_t) insn->ops[0].data)->prev_use = last_se; + insn->ops[0].data = NULL; DEBUG (2, { - fprintf (debug_file, " processing bb%lu insn: ", (unsigned long) bb_insn->bb->index); - MIR_output_insn (gen_ctx->ctx, debug_file, bb_insn->insn, curr_func_item->u.func, FALSE); + fprintf (debug_file, " Remove copy %-5lu", (unsigned long) ((bb_insn_t) insn->data)->index); + print_bb_insn (gen_ctx, insn->data, FALSE); }); - if (!MIR_branch_code_p (insn->code) || insn->code == MIR_JMP || insn->code == MIR_SWITCH) { - ccp_make_insn_update (gen_ctx, insn); // ??? should we process SWITCH as cond branch - return; - } - DEBUG (2, { fprintf (debug_file, "\n"); }); - if ((ccp_res = ccp_branch_update (gen_ctx, insn, &res)) == CCP_CONST) { - /* Remember about an edge to exit bb. First edge is always for - fall through and the 2nd edge is for jump bb. */ - gen_assert (DLIST_LENGTH (out_edge_t, bb->out_edges) >= 2); - e = res ? DLIST_EL (out_edge_t, bb->out_edges, 1) : DLIST_EL (out_edge_t, bb->out_edges, 0); - ccp_process_active_edge (gen_ctx, e); - } else if (ccp_res == CCP_VARYING) { /* activate all edges */ - for (e = DLIST_HEAD (out_edge_t, bb->out_edges); e != NULL; e = DLIST_NEXT (out_edge_t, e)) - ccp_process_active_edge (gen_ctx, e); - } + gen_delete_insn (gen_ctx, insn); } -static void ccp_process_bb (gen_ctx_t gen_ctx, bb_t bb) { - MIR_insn_t insn; - bb_insn_t bb_insn; - edge_t e; +/* we are at curr bb from start, return true if can go to start avoiding dst */ +static int reachable_without_visiting_bb_p (gen_ctx_t gen_ctx, bb_t curr, bb_t start, + bb_t exclude) { + if (curr == exclude || !bitmap_set_bit_p (temp_bitmap2, curr->index)) return FALSE; + for (edge_t e = DLIST_HEAD (out_edge_t, curr->out_edges); e != NULL; + e = DLIST_NEXT (out_edge_t, e)) + if (e->dst == start || reachable_without_visiting_bb_p (gen_ctx, e->dst, start, exclude)) + return TRUE; + return FALSE; +} - DEBUG (2, { fprintf (debug_file, " processing bb%lu\n", (unsigned long) bb->index); }); - for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; - bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) { - if ((insn = bb_insn->insn)->code == MIR_LABEL) continue; - if (insn->code != MIR_PHI) break; - DEBUG (2, { - gen_assert (insn->ops[0].mode == MIR_OP_REG); - fprintf (debug_file, - " processing phi of reg%lu(%s) in bb%lu:", (long unsigned) insn->ops[0].u.reg, - MIR_reg_name (gen_ctx->ctx, insn->ops[0].u.reg, curr_func_item->u.func), - (unsigned long) bb->index); - }); - ccp_make_insn_update (gen_ctx, bb_insn->insn); - } - if (!bitmap_set_bit_p (bb_visited, bb->index)) return; - for (; bb_insn != NULL; bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) { - ccp_process_insn (gen_ctx, bb_insn); - } - if ((bb_insn = DLIST_TAIL (bb_insn_t, bb->bb_insns)) == NULL - || !MIR_branch_code_p (bb_insn->insn->code) || bb_insn->insn->code == MIR_JMP - || bb_insn->insn->code == MIR_SWITCH) { - for (e = DLIST_HEAD (out_edge_t, bb->out_edges); e != NULL; e = DLIST_NEXT (out_edge_t, e)) { - gen_assert (!e->skipped_p); - if (!bitmap_bit_p (bb_visited, e->dst->index) && !e->dst->flag) { - e->dst->flag = TRUE; /* first process of dest which is not in ccp_bbs */ - VARR_PUSH (bb_t, ccp_bbs, e->dst); - } - ccp_process_active_edge (gen_ctx, e); - } - } +static int cycle_without_bb_visit_p (gen_ctx_t gen_ctx, bb_t start, bb_t exclude) { + bitmap_clear (temp_bitmap2); + return reachable_without_visiting_bb_p (gen_ctx, start, start, exclude); } -static void ccp_traverse (bb_t bb) { - bb->flag = TRUE; - for (edge_t e = DLIST_HEAD (out_edge_t, bb->out_edges); e != NULL; e = DLIST_NEXT (out_edge_t, e)) - if (!e->skipped_p && !e->dst->flag) - ccp_traverse (e->dst); /* visit unvisited active edge destination */ +static mem_expr_t find_first_available_mem_expr (mem_expr_t list, bitmap_t available_mem, + int any_p) { + for (mem_expr_t curr = list; curr != NULL; curr = curr->next) + if (bitmap_bit_p (available_mem, ((bb_insn_t) curr->insn->data)->mem_index) + && (any_p || curr->insn->ops[0].mode == MIR_OP_VAR_MEM)) + return curr; + return NULL; } -static int get_ccp_res_val (gen_ctx_t gen_ctx, MIR_insn_t insn, const_t *val) { - ccp_val_t ccp_val; - MIR_op_t op; +/* Memory displacement to prefer for memory address recalculation instead. */ +#ifndef TARGET_MAX_MEM_DISP +#define TARGET_MAX_MEM_DISP 127 +#endif - if (MIR_call_code_p (insn->code) || !get_ccp_res_op (gen_ctx, insn, 0, &op)) - return FALSE; /* call results always produce varying values */ - if (!var_insn_op_p (insn, 0)) return FALSE; - ccp_val = get_ccp_val (gen_ctx, insn->data); - if (ccp_val->val_kind != CCP_CONST) return FALSE; - *val = ccp_val->val; - return TRUE; -} +#ifndef TARGET_MIN_MEM_DISP +#define TARGET_MIN_MEM_DISP -128 +#endif -static void ccp_remove_insn_ssa_edges (gen_ctx_t gen_ctx, MIR_insn_t insn) { - ssa_edge_t ssa_edge; - for (size_t i = 0; i < insn->nops; i++) { - /* output operand refers to chain of ssa edges -- remove them all: */ - while ((ssa_edge = insn->ops[i].data) != NULL) remove_ssa_edge (gen_ctx, ssa_edge); +static void remove_unreachable_bb_edges (gen_ctx_t gen_ctx, bb_t bb, VARR (bb_t) * bbs) { + bb_t dst; + edge_t e, next_e; + + VARR_TRUNC (bb_t, bbs, 0); + VARR_PUSH (bb_t, bbs, bb); + while (VARR_LENGTH (bb_t, bbs) != 0) { + bb = VARR_POP (bb_t, bbs); + DEBUG (2, { + fprintf (debug_file, " Deleting output edges of unreachable bb%lu\n", + (unsigned long) bb->index); + }); + for (e = DLIST_HEAD (out_edge_t, bb->out_edges); e != NULL; e = next_e) { + next_e = DLIST_NEXT (out_edge_t, e); + remove_edge_phi_ops (e); + dst = e->dst; + dst->flag = TRUE; /* to recalculate dst mem_av_in */ + delete_edge (e); + if (dst->index > 2 && DLIST_HEAD (in_edge_t, dst->in_edges) == NULL) + VARR_PUSH (bb_t, bbs, dst); + } } } -static int ccp_modify (gen_ctx_t gen_ctx) { +static void gvn_modify (gen_ctx_t gen_ctx) { MIR_context_t ctx = gen_ctx->ctx; - bb_t bb, next_bb; - bb_insn_t bb_insn, next_bb_insn; - const_t val; - MIR_op_t op; - MIR_insn_t insn, prev_insn, first_insn; - ssa_edge_t se, next_se; - int res, change_p = FALSE; - long deleted_insns_num = 0, deleted_branches_num = 0; + bb_t bb; + bb_insn_t bb_insn, mem_bb_insn, new_bb_insn, new_bb_insn2, next_bb_insn, expr_bb_insn; + MIR_reg_t temp_reg; + long gvn_insns_num = 0, ccp_insns_num = 0, deleted_branches_num = 0; + bitmap_t curr_available_mem = temp_bitmap, removed_mem = temp_bitmap3; + MIR_func_t func = curr_func_item->u.func; -#ifndef NDEBUG - for (bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) - gen_assert (!bb->flag); -#endif - ccp_traverse (DLIST_HEAD (bb_t, curr_cfg->bbs)); /* entry */ - for (bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = next_bb) { - next_bb = DLIST_NEXT (bb_t, bb); - if (!bb->flag) { - change_p = TRUE; - DEBUG (2, { - fprintf (debug_file, " deleting unreachable bb%lu and its edges\n", - (unsigned long) bb->index); - }); - for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; - bb_insn = next_bb_insn) { - next_bb_insn = DLIST_NEXT (bb_insn_t, bb_insn); - insn = bb_insn->insn; - ccp_remove_insn_ssa_edges (gen_ctx, insn); - gen_delete_insn (gen_ctx, insn); - deleted_insns_num++; - } - delete_bb (gen_ctx, bb); + full_escape_p = FALSE; + VARR_TRUNC (mem_attr_t, mem_attrs, 0); + bitmap_clear (removed_mem); + for (size_t i = 0; i < VARR_LENGTH (bb_t, worklist); i++) + VARR_GET (bb_t, worklist, i)->flag = FALSE; + while (VARR_LENGTH (bb_t, worklist) != 0) { + bb = VARR_POP (bb_t, worklist); + DEBUG (2, { fprintf (debug_file, " BB%lu:\n", (unsigned long) bb->index); }); + if (bb->index > 2 && DLIST_HEAD (in_edge_t, bb->in_edges) == NULL) { + /* Unreachable bb because of branch transformation: remove output edges + recursively as it shorten phis in successors and this creates more opportunity for + optimizations. But don't remove insns as their output can be used in unreachable loops + (unreachable loops will be removed in jump optimization pass). */ + remove_unreachable_bb_edges (gen_ctx, bb, pending); continue; } - bb->flag = FALSE; /* reset for the future use */ + /* Recalculate mem_avin and mem_av_out: */ + if (DLIST_HEAD (in_edge_t, bb->in_edges) != NULL && bb->flag + && mem_av_con_func_n (gen_ctx, bb)) { + DEBUG (2, { fprintf (debug_file, " changed mem_avin\n"); }); + bitmap_and_compl (bb->in, bb->in, removed_mem); + if (mem_av_trans_func (gen_ctx, bb)) { + DEBUG (2, { fprintf (debug_file, " changed mem_avout\n"); }); + for (edge_t e = DLIST_HEAD (out_edge_t, bb->out_edges); e != NULL; + e = DLIST_NEXT (out_edge_t, e)) + e->dst->flag = TRUE; + } + } + bitmap_and_compl (curr_available_mem, bb->in, removed_mem); for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; bb_insn = next_bb_insn) { + expr_t e, new_e; + mem_expr_t prev_mem_expr, mem_expr; + MIR_op_t op; + int add_def_p, const_p, cont_p; + MIR_type_t type; + MIR_insn_code_t move_code; + MIR_insn_t mem_insn, new_insn, new_insn2, def_insn, after, insn = bb_insn->insn; + ssa_edge_t se, se2; + bb_insn_t def_bb_insn, new_bb_copy_insn; + int64_t val = 0, val2; + next_bb_insn = DLIST_NEXT (bb_insn_t, bb_insn); - if (get_ccp_res_val (gen_ctx, bb_insn->insn, &val) - && (bb_insn->insn->code != MIR_MOV - || (bb_insn->insn->ops[1].mode != MIR_OP_INT - && bb_insn->insn->ops[1].mode != MIR_OP_UINT))) { - gen_assert (!MIR_call_code_p (bb_insn->insn->code)); - change_p = TRUE; - DEBUG (2, { - fprintf (debug_file, " changing insn "); - MIR_output_insn (gen_ctx->ctx, debug_file, bb_insn->insn, curr_func_item->u.func, FALSE); - }); - op = val.uns_p ? MIR_new_uint_op (ctx, val.u.u) : MIR_new_int_op (ctx, val.u.i); -#ifndef NDEBUG - { - int out_p; + if (insn->code == MIR_MOV + && (insn->ops[1].mode == MIR_OP_INT || insn->ops[1].mode == MIR_OP_UINT)) { + bb_insn->gvn_val_const_p = TRUE; + bb_insn->gvn_val = insn->ops[1].u.i; + print_bb_insn_value (gen_ctx, bb_insn); + continue; + } + if (MIR_call_code_p (insn->code)) bitmap_clear (curr_available_mem); + if (!gvn_insn_p (insn)) continue; + const_p = FALSE; + switch (insn->code) { + case MIR_PHI: + const_p = gvn_phi_val (bb_insn, &val); + if (const_p) break; + if (insn->nops == 2 && insn->ops[0].mode == MIR_OP_VAR && insn->ops[1].mode == MIR_OP_VAR + && MIR_reg_hard_reg_name (ctx, insn->ops[0].u.var - MAX_HARD_REG, func) == NULL + && MIR_reg_hard_reg_name (ctx, insn->ops[1].u.var - MAX_HARD_REG, func) == NULL) { + remove_copy (gen_ctx, insn); + continue; + } + bb_insn->gvn_val_const_p = FALSE; + bb_insn->gvn_val = val; + print_bb_insn_value (gen_ctx, bb_insn); + continue; + case MIR_EXT8: GVN_EXT (int8_t); break; + case MIR_EXT16: GVN_EXT (int16_t); break; + case MIR_EXT32: GVN_EXT (int32_t); break; + case MIR_UEXT8: GVN_EXT (uint8_t); break; + case MIR_UEXT16: GVN_EXT (uint16_t); break; + case MIR_UEXT32: GVN_EXT (uint32_t); break; + + case MIR_NEG: GVN_IOP2 (-); break; + case MIR_NEGS: GVN_IOP2S (-); break; + + case MIR_MUL: GVN_IOP3 (*); break; + case MIR_MULS: GVN_IOP3S (*); break; + + case MIR_MULO: GVN_IOP3 (*); break; + case MIR_MULOS: GVN_IOP3S (*); break; + case MIR_UMULO: GVN_UOP3 (*); break; + case MIR_UMULOS: GVN_UOP3S (*); break; + + case MIR_DIV: GVN_IOP30 (/); break; + case MIR_DIVS: GVN_IOP3S0 (/); break; + case MIR_UDIV: GVN_UOP30 (/); break; + case MIR_UDIVS: GVN_UOP3S0 (/); break; + + case MIR_MOD: GVN_IOP30 (%); break; + case MIR_MODS: GVN_IOP3S0 (%); break; + case MIR_UMOD: GVN_UOP30 (%); break; + case MIR_UMODS: + GVN_UOP3S0 (%); + break; + + /* The following insn can be involved in addres calculation too: */ + case MIR_AND: GVN_IOP3 (&); goto set_alloca_flag; + case MIR_ANDS: GVN_IOP3S (&); goto set_alloca_flag; + case MIR_OR: GVN_IOP3 (|); goto set_alloca_flag; + case MIR_ORS: GVN_IOP3S (|); goto set_alloca_flag; + case MIR_XOR: GVN_IOP3 (^); goto set_alloca_flag; + case MIR_XORS: + GVN_IOP3S (^); + set_alloca_flag: + set_alloca_based_flag (bb_insn, FALSE); + break; - MIR_insn_op_mode (ctx, bb_insn->insn, 0, &out_p); /* result here is always 0-th op */ - gen_assert (out_p); + case MIR_LSH: GVN_IOP3 (<<); break; + case MIR_LSHS: GVN_IOP3S (<<); break; + case MIR_RSH: GVN_IOP3 (>>); break; + case MIR_RSHS: GVN_IOP3S (>>); break; + case MIR_URSH: GVN_UOP3 (>>); break; + case MIR_URSHS: GVN_UOP3S (>>); break; + + case MIR_EQ: GVN_ICMP (==); break; + case MIR_EQS: GVN_ICMPS (==); break; + case MIR_NE: GVN_ICMP (!=); break; + case MIR_NES: GVN_ICMPS (!=); break; + + case MIR_LT: GVN_ICMP (<); break; + case MIR_LTS: GVN_ICMPS (<); break; + case MIR_ULT: GVN_UCMP (<); break; + case MIR_ULTS: GVN_UCMPS (<); break; + case MIR_LE: GVN_ICMP (<=); break; + case MIR_LES: GVN_ICMPS (<=); break; + case MIR_ULE: GVN_UCMP (<=); break; + case MIR_ULES: GVN_UCMPS (<=); break; + case MIR_GT: GVN_ICMP (>); break; + case MIR_GTS: GVN_ICMPS (>); break; + case MIR_UGT: GVN_UCMP (>); break; + case MIR_UGTS: GVN_UCMPS (>); break; + case MIR_GE: GVN_ICMP (>=); break; + case MIR_GES: GVN_ICMPS (>=); break; + case MIR_UGE: GVN_UCMP (>=); break; + case MIR_UGES: + GVN_UCMPS (>=); + break; + /* special treatement for address canonization: */ + case MIR_ADD: + case MIR_ADDO: + set_alloca_based_flag (bb_insn, TRUE); + GVN_IOP3 (+); + if (!const_p) goto canon_expr; + break; + case MIR_ADDS: + case MIR_ADDOS: + set_alloca_based_flag (bb_insn, TRUE); + GVN_IOP3S (+); + if (!const_p) goto canon_expr; + break; + case MIR_SUB: + case MIR_SUBO: + set_alloca_based_flag (bb_insn, TRUE); + GVN_IOP3 (-); + if (!const_p) goto canon_expr; + break; + case MIR_SUBS: + case MIR_SUBOS: + set_alloca_based_flag (bb_insn, TRUE); + GVN_IOP3S (-); + if (!const_p) goto canon_expr; + break; + canon_expr: + cont_p = TRUE; + if ((insn->code == MIR_ADD || insn->code == MIR_ADDS) && (se = insn->ops[1].data) != NULL + && se->def->gvn_val_const_p + && ((se2 = insn->ops[2].data) == NULL || !se2->def->gvn_val_const_p)) { + MIR_op_t temp = insn->ops[2]; + insn->ops[2] = insn->ops[1]; + insn->ops[1] = temp; + se->use_op_num = 2; + se2->use_op_num = 1; + DEBUG (2, { + fprintf (debug_file, " exchange ops of insn"); + MIR_output_insn (ctx, debug_file, insn, func, TRUE); + }); } -#endif - /* remove edges whose def and use is the insn, e.g. for case "5: phi a,a #index 5,5" */ - for (se = bb_insn->insn->ops[0].data; se != NULL; se = next_se) { - next_se = se->next_use; - if (se->use == bb_insn) remove_ssa_edge (gen_ctx, se); + if (add_sub_const_insn_p (gen_ctx, insn, &val2) && (se = insn->ops[1].data) != NULL + && (def_insn = skip_moves (gen_ctx, se->def->insn)) != NULL + && add_sub_const_insn_p (gen_ctx, def_insn, &val)) { + /* r1=r0+const; ... r2=r1+const2 => + temp = r0; r1=r0+const; ... r2=r1+const2;r2=temp+(const+const2): */ + temp_reg = gen_new_temp_reg (gen_ctx, MIR_T_I64, func); + op = _MIR_new_var_op (ctx, temp_reg); + new_insn = MIR_new_insn (ctx, MIR_MOV, op, def_insn->ops[1]); + new_insn->ops[1].data = NULL; + gen_add_insn_before (gen_ctx, def_insn, new_insn); + new_bb_copy_insn = new_insn->data; + se = def_insn->ops[1].data; + def_bb_insn = se->def; /* ops[1] def */ + add_ssa_edge (gen_ctx, def_bb_insn, se->def_op_num, new_bb_copy_insn, 1); + copy_gvn_info (new_bb_copy_insn, def_bb_insn); + DEBUG (2, { + fprintf (debug_file, " adding insn "); + MIR_output_insn (ctx, debug_file, new_insn, func, FALSE); + fprintf (debug_file, " before def insn "); + MIR_output_insn (ctx, debug_file, def_insn, func, TRUE); + }); + print_bb_insn_value (gen_ctx, new_bb_copy_insn); + new_insn2 = NULL; + if (insn->code == MIR_ADDS || insn->code == MIR_SUBS) { + if ((uint32_t) val + (uint32_t) val2 == 0) { + new_insn = MIR_new_insn (ctx, MIR_MOV, insn->ops[0], op); + } else { + temp_reg = gen_new_temp_reg (gen_ctx, MIR_T_I64, func); + new_insn + = MIR_new_insn (ctx, MIR_MOV, _MIR_new_var_op (ctx, temp_reg), + MIR_new_int_op (ctx, (int32_t) ((uint32_t) val + (uint32_t) val2))); + new_insn2 + = MIR_new_insn (ctx, MIR_ADDS, insn->ops[0], op, _MIR_new_var_op (ctx, temp_reg)); + } + } else { + if ((uint64_t) val + (uint64_t) val2 == 0) { + new_insn = MIR_new_insn (ctx, MIR_MOV, insn->ops[0], op); + } else { + temp_reg = gen_new_temp_reg (gen_ctx, MIR_T_I64, func); + new_insn + = MIR_new_insn (ctx, MIR_MOV, _MIR_new_var_op (ctx, temp_reg), + MIR_new_int_op (ctx, (int64_t) ((uint64_t) val + (uint64_t) val2))); + new_insn2 + = MIR_new_insn (ctx, MIR_ADD, insn->ops[0], op, _MIR_new_var_op (ctx, temp_reg)); + } + } + new_bb_insn2 = NULL; + if (new_insn2 != NULL) { + gen_add_insn_after (gen_ctx, insn, new_insn2); + new_bb_insn2 = new_insn2->data; + } + gen_add_insn_after (gen_ctx, insn, new_insn); + new_bb_insn = new_insn->data; + if (new_insn2 != NULL) { + new_bb_insn->gvn_val_const_p = TRUE; + new_bb_insn->gvn_val = new_insn->ops[1].u.i; + add_ssa_edge (gen_ctx, new_bb_insn, 0, new_bb_insn2, 2); + } + redirect_def (gen_ctx, insn, (new_insn2 != NULL ? new_insn2 : new_insn), FALSE); + add_ssa_edge (gen_ctx, new_bb_copy_insn, 0, + (new_insn2 != NULL ? new_bb_insn2 : new_bb_insn), 1); + DEBUG (2, { + fprintf (debug_file, " adding insn after:"); + MIR_output_insn (ctx, debug_file, new_insn, func, TRUE); + if (new_insn2 != NULL) { + fprintf (debug_file, " adding 2nd insn after:"); + MIR_output_insn (ctx, debug_file, new_insn2, func, TRUE); + } + }); + if (new_insn2 != NULL) { /* start with modified add */ + next_bb_insn = new_bb_insn; + continue; + } + set_alloca_based_flag (new_bb_copy_insn, TRUE); + cont_p = new_insn->code != MIR_MOV || new_insn->ops[1].mode != MIR_OP_VAR; + if (!cont_p) set_alloca_based_flag (new_bb_insn, TRUE); + insn = new_insn; /* to consider new insn next */ + bb_insn = new_bb_insn; + next_bb_insn = DLIST_NEXT (bb_insn_t, bb_insn); } - insn = MIR_new_insn (ctx, MIR_MOV, bb_insn->insn->ops[0], op); /* copy ops[0].data too! */ - MIR_insert_insn_before (ctx, curr_func_item, bb_insn->insn, insn); - bb_insn->insn->ops[0].data = NULL; - ccp_remove_insn_ssa_edges (gen_ctx, bb_insn->insn); - MIR_remove_insn (ctx, curr_func_item, bb_insn->insn); - insn->data = bb_insn; - bb_insn->insn = insn; + if (cont_p) break; + /* fall through */ + case MIR_MOV: + case MIR_FMOV: + case MIR_DMOV: + case MIR_LDMOV: + if (insn->ops[0].mode == MIR_OP_VAR_MEM) { /* store */ + if ((se = insn->ops[1].data) != NULL && se->def->alloca_flag) full_escape_p = TRUE; + se = insn->ops[0].data; /* address def actually */ + mem_expr = find_mem_expr (gen_ctx, insn); + prev_mem_expr = find_first_available_mem_expr (mem_expr, curr_available_mem, FALSE); + /* If we can reach prev available store bb from itself w/o going through given bb then + it means it can be stores with different addresses and we just have the same + memory only for the last store and can not make dead store in prev expr bb. It + is also not worth to reuse stored value as it will create a move from some loop + containing prev expr bb and not containing given bb. Make new memory for such + case. */ + int new_mem_p + = (prev_mem_expr != NULL + && cycle_without_bb_visit_p (gen_ctx, ((bb_insn_t) prev_mem_expr->insn->data)->bb, + bb)); + prev_mem_expr = find_first_available_mem_expr (mem_expr, curr_available_mem, TRUE); + def_bb_insn = ((ssa_edge_t) insn->ops[1].data)->def; + if (new_mem_p || prev_mem_expr == NULL) { + new_mem_loc (gen_ctx, &insn->ops[0], se->def->alloca_flag); + } else if (prev_mem_expr->insn->ops[0].mode + == MIR_OP_VAR_MEM) { /* mem = x; ... ; mem=y */ + insn->ops[0].u.var_mem.nloc = prev_mem_expr->insn->ops[0].u.var_mem.nloc; + update_mem_loc_alloca_flag (gen_ctx, insn->ops[0].u.var_mem.nloc, se->def->alloca_flag); + } else { /* x = mem; ...; mem = y */ + gen_assert (prev_mem_expr->insn->ops[1].mode == MIR_OP_VAR_MEM); + insn->ops[0].u.var_mem.nloc = prev_mem_expr->insn->ops[1].u.var_mem.nloc; + update_mem_loc_alloca_flag (gen_ctx, insn->ops[0].u.var_mem.nloc, se->def->alloca_flag); + bb_insn_t prev_bb_insn = prev_mem_expr->insn->data; + if (def_bb_insn->gvn_val_const_p == prev_bb_insn->gvn_val_const_p + && def_bb_insn->gvn_val == prev_bb_insn->gvn_val) { /* x == y: remove insn */ + gen_assert (def_bb_insn->alloca_flag == prev_bb_insn->alloca_flag); + DEBUG (2, { + fprintf (debug_file, " deleting "); + print_bb_insn (gen_ctx, insn->data, TRUE); + }); + bitmap_clear_bit_p (curr_available_mem, bb_insn->mem_index); + bitmap_set_bit_p (removed_mem, bb_insn->mem_index); + ssa_delete_insn (gen_ctx, insn); + continue; + } + } + add_mem_insn (gen_ctx, insn); + update_mem_availability (gen_ctx, curr_available_mem, bb_insn); + copy_gvn_info (bb_insn, def_bb_insn); + print_bb_insn_value (gen_ctx, bb_insn); + continue; + } else if (insn->ops[1].mode == MIR_OP_VAR_MEM) { /* load */ + if (insn->ops[0].data == NULL) continue; /* dead load */ + se = insn->ops[1].data; /* address def actually */ + mem_expr = find_mem_expr (gen_ctx, insn); + mem_expr = find_first_available_mem_expr (mem_expr, curr_available_mem, TRUE); + if (mem_expr == NULL) { + new_mem_loc (gen_ctx, &insn->ops[1], se->def->alloca_flag); + add_mem_insn (gen_ctx, insn); + } else { + MIR_op_t *op_ref; + mem_insn = mem_expr->insn; + op_ref + = mem_insn->ops[0].mode == MIR_OP_VAR_MEM ? &mem_insn->ops[0] : &mem_insn->ops[1]; + insn->ops[1].u.var_mem.nloc = op_ref->u.var_mem.nloc; + update_mem_loc_alloca_flag (gen_ctx, op_ref->u.var_mem.nloc, se->def->alloca_flag); + if (!bitmap_bit_p (curr_available_mem, (mem_bb_insn = mem_insn->data)->mem_index) + /* last available load can become dead: */ + || (mem_insn->ops[1].mode == MIR_OP_VAR_MEM && mem_insn->ops[0].data == NULL)) { + add_mem_insn (gen_ctx, insn); + } else { /* (mem=x|x=mem); ...; r=mem => (mem=x|x=mem); t=x; ...; r=t */ + copy_gvn_info (bb_insn, mem_bb_insn); + print_bb_insn_value (gen_ctx, bb_insn); + temp_reg = mem_expr->temp_reg; + add_def_p = temp_reg == MIR_NON_VAR; + if (add_def_p) { + mem_expr->temp_reg = temp_reg + = get_expr_temp_reg (gen_ctx, mem_expr->insn, &mem_expr->temp_reg); + new_insn = MIR_new_insn (ctx, insn->code, _MIR_new_var_op (ctx, temp_reg), + op_ref == &mem_insn->ops[0] ? mem_insn->ops[1] + : mem_insn->ops[0]); + new_insn->ops[1].data = NULL; /* remove ssa edge taken from load/store op */ + gen_add_insn_after (gen_ctx, mem_insn, new_insn); + new_bb_insn = new_insn->data; + copy_gvn_info (new_bb_insn, mem_bb_insn); + se = op_ref == &mem_insn->ops[0] ? mem_insn->ops[1].data : mem_insn->ops[0].data; + add_ssa_edge (gen_ctx, se->def, se->def_op_num, new_bb_insn, 1); + DEBUG (2, { + fprintf (debug_file, " adding insn "); + MIR_output_insn (ctx, debug_file, new_insn, func, FALSE); + fprintf (debug_file, " after def insn "); + MIR_output_insn (ctx, debug_file, mem_insn, func, TRUE); + }); + } + bitmap_clear_bit_p (curr_available_mem, bb_insn->mem_index); + bitmap_set_bit_p (removed_mem, bb_insn->mem_index); + remove_ssa_edge ((ssa_edge_t) insn->ops[1].data); + insn->ops[1] = _MIR_new_var_op (ctx, temp_reg); /* changing mem */ + def_insn = DLIST_NEXT (MIR_insn_t, mem_insn); + add_ssa_edge (gen_ctx, def_insn->data, 0, bb_insn, 1); + gvn_insns_num++; + DEBUG (2, { + fprintf (debug_file, " changing curr insn to "); + MIR_output_insn (ctx, debug_file, insn, func, TRUE); + }); + continue; + } + } + update_mem_availability (gen_ctx, curr_available_mem, bb_insn); + } else if (move_p (insn) && (se = insn->ops[1].data) != NULL && !fake_insn_p (se->def) + && (se = se->def->insn->ops[se->def_op_num].data) != NULL && se->next_use == NULL + && MIR_reg_hard_reg_name (ctx, insn->ops[0].u.var - MAX_HARD_REG, func) == NULL + && MIR_reg_hard_reg_name (ctx, insn->ops[1].u.var - MAX_HARD_REG, func) + == NULL) { + /* one source for definition: remove copy */ + gen_assert (se->use == bb_insn && se->use_op_num == 1); + remove_copy (gen_ctx, insn); + continue; + } + break; + case MIR_BT: + case MIR_BTS: const_p = get_gvn_op (insn, 1, &val); break; + case MIR_BF: + case MIR_BFS: + const_p = get_gvn_op (insn, 1, &val); + if (const_p) val = !val; + break; + case MIR_BEQ: GVN_ICMP (==); break; + case MIR_BEQS: GVN_ICMPS (==); break; + case MIR_BNE: GVN_ICMP (!=); break; + case MIR_BNES: GVN_ICMPS (!=); break; + + case MIR_BLT: GVN_ICMP (<); break; + case MIR_BLTS: GVN_ICMPS (<); break; + case MIR_UBLT: GVN_UCMP (<); break; + case MIR_UBLTS: GVN_UCMPS (<); break; + case MIR_BLE: GVN_ICMP (<=); break; + case MIR_BLES: GVN_ICMPS (<=); break; + case MIR_UBLE: GVN_UCMP (<=); break; + case MIR_UBLES: GVN_UCMPS (<=); break; + case MIR_BGT: GVN_ICMP (>); break; + case MIR_BGTS: GVN_ICMPS (>); break; + case MIR_UBGT: GVN_UCMP (>); break; + case MIR_UBGTS: GVN_UCMPS (>); break; + case MIR_BGE: GVN_ICMP (>=); break; + case MIR_BGES: GVN_ICMPS (>=); break; + case MIR_UBGE: GVN_UCMP (>=); break; + case MIR_UBGES: GVN_UCMPS (>=); break; + default: break; + } + if (const_p) { + ccp_insns_num++; + print_bb_insn_value (gen_ctx, bb_insn); + if (MIR_any_branch_code_p (insn->code)) { + gen_assert (insn->code != MIR_SWITCH); + if (val == 0) { + DEBUG (2, { + fprintf (debug_file, " removing branch insn "); + MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, TRUE); + fprintf (debug_file, "\n"); + }); + ssa_delete_insn (gen_ctx, insn); + edge_t edge = DLIST_EL (out_edge_t, bb->out_edges, 1); + remove_edge_phi_ops (edge); + edge->dst->flag = TRUE; /* to recalculate dst mem_av_in */ + delete_edge (edge); + deleted_branches_num++; + } else { + new_insn = MIR_new_insn (ctx, MIR_JMP, insn->ops[0]); /* label is always 0-th op */ + DEBUG (2, { + fprintf (debug_file, " changing branch insn "); + MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, FALSE); + fprintf (debug_file, " onto jump insn "); + MIR_output_insn (ctx, debug_file, new_insn, curr_func_item->u.func, TRUE); + fprintf (debug_file, "\n"); + }); + MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + remove_insn_ssa_edges (insn); + MIR_remove_insn (ctx, curr_func_item, insn); + new_insn->data = bb_insn; + bb_insn->insn = new_insn; + edge_t edge = DLIST_EL (out_edge_t, bb->out_edges, 0); + remove_edge_phi_ops (edge); + edge->dst->flag = TRUE; /* to recalculate dst mem_av_in */ + delete_edge (edge); + } + } else { /* x=... and x is const => x=...; x=const */ + new_insn = MIR_new_insn (ctx, MIR_MOV, insn->ops[0], MIR_new_int_op (ctx, val)); + after = insn->code == MIR_PHI ? get_last_bb_phi_insn (insn) : insn; + gen_add_insn_after (gen_ctx, after, new_insn); + new_bb_insn = new_insn->data; + redirect_def (gen_ctx, insn, new_insn, FALSE); + new_bb_insn->gvn_val_const_p = TRUE; + new_bb_insn->gvn_val = val; + gvn_insns_num++; + DEBUG (2, { + fprintf (debug_file, " Adding insn after:"); + MIR_output_insn (ctx, debug_file, new_insn, func, TRUE); + }); + print_bb_insn_value (gen_ctx, new_bb_insn); + } + continue; + } + if (MIR_any_branch_code_p (insn->code) || insn->code == MIR_PHI) continue; + e = NULL; + if (move_p (insn)) { + def_bb_insn = ((ssa_edge_t) insn->ops[1].data)->def; + copy_gvn_info (bb_insn, def_bb_insn); + } else if (!MIR_overflow_insn_code_p (insn->code)) { + /* r=e; ...; x=e => r=e; t=r; ...; x=e; x=t */ + if (!find_expr (gen_ctx, insn, &e)) { + e = add_expr (gen_ctx, insn, FALSE); + DEBUG (2, { print_expr (gen_ctx, e, "Adding"); }); + } else if (move_code_p (insn->code) && insn->ops[1].mode == MIR_OP_VAR_MEM + && !bitmap_bit_p (curr_available_mem, ((bb_insn_t) e->insn->data)->mem_index)) { + e = add_expr (gen_ctx, insn, TRUE); + DEBUG (2, { print_expr (gen_ctx, e, "Replacing"); }); + } + bb_insn->gvn_val_const_p = FALSE; + bb_insn->gvn_val = e->num; + bb_insn->alloca_flag = ((bb_insn_t) e->insn->data)->alloca_flag; + } + print_bb_insn_value (gen_ctx, bb_insn); + if (e == NULL || e->insn == insn || (imm_move_p (insn) && insn->ops[1].mode != MIR_OP_REF)) + continue; + if (MIR_addr_code_p (insn->code)) { + continue; + } else if ((insn->code == MIR_ADD || insn->code == MIR_SUB) + && (se = insn->ops[0].data) != NULL && se->next_use == NULL + && se->use->insn->ops[se->use_op_num].mode == MIR_OP_VAR_MEM + && (((se2 = insn->ops[2].data) != NULL && imm_move_p (se2->def->insn)) + || (insn->code == MIR_ADD && (se2 = insn->ops[1].data) != NULL + && imm_move_p (se2->def->insn)))) { + /* Do not recalculate reg + const if it is only used in address: */ + int64_t disp = se2->def->insn->ops[1].u.i; + if (insn->code == MIR_SUB) disp = -disp; + if (TARGET_MIN_MEM_DISP <= disp && disp <= TARGET_MAX_MEM_DISP) continue; + } + expr_bb_insn = e->insn->data; + if (bb->index != expr_bb_insn->bb->index + && !bitmap_bit_p (bb->dom_in, expr_bb_insn->bb->index)) + continue; + add_def_p = e->temp_reg == MIR_NON_VAR; + temp_reg = get_expr_temp_reg (gen_ctx, e->insn, &e->temp_reg); + op = _MIR_new_var_op (ctx, temp_reg); + type = MIR_reg_type (ctx, temp_reg - MAX_HARD_REG, func); +#ifndef NDEBUG + int out_p; + MIR_insn_op_mode (ctx, insn, 0, &out_p); /* result here is always 0-th op */ + gen_assert (out_p); +#endif + move_code = get_move_code (type); + if (add_def_p) { + gen_assert (e->insn->ops[0].mode == MIR_OP_VAR); + new_insn = MIR_new_insn (ctx, move_code, op, _MIR_new_var_op (ctx, e->insn->ops[0].u.var)); + gen_add_insn_after (gen_ctx, e->insn, new_insn); + new_bb_insn = new_insn->data; + redirect_def (gen_ctx, e->insn, new_insn, TRUE); + if (!find_expr (gen_ctx, new_insn, &new_e)) new_e = add_expr (gen_ctx, new_insn, FALSE); + new_bb_insn->gvn_val_const_p = FALSE; + new_bb_insn->gvn_val = e->num; + new_bb_insn->alloca_flag = ((bb_insn_t) e->insn->data)->alloca_flag; DEBUG (2, { - fprintf (debug_file, " on insn "); - MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, TRUE); + fprintf (debug_file, " adding insn "); + MIR_output_insn (ctx, debug_file, new_insn, func, FALSE); + fprintf (debug_file, " after def insn "); + MIR_output_insn (ctx, debug_file, e->insn, func, TRUE); }); } - } - if ((bb_insn = DLIST_TAIL (bb_insn_t, bb->bb_insns)) == NULL) continue; - insn = bb_insn->insn; - first_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns)->insn; - if (first_insn->code == MIR_LABEL && (prev_insn = DLIST_PREV (MIR_insn_t, first_insn)) != NULL - && prev_insn->code == MIR_JMP && prev_insn->ops[0].u.label == first_insn) { - DEBUG (2, { - fprintf (debug_file, " removing useless jump insn "); - MIR_output_insn (ctx, debug_file, prev_insn, curr_func_item->u.func, TRUE); - fprintf (debug_file, "\n"); - }); - ccp_remove_insn_ssa_edges (gen_ctx, prev_insn); - gen_delete_insn (gen_ctx, prev_insn); - deleted_branches_num++; - } - if (!MIR_branch_code_p (insn->code) || insn->code == MIR_JMP || insn->code == MIR_SWITCH - || ccp_branch_update (gen_ctx, insn, &res) != CCP_CONST) - continue; - change_p = TRUE; - if (!res) { - edge_t e; - - DEBUG (2, { - fprintf (debug_file, " removing branch insn "); - MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, TRUE); - fprintf (debug_file, "\n"); - }); - ccp_remove_insn_ssa_edges (gen_ctx, insn); - gen_delete_insn (gen_ctx, insn); - if ((e = DLIST_EL (out_edge_t, bb->out_edges, 1)) != NULL) - delete_edge (e); /* e can be arleady deleted by previous removing an unreachable BB */ - deleted_branches_num++; - } else { - insn = MIR_new_insn (ctx, MIR_JMP, insn->ops[0]); /* label is always 0-th op */ + new_insn = MIR_new_insn (ctx, move_code, insn->ops[0], op); + gen_add_insn_after (gen_ctx, insn, new_insn); + new_bb_insn = new_insn->data; + redirect_def (gen_ctx, insn, new_insn, FALSE); + def_insn = DLIST_NEXT (MIR_insn_t, e->insn); + add_ssa_edge (gen_ctx, def_insn->data, 0, new_insn->data, 1); + if (!find_expr (gen_ctx, new_insn, &new_e)) new_e = add_expr (gen_ctx, new_insn, FALSE); + new_bb_insn->gvn_val_const_p = FALSE; + new_bb_insn->gvn_val = e->num; + new_bb_insn->alloca_flag = ((bb_insn_t) e->insn->data)->alloca_flag; + gvn_insns_num++; DEBUG (2, { - fprintf (debug_file, " changing branch insn "); - MIR_output_insn (ctx, debug_file, bb_insn->insn, curr_func_item->u.func, FALSE); - fprintf (debug_file, " onto jump insn "); - MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, TRUE); - fprintf (debug_file, "\n"); + fprintf (debug_file, " adding insn "); + MIR_output_insn (ctx, debug_file, new_insn, func, FALSE); + fprintf (debug_file, " after use insn "); + MIR_output_insn (ctx, debug_file, insn, func, TRUE); }); - MIR_insert_insn_before (ctx, curr_func_item, bb_insn->insn, insn); - ccp_remove_insn_ssa_edges (gen_ctx, bb_insn->insn); - MIR_remove_insn (ctx, curr_func_item, bb_insn->insn); - insn->data = bb_insn; - bb_insn->insn = insn; - delete_edge (DLIST_EL (out_edge_t, bb->out_edges, 0)); } } DEBUG (1, { - fprintf (debug_file, "%5ld deleted CCP insns + %ld deleted branches\n", deleted_insns_num, - deleted_branches_num); + fprintf (debug_file, + "%5ld found GVN redundant insns, %ld ccp insns, %ld deleted " + "branches\n", + gvn_insns_num, ccp_insns_num, deleted_branches_num); }); - return change_p; } -static int ccp (gen_ctx_t gen_ctx) { /* conditional constant propagation */ - DEBUG (2, { fprintf (debug_file, " CCP analysis:\n"); }); - curr_ccp_run++; - bb_visited = temp_bitmap; - initiate_ccp_info (gen_ctx); - while (VARR_LENGTH (bb_t, ccp_bbs) != 0 || VARR_LENGTH (bb_insn_t, ccp_insns) != 0) { - while (VARR_LENGTH (bb_t, ccp_bbs) != 0) { - bb_t bb = VARR_POP (bb_t, ccp_bbs); - - bb->flag = FALSE; - ccp_process_bb (gen_ctx, bb); - } - while (VARR_LENGTH (bb_insn_t, ccp_insns) != 0) { - bb_insn_t bb_insn = VARR_POP (bb_insn_t, ccp_insns); - - gen_assert (bb_insn->flag); - bb_insn->flag = FALSE; - ccp_process_insn (gen_ctx, bb_insn); - } - } - DEBUG (2, { fprintf (debug_file, " CCP modification:\n"); }); - return ccp_modify (gen_ctx); +static void gvn (gen_ctx_t gen_ctx) { + calculate_memory_availability (gen_ctx); + calculate_dominators (gen_ctx); + VARR_TRUNC (bb_t, worklist, 0); + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) + VARR_PUSH (bb_t, worklist, bb); + qsort (VARR_ADDR (bb_t, worklist), VARR_LENGTH (bb_t, worklist), sizeof (bb_t), post_cmp); + gvn_modify (gen_ctx); } -static void init_ccp (gen_ctx_t gen_ctx) { - gen_ctx->ccp_ctx = gen_malloc (gen_ctx, sizeof (struct ccp_ctx)); - curr_ccp_run = 0; - VARR_CREATE (bb_t, ccp_bbs, 256); - VARR_CREATE (bb_insn_t, ccp_insns, 256); - VARR_CREATE (ccp_val_t, ccp_vals, 256); +static void gvn_clear (gen_ctx_t gen_ctx) { + HTAB_CLEAR (expr_t, expr_tab); + while (VARR_LENGTH (expr_t, exprs) != 0) free (VARR_POP (expr_t, exprs)); + HTAB_CLEAR (mem_expr_t, mem_expr_tab); + while (VARR_LENGTH (mem_expr_t, mem_exprs) != 0) free (VARR_POP (mem_expr_t, mem_exprs)); } -static void finish_ccp (gen_ctx_t gen_ctx) { - ccp_val_t ccp_val; +static void init_gvn (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; - VARR_DESTROY (bb_t, ccp_bbs); - VARR_DESTROY (bb_insn_t, ccp_insns); - while (VARR_LENGTH (ccp_val_t, ccp_vals) != 0) - if ((ccp_val = VARR_POP (ccp_val_t, ccp_vals)) != NULL) free (ccp_val); - VARR_DESTROY (ccp_val_t, ccp_vals); - free (gen_ctx->ccp_ctx); - gen_ctx->ccp_ctx = NULL; + gen_ctx->gvn_ctx = gen_malloc (gen_ctx, sizeof (struct gvn_ctx)); + VARR_CREATE (expr_t, exprs, 512); + HTAB_CREATE (expr_t, expr_tab, 1024, expr_hash, expr_eq, gen_ctx); + temp_mem_insn + = MIR_new_insn (ctx, MIR_MOV, + _MIR_new_var_mem_op (ctx, MIR_T_I64, 0, MIR_NON_VAR, MIR_NON_VAR, 0), + _MIR_new_var_op (ctx, 0)); + VARR_CREATE (mem_expr_t, mem_exprs, 256); + HTAB_CREATE (mem_expr_t, mem_expr_tab, 512, mem_expr_hash, mem_expr_eq, gen_ctx); + VARR_CREATE (insn_nop_pair_t, insn_nop_pairs, 16); } -#undef live_in -#undef live_out +static void finish_gvn (gen_ctx_t gen_ctx) { + VARR_DESTROY (expr_t, exprs); + HTAB_DESTROY (expr_t, expr_tab); + free (temp_mem_insn); /* ??? */ + VARR_DESTROY (mem_expr_t, mem_exprs); + HTAB_DESTROY (mem_expr_t, mem_expr_tab); + VARR_DESTROY (insn_nop_pair_t, insn_nop_pairs); + free (gen_ctx->gvn_ctx); + gen_ctx->gvn_ctx = NULL; +} -/* New Page */ +/* New page */ -#define live_in in -#define live_out out -#define live_kill kill -#define live_gen gen +/* Dead store elimination */ -/* Life analysis */ -static void live_con_func_0 (bb_t bb) { bitmap_clear (bb->live_in); } +#define mem_live_in in +#define mem_live_out out +#define mem_live_gen gen +#define mem_live_kill kill -static int live_con_func_n (gen_ctx_t gen_ctx, bb_t bb) { +static void mem_live_con_func_0 (bb_t bb) { + if (bb->index != 1) bitmap_clear (bb->mem_live_in); +} + +static int mem_live_con_func_n (gen_ctx_t gen_ctx MIR_UNUSED, bb_t bb) { edge_t e; int change_p = FALSE; for (e = DLIST_HEAD (out_edge_t, bb->out_edges); e != NULL; e = DLIST_NEXT (out_edge_t, e)) - change_p |= bitmap_ior (bb->live_out, bb->live_out, e->dst->live_in); + change_p |= bitmap_ior (bb->mem_live_out, bb->mem_live_out, e->dst->mem_live_in); return change_p; } -static int live_trans_func (gen_ctx_t gen_ctx, bb_t bb) { - return bitmap_ior_and_compl (bb->live_in, bb->live_gen, bb->live_out, bb->live_kill); +static int mem_live_trans_func (gen_ctx_t gen_ctx MIR_UNUSED, bb_t bb) { + return bitmap_ior_and_compl (bb->mem_live_in, bb->mem_live_gen, bb->mem_live_out, + bb->mem_live_kill); } -static int bb_loop_level (bb_t bb) { - loop_node_t loop_node; - int level = -1; +static int alloca_arg_p (gen_ctx_t gen_ctx MIR_UNUSED, MIR_insn_t call_insn) { + MIR_proto_t proto; + ssa_edge_t se; - for (loop_node = bb->loop_node; loop_node->parent != NULL; loop_node = loop_node->parent) level++; - gen_assert (level >= 0); - return level; + gen_assert (MIR_call_code_p (call_insn->code) && call_insn->ops[0].mode == MIR_OP_REF + && call_insn->ops[0].u.ref->item_type == MIR_proto_item); + proto = call_insn->ops[0].u.ref->u.proto; + for (size_t i = proto->nres + 1; i < call_insn->nops; i++) { + if (call_insn->ops[i].mode != MIR_OP_VAR && call_insn->ops[i].mode != MIR_OP_VAR_MEM) continue; + if ((se = call_insn->ops[i].data) == NULL) continue; + if ((se->def->alloca_flag & MUST_ALLOCA) || (se->def->alloca_flag & MAY_ALLOCA)) return TRUE; + } + return FALSE; } -static void increase_pressure (int int_p, bb_t bb, int *int_pressure, int *fp_pressure) { - if (int_p) { - if (bb->max_int_pressure < ++(*int_pressure)) bb->max_int_pressure = *int_pressure; +static void update_call_mem_live (gen_ctx_t gen_ctx, bitmap_t mem_live, MIR_insn_t call_insn) { + gen_assert (MIR_call_code_p (call_insn->code)); + gen_assert (call_insn->ops[0].mode == MIR_OP_REF + && call_insn->ops[0].u.ref->item_type == MIR_proto_item); + if (full_escape_p || alloca_arg_p (gen_ctx, call_insn)) { + bitmap_set_bit_range_p (mem_live, 1, VARR_LENGTH (mem_attr_t, mem_attrs)); } else { - if (bb->max_fp_pressure < ++(*fp_pressure)) bb->max_fp_pressure = *fp_pressure; + mem_attr_t *mem_attr_addr = VARR_ADDR (mem_attr_t, mem_attrs); + + for (size_t i = 1; i < VARR_LENGTH (mem_attr_t, mem_attrs); i++) + if (!mem_attr_addr[i].alloca_flag) bitmap_set_bit_p (mem_live, i); } } -static int int_var_type_p (gen_ctx_t gen_ctx, MIR_reg_t var) { - if (!var_is_reg_p (var)) return target_hard_reg_type_ok_p (var, MIR_T_I32); - return MIR_int_type_p ( - MIR_reg_type (gen_ctx->ctx, var2reg (gen_ctx, var), curr_func_item->u.func)); +static int alloca_mem_intersect_p (gen_ctx_t gen_ctx, uint32_t nloc1, MIR_type_t type1, + uint32_t nloc2, MIR_type_t type2) { + MIR_context_t ctx = gen_ctx->ctx; + mem_attr_t *mem_attr_ref1 = &VARR_ADDR (mem_attr_t, mem_attrs)[nloc1]; + mem_attr_t *mem_attr_ref2 = &VARR_ADDR (mem_attr_t, mem_attrs)[nloc2]; + int64_t disp1, disp2, size1, size2; + + gen_assert (nloc1 != 0 && nloc2 != 0); + if (!mem_attr_ref1->disp_def_p || !mem_attr_ref2->disp_def_p) return TRUE; + if (mem_attr_ref1->def_insn == NULL || mem_attr_ref1->def_insn != mem_attr_ref2->def_insn) + return TRUE; + disp1 = mem_attr_ref1->disp; + disp2 = mem_attr_ref2->disp; + size1 = _MIR_type_size (ctx, type1); + size2 = _MIR_type_size (ctx, type2); + if (disp2 <= disp1 && disp1 < disp2 + size2) return TRUE; + return disp1 <= disp2 && disp2 < disp1 + size1; +} + +static void make_live_from_mem (gen_ctx_t gen_ctx, MIR_op_t *mem_ref, bitmap_t gen, bitmap_t kill, + int must_alloca_p) { + mem_attr_t *mem_attr_addr = VARR_ADDR (mem_attr_t, mem_attrs); + + gen_assert (mem_ref->mode == MIR_OP_VAR_MEM); + for (size_t i = 1; i < VARR_LENGTH (mem_attr_t, mem_attrs); i++) { + if (!may_alias_p (mem_ref->u.var_mem.alias, mem_attr_addr[i].alias, mem_ref->u.var_mem.nonalias, + mem_attr_addr[i].nonalias)) + continue; + if (must_alloca_p && (mem_attr_addr[i].alloca_flag & MUST_ALLOCA) + && !alloca_mem_intersect_p (gen_ctx, mem_ref->u.var_mem.nloc, mem_ref->u.var_mem.type, + (uint32_t) i, mem_attr_addr[i].type)) + continue; + /* all aliased but unintersected must alloca: */ + bitmap_set_bit_p (gen, i); + if (kill != NULL) bitmap_clear_bit_p (kill, i); + } } -static MIR_insn_t initiate_bb_live_info (gen_ctx_t gen_ctx, MIR_insn_t bb_tail_insn, int moves_p, - uint32_t *mvs_num) { +static MIR_insn_t initiate_bb_mem_live_info (gen_ctx_t gen_ctx, MIR_insn_t bb_tail_insn) { bb_t bb = get_insn_bb (gen_ctx, bb_tail_insn); MIR_insn_t insn; - size_t niter, passed_mem_num, bb_freq; - MIR_reg_t var, early_clobbered_hard_reg1, early_clobbered_hard_reg2; - int op_num, out_p, mem_p, int_p = FALSE; - int bb_int_pressure, bb_fp_pressure; - mv_t mv; - reg_info_t *breg_infos; - insn_var_iterator_t insn_var_iter; + ssa_edge_t se; + uint32_t nloc; - gen_assert (!moves_p || optimize_level != 0); - breg_infos = VARR_ADDR (reg_info_t, curr_cfg->breg_info); - bb_freq = 1; - if (moves_p) - for (int i = bb_loop_level (bb); i > 0; i--) - if (bb_freq < SIZE_MAX / 8) bb_freq *= 5; - bb->max_int_pressure = bb_int_pressure = bb->max_fp_pressure = bb_fp_pressure = 0; for (insn = bb_tail_insn; insn != NULL && get_insn_bb (gen_ctx, insn) == bb; insn = DLIST_PREV (MIR_insn_t, insn)) { - if (MIR_call_code_p (insn->code)) { - bitmap_ior (bb->live_kill, bb->live_kill, call_used_hard_regs[MIR_T_UNDEF]); - bitmap_and_compl (bb->live_gen, bb->live_gen, call_used_hard_regs[MIR_T_UNDEF]); - } - /* Process output ops on 0-th iteration, then input ops. */ - for (niter = 0; niter <= 1; niter++) { - FOREACH_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (!out_p && niter != 0) { - if (bitmap_set_bit_p (bb->live_gen, var) && optimize_level != 0) - increase_pressure (int_var_type_p (gen_ctx, var), bb, &bb_int_pressure, - &bb_fp_pressure); - } else if (niter == 0) { - if (bitmap_clear_bit_p (bb->live_gen, var) && optimize_level != 0) - (int_var_type_p (gen_ctx, var) ? bb_int_pressure-- : bb_fp_pressure--); - bitmap_set_bit_p (bb->live_kill, var); - } - if (var_is_reg_p (var)) { - if (breg_infos[var2breg (gen_ctx, var)].freq < LONG_MAX - bb_freq) - breg_infos[var2breg (gen_ctx, var)].freq += bb_freq; - else - breg_infos[var2breg (gen_ctx, var)].freq = LONG_MAX; - } - } - } - target_get_early_clobbered_hard_regs (insn, &early_clobbered_hard_reg1, - &early_clobbered_hard_reg2); - if (early_clobbered_hard_reg1 != MIR_NON_HARD_REG) { - if (optimize_level != 0) { - int_p = int_var_type_p (gen_ctx, early_clobbered_hard_reg1); - increase_pressure (int_p, bb, &bb_int_pressure, &bb_fp_pressure); + if (MIR_call_code_p (insn->code)) update_call_mem_live (gen_ctx, bb->mem_live_gen, insn); + if (!move_code_p (insn->code)) continue; + if (insn->ops[0].mode == MIR_OP_VAR_MEM) { /* store */ + if ((nloc = insn->ops[0].u.var_mem.nloc) != 0) { + bitmap_clear_bit_p (bb->mem_live_gen, nloc); + bitmap_set_bit_p (bb->mem_live_kill, nloc); } - bitmap_clear_bit_p (bb->live_gen, early_clobbered_hard_reg1); - bitmap_set_bit_p (bb->live_kill, early_clobbered_hard_reg1); - if (optimize_level != 0) (int_p ? bb_int_pressure-- : bb_fp_pressure--); - } - if (early_clobbered_hard_reg2 != MIR_NON_HARD_REG) { - if (optimize_level != 0) { - int_p = int_var_type_p (gen_ctx, early_clobbered_hard_reg2); - increase_pressure (int_p, bb, &bb_int_pressure, &bb_fp_pressure); + } else if (insn->ops[1].mode == MIR_OP_VAR_MEM) { /* load */ + if ((nloc = insn->ops[1].u.var_mem.nloc) != 0) { + bitmap_set_bit_p (bb->mem_live_gen, nloc); + bitmap_clear_bit_p (bb->mem_live_kill, nloc); + se = insn->ops[1].data; + make_live_from_mem (gen_ctx, &insn->ops[1], bb->mem_live_gen, bb->mem_live_kill, + se != NULL && (se->def->alloca_flag & MUST_ALLOCA)); + } else { + bitmap_set_bit_range_p (bb->mem_live_gen, 1, VARR_LENGTH (mem_attr_t, mem_attrs)); } - bitmap_clear_bit_p (bb->live_gen, early_clobbered_hard_reg2); - bitmap_set_bit_p (bb->live_kill, early_clobbered_hard_reg2); - if (optimize_level != 0) (int_p ? bb_int_pressure-- : bb_fp_pressure--); - } - if (MIR_call_code_p (insn->code)) { - bitmap_t reg_args; - - if (optimize_level != 0) - bitmap_ior (bb->live_gen, bb->live_gen, ((bb_insn_t) insn->data)->call_hard_reg_args); - else if ((reg_args = ((insn_data_t) insn->data)->u.call_hard_reg_args) != NULL) - bitmap_ior (bb->live_gen, bb->live_gen, reg_args); - } - if (moves_p && move_p (insn)) { - mv = get_free_move (gen_ctx); - mv->bb_insn = insn->data; - mv->freq = bb_freq; - if (insn->ops[0].mode == MIR_OP_REG) - DLIST_APPEND (dst_mv_t, breg_infos[reg2breg (gen_ctx, insn->ops[0].u.reg)].dst_moves, mv); - if (insn->ops[1].mode == MIR_OP_REG) - DLIST_APPEND (src_mv_t, breg_infos[reg2breg (gen_ctx, insn->ops[1].u.reg)].src_moves, mv); - (*mvs_num)++; - DEBUG (2, { - fprintf (debug_file, " move with freq %10lu:", (unsigned long) mv->freq); - MIR_output_insn (gen_ctx->ctx, debug_file, insn, curr_func_item->u.func, TRUE); - }); } } return insn; } -static void initiate_live_info (gen_ctx_t gen_ctx, int moves_p) { - MIR_reg_t nregs, n; - mv_t mv, next_mv; - reg_info_t ri; - uint32_t mvs_num = 0; +static void initiate_mem_live_info (gen_ctx_t gen_ctx) { + bb_t exit_bb = DLIST_EL (bb_t, curr_cfg->bbs, 1); + mem_attr_t *mem_attr_addr; - for (mv = DLIST_HEAD (mv_t, curr_cfg->used_moves); mv != NULL; mv = next_mv) { - next_mv = DLIST_NEXT (mv_t, mv); - free_move (gen_ctx, mv); - } - VARR_TRUNC (reg_info_t, curr_cfg->breg_info, 0); - nregs = get_nregs (gen_ctx); - for (n = 0; n < nregs; n++) { - ri.freq = ri.thread_freq = 0; - ri.live_length = 0; - ri.thread_first = n; - ri.thread_next = MIR_MAX_REG_NUM; - DLIST_INIT (dst_mv_t, ri.dst_moves); - DLIST_INIT (src_mv_t, ri.src_moves); - VARR_PUSH (reg_info_t, curr_cfg->breg_info, ri); - } for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { - gen_assert (bb != NULL && bb->live_in != NULL && bb->live_out != NULL && bb->live_gen != NULL - && bb->live_kill != NULL); - bitmap_clear (bb->live_in); - bitmap_clear (bb->live_out); - bitmap_clear (bb->live_gen); - bitmap_clear (bb->live_kill); + gen_assert (bb->mem_live_in != NULL && bb->mem_live_out != NULL && bb->mem_live_gen != NULL + && bb->mem_live_kill != NULL); + bitmap_clear (bb->mem_live_in); + bitmap_clear (bb->mem_live_out); + bitmap_clear (bb->mem_live_gen); + bitmap_clear (bb->mem_live_kill); } for (MIR_insn_t tail = DLIST_TAIL (MIR_insn_t, curr_func_item->u.func->insns); tail != NULL;) - tail = initiate_bb_live_info (gen_ctx, tail, moves_p, &mvs_num); - if (moves_p) curr_cfg->non_conflicting_moves = mvs_num; + tail = initiate_bb_mem_live_info (gen_ctx, tail); + mem_attr_addr = VARR_ADDR (mem_attr_t, mem_attrs); + for (size_t i = 1; i < VARR_LENGTH (mem_attr_t, mem_attrs); i++) { + if (mem_attr_addr[i].alloca_flag & MUST_ALLOCA) continue; /* skip alloca memory */ + bitmap_set_bit_p (exit_bb->mem_live_in, i); + bitmap_set_bit_p (exit_bb->mem_live_out, i); + } } -static void update_bb_pressure (gen_ctx_t gen_ctx) { - size_t nel; - bitmap_iterator_t bi; - - for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { - int int_pressure = bb->max_int_pressure, fp_pressure = bb->max_fp_pressure; - - FOREACH_BITMAP_BIT (bi, bb->live_out, nel) { - increase_pressure (int_var_type_p (gen_ctx, (MIR_reg_t) nel), bb, &int_pressure, - &fp_pressure); - } - } +static void print_mem_bb_live_info (gen_ctx_t gen_ctx, bb_t bb) { + fprintf (debug_file, "BB %3lu:\n", (unsigned long) bb->index); + output_bitmap (gen_ctx, " Mem live in:", bb->mem_live_in, FALSE, NULL); + output_bitmap (gen_ctx, " Mem live out:", bb->mem_live_out, FALSE, NULL); + output_bitmap (gen_ctx, " Mem live gen:", bb->mem_live_gen, FALSE, NULL); + output_bitmap (gen_ctx, " Mem live kill:", bb->mem_live_kill, FALSE, NULL); } -static void calculate_func_cfg_live_info (gen_ctx_t gen_ctx, int moves_p) { - initiate_live_info (gen_ctx, moves_p); - solve_dataflow (gen_ctx, FALSE, live_con_func_0, live_con_func_n, live_trans_func); - if (optimize_level != 0) update_bb_pressure (gen_ctx); +static void calculate_mem_live_info (gen_ctx_t gen_ctx) { + initiate_mem_live_info (gen_ctx); + solve_dataflow (gen_ctx, FALSE, mem_live_con_func_0, mem_live_con_func_n, mem_live_trans_func); + DEBUG (2, { + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) + print_mem_bb_live_info (gen_ctx, bb); + }); } -static void add_bb_insn_dead_vars (gen_ctx_t gen_ctx) { +static void dse (gen_ctx_t gen_ctx) { MIR_insn_t insn; + uint32_t nloc; + long dead_stores_num = 0; + ssa_edge_t se; bb_insn_t bb_insn, prev_bb_insn; - size_t passed_mem_num; - MIR_reg_t var, early_clobbered_hard_reg1, early_clobbered_hard_reg2; - int op_num, out_p, mem_p; - bitmap_t live; - insn_var_iterator_t insn_var_iter; + bitmap_t live = temp_bitmap; - live = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); + calculate_mem_live_info (gen_ctx); for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { - bitmap_copy (live, bb->live_out); + bitmap_copy (live, bb->mem_live_out); for (bb_insn = DLIST_TAIL (bb_insn_t, bb->bb_insns); bb_insn != NULL; bb_insn = prev_bb_insn) { prev_bb_insn = DLIST_PREV (bb_insn_t, bb_insn); - clear_bb_insn_dead_vars (gen_ctx, bb_insn); insn = bb_insn->insn; - FOREACH_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (out_p) bitmap_clear_bit_p (live, var); - } - if (MIR_call_code_p (insn->code)) - bitmap_and_compl (live, live, call_used_hard_regs[MIR_T_UNDEF]); - FOREACH_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (out_p) continue; - if (bitmap_set_bit_p (live, var)) add_bb_insn_dead_var (gen_ctx, bb_insn, var); + if (MIR_call_code_p (insn->code)) update_call_mem_live (gen_ctx, live, insn); + if (!move_code_p (insn->code)) continue; + if (insn->ops[0].mode == MIR_OP_VAR_MEM) { /* store */ + if ((nloc = insn->ops[0].u.var_mem.nloc) != 0) { + if (!bitmap_clear_bit_p (live, nloc)) { + DEBUG (2, { + fprintf (debug_file, "Removing dead store "); + print_bb_insn (gen_ctx, bb_insn, FALSE); + }); + ssa_delete_insn (gen_ctx, insn); + dead_stores_num++; + } + } + } else if (insn->ops[1].mode == MIR_OP_VAR_MEM) { /* load */ + if ((nloc = insn->ops[1].u.var_mem.nloc) != 0) { + bitmap_set_bit_p (live, nloc); + se = insn->ops[1].data; + make_live_from_mem (gen_ctx, &insn->ops[1], live, NULL, + se != NULL && (se->def->alloca_flag & MUST_ALLOCA)); + } else { + bitmap_set_bit_range_p (live, 1, VARR_LENGTH (mem_attr_t, mem_attrs)); + } } - target_get_early_clobbered_hard_regs (insn, &early_clobbered_hard_reg1, - &early_clobbered_hard_reg2); - if (early_clobbered_hard_reg1 != MIR_NON_HARD_REG) - bitmap_clear_bit_p (live, early_clobbered_hard_reg1); - if (early_clobbered_hard_reg2 != MIR_NON_HARD_REG) - bitmap_clear_bit_p (live, early_clobbered_hard_reg2); - if (MIR_call_code_p (insn->code)) bitmap_ior (live, live, bb_insn->call_hard_reg_args); } } - bitmap_destroy (live); + DEBUG (1, { fprintf (debug_file, "%5ld removed dead stores\n", dead_stores_num); }); } -typedef struct live_range *live_range_t; /* vars */ +#undef mem_live_in +#undef mem_live_out +#undef mem_live_gen +#undef mem_live_kill -struct live_range { - int start, finish; - live_range_t next; -}; +/* New Page */ -DEF_VARR (live_range_t); +/* SSA dead code elimination */ + +static int reachable_bo_exists_p (bb_insn_t bb_insn) { + for (; bb_insn != NULL; bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) + if (bb_insn->insn->code == MIR_BO || bb_insn->insn->code == MIR_UBO + || bb_insn->insn->code == MIR_BNO || bb_insn->insn->code == MIR_UBNO) + return TRUE; + else if (bb_insn->insn->code != MIR_MOV && bb_insn->insn->code != MIR_EXT32 + && bb_insn->insn->code != MIR_UEXT32) + break; + return FALSE; +} + +static int ssa_dead_insn_p (gen_ctx_t gen_ctx, bb_insn_t bb_insn) { + MIR_insn_t insn = bb_insn->insn; + int op_num, output_exists_p = FALSE; + MIR_reg_t var; + insn_var_iterator_t iter; + ssa_edge_t ssa_edge; + + /* check control insns with possible output: */ + if (MIR_call_code_p (insn->code) || insn->code == MIR_ALLOCA || insn->code == MIR_BSTART + || insn->code == MIR_VA_START || insn->code == MIR_VA_ARG + || (insn->nops > 0 && insn->ops[0].mode == MIR_OP_VAR + && (insn->ops[0].u.var == FP_HARD_REG || insn->ops[0].u.var == SP_HARD_REG))) + return FALSE; + if (fake_insn_p (bb_insn)) return FALSE; + FOREACH_OUT_INSN_VAR (gen_ctx, iter, insn, var, op_num) { + output_exists_p = TRUE; + if (insn->ops[op_num].mode == MIR_OP_VAR_MEM || (ssa_edge = insn->ops[op_num].data) != NULL) + return FALSE; + } + if (!MIR_overflow_insn_code_p (insn->code) + || !reachable_bo_exists_p (DLIST_NEXT (bb_insn_t, bb_insn))) + return output_exists_p; + return FALSE; +} + +static int ssa_delete_insn_if_dead_p (gen_ctx_t gen_ctx, bb_insn_t bb_insn) { + if (!ssa_dead_insn_p (gen_ctx, bb_insn)) return FALSE; + DEBUG (2, { + fprintf (debug_file, " deleting now dead insn "); + print_bb_insn (gen_ctx, bb_insn, FALSE); + }); + ssa_delete_insn (gen_ctx, bb_insn->insn); + return TRUE; +} + +static void ssa_dead_code_elimination (gen_ctx_t gen_ctx) { + MIR_insn_t insn; + bb_insn_t bb_insn, def; + int op_num; + MIR_reg_t var; + insn_var_iterator_t iter; + ssa_edge_t ssa_edge; + long dead_insns_num = 0; + + DEBUG (2, { fprintf (debug_file, "+++++++++++++Dead code elimination:\n"); }); + VARR_TRUNC (bb_insn_t, temp_bb_insns, 0); + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) + for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; + bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) + if (ssa_dead_insn_p (gen_ctx, bb_insn)) VARR_PUSH (bb_insn_t, temp_bb_insns, bb_insn); + while (VARR_LENGTH (bb_insn_t, temp_bb_insns) != 0) { + bb_insn = VARR_POP (bb_insn_t, temp_bb_insns); + insn = bb_insn->insn; + DEBUG (2, { + fprintf (debug_file, " Removing dead insn %-5lu", (unsigned long) bb_insn->index); + print_bb_insn (gen_ctx, bb_insn, FALSE); + }); + FOREACH_IN_INSN_VAR (gen_ctx, iter, insn, var, op_num) { + if ((ssa_edge = insn->ops[op_num].data) == NULL) continue; + def = ssa_edge->def; + remove_ssa_edge (ssa_edge); + if (ssa_dead_insn_p (gen_ctx, def)) VARR_PUSH (bb_insn_t, temp_bb_insns, def); + } + gen_delete_insn (gen_ctx, insn); + dead_insns_num++; + } + DEBUG (1, { fprintf (debug_file, "%5ld removed SSA dead insns\n", dead_insns_num); }); +} + +/* New Page */ + +/* Loop invariant motion */ + +static edge_t find_loop_entry_edge (bb_t loop_entry) { + edge_t e, entry_e = NULL; + bb_insn_t head, tail; + + for (e = DLIST_HEAD (in_edge_t, loop_entry->in_edges); e != NULL; e = DLIST_NEXT (in_edge_t, e)) { + if (e->back_edge_p) continue; + if (entry_e != NULL) return NULL; + entry_e = e; + } + if (entry_e == NULL) return NULL; /* unreachable loop */ + tail = DLIST_TAIL (bb_insn_t, entry_e->src->bb_insns); + head = DLIST_HEAD (bb_insn_t, entry_e->dst->bb_insns); + if (tail != NULL && head != NULL && DLIST_NEXT (MIR_insn_t, tail->insn) != head->insn) + return NULL; /* not fall through */ + return entry_e; +} + +static void create_preheader_from_edge (gen_ctx_t gen_ctx, edge_t e, loop_node_t loop) { + bb_t new_bb = create_bb (gen_ctx, NULL), prev_bb; + loop_node_t bb_loop_node = create_loop_node (gen_ctx, new_bb), parent = loop->parent; + + add_new_bb (gen_ctx, new_bb); + DLIST_REMOVE (bb_t, curr_cfg->bbs, new_bb); + DLIST_INSERT_BEFORE (bb_t, curr_cfg->bbs, e->dst, new_bb); /* insert before loop entry */ + gen_assert (parent != NULL); + if ((prev_bb = DLIST_PREV (bb_t, e->dst)) != NULL && prev_bb->loop_node->parent == parent) + DLIST_INSERT_AFTER (loop_node_t, parent->children, prev_bb->loop_node, bb_loop_node); + else if (e->src->loop_node->parent == parent) + DLIST_INSERT_AFTER (loop_node_t, parent->children, e->src->loop_node, bb_loop_node); + else + DLIST_APPEND (loop_node_t, parent->children, bb_loop_node); + bb_loop_node->parent = parent; + bb_loop_node->u.preheader_loop = loop; + loop->u.preheader = bb_loop_node; + create_edge (gen_ctx, e->src, new_bb, TRUE, FALSE); /* fall through should be the 1st edge */ + create_edge (gen_ctx, new_bb, e->dst, TRUE, FALSE); + delete_edge (e); +} + +static void licm_add_loop_preheaders (gen_ctx_t gen_ctx, loop_node_t loop) { + int subloop_p = FALSE; + bb_insn_t bb_insn; + edge_t e; + + for (loop_node_t node = DLIST_HEAD (loop_node_t, loop->children); node != NULL; + node = DLIST_NEXT (loop_node_t, node)) + if (node->bb == NULL) { + subloop_p = TRUE; + licm_add_loop_preheaders (gen_ctx, node); /* process sub-loops */ + } + /* See loop_licm where we process only the nested loops: */ + if (subloop_p || loop == curr_cfg->root_loop_node) return; + loop->u.preheader = NULL; + if ((e = find_loop_entry_edge (loop->entry->bb)) == NULL) return; + if ((bb_insn = DLIST_TAIL (bb_insn_t, e->src->bb_insns)) == NULL || bb_insn->insn->code == MIR_JMP + || !MIR_any_branch_code_p (bb_insn->insn->code)) { + loop->u.preheader = e->src->loop_node; /* The preheader already exists */ + e->src->loop_node->u.preheader_loop = loop; /* The preheader already exists */ + } else { + create_preheader_from_edge (gen_ctx, e, loop); + } +} + +static int loop_invariant_p (gen_ctx_t gen_ctx, loop_node_t loop, bb_insn_t bb_insn, + bitmap_t loop_invariant_insn_bitmap) { + MIR_insn_t insn = bb_insn->insn; + bb_t bb = bb_insn->bb; + loop_node_t curr_loop; + int op_num; + MIR_reg_t var; + ssa_edge_t se; + insn_var_iterator_t iter; + + if (MIR_any_branch_code_p (insn->code) || insn->code == MIR_PHI || insn->code == MIR_RET + || insn->code == MIR_JRET || insn->code == MIR_LABEL || MIR_call_code_p (insn->code) + || insn->code == MIR_ALLOCA || insn->code == MIR_BSTART || insn->code == MIR_BEND + || insn->code == MIR_VA_START || insn->code == MIR_VA_ARG || insn->code == MIR_VA_BLOCK_ARG + || insn->code == MIR_VA_END + /* possible exception insns: */ + || insn->code == MIR_DIV || insn->code == MIR_DIVS || insn->code == MIR_UDIV + || insn->code == MIR_UDIVS || insn->code == MIR_MOD || insn->code == MIR_MODS + || insn->code == MIR_UMOD || insn->code == MIR_UMODS) + return FALSE; + for (size_t i = 0; i < insn->nops; i++) { + if (insn->ops[i].mode == MIR_OP_VAR_MEM) return FALSE; + if (insn->ops[i].mode == MIR_OP_VAR && bitmap_bit_p (tied_regs, insn->ops[i].u.var)) + return FALSE; + } + FOREACH_IN_INSN_VAR (gen_ctx, iter, insn, var, op_num) { + se = insn->ops[op_num].data; + gen_assert (se != NULL); + bb_insn = se->def; + if (loop_invariant_insn_bitmap != NULL + && bitmap_bit_p (loop_invariant_insn_bitmap, bb_insn->index)) + continue; + bb = bb_insn->bb; + for (curr_loop = loop->parent; curr_loop != NULL; curr_loop = curr_loop->parent) + if (curr_loop == bb->loop_node->parent) break; + if (curr_loop == NULL) return FALSE; + } + return TRUE; +} + +static void licm_move_insn (gen_ctx_t gen_ctx, bb_insn_t bb_insn, bb_t to, bb_insn_t before) { + MIR_context_t ctx = gen_ctx->ctx; + bb_t bb = bb_insn->bb; + MIR_insn_t insn = bb_insn->insn; + bb_insn_t last = DLIST_TAIL (bb_insn_t, to->bb_insns); + + gen_assert (before != NULL); + DLIST_REMOVE (bb_insn_t, bb->bb_insns, bb_insn); + DLIST_REMOVE (MIR_insn_t, curr_func_item->u.func->insns, insn); + if (last != NULL && last->insn->code == MIR_JMP) { + DLIST_INSERT_BEFORE (bb_insn_t, to->bb_insns, last, bb_insn); + MIR_insert_insn_before (ctx, curr_func_item, last->insn, insn); + } else { + DLIST_APPEND (bb_insn_t, to->bb_insns, bb_insn); + MIR_insert_insn_before (ctx, curr_func_item, before->insn, insn); + } + bb_insn->bb = to; +} + +static void mark_as_moved (gen_ctx_t gen_ctx, bb_insn_t bb_insn, + bitmap_t loop_invariant_bb_insn_bitmap, + bitmap_t bb_insns_to_move_bitmap) { + int op_num; + MIR_reg_t var; + insn_var_iterator_t iter; + ssa_edge_t se; + + VARR_TRUNC (bb_insn_t, temp_bb_insns2, 0); + VARR_PUSH (bb_insn_t, temp_bb_insns2, bb_insn); + gen_assert (bitmap_bit_p (loop_invariant_bb_insn_bitmap, bb_insn->index)); + while (VARR_LENGTH (bb_insn_t, temp_bb_insns2) != 0) { + bb_insn = VARR_POP (bb_insn_t, temp_bb_insns2); + bitmap_set_bit_p (bb_insns_to_move_bitmap, bb_insn->index); + FOREACH_IN_INSN_VAR (gen_ctx, iter, bb_insn->insn, var, op_num) + if ((se = bb_insn->insn->ops[op_num].data) != NULL + && bitmap_bit_p (loop_invariant_bb_insn_bitmap, bb_insn->index)) + VARR_PUSH (bb_insn_t, temp_bb_insns2, se->def); + } +} + +static int non_invariant_use_p (gen_ctx_t gen_ctx, bb_insn_t bb_insn, + bitmap_t loop_invariant_bb_insn_bitmap) { + int op_num; + MIR_reg_t var; + insn_var_iterator_t iter; + ssa_edge_t se; + + FOREACH_OUT_INSN_VAR (gen_ctx, iter, bb_insn->insn, var, op_num) { + for (se = bb_insn->insn->ops[op_num].data; se != NULL; se = se->next_use) + if (!bitmap_bit_p (loop_invariant_bb_insn_bitmap, se->use->index)) return TRUE; + } + return FALSE; +} + +static int expensive_insn_p (MIR_insn_t insn) { + return insn->code == MIR_MUL || insn->code == MIR_MULS; +} + +static int loop_licm (gen_ctx_t gen_ctx, loop_node_t loop) { + MIR_insn_t insn; + bb_insn_t bb_insn; + loop_node_t node; + int subloop_p = FALSE, move_p = FALSE, op_num; + MIR_reg_t var; + ssa_edge_t se; + insn_var_iterator_t iter; + bitmap_t loop_invariant_bb_insn_bitmap = temp_bitmap; + bitmap_t bb_insns_to_move_bitmap = temp_bitmap2; + VARR (bb_insn_t) *loop_invariant_bb_insns = temp_bb_insns; + + for (node = DLIST_HEAD (loop_node_t, loop->children); node != NULL; + node = DLIST_NEXT (loop_node_t, node)) + if (node->bb == NULL) { + subloop_p = TRUE; + if (loop_licm (gen_ctx, node)) move_p = TRUE; /* process sub-loops first */ + } + if (subloop_p || curr_cfg->root_loop_node == loop || loop->u.preheader == NULL) + return move_p; /* e.g. root or unreachable root */ + DEBUG (2, { + fprintf (debug_file, "Processing Loop%3lu for loop invariant motion:\n", + (unsigned long) loop->index); + }); + VARR_TRUNC (bb_insn_t, loop_invariant_bb_insns, 0); + bitmap_clear (loop_invariant_bb_insn_bitmap); + for (node = DLIST_HEAD (loop_node_t, loop->children); node != NULL; + node = DLIST_NEXT (loop_node_t, node)) { + if (node->bb == NULL) continue; /* skip subloops */ + for (bb_insn = DLIST_HEAD (bb_insn_t, node->bb->bb_insns); bb_insn != NULL; + bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) + if (loop_invariant_p (gen_ctx, loop, bb_insn, NULL)) { /* Push start invariants */ + VARR_PUSH (bb_insn_t, loop_invariant_bb_insns, bb_insn); + bitmap_set_bit_p (loop_invariant_bb_insn_bitmap, bb_insn->index); + } + } + for (size_t i = 0; i < VARR_LENGTH (bb_insn_t, loop_invariant_bb_insns); i++) { + /* Add insns becoming invariant if we move its inputs: */ + bb_insn = VARR_GET (bb_insn_t, loop_invariant_bb_insns, i); + insn = bb_insn->insn; + FOREACH_OUT_INSN_VAR (gen_ctx, iter, insn, var, op_num) { + for (se = insn->ops[op_num].data; se != NULL; se = se->next_use) { + if (loop_invariant_p (gen_ctx, loop, se->use, loop_invariant_bb_insn_bitmap) + && bitmap_set_bit_p (loop_invariant_bb_insn_bitmap, se->use->index)) + VARR_PUSH (bb_insn_t, loop_invariant_bb_insns, se->use); + } + } + } + bitmap_clear (bb_insns_to_move_bitmap); + for (int i = (int) VARR_LENGTH (bb_insn_t, loop_invariant_bb_insns) - 1; i >= 0; i--) { + bb_insn = VARR_GET (bb_insn_t, loop_invariant_bb_insns, i); + insn = bb_insn->insn; + DEBUG (2, { + fprintf (debug_file, " Considering invariant "); + print_bb_insn (gen_ctx, bb_insn, FALSE); + }); + if (bitmap_bit_p (bb_insns_to_move_bitmap, bb_insn->index)) { + DEBUG (2, { fprintf (debug_file, " -- already marked as moved\n"); }); + continue; + } + if (expensive_insn_p (insn)) { + DEBUG (2, { fprintf (debug_file, " -- marked as moved becuase it is costly\n"); }); + mark_as_moved (gen_ctx, bb_insn, loop_invariant_bb_insn_bitmap, bb_insns_to_move_bitmap); + continue; + } + int can_be_moved = TRUE, input_var_p = FALSE; + FOREACH_IN_INSN_VAR (gen_ctx, iter, insn, var, op_num) { + input_var_p = TRUE; + if ((se = insn->ops[op_num].data) != NULL + && bitmap_bit_p (loop_invariant_bb_insn_bitmap, se->def->index) + && !bitmap_bit_p (bb_insns_to_move_bitmap, se->def->index) + && non_invariant_use_p (gen_ctx, se->def, loop_invariant_bb_insn_bitmap)) { + can_be_moved = FALSE; + break; + } + } + DEBUG (2, { + if (input_var_p) + fprintf (debug_file, " -- %s be moved because reg presure consideration\n", + can_be_moved ? "can" : "can't"); + else + fprintf (debug_file, " -- can't be moved because single insn\n"); + }); + if (can_be_moved && input_var_p) + mark_as_moved (gen_ctx, bb_insn, loop_invariant_bb_insn_bitmap, bb_insns_to_move_bitmap); + } + for (size_t i = 0; i < VARR_LENGTH (bb_insn_t, loop_invariant_bb_insns); i++) { + bb_insn = VARR_GET (bb_insn_t, loop_invariant_bb_insns, i); + if (!bitmap_bit_p (bb_insns_to_move_bitmap, bb_insn->index)) continue; + insn = bb_insn->insn; + DEBUG (2, { + fprintf (debug_file, " Move invariant (target bb%lu) %-5lu", + (unsigned long) loop->u.preheader->bb->index, (unsigned long) bb_insn->index); + print_bb_insn (gen_ctx, bb_insn, FALSE); + }); + licm_move_insn (gen_ctx, bb_insn, loop->u.preheader->bb, + DLIST_HEAD (bb_insn_t, loop->entry->bb->bb_insns)); + move_p = TRUE; + } + return move_p; +} + +static int licm (gen_ctx_t gen_ctx) { + loop_node_t node; + for (node = DLIST_HEAD (loop_node_t, curr_cfg->root_loop_node->children); node != NULL; + node = DLIST_NEXT (loop_node_t, node)) + if (node->bb == NULL) break; + if (node == NULL) return FALSE; /* no loops */ + licm_add_loop_preheaders (gen_ctx, curr_cfg->root_loop_node); + return loop_licm (gen_ctx, curr_cfg->root_loop_node); +} + +/* New Page */ + +/* Pressure relief */ + +static int pressure_relief (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_func_t func = curr_func_item->u.func; + MIR_insn_t insn; + bb_insn_t bb_insn, next_bb_insn, use; + loop_node_t loop; + ssa_edge_t se; + int moved_p = FALSE; + + DEBUG (2, { fprintf (debug_file, "+++++++++++++Pressure Relief:\n"); }); + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) + for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; bb_insn = next_bb_insn) { + next_bb_insn = DLIST_NEXT (bb_insn_t, bb_insn); + insn = bb_insn->insn; + if (!move_code_p (insn->code) || insn->ops[0].mode != MIR_OP_VAR + || insn->ops[1].mode == MIR_OP_VAR || insn->ops[1].mode == MIR_OP_VAR_MEM + || (se = insn->ops[0].data) == NULL || se->next_use != NULL || (use = se->use)->bb == bb + || use->insn->code == MIR_PHI) + continue; + if ((loop = use->bb->loop_node) != NULL) { + for (loop = loop->parent; loop != NULL; loop = loop->parent) + if (loop == bb->loop_node->parent) break; + if (loop != NULL) continue; /* avoid move into a loop */ + } + /* One use in another BB: move closer */ + DEBUG (2, { + fprintf (debug_file, " Move insn %-5lu", (unsigned long) bb_insn->index); + MIR_output_insn (ctx, debug_file, insn, func, FALSE); + fprintf (debug_file, " before insn %-5lu", (unsigned long) use->index); + MIR_output_insn (ctx, debug_file, use->insn, func, TRUE); + }); + gen_move_insn_before (gen_ctx, use->insn, insn); + moved_p = TRUE; + } + return moved_p; +} + +/* New Page */ + +/* SSA insn combining is done on conventional SSA (to work with move insns) in reverse insn order. + We combine addresses and cmp and branch case. Copy prop before permits to ignore moves for + combining. It is the last SSA pass as it makes ssa edges unreachable from uses (in + mem[base,index] case). Advantages in comparison with combining after RA: + o no artificial dependencies on a hard reg assigned to different regs + o no missed dependencies on spilled regs */ + +typedef struct addr_info { + MIR_type_t type; + MIR_disp_t disp; + MIR_op_t *base, *index; /* var operands can be used for memory base and index */ + MIR_scale_t scale; +} addr_info_t; + +static int get_int_const (gen_ctx_t gen_ctx, MIR_op_t *op_ref, int64_t *c) { + if (op_ref->mode == MIR_OP_VAR) { + ssa_edge_t se = op_ref->data; + if (se == NULL || se->def->insn->code != MIR_MOV) return FALSE; + op_ref = &se->def->insn->ops[1]; + } + if (op_ref->mode == MIR_OP_INT || op_ref->mode == MIR_OP_UINT) { + *c = op_ref->u.i; + } else if (op_ref->mode == MIR_OP_REF && op_ref->u.ref->item_type != MIR_func_item) { + *c = get_ref_value (gen_ctx, op_ref); + } else { + return FALSE; + } + return TRUE; +} + +static int cycle_phi_p (bb_insn_t bb_insn) { /* we are not in pure SSA at this stage */ + ssa_edge_t se; + if (bb_insn->insn->code != MIR_PHI) return FALSE; + for (size_t i = 1; i < bb_insn->insn->nops; i++) + if ((se = bb_insn->insn->ops[i].data) != NULL && se->def->bb == bb_insn->bb) return TRUE; + return FALSE; +} + +static int var_plus_const (gen_ctx_t gen_ctx, ssa_edge_t se, bb_t from_bb, MIR_op_t **var_op_ref, + int64_t *c) { + if (se == NULL) return FALSE; /* e.g. for arg */ + gen_assert (*var_op_ref != NULL && (*var_op_ref)->mode == MIR_OP_VAR); + MIR_reg_t reg = (*var_op_ref)->u.var - MAX_HARD_REG; + if (MIR_reg_hard_reg_name (gen_ctx->ctx, reg, curr_func_item->u.func) != NULL) return FALSE; + MIR_insn_t insn = se->def->insn; + MIR_op_t *res_ref = NULL; + *c = 0; + if (insn->code == MIR_MOV && insn->ops[1].mode == MIR_OP_VAR) { + res_ref = &insn->ops[1]; + } else if ((insn->code == MIR_ADD || insn->code == MIR_SUB) && insn->ops[1].mode == MIR_OP_VAR + && get_int_const (gen_ctx, &insn->ops[2], c)) { + res_ref = &insn->ops[1]; + if (insn->code == MIR_SUB) *c = -*c; + } else if (insn->code == MIR_ADD && insn->ops[2].mode == MIR_OP_VAR + && get_int_const (gen_ctx, &insn->ops[1], c)) { + res_ref = &insn->ops[2]; + } else { + return FALSE; + } + if ((se = res_ref->data) != NULL && se->def->bb != from_bb && cycle_phi_p (se->def)) return FALSE; + *var_op_ref = res_ref; + return TRUE; +} + +static int var_mult_const (gen_ctx_t gen_ctx, ssa_edge_t se, bb_t from_bb, MIR_op_t **var_op_ref, + int64_t *c) { + if (se == NULL) return FALSE; /* e.g. for arg */ + gen_assert (*var_op_ref != NULL && (*var_op_ref)->mode == MIR_OP_VAR); + MIR_reg_t reg = (*var_op_ref)->u.var - MAX_HARD_REG; + if (MIR_reg_hard_reg_name (gen_ctx->ctx, reg, curr_func_item->u.func) != NULL) return FALSE; + MIR_insn_t insn = se->def->insn; + MIR_op_t *res_ref = NULL; + *c = 0; + if ((insn->code == MIR_MUL || insn->code == MIR_LSH) && insn->ops[1].mode == MIR_OP_VAR + && get_int_const (gen_ctx, &insn->ops[2], c)) { + res_ref = &insn->ops[1]; + if (insn->code == MIR_LSH) { + if (*c < 0 || *c >= (int) sizeof (int64_t) * 8) + res_ref = NULL; + else + *c = (int64_t) 1 << *c; + } + } else if (insn->code == MIR_MUL && insn->ops[2].mode == MIR_OP_VAR + && get_int_const (gen_ctx, &insn->ops[1], c)) { + res_ref = &insn->ops[2]; + } + if (res_ref == NULL) return FALSE; + if (*c < 0 || *c > MIR_MAX_SCALE) return FALSE; + if ((se = res_ref->data) != NULL && se->def->bb != from_bb && cycle_phi_p (se->def)) return FALSE; + *var_op_ref = res_ref; + return TRUE; +} + +static int var_plus_var (gen_ctx_t gen_ctx, ssa_edge_t se, bb_t from_bb, MIR_op_t **var_op_ref1, + MIR_op_t **var_op_ref2) { + if (se == NULL) return FALSE; /* e.g. for arg */ + gen_assert (*var_op_ref1 != NULL && (*var_op_ref1)->mode == MIR_OP_VAR && *var_op_ref2 == NULL); + MIR_reg_t reg = (*var_op_ref1)->u.var - MAX_HARD_REG; + if (MIR_reg_hard_reg_name (gen_ctx->ctx, reg, curr_func_item->u.func) != NULL) return FALSE; + MIR_insn_t insn = se->def->insn; + if (insn->code != MIR_ADD || insn->ops[1].mode != MIR_OP_VAR || insn->ops[2].mode != MIR_OP_VAR) + return FALSE; + if ((se = insn->ops[1].data) != NULL && se->def->bb != from_bb && cycle_phi_p (se->def)) + return FALSE; + if ((se = insn->ops[2].data) != NULL && se->def->bb != from_bb && cycle_phi_p (se->def)) + return FALSE; + *var_op_ref1 = &insn->ops[1]; + *var_op_ref2 = &insn->ops[2]; + return TRUE; +} + +static int addr_info_eq_p (addr_info_t *addr1, addr_info_t *addr2) { + return (addr1->type == addr2->type && addr1->disp == addr2->disp && addr1->base == addr2->base + && addr1->index == addr2->index && addr1->scale == addr2->scale); +} + +static int addr_info_ok_p (gen_ctx_t gen_ctx, addr_info_t *addr) { + MIR_op_t mem_op + = _MIR_new_var_mem_op (gen_ctx->ctx, addr->type, addr->disp, + addr->base == NULL ? MIR_NON_VAR : addr->base->u.var, + addr->index == NULL ? MIR_NON_VAR : addr->index->u.var, addr->scale); + return target_memory_ok_p (gen_ctx, &mem_op); +} + +static int update_addr_p (gen_ctx_t gen_ctx, bb_t from_bb, MIR_op_t *mem_op_ref, + MIR_op_t *temp_op_ref, addr_info_t *addr_info) { + int temp_int, stop_base_p = FALSE, stop_index_p = TRUE, temp_stop_index_p; + int64_t c; + addr_info_t temp_addr_info; + + gen_assert (mem_op_ref->mode == MIR_OP_VAR_MEM && mem_op_ref->u.var_mem.index == MIR_NON_VAR); + if (mem_op_ref->u.var_mem.base == MIR_NON_VAR) return FALSE; + *temp_op_ref = _MIR_new_var_op (gen_ctx->ctx, mem_op_ref->u.var_mem.base); + temp_op_ref->data = mem_op_ref->data; + addr_info->type = mem_op_ref->u.var_mem.type; + addr_info->disp = mem_op_ref->u.var_mem.disp; + addr_info->scale = 1; + addr_info->base = temp_op_ref; + addr_info->index = NULL; + for (int change_p = FALSE;;) { + temp_addr_info = *addr_info; + temp_stop_index_p = stop_index_p; + if (!stop_base_p) { + if (var_plus_const (gen_ctx, addr_info->base->data, from_bb, &addr_info->base, &c)) { + addr_info->disp += c; + } else if (addr_info->scale == 1 + && var_mult_const (gen_ctx, addr_info->base->data, from_bb, &addr_info->base, + &c)) { + if (c != 1) { + SWAP (addr_info->base, addr_info->index, temp_op_ref); + SWAP (stop_base_p, stop_index_p, temp_int); + addr_info->scale = (MIR_scale_t) c; + } + } else if (addr_info->index == NULL + && var_plus_var (gen_ctx, addr_info->base->data, from_bb, &addr_info->base, + &addr_info->index)) { + stop_index_p = FALSE; + } + } + if (!addr_info_eq_p (addr_info, &temp_addr_info) && addr_info_ok_p (gen_ctx, addr_info)) { + change_p = TRUE; + continue; + } + *addr_info = temp_addr_info; + stop_index_p = temp_stop_index_p; + stop_base_p = TRUE; + if (stop_index_p) return change_p; + if (var_plus_const (gen_ctx, addr_info->index->data, from_bb, &addr_info->index, &c)) { + addr_info->disp += c * addr_info->scale; + } else if (var_mult_const (gen_ctx, addr_info->index->data, from_bb, &addr_info->index, &c)) { + addr_info->scale *= (MIR_scale_t) c; + } else { + gen_assert (addr_info->base != NULL || addr_info->scale != 1); + } + if (!addr_info_eq_p (addr_info, &temp_addr_info) && addr_info_ok_p (gen_ctx, addr_info)) { + change_p = TRUE; + continue; + } + *addr_info = temp_addr_info; + return change_p; + } +} + +static MIR_insn_code_t get_combined_br_code (int true_p, MIR_insn_code_t cmp_code) { + switch (cmp_code) { + case MIR_EQ: return true_p ? MIR_BEQ : MIR_BNE; + case MIR_EQS: return true_p ? MIR_BEQS : MIR_BNES; + case MIR_NE: return true_p ? MIR_BNE : MIR_BEQ; + case MIR_NES: return true_p ? MIR_BNES : MIR_BEQS; + case MIR_LT: return true_p ? MIR_BLT : MIR_BGE; + case MIR_LTS: return true_p ? MIR_BLTS : MIR_BGES; + case MIR_ULT: return true_p ? MIR_UBLT : MIR_UBGE; + case MIR_ULTS: return true_p ? MIR_UBLTS : MIR_UBGES; + case MIR_LE: return true_p ? MIR_BLE : MIR_BGT; + case MIR_LES: return true_p ? MIR_BLES : MIR_BGTS; + case MIR_ULE: return true_p ? MIR_UBLE : MIR_UBGT; + case MIR_ULES: return true_p ? MIR_UBLES : MIR_UBGTS; + case MIR_GT: return true_p ? MIR_BGT : MIR_BLE; + case MIR_GTS: return true_p ? MIR_BGTS : MIR_BLES; + case MIR_UGT: return true_p ? MIR_UBGT : MIR_UBLE; + case MIR_UGTS: return true_p ? MIR_UBGTS : MIR_UBLES; + case MIR_GE: return true_p ? MIR_BGE : MIR_BLT; + case MIR_GES: return true_p ? MIR_BGES : MIR_BLTS; + case MIR_UGE: return true_p ? MIR_UBGE : MIR_UBLT; + case MIR_UGES: + return true_p ? MIR_UBGES : MIR_UBLTS; + /* Cannot revert in the false case for IEEE754: */ + case MIR_FEQ: return true_p ? MIR_FBEQ : MIR_INSN_BOUND; + case MIR_DEQ: return true_p ? MIR_DBEQ : MIR_INSN_BOUND; + case MIR_LDEQ: return true_p ? MIR_LDBEQ : MIR_INSN_BOUND; + case MIR_FNE: return true_p ? MIR_FBNE : MIR_INSN_BOUND; + case MIR_DNE: return true_p ? MIR_DBNE : MIR_INSN_BOUND; + case MIR_LDNE: return true_p ? MIR_LDBNE : MIR_INSN_BOUND; + case MIR_FLT: return true_p ? MIR_FBLT : MIR_INSN_BOUND; + case MIR_DLT: return true_p ? MIR_DBLT : MIR_INSN_BOUND; + case MIR_LDLT: return true_p ? MIR_LDBLT : MIR_INSN_BOUND; + case MIR_FLE: return true_p ? MIR_FBLE : MIR_INSN_BOUND; + case MIR_DLE: return true_p ? MIR_DBLE : MIR_INSN_BOUND; + case MIR_LDLE: return true_p ? MIR_LDBLE : MIR_INSN_BOUND; + case MIR_FGT: return true_p ? MIR_FBGT : MIR_INSN_BOUND; + case MIR_DGT: return true_p ? MIR_DBGT : MIR_INSN_BOUND; + case MIR_LDGT: return true_p ? MIR_LDBGT : MIR_INSN_BOUND; + case MIR_FGE: return true_p ? MIR_FBGE : MIR_INSN_BOUND; + case MIR_DGE: return true_p ? MIR_DBGE : MIR_INSN_BOUND; + case MIR_LDGE: return true_p ? MIR_LDBGE : MIR_INSN_BOUND; + default: return MIR_INSN_BOUND; + } +} + +static bb_insn_t combine_branch_and_cmp (gen_ctx_t gen_ctx, bb_insn_t bb_insn) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_insn_t def_insn, new_insn, insn = bb_insn->insn; + bb_t bb = bb_insn->bb; + bb_insn_t def_bb_insn; + ssa_edge_t se; + MIR_insn_code_t code = insn->code; + MIR_op_t *op_ref; + + if (code != MIR_BT && code != MIR_BF && code != MIR_BTS && code != MIR_BFS) return NULL; + op_ref = &insn->ops[1]; + if (op_ref->mode != MIR_OP_VAR || (se = op_ref->data) == NULL) return NULL; + def_bb_insn = se->def; + def_insn = def_bb_insn->insn; + if ((code = get_combined_br_code (code == MIR_BT || code == MIR_BTS, def_insn->code)) + == MIR_INSN_BOUND) + return NULL; + new_insn = MIR_new_insn (ctx, code, insn->ops[0], def_insn->ops[1], def_insn->ops[2]); + new_insn->ops[1].data = new_insn->ops[2].data = NULL; + /* don't use gen_add_insn_before as it checks adding branch after branch: */ + MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + ssa_delete_insn (gen_ctx, insn); + bb_insn = add_new_bb_insn (gen_ctx, new_insn, bb, TRUE); + se = def_insn->ops[1].data; + if (se != NULL) add_ssa_edge (gen_ctx, se->def, se->def_op_num, bb_insn, 1); + se = def_insn->ops[2].data; + if (se != NULL) add_ssa_edge (gen_ctx, se->def, se->def_op_num, bb_insn, 2); + DEBUG (2, { + fprintf (debug_file, " changing to "); + print_bb_insn (gen_ctx, bb_insn, TRUE); + }); + ssa_delete_insn_if_dead_p (gen_ctx, def_bb_insn); + return bb_insn; +} + +static void ssa_combine (gen_ctx_t gen_ctx) { // tied reg, alias ??? + MIR_op_t temp_op; + MIR_insn_t insn; + bb_insn_t bb_insn, prev_bb_insn, new_bb_insn; + ssa_edge_t se; + addr_info_t addr_info; + + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + DEBUG (2, { fprintf (debug_file, "Processing bb%lu\n", (unsigned long) bb->index); }); + for (bb_insn = DLIST_TAIL (bb_insn_t, bb->bb_insns); bb_insn != NULL; bb_insn = prev_bb_insn) { + prev_bb_insn = DLIST_PREV (bb_insn_t, bb_insn); + insn = bb_insn->insn; + /* not all insn is deleted if we use addr defs from other bbs */ + if (ssa_delete_insn_if_dead_p (gen_ctx, bb_insn)) continue; + if (insn->code == MIR_LABEL || MIR_call_code_p (insn->code)) continue; + DEBUG (2, { + fprintf (debug_file, " combining insn "); + print_bb_insn (gen_ctx, bb_insn, FALSE); + }); + if ((new_bb_insn = combine_branch_and_cmp (gen_ctx, bb_insn)) != NULL) { + bb_insn = new_bb_insn; + prev_bb_insn = DLIST_PREV (bb_insn_t, bb_insn); + insn = bb_insn->insn; + } + for (size_t i = 0; i < insn->nops; i++) { + if (insn->ops[i].mode != MIR_OP_VAR_MEM) continue; + if (!update_addr_p (gen_ctx, bb, &insn->ops[i], &temp_op, &addr_info)) continue; + remove_ssa_edge (insn->ops[i].data); + insn->ops[i].u.var_mem.disp = addr_info.disp; + insn->ops[i].u.var_mem.base = insn->ops[i].u.var_mem.index = MIR_NON_VAR; + if (addr_info.base != NULL) { + insn->ops[i].u.var_mem.base = addr_info.base->u.var; + if ((se = addr_info.base->data) != NULL) + add_ssa_edge (gen_ctx, se->def, se->def_op_num, bb_insn, (int) i); + } + if (addr_info.index != NULL) { + insn->ops[i].u.var_mem.index = addr_info.index->u.var; + if ((se = addr_info.index->data) != NULL) + add_ssa_edge_dup (gen_ctx, se->def, se->def_op_num, bb_insn, (int) i); + } + insn->ops[i].u.var_mem.scale = addr_info.scale; + DEBUG (2, { + fprintf (debug_file, " changing mem op %lu to ", (unsigned long) i); + print_insn (gen_ctx, insn, TRUE); + }); + } + } + } +} + +/* New Page */ + +/* Live and live range analysis: */ + +#define live_in in +#define live_out out +#define live_kill kill +#define live_gen gen + +typedef struct lr_bb *lr_bb_t; +struct lr_bb { + bb_t bb; + lr_bb_t next; +}; + +typedef struct live_range *live_range_t; /* vars */ +struct live_range { + lr_bb_t lr_bb; /* first BB which is entirely in this range, NULL otherwise */ + int start, finish; + int ref_cost; + /* to smaller start and finish, but still this start can be equal to the next finish: */ + live_range_t next; +}; + +DEF_VARR (live_range_t); +DEF_VARR (MIR_reg_t); + +struct lr_ctx { + int ssa_live_info_p; /* TRUE if found PHIs */ + int scan_vars_num; /* vars considered for live analysis: 0 means all vars */ + VARR (int) * var_to_scan_var_map; /* if var is less than the map size: its live_var or -1 */ + VARR (MIR_reg_t) * scan_var_to_var_map; /* of size scan_vars_num */ + live_range_t free_lr_list; + lr_bb_t free_lr_bb_list; + int curr_point; + bitmap_t live_vars, referenced_vars; + bitmap_t points_with_born_vars, points_with_dead_vars, points_with_born_or_dead_vars; + VARR (live_range_t) * var_live_ranges; + VARR (int) * point_map; +}; + +#define ssa_live_info_p gen_ctx->lr_ctx->ssa_live_info_p +#define scan_vars_num gen_ctx->lr_ctx->scan_vars_num +#define var_to_scan_var_map gen_ctx->lr_ctx->var_to_scan_var_map +#define scan_var_to_var_map gen_ctx->lr_ctx->scan_var_to_var_map +#define free_lr_list gen_ctx->lr_ctx->free_lr_list +#define free_lr_bb_list gen_ctx->lr_ctx->free_lr_bb_list +#define curr_point gen_ctx->lr_ctx->curr_point +#define live_vars gen_ctx->lr_ctx->live_vars +#define referenced_vars gen_ctx->lr_ctx->referenced_vars +#define points_with_born_vars gen_ctx->lr_ctx->points_with_born_vars +#define points_with_dead_vars gen_ctx->lr_ctx->points_with_dead_vars +#define points_with_born_or_dead_vars gen_ctx->lr_ctx->points_with_born_or_dead_vars +#define var_live_ranges gen_ctx->lr_ctx->var_live_ranges +#define point_map gen_ctx->lr_ctx->point_map + +static int var_to_scan_var (gen_ctx_t gen_ctx, MIR_reg_t var) { + if (scan_vars_num == 0) return (int) var; + if (VARR_LENGTH (int, var_to_scan_var_map) <= var) return -1; + return VARR_GET (int, var_to_scan_var_map, var); +} + +static MIR_reg_t scan_var_to_var (gen_ctx_t gen_ctx, int scan_var) { + if (scan_vars_num == 0) return (MIR_reg_t) scan_var; + gen_assert (scan_var >= 0 && (int) VARR_LENGTH (MIR_reg_t, scan_var_to_var_map) > scan_var); + return VARR_GET (MIR_reg_t, scan_var_to_var_map, scan_var); +} + +/* Life analysis */ +static void live_con_func_0 (bb_t bb MIR_UNUSED) {} + +static int live_con_func_n (gen_ctx_t gen_ctx, bb_t bb) { + MIR_op_t *op_ref; + bb_insn_t bb_insn; + edge_t e, e2; + int n, change_p = FALSE; + + for (e = DLIST_HEAD (out_edge_t, bb->out_edges); e != NULL; e = DLIST_NEXT (out_edge_t, e)) { + change_p |= bitmap_ior (bb->live_out, bb->live_out, e->dst->live_in); + if (ssa_live_info_p) { + for (bb_insn = DLIST_HEAD (bb_insn_t, e->dst->bb_insns); bb_insn != NULL; + bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) + if (bb_insn->insn->code != MIR_LABEL) break; + if (bb_insn == NULL || bb_insn->insn->code != MIR_PHI) continue; /* no phis in dst */ + for (n = 1, e2 = DLIST_HEAD (in_edge_t, e->dst->in_edges); e2 != NULL; + e2 = DLIST_NEXT (in_edge_t, e2), n++) + if (e2 == e) break; + gen_assert (e2 == e); + for (;;) { + op_ref = &bb_insn->insn->ops[n]; + if (op_ref->mode == MIR_OP_VAR) + change_p |= bitmap_set_bit_p (bb->live_out, var_to_scan_var (gen_ctx, op_ref->u.var)); + if ((bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) == NULL || bb_insn->insn->code != MIR_PHI) + break; + } + } + } + return change_p; +} + +static int live_trans_func (gen_ctx_t gen_ctx MIR_UNUSED, bb_t bb) { + return bitmap_ior_and_compl (bb->live_in, bb->live_gen, bb->live_out, bb->live_kill); +} + +static int bb_loop_level (bb_t bb) { + loop_node_t loop_node; + int level = -1; + + for (loop_node = bb->loop_node; loop_node->parent != NULL; loop_node = loop_node->parent) level++; + gen_assert (level >= 0); + return level; +} + +static void increase_pressure (int int_p, bb_t bb, int *int_pressure, int *fp_pressure) { + if (int_p) { + if (bb->max_int_pressure < ++(*int_pressure)) bb->max_int_pressure = *int_pressure; + } else { + if (bb->max_fp_pressure < ++(*fp_pressure)) bb->max_fp_pressure = *fp_pressure; + } +} + +static int int_var_type_p (gen_ctx_t gen_ctx, MIR_reg_t var) { + if (var <= MAX_HARD_REG) return target_hard_reg_type_ok_p (var, MIR_T_I32); + return MIR_int_type_p (MIR_reg_type (gen_ctx->ctx, var - MAX_HARD_REG, curr_func_item->u.func)); +} + +static MIR_insn_t initiate_bb_live_info (gen_ctx_t gen_ctx, MIR_insn_t bb_tail_insn, int freq_p) { + bb_t bb = get_insn_bb (gen_ctx, bb_tail_insn); + MIR_insn_t insn; + long bb_freq; + MIR_reg_t var, early_clobbered_hard_reg1, early_clobbered_hard_reg2; + int scan_var, op_num, int_p = FALSE; + int bb_int_pressure, bb_fp_pressure; + reg_info_t *reg_infos; + insn_var_iterator_t insn_var_iter; + bitmap_t global_hard_regs + = _MIR_get_module_global_var_hard_regs (gen_ctx->ctx, curr_func_item->module); + + reg_infos = VARR_ADDR (reg_info_t, curr_cfg->reg_info); + bb_freq = 1; + if (optimize_level != 0 && freq_p) + for (int i = bb_loop_level (bb); i > 0; i--) + if (bb_freq < LONG_MAX / 8) bb_freq *= LOOP_COST_FACTOR; + bb->max_int_pressure = bb_int_pressure = bb->max_fp_pressure = bb_fp_pressure = 0; + for (insn = bb_tail_insn; insn != NULL && get_insn_bb (gen_ctx, insn) == bb; + insn = DLIST_PREV (MIR_insn_t, insn)) { + if (insn->code == MIR_PHI) { + ssa_live_info_p = TRUE; + var = insn->ops[0].u.var; + if ((scan_var = var_to_scan_var (gen_ctx, var)) < 0) continue; + if (bitmap_clear_bit_p (bb->live_gen, scan_var) && optimize_level != 0) + (int_var_type_p (gen_ctx, var) ? bb_int_pressure-- : bb_fp_pressure--); + bitmap_set_bit_p (bb->live_kill, scan_var); + continue; + } + if (MIR_call_code_p (insn->code) && scan_vars_num == 0) { + bitmap_ior (bb->live_kill, bb->live_kill, call_used_hard_regs[MIR_T_UNDEF]); + if (global_hard_regs != NULL) + bitmap_ior_and_compl (bb->live_gen, global_hard_regs, bb->live_gen, + call_used_hard_regs[MIR_T_UNDEF]); + else + bitmap_and_compl (bb->live_gen, bb->live_gen, call_used_hard_regs[MIR_T_UNDEF]); + } + FOREACH_OUT_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num) { /* output vars */ + if ((scan_var = var_to_scan_var (gen_ctx, var)) < 0) continue; + if (bitmap_clear_bit_p (bb->live_gen, scan_var) && optimize_level != 0) + (int_var_type_p (gen_ctx, var) ? bb_int_pressure-- : bb_fp_pressure--); + bitmap_set_bit_p (bb->live_kill, scan_var); + if (freq_p && var > MAX_HARD_REG) + reg_infos[var].freq + = reg_infos[var].freq < LONG_MAX - bb_freq ? reg_infos[var].freq + bb_freq : LONG_MAX; + } + FOREACH_IN_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num) { /* input vars */ + if ((scan_var = var_to_scan_var (gen_ctx, var)) < 0) continue; + if (bitmap_set_bit_p (bb->live_gen, scan_var) && optimize_level != 0) + increase_pressure (int_var_type_p (gen_ctx, var), bb, &bb_int_pressure, &bb_fp_pressure); + if (freq_p && var > MAX_HARD_REG) + reg_infos[var].freq + = reg_infos[var].freq < LONG_MAX - bb_freq ? reg_infos[var].freq + bb_freq : LONG_MAX; + } + if (scan_vars_num != 0) continue; + target_get_early_clobbered_hard_regs (insn, &early_clobbered_hard_reg1, + &early_clobbered_hard_reg2); + if (early_clobbered_hard_reg1 != MIR_NON_VAR) { + if (optimize_level != 0) { + int_p = int_var_type_p (gen_ctx, early_clobbered_hard_reg1); + increase_pressure (int_p, bb, &bb_int_pressure, &bb_fp_pressure); + } + bitmap_clear_bit_p (bb->live_gen, early_clobbered_hard_reg1); + bitmap_set_bit_p (bb->live_kill, early_clobbered_hard_reg1); + if (optimize_level != 0) (int_p ? bb_int_pressure-- : bb_fp_pressure--); + } + if (early_clobbered_hard_reg2 != MIR_NON_VAR) { + if (optimize_level != 0) { + int_p = int_var_type_p (gen_ctx, early_clobbered_hard_reg2); + increase_pressure (int_p, bb, &bb_int_pressure, &bb_fp_pressure); + } + bitmap_clear_bit_p (bb->live_gen, early_clobbered_hard_reg2); + bitmap_set_bit_p (bb->live_kill, early_clobbered_hard_reg2); + if (optimize_level != 0) (int_p ? bb_int_pressure-- : bb_fp_pressure--); + } + if (MIR_call_code_p (insn->code)) { + bitmap_t reg_args; + + if (optimize_level != 0) + bitmap_ior (bb->live_gen, bb->live_gen, ((bb_insn_t) insn->data)->call_hard_reg_args); + else if ((reg_args = ((insn_data_t) insn->data)->u.call_hard_reg_args) != NULL) + bitmap_ior (bb->live_gen, bb->live_gen, reg_args); + } + } + return insn; +} + +static void initiate_live_info (gen_ctx_t gen_ctx, int freq_p) { + MIR_reg_t max_var, n; + reg_info_t ri; + bitmap_t global_hard_regs + = _MIR_get_module_global_var_hard_regs (gen_ctx->ctx, curr_func_item->module); + + VARR_TRUNC (reg_info_t, curr_cfg->reg_info, 0); + max_var = get_max_var (gen_ctx); + for (n = 0; n <= max_var; n++) { + ri.freq = 0; + ri.live_length = 0; + VARR_PUSH (reg_info_t, curr_cfg->reg_info, ri); + } + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + gen_assert (bb != NULL && bb->live_in != NULL && bb->live_out != NULL && bb->live_gen != NULL + && bb->live_kill != NULL); + bitmap_clear (bb->live_in); + bitmap_clear (bb->live_out); + bitmap_clear (bb->live_gen); + bitmap_clear (bb->live_kill); + } + if (global_hard_regs != NULL && scan_vars_num == 0) /* exit bb */ + bitmap_copy (DLIST_EL (bb_t, curr_cfg->bbs, 1)->live_out, global_hard_regs); + for (MIR_insn_t tail = DLIST_TAIL (MIR_insn_t, curr_func_item->u.func->insns); tail != NULL;) + tail = initiate_bb_live_info (gen_ctx, tail, freq_p); +} + +static void update_bb_pressure (gen_ctx_t gen_ctx) { + size_t nel; + bitmap_iterator_t bi; + + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + int int_pressure = bb->max_int_pressure, fp_pressure = bb->max_fp_pressure; + + FOREACH_BITMAP_BIT (bi, bb->live_out, nel) { + increase_pressure (int_var_type_p (gen_ctx, (MIR_reg_t) nel), bb, &int_pressure, + &fp_pressure); + } + } +} + +static void calculate_func_cfg_live_info (gen_ctx_t gen_ctx, int freq_p) { + ssa_live_info_p = FALSE; + initiate_live_info (gen_ctx, freq_p); + solve_dataflow (gen_ctx, FALSE, live_con_func_0, live_con_func_n, live_trans_func); + if (optimize_level != 0) update_bb_pressure (gen_ctx); +} + +static void consider_all_live_vars (gen_ctx_t gen_ctx) { scan_vars_num = 0; } + +#ifndef MIR_MAX_COALESCE_VARS +#define MIR_MAX_COALESCE_VARS 10000 /* 10K means about 8MB for conflict matrix */ +#endif + +static void collect_scan_var (gen_ctx_t gen_ctx, MIR_reg_t var) { + if (!bitmap_set_bit_p (temp_bitmap, var)) return; + if (scan_vars_num >= MIR_MAX_COALESCE_VARS) return; + while (VARR_LENGTH (int, var_to_scan_var_map) <= var) VARR_PUSH (int, var_to_scan_var_map, -1); + VARR_PUSH (MIR_reg_t, scan_var_to_var_map, var); + VARR_SET (int, var_to_scan_var_map, var, scan_vars_num++); +} + +static void scan_collected_moves (gen_ctx_t gen_ctx); + +static int consider_move_vars_only (gen_ctx_t gen_ctx) { + VARR_TRUNC (int, var_to_scan_var_map, 0); + VARR_TRUNC (MIR_reg_t, scan_var_to_var_map, 0); + bitmap_clear (temp_bitmap); + scan_vars_num = 0; + scan_collected_moves (gen_ctx); + return scan_vars_num > 0; +} + +static void add_bb_insn_dead_vars (gen_ctx_t gen_ctx) { + MIR_insn_t insn; + bb_insn_t bb_insn, prev_bb_insn; + MIR_reg_t var, early_clobbered_hard_reg1, early_clobbered_hard_reg2; + int scan_var, op_num; + bitmap_t live; + insn_var_iterator_t insn_var_iter; + + /* we need all var analysis and bb insns to keep dead var info */ + gen_assert (optimize_level > 0); + live = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + bitmap_copy (live, bb->live_out); + for (bb_insn = DLIST_TAIL (bb_insn_t, bb->bb_insns); bb_insn != NULL; bb_insn = prev_bb_insn) { + prev_bb_insn = DLIST_PREV (bb_insn_t, bb_insn); + clear_bb_insn_dead_vars (gen_ctx, bb_insn); + insn = bb_insn->insn; + FOREACH_OUT_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num) { + if ((scan_var = var_to_scan_var (gen_ctx, var)) < 0) continue; + bitmap_clear_bit_p (live, scan_var); + } + if (scan_vars_num == 0 && MIR_call_code_p (insn->code)) + bitmap_and_compl (live, live, call_used_hard_regs[MIR_T_UNDEF]); + FOREACH_IN_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num) { + if ((scan_var = var_to_scan_var (gen_ctx, var)) < 0) continue; + if (bitmap_set_bit_p (live, scan_var)) add_bb_insn_dead_var (gen_ctx, bb_insn, var); + } + if (scan_vars_num != 0) continue; + target_get_early_clobbered_hard_regs (insn, &early_clobbered_hard_reg1, + &early_clobbered_hard_reg2); + if (early_clobbered_hard_reg1 != MIR_NON_VAR) + bitmap_clear_bit_p (live, early_clobbered_hard_reg1); + if (early_clobbered_hard_reg2 != MIR_NON_VAR) + bitmap_clear_bit_p (live, early_clobbered_hard_reg2); + if (MIR_call_code_p (insn->code)) bitmap_ior (live, live, bb_insn->call_hard_reg_args); + } + } + bitmap_destroy (live); +} + +#if !MIR_NO_GEN_DEBUG +static void output_bb_border_live_info (gen_ctx_t gen_ctx, bb_t bb) { + MIR_reg_t *map = scan_vars_num == 0 ? NULL : VARR_ADDR (MIR_reg_t, scan_var_to_var_map); + output_bitmap (gen_ctx, " live_in:", bb->live_in, TRUE, map); + output_bitmap (gen_ctx, " live_out:", bb->live_out, TRUE, map); +} + +static void output_bb_live_info (gen_ctx_t gen_ctx, bb_t bb) { + MIR_reg_t *map = scan_vars_num == 0 ? NULL : VARR_ADDR (MIR_reg_t, scan_var_to_var_map); + output_bb_border_live_info (gen_ctx, bb); + output_bitmap (gen_ctx, " live_gen:", bb->live_gen, TRUE, map); + output_bitmap (gen_ctx, " live_kill:", bb->live_kill, TRUE, map); +} +#endif + +static void print_live_info (gen_ctx_t gen_ctx, const char *title, int dead_var_p, int pressure_p) { + DEBUG (2, { + if (dead_var_p) add_bb_insn_dead_vars (gen_ctx); + fprintf (debug_file, "+++++++++++++%s:\n", title); + print_loop_tree (gen_ctx, TRUE); + print_CFG (gen_ctx, TRUE, pressure_p, TRUE, TRUE, output_bb_live_info); + }); +} + +#undef live_kill +#undef live_gen + +static lr_bb_t create_lr_bb (gen_ctx_t gen_ctx, bb_t bb, lr_bb_t next) { + lr_bb_t lr_bb; + if ((lr_bb = free_lr_bb_list) != NULL) { + free_lr_bb_list = free_lr_bb_list->next; + } else { + lr_bb = gen_malloc (gen_ctx, sizeof (struct lr_bb)); + } + lr_bb->bb = bb; + lr_bb->next = next; + return lr_bb; +} + +static void free_lr_bbs (gen_ctx_t gen_ctx, lr_bb_t list) { + for (lr_bb_t lr_bb = list; lr_bb != NULL; lr_bb = list) { + list = lr_bb->next; + lr_bb->next = free_lr_bb_list; + free_lr_bb_list = lr_bb; + } +} + +static void init_lr_bbs (gen_ctx_t gen_ctx) { free_lr_bb_list = NULL; } +static void finish_lr_bbs (gen_ctx_t gen_ctx) { + for (lr_bb_t lr_bb = free_lr_bb_list; lr_bb != NULL; lr_bb = free_lr_bb_list) { + free_lr_bb_list = lr_bb->next; + free (lr_bb); + } +} + +static void free_one_live_range (gen_ctx_t gen_ctx, live_range_t lr) { + free_lr_bbs (gen_ctx, lr->lr_bb); + lr->next = free_lr_list; + free_lr_list = lr; +} + +static void free_live_ranges (gen_ctx_t gen_ctx, live_range_t list) { + for (live_range_t lr = list; lr != NULL; lr = list) { + list = lr->next; + free_one_live_range (gen_ctx, lr); + } +} + +static live_range_t create_live_range (gen_ctx_t gen_ctx, int start, int finish, + live_range_t next) { + live_range_t lr; + if ((lr = free_lr_list) != NULL) { + free_lr_list = free_lr_list->next; + } else { + lr = gen_malloc (gen_ctx, sizeof (struct live_range)); + } + gen_assert (start >= 0); + gen_assert (finish < 0 || start <= finish); + lr->start = start; + lr->finish = finish; + lr->ref_cost = 1; + lr->next = next; + lr->lr_bb = NULL; + return lr; +} + +static void move_lr_bbs (live_range_t from, live_range_t to) { + lr_bb_t lr_bb, next_lr_bb; + for (lr_bb = from->lr_bb; lr_bb != NULL; lr_bb = from->lr_bb) { + next_lr_bb = lr_bb->next; + lr_bb->next = to->lr_bb; + to->lr_bb = lr_bb; + from->lr_bb = next_lr_bb; + } +} + +static void init_lrs (gen_ctx_t gen_ctx) { free_lr_list = NULL; } +static void finish_lrs (gen_ctx_t gen_ctx) { + for (live_range_t lr = free_lr_list; lr != NULL; lr = free_lr_list) { + free_lr_list = lr->next; + free (lr); + } +} + +static inline int make_var_dead (gen_ctx_t gen_ctx, MIR_reg_t var, int scan_var, int point, + int insn_p) { + live_range_t lr; + if (insn_p && scan_vars_num == 0) bitmap_set_bit_p (referenced_vars, var); + lr = VARR_GET (live_range_t, var_live_ranges, var); + if (bitmap_clear_bit_p (live_vars, scan_var)) { + lr->finish = point; + } else { + /* insn with unused result: result still needs a hard register */ + VARR_SET (live_range_t, var_live_ranges, var, create_live_range (gen_ctx, point, point, lr)); + } + return TRUE; +} + +static inline int make_var_live (gen_ctx_t gen_ctx, MIR_reg_t var, int scan_var, int point, + int insn_p) { + live_range_t lr; + lr = VARR_GET (live_range_t, var_live_ranges, var); + if (insn_p && scan_vars_num == 0) bitmap_set_bit_p (referenced_vars, var); + if (!bitmap_set_bit_p (live_vars, scan_var)) return FALSE; + /* Always start new live range for starting living at bb end or if + the last live range is covering a whole bb: */ + if (!insn_p || lr == NULL || lr->lr_bb != NULL + || (lr->finish != point && lr->finish + 1 != point)) { + VARR_SET (live_range_t, var_live_ranges, var, create_live_range (gen_ctx, point, -1, lr)); + } + return TRUE; +} + +static void add_lr_bb (gen_ctx_t gen_ctx, MIR_reg_t var, bb_t bb) { + live_range_t lr = VARR_GET (live_range_t, var_live_ranges, var); + gen_assert (lr != NULL && lr->lr_bb == NULL); + lr->lr_bb = create_lr_bb (gen_ctx, bb, NULL); +} + +#if !MIR_NO_GEN_DEBUG +static void print_live_range (gen_ctx_t gen_ctx, live_range_t lr) { + fprintf (debug_file, " [%d..%d]", lr->start, lr->finish); + if (lr->lr_bb == NULL) return; + for (lr_bb_t lr_bb = lr->lr_bb; lr_bb != NULL; lr_bb = lr_bb->next) + fprintf (debug_file, "%cbb%lu", lr_bb == lr->lr_bb ? '(' : ' ', + (long unsigned) lr_bb->bb->index); + fprintf (debug_file, ")"); +} + +static void print_live_ranges (gen_ctx_t gen_ctx, live_range_t lr) { + for (; lr != NULL; lr = lr->next) print_live_range (gen_ctx, lr); + fprintf (debug_file, "\n"); +} + +static void print_all_live_ranges (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; + live_range_t lr; + + fprintf (debug_file, "+++++++++++++Live ranges:\n"); + for (size_t i = 0; i < VARR_LENGTH (live_range_t, var_live_ranges); i++) { + if ((lr = VARR_GET (live_range_t, var_live_ranges, i)) == NULL) continue; + fprintf (debug_file, "%lu", (unsigned long) i); + if (scan_vars_num != 0) + fprintf (debug_file, " (%lu)", (unsigned long) var_to_scan_var (gen_ctx, (MIR_reg_t) i)); + if (i > MAX_HARD_REG) + fprintf (debug_file, " (%s:%s)", + MIR_type_str (ctx, MIR_reg_type (ctx, (MIR_reg_t) (i - MAX_HARD_REG), + curr_func_item->u.func)), + MIR_reg_name (ctx, (MIR_reg_t) (i - MAX_HARD_REG), curr_func_item->u.func)); + fprintf (debug_file, ":"); + print_live_ranges (gen_ctx, lr); + } +} +#endif + +static void shrink_live_ranges (gen_ctx_t gen_ctx) { + size_t p; + long int pn, rn, old_rn; + live_range_t lr, prev_lr, next_lr; + int born_p, dead_p, prev_dead_p; + bitmap_iterator_t bi; + + bitmap_clear (points_with_born_vars); + bitmap_clear (points_with_dead_vars); + for (size_t i = 0; i < VARR_LENGTH (live_range_t, var_live_ranges); i++) { + for (lr = VARR_GET (live_range_t, var_live_ranges, i); lr != NULL; lr = lr->next) { + gen_assert (lr->start <= lr->finish); + bitmap_set_bit_p (points_with_born_vars, lr->start); + bitmap_set_bit_p (points_with_dead_vars, lr->finish); + } + } + + VARR_TRUNC (int, point_map, 0); + for (int i = 0; i <= curr_point; i++) VARR_PUSH (int, point_map, 0); + bitmap_ior (points_with_born_or_dead_vars, points_with_born_vars, points_with_dead_vars); + pn = -1; + prev_dead_p = TRUE; + FOREACH_BITMAP_BIT (bi, points_with_born_or_dead_vars, p) { + born_p = bitmap_bit_p (points_with_born_vars, p); + dead_p = bitmap_bit_p (points_with_dead_vars, p); + assert (born_p || dead_p); + if (!prev_dead_p || !born_p) /* 1st point is always a born */ + VARR_SET (int, point_map, p, pn); + else + VARR_SET (int, point_map, p, ++pn); + prev_dead_p = dead_p; + } + pn++; + + old_rn = rn = 0; + for (size_t i = 0; i < VARR_LENGTH (live_range_t, var_live_ranges); i++) { + for (lr = VARR_GET (live_range_t, var_live_ranges, i), prev_lr = NULL; lr != NULL; + lr = next_lr) { + old_rn++; + next_lr = lr->next; + lr->start = VARR_GET (int, point_map, lr->start); + lr->finish = VARR_GET (int, point_map, lr->finish); + if (prev_lr == NULL || (prev_lr->start != lr->finish && prev_lr->start != lr->finish + 1) + || (prev_lr->lr_bb != NULL && lr->lr_bb == NULL) + || (prev_lr->lr_bb == NULL && lr->lr_bb != NULL)) { + rn++; + prev_lr = lr; + continue; + } + prev_lr->start = lr->start; + prev_lr->next = next_lr; + move_lr_bbs (lr, prev_lr); + free_one_live_range (gen_ctx, lr); + } + } + DEBUG (2, { + fprintf (debug_file, "Compressing live range points: from %d to %ld - %ld%%\n", curr_point, pn, + curr_point == 0 ? 100 : 100 * pn / curr_point); + if (rn != old_rn) + fprintf (debug_file, "Compressing live ranges: from %ld to %ld - %ld%%\n", old_rn, rn, + rn == 0 ? 100 : 100 * rn / old_rn); + }); + curr_point = pn; + DEBUG (2, { + fprintf (debug_file, "Ranges after the compression:\n"); + print_all_live_ranges (gen_ctx); + }); +} + +#define spill_gen gen /* pseudo regs fully spilled in BB */ +#define spill_kill kill /* pseudo regs referenced in the BB */ + +static void process_bb_ranges (gen_ctx_t gen_ctx, bb_t bb, MIR_insn_t start_insn, + MIR_insn_t tail_insn) { + MIR_reg_t var, reg, early_clobbered_hard_reg1, early_clobbered_hard_reg2; + size_t nel; + int scan_var, op_num, incr_p; + bitmap_iterator_t bi; + insn_var_iterator_t insn_var_iter; + + DEBUG (2, { + fprintf (debug_file, " ------BB%u end: point=%d\n", (unsigned) bb->index, curr_point); + }); + bitmap_clear (referenced_vars); + bitmap_clear (live_vars); + if (bb->live_out != NULL) FOREACH_BITMAP_BIT (bi, bb->live_out, nel) { + make_var_live (gen_ctx, scan_var_to_var (gen_ctx, (int) nel), (int) nel, curr_point, FALSE); + } + for (MIR_insn_t insn = tail_insn;; insn = DLIST_PREV (MIR_insn_t, insn)) { + DEBUG (2, { + fprintf (debug_file, " p%-5d", curr_point); + MIR_output_insn (gen_ctx->ctx, debug_file, insn, curr_func_item->u.func, TRUE); + }); + if (insn->code == MIR_PHI) { + if ((scan_var = var_to_scan_var (gen_ctx, insn->ops[0].u.var)) >= 0) { + make_var_dead (gen_ctx, insn->ops[0].u.var, scan_var, curr_point, TRUE); + curr_point++; + } + if (insn == start_insn) break; + continue; + } + incr_p = FALSE; + FOREACH_OUT_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num) { + if ((scan_var = var_to_scan_var (gen_ctx, var)) < 0) continue; + incr_p |= make_var_dead (gen_ctx, var, scan_var, curr_point, TRUE); + } + if (scan_vars_num == 0 && MIR_call_code_p (insn->code)) { + if (incr_p) curr_point++; + incr_p = FALSE; + FOREACH_BITMAP_BIT (bi, call_used_hard_regs[MIR_T_UNDEF], nel) { + make_var_dead (gen_ctx, (MIR_reg_t) nel, (int) nel, curr_point, TRUE); + incr_p = TRUE; + } + bitmap_t args = (optimize_level > 0 ? ((bb_insn_t) insn->data)->call_hard_reg_args + : ((insn_data_t) insn->data)->u.call_hard_reg_args); + if (args != NULL) { + FOREACH_BITMAP_BIT (bi, args, nel) { + make_var_live (gen_ctx, (MIR_reg_t) nel, (int) nel, curr_point, TRUE); + } + } + FOREACH_BITMAP_BIT (bi, live_vars, nel) { + MIR_reg_t live_reg; + + if (nel <= MAX_HARD_REG) continue; + live_reg = (MIR_reg_t) nel; + bitmap_set_bit_p (curr_cfg->call_crossed_regs, live_reg); + } + } + if (incr_p) curr_point++; + incr_p = FALSE; + FOREACH_IN_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num) { + if ((scan_var = var_to_scan_var (gen_ctx, var)) < 0) continue; + incr_p |= make_var_live (gen_ctx, var, scan_var, curr_point, TRUE); + } + if (scan_vars_num == 0) { + target_get_early_clobbered_hard_regs (insn, &early_clobbered_hard_reg1, + &early_clobbered_hard_reg2); + if ((reg = early_clobbered_hard_reg1) != MIR_NON_VAR) { + incr_p |= make_var_live (gen_ctx, reg, (int) reg, curr_point, TRUE); + incr_p |= make_var_dead (gen_ctx, reg, (int) reg, curr_point, TRUE); + } + if ((reg = early_clobbered_hard_reg2) != MIR_NON_VAR) { + incr_p |= make_var_live (gen_ctx, reg, (int) reg, curr_point, TRUE); + incr_p |= make_var_dead (gen_ctx, reg, (int) reg, curr_point, TRUE); + } + } + if (incr_p) curr_point++; + if (insn == start_insn) break; + } + gen_assert (bitmap_equal_p (bb->live_in, live_vars)); + FOREACH_BITMAP_BIT (bi, bb->live_in, nel) { + make_var_dead (gen_ctx, scan_var_to_var (gen_ctx, (int) nel), (int) nel, curr_point, FALSE); + if (scan_vars_num == 0 && !bitmap_bit_p (referenced_vars, nel)) + add_lr_bb (gen_ctx, (MIR_reg_t) nel, bb); + } + if (scan_vars_num == 0) { /* setup spill info for RA */ + bitmap_clear (bb->spill_gen); + bitmap_clear (bb->spill_kill); + FOREACH_BITMAP_BIT (bi, referenced_vars, nel) { + if (nel > MAX_HARD_REG) bitmap_set_bit_p (bb->spill_kill, nel); + } + } + if (!bitmap_empty_p (bb->live_in)) curr_point++; +} + +#undef spill_gen +#undef spill_kill + +static void build_live_ranges (gen_ctx_t gen_ctx) { + size_t i; + MIR_reg_t max_var; + MIR_insn_t insn, next_insn, head_insn; + bb_t bb; + + curr_point = 0; + max_var = get_max_var (gen_ctx); + gen_assert (VARR_LENGTH (live_range_t, var_live_ranges) == 0); + for (i = 0; i <= max_var; i++) VARR_PUSH (live_range_t, var_live_ranges, NULL); + if (optimize_level == 0) { + for (head_insn = insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); insn != NULL; + insn = next_insn) { + next_insn = DLIST_NEXT (MIR_insn_t, insn); + bb = get_insn_bb (gen_ctx, head_insn); + if (next_insn == NULL || bb != get_insn_bb (gen_ctx, next_insn)) { + process_bb_ranges (gen_ctx, bb, head_insn, insn); + head_insn = next_insn; + } + } + } else { + VARR_TRUNC (bb_t, worklist, 0); + for (bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) + VARR_PUSH (bb_t, worklist, bb); + if (optimize_level <= 1) /* arrange BBs in PO (post order) for more compact ranges: */ + qsort (VARR_ADDR (bb_t, worklist), VARR_LENGTH (bb_t, worklist), sizeof (bb), post_cmp); + for (i = 0; i < VARR_LENGTH (bb_t, worklist); i++) { + bb = VARR_GET (bb_t, worklist, i); + if (DLIST_HEAD (bb_insn_t, bb->bb_insns) == NULL) continue; + process_bb_ranges (gen_ctx, bb, DLIST_HEAD (bb_insn_t, bb->bb_insns)->insn, + DLIST_TAIL (bb_insn_t, bb->bb_insns)->insn); + } + } + DEBUG (2, { print_all_live_ranges (gen_ctx); }); + shrink_live_ranges (gen_ctx); +} + +static void free_func_live_ranges (gen_ctx_t gen_ctx) { + size_t i; + + for (i = 0; i < VARR_LENGTH (live_range_t, var_live_ranges); i++) + free_live_ranges (gen_ctx, VARR_GET (live_range_t, var_live_ranges, i)); + VARR_TRUNC (live_range_t, var_live_ranges, 0); +} + +static void init_live_ranges (gen_ctx_t gen_ctx) { + gen_ctx->lr_ctx = gen_malloc (gen_ctx, sizeof (struct lr_ctx)); + VARR_CREATE (int, var_to_scan_var_map, 0); + VARR_CREATE (MIR_reg_t, scan_var_to_var_map, 0); + init_lr_bbs (gen_ctx); + init_lrs (gen_ctx); + VARR_CREATE (live_range_t, var_live_ranges, 0); + VARR_CREATE (int, point_map, 1024); + live_vars = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); + referenced_vars = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); + points_with_born_vars = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); + points_with_dead_vars = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); + points_with_born_or_dead_vars = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); +} + +static void finish_live_ranges (gen_ctx_t gen_ctx) { + VARR_DESTROY (live_range_t, var_live_ranges); + VARR_DESTROY (int, point_map); + bitmap_destroy (live_vars); + bitmap_destroy (referenced_vars); + bitmap_destroy (points_with_born_vars); + bitmap_destroy (points_with_dead_vars); + bitmap_destroy (points_with_born_or_dead_vars); + finish_lrs (gen_ctx); + finish_lr_bbs (gen_ctx); + VARR_DESTROY (int, var_to_scan_var_map); + VARR_DESTROY (MIR_reg_t, scan_var_to_var_map); + free (gen_ctx->lr_ctx); + gen_ctx->lr_ctx = NULL; +} + +/* New page */ + +/* Jump optimizations */ + +/* Remove empty blocks, branches to next insn, change branches to + jumps. ??? consider switch as a branch. */ +static void jump_opt (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; + bb_t bb, next_bb; + int maybe_unreachable_bb_p = FALSE; + long bb_deleted_insns_num; + + if ((bb_deleted_insns_num = remove_unreachable_bbs (gen_ctx)) != 0) { + DEBUG (1, { fprintf (debug_file, "%ld deleted unrechable bb insns\n", bb_deleted_insns_num); }); + } + bitmap_clear (temp_bitmap); + for (bb = DLIST_EL (bb_t, curr_cfg->bbs, 2); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + bb_insn_t bb_insn; + int i, start_nop, bound_nop; + + if ((bb_insn = DLIST_TAIL (bb_insn_t, bb->bb_insns)) == NULL) continue; + if (bb_insn->insn->code == MIR_SWITCH) { + start_nop = 1; + bound_nop = bb_insn->insn->nops; + } else if (MIR_branch_code_p (bb_insn->insn->code)) { + start_nop = 0; + bound_nop = 1; + } else { + continue; + } + for (i = start_nop; i < bound_nop; i++) + bitmap_set_bit_p (temp_bitmap, bb_insn->insn->ops[i].u.label->ops[0].u.u); + } + for (bb = DLIST_EL (bb_t, curr_cfg->bbs, 2); bb != NULL; bb = next_bb) { + edge_t e, out_e; + bb_insn_t label_bb_insn, last_label_bb_insn, bb_insn = DLIST_TAIL (bb_insn_t, bb->bb_insns); + MIR_insn_t insn, next_insn, last_label; + + next_bb = DLIST_NEXT (bb_t, bb); + if (bb->index != 2 /* BB2 will be used for machinize */ + && (e = DLIST_HEAD (in_edge_t, bb->in_edges)) != NULL && DLIST_NEXT (in_edge_t, e) == NULL + && (bb_insn == NULL + || ((insn = bb_insn->insn)->code == MIR_LABEL && DLIST_NEXT (bb_insn_t, bb_insn) == NULL + && DLIST_PREV (bb_insn_t, bb_insn) == NULL + && !bitmap_bit_p (temp_bitmap, insn->ops[0].u.u)))) { + /* empty bb or bb with the only label which can be removed. we can have more one the same + dest edge (e.g. when removed cond branch to the next insn). */ + out_e = DLIST_HEAD (out_edge_t, bb->out_edges); + gen_assert (out_e != NULL); + e->dst = out_e->dst; + DLIST_REMOVE (in_edge_t, bb->in_edges, e); + DLIST_INSERT_BEFORE (in_edge_t, out_e->dst->in_edges, out_e, e); + gen_assert (DLIST_HEAD (in_edge_t, bb->in_edges) == NULL); + /* Don't shorten phis in dest bbs. We don't care about SSA in this kind of bb. */ + remove_bb (gen_ctx, bb); + continue; + } + if (bb_insn == NULL) continue; + insn = bb_insn->insn; + if (!MIR_branch_code_p (insn->code)) continue; + DEBUG (2, { fprintf (debug_file, " BB%lu:\n", (unsigned long) bb->index); }); + gen_assert (insn->ops[0].mode == MIR_OP_LABEL); + if ((next_insn = DLIST_NEXT (MIR_insn_t, insn)) != NULL && next_insn->code == MIR_LABEL + && next_insn == insn->ops[0].u.label) { + DEBUG (2, { + fprintf (debug_file, " Removing trivial branch insn "); + MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, TRUE); + }); + out_e = DLIST_HEAD (out_edge_t, bb->out_edges); + out_e->fall_through_p = TRUE; + e = DLIST_NEXT (out_edge_t, out_e); + gen_assert (e == NULL || DLIST_NEXT (out_edge_t, e) == NULL); + if (e != NULL) delete_edge (e); + gen_delete_insn (gen_ctx, insn); + next_bb = bb; /* bb can became empty after removing jump. */ + } else { + for (;;) { + for (last_label = insn->ops[0].u.label; + (next_insn = DLIST_NEXT (MIR_insn_t, last_label)) != NULL + && next_insn->code == MIR_LABEL;) + last_label = next_insn; + if ((next_insn = DLIST_NEXT (MIR_insn_t, last_label)) != NULL + && next_insn->code == MIR_JMP) { + last_label = next_insn->ops[0].u.label; + } + if (insn->ops[0].u.label == last_label) break; + DEBUG (2, { + fprintf (debug_file, " Changing label in branch insn "); + MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, FALSE); + }); + label_bb_insn = insn->ops[0].u.label->data; + insn->ops[0].u.label = last_label; + last_label_bb_insn = last_label->data; + gen_assert (label_bb_insn->bb != last_label_bb_insn->bb); + e = find_edge (bb, label_bb_insn->bb); + e->dst = last_label_bb_insn->bb; + DLIST_REMOVE (in_edge_t, label_bb_insn->bb->in_edges, e); + /* We don't need to keep the edge order as we are already off SSA: */ + DLIST_APPEND (in_edge_t, e->dst->in_edges, e); + DEBUG (2, { + fprintf (debug_file, " , result insn "); + MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, TRUE); + }); + maybe_unreachable_bb_p = TRUE; + } + } + } + /* Don't shorten phis in dest bbs. We don't care about SSA for new trivial unreachable bbs. */ + if (maybe_unreachable_bb_p) remove_unreachable_bbs (gen_ctx); + enumerate_bbs (gen_ctx); +} -struct lr_ctx { - int curr_point; - bitmap_t live_vars, born_vars, dead_vars, born_or_dead_vars; - VARR (live_range_t) * var_live_ranges; - VARR (int) * point_map; +/* New Page */ +/* Aggressive register coalescing */ + +typedef struct mv { + bb_insn_t bb_insn; + size_t freq; +} mv_t; + +DEF_VARR (mv_t); + +struct coalesce_ctx { + VARR (mv_t) * moves; + /* the first and the next res in the coalesced regs group */ + VARR (MIR_reg_t) * first_coalesced_reg, *next_coalesced_reg; + bitmap_t conflict_matrix; }; -#define curr_point gen_ctx->lr_ctx->curr_point -#define live_vars gen_ctx->lr_ctx->live_vars -#define born_vars gen_ctx->lr_ctx->born_vars -#define dead_vars gen_ctx->lr_ctx->dead_vars -#define born_or_dead_vars gen_ctx->lr_ctx->born_or_dead_vars -#define var_live_ranges gen_ctx->lr_ctx->var_live_ranges -#define point_map gen_ctx->lr_ctx->point_map +#define moves gen_ctx->coalesce_ctx->moves +#define first_coalesced_reg gen_ctx->coalesce_ctx->first_coalesced_reg +#define next_coalesced_reg gen_ctx->coalesce_ctx->next_coalesced_reg +#define conflict_matrix gen_ctx->coalesce_ctx->conflict_matrix -static live_range_t create_live_range (gen_ctx_t gen_ctx, int start, int finish, - live_range_t next) { - live_range_t lr = gen_malloc (gen_ctx, sizeof (struct live_range)); +static void set_scan_var_conflict (gen_ctx_t gen_ctx, int scan_var1, int scan_var2) { + int temp_scan_var; + if (scan_var1 > scan_var2) SWAP (scan_var1, scan_var2, temp_scan_var); + bitmap_set_bit_p (conflict_matrix, scan_var1 * scan_vars_num + scan_var2); +} - gen_assert (finish < 0 || start <= finish); - lr->start = start; - lr->finish = finish; - lr->next = next; - return lr; +static int scan_var_conflict_p (gen_ctx_t gen_ctx, int scan_var1, int scan_var2) { + int temp_scan_var; + if (scan_var1 > scan_var2) SWAP (scan_var1, scan_var2, temp_scan_var); + return bitmap_bit_p (conflict_matrix, scan_var1 * scan_vars_num + scan_var2); } -static void destroy_live_range (live_range_t lr) { - live_range_t next_lr; +static void process_bb_conflicts (gen_ctx_t gen_ctx, bb_t bb, MIR_insn_t start_insn, + MIR_insn_t tail_insn) { + MIR_reg_t var; + size_t nel; + int scan_var, ignore_scan_var, live_scan_var, op_num; + bitmap_iterator_t bi; + insn_var_iterator_t insn_var_iter; - for (; lr != NULL; lr = next_lr) { - next_lr = lr->next; - free (lr); + bitmap_clear (live_vars); + if (bb->live_out != NULL) FOREACH_BITMAP_BIT (bi, bb->live_out, nel) { + bitmap_set_bit_p (live_vars, nel); + } + for (MIR_insn_t insn = tail_insn;; insn = DLIST_PREV (MIR_insn_t, insn)) { + ignore_scan_var = -1; + if (move_p (insn)) ignore_scan_var = var_to_scan_var (gen_ctx, insn->ops[1].u.var); + FOREACH_OUT_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num) { + if ((scan_var = var_to_scan_var (gen_ctx, var)) < 0) continue; + FOREACH_BITMAP_BIT (bi, live_vars, nel) { + live_scan_var = (MIR_reg_t) nel; + if (live_scan_var != scan_var && live_scan_var != ignore_scan_var) + set_scan_var_conflict (gen_ctx, scan_var, live_scan_var); + } + bitmap_clear_bit_p (live_vars, scan_var); + } + FOREACH_IN_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num) { + if ((scan_var = var_to_scan_var (gen_ctx, var)) >= 0) bitmap_set_bit_p (live_vars, scan_var); + } + if (insn == start_insn) break; } + gen_assert (bitmap_equal_p (bb->live_in, live_vars)); } -static inline int make_var_dead (gen_ctx_t gen_ctx, MIR_reg_t var, int point) { - live_range_t lr; - - if (bitmap_clear_bit_p (live_vars, var)) { - lr = VARR_GET (live_range_t, var_live_ranges, var); - lr->finish = point; - } else { /* insn with unused result: result still needs a register */ - VARR_SET (live_range_t, var_live_ranges, var, - create_live_range (gen_ctx, point, point, - VARR_GET (live_range_t, var_live_ranges, var))); +static void build_conflict_matrix (gen_ctx_t gen_ctx) { + bitmap_clear (conflict_matrix); + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + if (DLIST_HEAD (bb_insn_t, bb->bb_insns) == NULL) continue; + process_bb_conflicts (gen_ctx, bb, DLIST_HEAD (bb_insn_t, bb->bb_insns)->insn, + DLIST_TAIL (bb_insn_t, bb->bb_insns)->insn); } - return TRUE; + DEBUG (2, { + fprintf (debug_file, " Conflict matrix size=%lu, scan vars = %d\n", + (unsigned long) bitmap_size (conflict_matrix), scan_vars_num); + }); } -static inline int make_var_live (gen_ctx_t gen_ctx, MIR_reg_t var, int point) { - live_range_t lr; - - if (!bitmap_set_bit_p (live_vars, var)) return FALSE; - if ((lr = VARR_GET (live_range_t, var_live_ranges, var)) == NULL - || (lr->finish != point && lr->finish + 1 != point)) - VARR_SET (live_range_t, var_live_ranges, var, create_live_range (gen_ctx, point, -1, lr)); +static int substitute_reg (gen_ctx_t gen_ctx, MIR_reg_t *reg) { + if (*reg == MIR_NON_VAR || VARR_GET (MIR_reg_t, first_coalesced_reg, *reg) == *reg) return FALSE; + *reg = VARR_GET (MIR_reg_t, first_coalesced_reg, *reg); return TRUE; } -#if !MIR_NO_GEN_DEBUG -static void print_live_ranges (gen_ctx_t gen_ctx) { - MIR_context_t ctx = gen_ctx->ctx; - live_range_t lr; +static int mv_freq_cmp (const void *v1p, const void *v2p) { + const mv_t *mv1 = (const mv_t *) v1p; + const mv_t *mv2 = (const mv_t *) v2p; - fprintf (debug_file, "+++++++++++++Live ranges:\n"); - gen_assert (get_nvars (gen_ctx) == VARR_LENGTH (live_range_t, var_live_ranges)); - for (size_t i = 0; i < VARR_LENGTH (live_range_t, var_live_ranges); i++) { - if ((lr = VARR_GET (live_range_t, var_live_ranges, i)) == NULL) continue; - fprintf (debug_file, "%lu", (unsigned long) i); - if (var_is_reg_p (i)) - fprintf (debug_file, " (%s:%s)", - MIR_type_str (ctx, MIR_reg_type (ctx, var2reg (gen_ctx, i), curr_func_item->u.func)), - MIR_reg_name (ctx, var2reg (gen_ctx, i), curr_func_item->u.func)); - fprintf (debug_file, ":"); - for (; lr != NULL; lr = lr->next) fprintf (debug_file, " [%d..%d]", lr->start, lr->finish); - fprintf (debug_file, "\n"); - } + if (mv1->freq > mv2->freq) return -1; + if (mv1->freq < mv2->freq) return 1; + return (long) mv1->bb_insn->index - (long) mv2->bb_insn->index; } -#endif - -static void shrink_live_ranges (gen_ctx_t gen_ctx) { - size_t p; - long int n; - live_range_t lr, prev_lr, next_lr; - int born_p, dead_p, prev_dead_p; - bitmap_iterator_t bi; - bitmap_clear (born_vars); - bitmap_clear (dead_vars); - for (size_t i = 0; i < VARR_LENGTH (live_range_t, var_live_ranges); i++) { - for (lr = VARR_GET (live_range_t, var_live_ranges, i); lr != NULL; lr = lr->next) { - gen_assert (lr->start <= lr->finish); - bitmap_set_bit_p (born_vars, lr->start); - bitmap_set_bit_p (dead_vars, lr->finish); +static int var_conflict_p (gen_ctx_t gen_ctx, MIR_reg_t var1, MIR_reg_t var2) { + gen_assert (var1 == VARR_GET (MIR_reg_t, first_coalesced_reg, var1)); + gen_assert (var2 == VARR_GET (MIR_reg_t, first_coalesced_reg, var2)); + for (MIR_reg_t last_reg1 = var1, reg1 = VARR_GET (MIR_reg_t, next_coalesced_reg, var1);; + reg1 = VARR_GET (MIR_reg_t, next_coalesced_reg, reg1)) { + int scan_var1 = var_to_scan_var (gen_ctx, reg1); + for (MIR_reg_t last_reg2 = var2, reg2 = VARR_GET (MIR_reg_t, next_coalesced_reg, var2);; + reg2 = VARR_GET (MIR_reg_t, next_coalesced_reg, reg2)) { + int scan_var2 = var_to_scan_var (gen_ctx, reg2); + if (scan_var_conflict_p (gen_ctx, scan_var1, scan_var2)) return TRUE; + if (reg2 == last_reg2) break; } + if (reg1 == last_reg1) break; } + return FALSE; +} - VARR_TRUNC (int, point_map, 0); - for (size_t i = 0; i <= curr_point; i++) VARR_PUSH (int, point_map, 0); - bitmap_ior (born_or_dead_vars, born_vars, dead_vars); - n = -1; - prev_dead_p = TRUE; - FOREACH_BITMAP_BIT (bi, born_or_dead_vars, p) { - born_p = bitmap_bit_p (born_vars, p); - dead_p = bitmap_bit_p (dead_vars, p); - assert (born_p || dead_p); - if (!prev_dead_p || !born_p) /* 1st point is always a born */ - VARR_SET (int, point_map, p, n); - else - VARR_SET (int, point_map, p, ++n); - prev_dead_p = dead_p; +/* Merge two sets of coalesced regs given correspondingly by regs REG1 and REG2. */ +static void merge_regs (gen_ctx_t gen_ctx, MIR_reg_t reg1, MIR_reg_t reg2) { + MIR_reg_t reg, first, first2, last, next, temp; + + first = VARR_GET (MIR_reg_t, first_coalesced_reg, reg1); + if ((first2 = VARR_GET (MIR_reg_t, first_coalesced_reg, reg2)) == first) return; + if (MIR_reg_hard_reg_name (gen_ctx->ctx, first2 - MAX_HARD_REG, curr_func_item->u.func) != NULL + || (MIR_reg_hard_reg_name (gen_ctx->ctx, first - MAX_HARD_REG, curr_func_item->u.func) == NULL + && first2 < first)) { + SWAP (first, first2, temp); + SWAP (reg1, reg2, temp); + } + for (last = reg2, reg = VARR_GET (MIR_reg_t, next_coalesced_reg, reg2);; + reg = VARR_GET (MIR_reg_t, next_coalesced_reg, reg)) { + VARR_SET (MIR_reg_t, first_coalesced_reg, reg, first); + if (reg == reg2) break; + last = reg; } + next = VARR_GET (MIR_reg_t, next_coalesced_reg, first); + VARR_SET (MIR_reg_t, next_coalesced_reg, first, reg2); + VARR_SET (MIR_reg_t, next_coalesced_reg, last, next); +} - n++; - DEBUG (2, { - fprintf (debug_file, "Compressing live ranges: from %d to %ld - %ld%%\n", curr_point, n, - curr_point == 0 ? 100 : 100 * n / curr_point); - }); - curr_point = n; +static void update_bitmap_after_coalescing (gen_ctx_t gen_ctx, bitmap_t bm) { + MIR_reg_t reg, first_reg; + size_t nel; + bitmap_iterator_t bi; - for (size_t i = 0; i < VARR_LENGTH (live_range_t, var_live_ranges); i++) { - for (lr = VARR_GET (live_range_t, var_live_ranges, i), prev_lr = NULL; lr != NULL; - lr = next_lr) { - next_lr = lr->next; - lr->start = VARR_GET (int, point_map, lr->start); - lr->finish = VARR_GET (int, point_map, lr->finish); - if (prev_lr == NULL || prev_lr->start > lr->finish + 1) { - prev_lr = lr; - continue; + FOREACH_BITMAP_BIT (bi, bm, nel) { + reg = (MIR_reg_t) nel; + if (reg <= MAX_HARD_REG) continue; + if ((first_reg = VARR_GET (MIR_reg_t, first_coalesced_reg, reg)) == reg) continue; + bitmap_clear_bit_p (bm, reg); + bitmap_set_bit_p (bm, first_reg); + } +} + +static void collect_moves (gen_ctx_t gen_ctx) { + gen_assert (optimize_level > 0); + VARR_TRUNC (mv_t, moves, 0); + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + for (bb_insn_t bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; + bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) { + MIR_insn_t insn = bb_insn->insn; + mv_t mv; + + if (move_p (insn)) { + mv.bb_insn = bb_insn; + mv.freq = 1; + for (int i = bb_loop_level (bb); i > 0; i--) + if (mv.freq < SIZE_MAX / 8) mv.freq *= LOOP_COST_FACTOR; + VARR_PUSH (mv_t, moves, mv); } - prev_lr->start = lr->start; - prev_lr->next = next_lr; - free (lr); } } - DEBUG (2, { - fprintf (debug_file, "Ranges after the compression:\n"); - print_live_ranges (gen_ctx); - }); + qsort (VARR_ADDR (mv_t, moves), VARR_LENGTH (mv_t, moves), sizeof (mv_t), mv_freq_cmp); } -static void build_live_ranges (gen_ctx_t gen_ctx) { - MIR_insn_t insn; - MIR_reg_t var, nvars, early_clobbered_hard_reg1, early_clobbered_hard_reg2; - size_t i, nel, passed_mem_num; - int op_num, incr_p, out_p, mem_p; - bitmap_iterator_t bi; - insn_var_iterator_t insn_var_iter; +static void scan_collected_moves (gen_ctx_t gen_ctx) { + for (size_t i = 0; i < VARR_LENGTH (mv_t, moves); i++) { + mv_t mv = VARR_GET (mv_t, moves, i); + MIR_insn_t insn = mv.bb_insn->insn; + collect_scan_var (gen_ctx, insn->ops[0].u.var); + collect_scan_var (gen_ctx, insn->ops[1].u.var); + } +} - curr_point = 0; - nvars = get_nvars (gen_ctx); - gen_assert (VARR_LENGTH (live_range_t, var_live_ranges) == 0); - for (i = 0; i < nvars; i++) VARR_PUSH (live_range_t, var_live_ranges, NULL); - for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { - DEBUG (2, { - fprintf (debug_file, " ------BB%u end: point=%d\n", (unsigned) bb->index, curr_point); - }); - bitmap_clear (live_vars); - if (bb->live_out != NULL) FOREACH_BITMAP_BIT (bi, bb->live_out, nel) { - make_var_live (gen_ctx, nel, curr_point); - } - for (bb_insn_t bb_insn = DLIST_TAIL (bb_insn_t, bb->bb_insns); bb_insn != NULL; - bb_insn = DLIST_PREV (bb_insn_t, bb_insn)) { - insn = bb_insn->insn; +static void coalesce (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_reg_t reg, sreg, dreg, first_reg, first_sreg, first_dreg, sreg_var, dreg_var; + MIR_insn_t insn, new_insn, next_insn; + bb_t bb; + bb_insn_t bb_insn; + mv_t mv; + size_t nops; + int coalesced_moves = 0, change_p; + + gen_assert (optimize_level > 0); + VARR_TRUNC (MIR_reg_t, first_coalesced_reg, 0); + VARR_TRUNC (MIR_reg_t, next_coalesced_reg, 0); + for (MIR_reg_t i = 0; i <= curr_cfg->max_var; i++) { + VARR_PUSH (MIR_reg_t, first_coalesced_reg, i); + VARR_PUSH (MIR_reg_t, next_coalesced_reg, i); + } + build_conflict_matrix (gen_ctx); + /* Coalesced moves, most frequently executed first. */ + for (size_t i = 0; i < VARR_LENGTH (mv_t, moves); i++) { + mv = VARR_GET (mv_t, moves, i); + bb_insn = mv.bb_insn; + insn = bb_insn->insn; + dreg = insn->ops[0].u.var; + sreg = insn->ops[1].u.var; + gen_assert (sreg > MAX_HARD_REG && dreg > MAX_HARD_REG); + first_sreg = VARR_GET (MIR_reg_t, first_coalesced_reg, sreg); + first_dreg = VARR_GET (MIR_reg_t, first_coalesced_reg, dreg); + if (first_sreg == first_dreg) { + coalesced_moves++; DEBUG (2, { - fprintf (debug_file, " p%-5d", curr_point); + fprintf (debug_file, "Coalescing move r%d-r%d (freq=%llud):", sreg, dreg, + (unsigned long long) mv.freq); print_bb_insn (gen_ctx, bb_insn, TRUE); }); - incr_p = FALSE; - FOREACH_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (out_p) incr_p |= make_var_dead (gen_ctx, var, curr_point); - } - if (MIR_call_code_p (insn->code)) { - FOREACH_BITMAP_BIT (bi, call_used_hard_regs[MIR_T_UNDEF], nel) { - make_var_dead (gen_ctx, nel, curr_point); - } - FOREACH_BITMAP_BIT (bi, bb_insn->call_hard_reg_args, nel) { - make_var_live (gen_ctx, nel, curr_point); - } - FOREACH_BITMAP_BIT (bi, live_vars, nel) { - MIR_reg_t breg; - - if (!var_is_reg_p (nel)) continue; - breg = reg2breg (gen_ctx, var2reg (gen_ctx, nel)); - bitmap_set_bit_p (curr_cfg->call_crossed_bregs, breg); - } - } - if (incr_p) curr_point++; - incr_p = FALSE; - FOREACH_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (!out_p) incr_p |= make_var_live (gen_ctx, var, curr_point); - } - target_get_early_clobbered_hard_regs (insn, &early_clobbered_hard_reg1, - &early_clobbered_hard_reg2); - if (early_clobbered_hard_reg1 != MIR_NON_HARD_REG) { - incr_p |= make_var_live (gen_ctx, early_clobbered_hard_reg1, curr_point); - incr_p |= make_var_dead (gen_ctx, early_clobbered_hard_reg1, curr_point); + } else { + sreg_var = first_sreg; + dreg_var = first_dreg; + if (!var_conflict_p (gen_ctx, sreg_var, dreg_var) + && (MIR_reg_hard_reg_name (ctx, first_sreg - MAX_HARD_REG, curr_func_item->u.func) == NULL + || MIR_reg_hard_reg_name (ctx, first_dreg - MAX_HARD_REG, curr_func_item->u.func) + == NULL)) { + coalesced_moves++; + DEBUG (2, { + fprintf (debug_file, "Coalescing move r%d-r%d (freq=%llu):", sreg, dreg, + (unsigned long long) mv.freq); + print_bb_insn (gen_ctx, bb_insn, TRUE); + }); + merge_regs (gen_ctx, sreg, dreg); } - if (early_clobbered_hard_reg2 != MIR_NON_HARD_REG) { - incr_p |= make_var_live (gen_ctx, early_clobbered_hard_reg2, curr_point); - incr_p |= make_var_dead (gen_ctx, early_clobbered_hard_reg2, curr_point); + } + } + reg_info_t *reg_infos = VARR_ADDR (reg_info_t, curr_cfg->reg_info); + for (reg = MAX_HARD_REG + 1; reg <= curr_cfg->max_var; reg++) { + if ((first_reg = VARR_GET (MIR_reg_t, first_coalesced_reg, reg)) == reg) continue; + reg_infos[first_reg].freq += reg_infos[reg].freq; + reg_infos[reg].freq = 0; + } + for (size_t i = 0; i < VARR_LENGTH (mv_t, moves); i++) { + mv = VARR_GET (mv_t, moves, i); + bb_insn = mv.bb_insn; + bb = bb_insn->bb; + insn = bb_insn->insn; + dreg = insn->ops[0].u.var; + sreg = insn->ops[1].u.var; + gen_assert (sreg > MAX_HARD_REG && dreg > MAX_HARD_REG); + first_reg = VARR_GET (MIR_reg_t, first_coalesced_reg, sreg); + if (first_reg != VARR_GET (MIR_reg_t, first_coalesced_reg, dreg)) continue; + if (DLIST_TAIL (bb_insn_t, bb->bb_insns) == bb_insn + && DLIST_HEAD (bb_insn_t, bb->bb_insns) == bb_insn) { /* bb is becoming empty */ + new_insn = MIR_new_label (ctx); + MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + add_new_bb_insn (gen_ctx, new_insn, bb, FALSE); + DEBUG (2, { + fprintf (debug_file, "Adding label for becoming empty BB "); + MIR_output_insn (ctx, debug_file, new_insn, curr_func_item->u.func, TRUE); + }); + } + DEBUG (2, { + fprintf (debug_file, "Deleting coalesced move "); + MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, TRUE); + }); + gen_delete_insn (gen_ctx, insn); + } + for (insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); insn != NULL; + insn = next_insn) { + next_insn = DLIST_NEXT (MIR_insn_t, insn); + nops = MIR_insn_nops (ctx, insn); + change_p = FALSE; + for (size_t i = 0; i < nops; i++) { + MIR_op_t *op = &insn->ops[i]; + switch (op->mode) { + case MIR_OP_VAR: change_p = substitute_reg (gen_ctx, &op->u.var) || change_p; break; + case MIR_OP_VAR_MEM: + change_p = substitute_reg (gen_ctx, &op->u.var_mem.base) || change_p; + change_p = substitute_reg (gen_ctx, &op->u.var_mem.index) || change_p; + break; + default: /* do nothing */ break; } - if (incr_p) curr_point++; } - gen_assert (bitmap_equal_p (live_vars, bb->live_in)); - FOREACH_BITMAP_BIT (bi, bb->live_in, nel) { make_var_dead (gen_ctx, nel, curr_point); } - if (!bitmap_empty_p (bb->live_in)) curr_point++; + if (change_p) + for (dead_var_t dv = DLIST_HEAD (dead_var_t, ((bb_insn_t) insn->data)->insn_dead_vars); + dv != NULL; dv = DLIST_NEXT (dead_var_t, dv)) { + if (dv->var <= MAX_HARD_REG) continue; + reg = dv->var; + if ((first_reg = VARR_GET (MIR_reg_t, first_coalesced_reg, reg)) != reg) + dv->var = first_reg; + } } - DEBUG (2, { print_live_ranges (gen_ctx); }); - shrink_live_ranges (gen_ctx); + /* Update live_in & live_out */ + for (bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + update_bitmap_after_coalescing (gen_ctx, bb->live_in); + update_bitmap_after_coalescing (gen_ctx, bb->live_out); + } + DEBUG (1, { + int moves_num = (int) VARR_LENGTH (mv_t, moves); + if (coalesced_moves != 0) { + fprintf (debug_file, "Coalesced Moves = %d out of %d moves (%.1f%%)\n", coalesced_moves, + moves_num, 100.0 * coalesced_moves / moves_num); + } + }); } +#undef live_in +#undef live_out -static void destroy_func_live_ranges (gen_ctx_t gen_ctx) { - size_t i; +static void init_coalesce (gen_ctx_t gen_ctx) { + gen_ctx->coalesce_ctx = gen_malloc (gen_ctx, sizeof (struct coalesce_ctx)); + VARR_CREATE (mv_t, moves, 0); + VARR_CREATE (MIR_reg_t, first_coalesced_reg, 0); + VARR_CREATE (MIR_reg_t, next_coalesced_reg, 0); + conflict_matrix = bitmap_create (); +} - for (i = 0; i < VARR_LENGTH (live_range_t, var_live_ranges); i++) - destroy_live_range (VARR_GET (live_range_t, var_live_ranges, i)); - VARR_TRUNC (live_range_t, var_live_ranges, 0); +static void finish_coalesce (gen_ctx_t gen_ctx) { + VARR_DESTROY (mv_t, moves); + VARR_DESTROY (MIR_reg_t, first_coalesced_reg); + VARR_DESTROY (MIR_reg_t, next_coalesced_reg); + bitmap_destroy (conflict_matrix); + free (gen_ctx->coalesce_ctx); + gen_ctx->coalesce_ctx = NULL; } -#if !MIR_NO_GEN_DEBUG -static void output_bb_live_info (gen_ctx_t gen_ctx, bb_t bb) { - output_bitmap (gen_ctx, " live_in:", bb->live_in, TRUE); - output_bitmap (gen_ctx, " live_out:", bb->live_out, TRUE); - output_bitmap (gen_ctx, " live_gen:", bb->live_gen, TRUE); - output_bitmap (gen_ctx, " live_kill:", bb->live_kill, TRUE); +/* New page */ + +static void add_reload (gen_ctx_t gen_ctx, MIR_insn_t anchor, const MIR_op_t *op1, + const MIR_op_t *op2, MIR_type_t type, int to_p) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_insn_t new_insn; + MIR_insn_code_t move_code = get_move_code (type); + if (to_p) { + new_insn = MIR_new_insn (ctx, move_code, *op1, *op2); + gen_add_insn_after (gen_ctx, anchor, new_insn); + } else { + new_insn = MIR_new_insn (ctx, move_code, *op2, *op1); + gen_add_insn_before (gen_ctx, anchor, new_insn); + } + DEBUG (2, { + fprintf (debug_file, " Add %s insn", (to_p ? "after" : "before")); + MIR_output_insn (ctx, debug_file, anchor, curr_func_item->u.func, FALSE); + fprintf (debug_file, ": "); + MIR_output_insn (ctx, debug_file, new_insn, curr_func_item->u.func, TRUE); + }); } -#endif -static void init_live_ranges (gen_ctx_t gen_ctx) { - gen_ctx->lr_ctx = gen_malloc (gen_ctx, sizeof (struct lr_ctx)); - VARR_CREATE (live_range_t, var_live_ranges, 0); - VARR_CREATE (int, point_map, 1024); - live_vars = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); - born_vars = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); - dead_vars = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); - born_or_dead_vars = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); +static void add_inout_reloads (gen_ctx_t gen_ctx, MIR_insn_t insn) { + MIR_context_t ctx = gen_ctx->ctx; + int out_p; + MIR_op_mode_t mode; + MIR_type_t type; + MIR_reg_t temp_reg; + MIR_op_t temp_op; + + gen_assert (MIR_insn_nops (ctx, insn) >= 2 && !MIR_call_code_p (insn->code) + && insn->code != MIR_RET); + MIR_insn_op_mode (ctx, insn, 1, &out_p); + gen_assert (!out_p); + mode = MIR_insn_op_mode (ctx, insn, 0, &out_p); + gen_assert (out_p && mode == MIR_insn_op_mode (ctx, insn, 1, &out_p) && !out_p); + type = mode2type (mode); + temp_reg = gen_new_temp_reg (gen_ctx, type, curr_func_item->u.func); + temp_op = _MIR_new_var_op (ctx, temp_reg); + add_reload (gen_ctx, insn, &insn->ops[1], &temp_op, type, FALSE); + add_reload (gen_ctx, insn, &insn->ops[0], &temp_op, type, TRUE); + insn->ops[0] = insn->ops[1] = temp_op; } -static void finish_live_ranges (gen_ctx_t gen_ctx) { - VARR_DESTROY (live_range_t, var_live_ranges); - VARR_DESTROY (int, point_map); - bitmap_destroy (live_vars); - bitmap_destroy (born_vars); - bitmap_destroy (dead_vars); - bitmap_destroy (born_or_dead_vars); - free (gen_ctx->lr_ctx); - gen_ctx->lr_ctx = NULL; +static void make_io_dup_op_insns (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_func_t func; + MIR_insn_t insn, next_insn; + MIR_insn_code_t code; + MIR_op_t mem_op, *mem_op_ref; + MIR_op_mode_t mode; + MIR_type_t type; + MIR_reg_t temp_reg, type_regs[MIR_T_BOUND]; + size_t i; + int out_p; + + gen_assert (curr_func_item->item_type == MIR_func_item); + func = curr_func_item->u.func; + for (i = 0; target_io_dup_op_insn_codes[i] != MIR_INSN_BOUND; i++) + bitmap_set_bit_p (insn_to_consider, target_io_dup_op_insn_codes[i]); + for (type = 0; type < MIR_T_BOUND; type++) type_regs[type] = MIR_NON_VAR; + for (insn = DLIST_HEAD (MIR_insn_t, func->insns); insn != NULL; insn = next_insn) { + next_insn = DLIST_NEXT (MIR_insn_t, insn); + code = insn->code; + if (code == MIR_LABEL || MIR_addr_code_p (code) || code == MIR_USE) continue; + if (bitmap_bit_p (insn_to_consider, code) && !MIR_op_eq_p (ctx, insn->ops[0], insn->ops[1])) + add_inout_reloads (gen_ctx, insn); + if (target_insn_ok_p (gen_ctx, insn)) continue; + for (i = 0; i < insn->nops; i++) { /* try to change one non-dup mem op to reg */ + if (insn->ops[i].mode != MIR_OP_VAR_MEM) continue; + if (bitmap_bit_p (insn_to_consider, code) && (i == 0 || i == 1)) continue; + mode = MIR_insn_op_mode (ctx, insn, i, &out_p); + type = mode2type (mode); + /* we don't use hard regs for this type: */ + if (get_temp_hard_reg (type, TRUE) == MIR_NON_VAR) continue; + if (type_regs[type] == MIR_NON_VAR) + type_regs[type] = gen_new_temp_reg (gen_ctx, type, curr_func_item->u.func); + mem_op = insn->ops[i]; + insn->ops[i] = _MIR_new_var_op (ctx, type_regs[type]); + if (target_insn_ok_p (gen_ctx, insn)) { + add_reload (gen_ctx, insn, &mem_op, &insn->ops[i], type, out_p); + type_regs[type] = MIR_NON_VAR; + break; + } + insn->ops[i] = mem_op; + } + if (i < insn->nops) continue; + if (bitmap_bit_p (insn_to_consider, code) && insn->ops[0].mode == MIR_OP_VAR_MEM) { + add_inout_reloads (gen_ctx, insn); + if (target_insn_ok_p (gen_ctx, insn)) continue; + } + for (i = 0; i < insn->nops; i++) { /* change all non-dup mem ops to pseudo */ + if (insn->ops[i].mode != MIR_OP_VAR_MEM) continue; + if (bitmap_bit_p (insn_to_consider, code) && (i == 0 || i == 1)) continue; + mode = MIR_insn_op_mode (ctx, insn, i, &out_p); + type = mode2type (mode); + /* we don't use hard regs for this type: */ + if (get_temp_hard_reg (type, TRUE) == MIR_NON_VAR) continue; + temp_reg = gen_new_temp_reg (gen_ctx, type, curr_func_item->u.func); + mem_op_ref = &insn->ops[i]; + insn->ops[i] = _MIR_new_var_op (ctx, temp_reg); + add_reload (gen_ctx, insn, mem_op_ref, &insn->ops[i], type, out_p); + } + /* target_insn_ok_p still can return FALSE here for insn which will be converted to builtin */ + } } /* New Page */ /* Register allocation */ -DEF_VARR (MIR_reg_t); +typedef struct allocno_info { + MIR_reg_t reg; + int tied_reg_p; + reg_info_t *reg_infos; +} allocno_info_t; + +DEF_VARR (allocno_info_t); + +typedef struct spill_cache_el { + uint32_t age; + MIR_reg_t slot; +} spill_cache_el_t; -typedef struct breg_info { - MIR_reg_t breg; - reg_info_t *breg_infos; -} breg_info_t; +DEF_VARR (spill_cache_el_t); -DEF_VARR (breg_info_t); DEF_VARR (bitmap_t); +typedef struct lr_gap { + int16_t hreg; /* key, hard reg assigned to reg */ + int16_t type; /* type of reg */ + MIR_reg_t reg; /* reg of the gap lr */ + live_range_t lr; /* the gap lr, lr->start is another key */ +} lr_gap_t; + +DEF_VARR (lr_gap_t); +DEF_HTAB (lr_gap_t); + +typedef struct spill_el { + MIR_reg_t reg; + signed char spill_p, edge_p, bb_end_p /* used only for !edge_p */; + union { + edge_t e; + bb_t bb; + } u; +} spill_el_t; + +DEF_VARR (spill_el_t); + +typedef struct insn_reload { + MIR_type_t type; + MIR_reg_t var, hreg; +} insn_reload_t; + +#define MAX_INSN_RELOADS (2 * 4) /* 2 temp regs * 4 types */ struct ra_ctx { - VARR (MIR_reg_t) * breg_renumber; - VARR (breg_info_t) * sorted_bregs; - VARR (bitmap_t) * used_locs; /* indexed by bb or point */ + MIR_reg_t start_mem_loc; + VARR (MIR_reg_t) * reg_renumber; + VARR (allocno_info_t) * sorted_regs; + VARR (bitmap_t) * used_locs, *busy_used_locs; /* indexed by bb or point */ VARR (bitmap_t) * var_bbs; - bitmap_t conflict_locs; - reg_info_t *curr_breg_infos; - VARR (size_t) * loc_profits; - VARR (size_t) * loc_profit_ages; - size_t curr_age; + VARR (lr_gap_t) * spill_gaps, *curr_gaps; /* used to find live ranges to spill */ + bitmap_t lr_gap_bitmaps[MAX_HARD_REG + 1]; + HTAB (lr_gap_t) * lr_gap_tab; + VARR (spill_el_t) * spill_els; + VARR (spill_cache_el_t) * spill_cache; + uint32_t spill_cache_age; + bitmap_t conflict_locs1; + reg_info_t *curr_reg_infos; + int in_reloads_num, out_reloads_num; + insn_reload_t in_reloads[MAX_INSN_RELOADS], out_reloads[MAX_INSN_RELOADS]; }; -#define breg_renumber gen_ctx->ra_ctx->breg_renumber -#define sorted_bregs gen_ctx->ra_ctx->sorted_bregs +#define start_mem_loc gen_ctx->ra_ctx->start_mem_loc +#define reg_renumber gen_ctx->ra_ctx->reg_renumber +#define sorted_regs gen_ctx->ra_ctx->sorted_regs #define used_locs gen_ctx->ra_ctx->used_locs +#define busy_used_locs gen_ctx->ra_ctx->busy_used_locs #define var_bbs gen_ctx->ra_ctx->var_bbs -#define conflict_locs gen_ctx->ra_ctx->conflict_locs -#define curr_breg_infos gen_ctx->ra_ctx->curr_breg_infos -#define loc_profits gen_ctx->ra_ctx->loc_profits -#define loc_profit_ages gen_ctx->ra_ctx->loc_profit_ages -#define curr_age gen_ctx->ra_ctx->curr_age - -static void fast_assign (gen_ctx_t gen_ctx) { - MIR_reg_t loc, curr_loc, best_loc, i, reg, breg, var, nregs = get_nregs (gen_ctx); - MIR_type_t type; - int slots_num; - int k; - bitmap_t bm; - bitmap_t *used_locs_addr; - size_t nel; - bitmap_iterator_t bi; +#define spill_gaps gen_ctx->ra_ctx->spill_gaps +#define curr_gaps gen_ctx->ra_ctx->curr_gaps +#define lr_gap_bitmaps gen_ctx->ra_ctx->lr_gap_bitmaps +#define lr_gap_tab gen_ctx->ra_ctx->lr_gap_tab +#define spill_els gen_ctx->ra_ctx->spill_els +#define spill_cache gen_ctx->ra_ctx->spill_cache +#define spill_cache_age gen_ctx->ra_ctx->spill_cache_age +#define conflict_locs1 gen_ctx->ra_ctx->conflict_locs1 +#define curr_reg_infos gen_ctx->ra_ctx->curr_reg_infos +#define in_reloads_num gen_ctx->ra_ctx->in_reloads_num +#define out_reloads_num gen_ctx->ra_ctx->out_reloads_num +#define in_reloads gen_ctx->ra_ctx->in_reloads +#define out_reloads gen_ctx->ra_ctx->out_reloads + +/* Priority RA */ - func_stack_slots_num = 0; - if (nregs == 0) return; - for (size_t n = 0; n < nregs + MAX_HARD_REG + 1 && n < VARR_LENGTH (bitmap_t, var_bbs); n++) - bitmap_clear (VARR_GET (bitmap_t, var_bbs, n)); - while (VARR_LENGTH (bitmap_t, var_bbs) < nregs + MAX_HARD_REG + 1) { - bm = bitmap_create2 (curr_bb_index); - VARR_PUSH (bitmap_t, var_bbs, bm); - } - for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { - bitmap_ior (temp_bitmap, bb->live_in, bb->live_out); - bitmap_ior (temp_bitmap, temp_bitmap, bb->gen); - bitmap_ior (temp_bitmap, temp_bitmap, bb->kill); - FOREACH_BITMAP_BIT (bi, temp_bitmap, nel) { - bitmap_set_bit_p (VARR_GET (bitmap_t, var_bbs, nel), bb->index); - } - } - VARR_TRUNC (MIR_reg_t, breg_renumber, 0); - for (i = 0; i < nregs; i++) VARR_PUSH (MIR_reg_t, breg_renumber, MIR_NON_HARD_REG); - for (size_t n = 0; n < curr_bb_index && n < VARR_LENGTH (bitmap_t, used_locs); n++) - bitmap_clear (VARR_GET (bitmap_t, used_locs, n)); - while (VARR_LENGTH (bitmap_t, used_locs) < curr_bb_index) { - bm = bitmap_create2 (2 * MAX_HARD_REG + 1); - VARR_PUSH (bitmap_t, used_locs, bm); +#define live_in in +#define live_out out +#define spill_gen gen /* pseudo regs fully spilled in BB, for them spill_kill is false */ +#define spill_kill kill /* pseudo regs referenced in the BB and should use assigned hreg */ + +static htab_hash_t lr_gap_hash (lr_gap_t el, void *arg MIR_UNUSED) { + return (htab_hash_t) mir_hash_finish ( + mir_hash_step (mir_hash_step (mir_hash_init (0x88), (uint64_t) el.hreg), + (uint64_t) el.lr->start)); +} + +static int lr_gap_eq (lr_gap_t el1, lr_gap_t el2, void *arg MIR_UNUSED) { + return el1.hreg == el2.hreg && el1.lr->start == el2.lr->start; +} + +static void insert_lr_gap (gen_ctx_t gen_ctx, int hreg, MIR_type_t type, MIR_reg_t reg, + live_range_t lr) { + lr_gap_t el = {hreg, type, reg, lr}, tab_el; + gen_assert (lr->lr_bb != NULL); + if (lr_gap_bitmaps[hreg] == NULL) lr_gap_bitmaps[hreg] = bitmap_create2 (3 * lr->start / 2); + bitmap_set_bit_p (lr_gap_bitmaps[hreg], lr->start); + HTAB_DO (lr_gap_t, lr_gap_tab, el, HTAB_INSERT, tab_el); +} + +static void delete_lr_gap (gen_ctx_t gen_ctx, int hreg, live_range_t lr) { + lr_gap_t el, tab_el; + gen_assert (lr->lr_bb != NULL && lr_gap_bitmaps[hreg] != NULL); + bitmap_clear_bit_p (lr_gap_bitmaps[hreg], lr->start); + el.hreg = hreg; + el.lr = lr; + HTAB_DO (lr_gap_t, lr_gap_tab, el, HTAB_DELETE, tab_el); +} + +static inline int find_lr_gap (gen_ctx_t gen_ctx, int hreg, int point, lr_gap_t *tab_el) { + struct live_range lr; + lr_gap_t el; + if (lr_gap_bitmaps[hreg] == NULL || !bitmap_bit_p (lr_gap_bitmaps[hreg], point)) return FALSE; + el.hreg = hreg; + lr.start = point; + el.lr = &lr; + return HTAB_DO (lr_gap_t, lr_gap_tab, el, HTAB_FIND, *tab_el); +} + +static void init_lr_gap_tab (gen_ctx_t gen_ctx) { + for (int i = 0; i <= MAX_HARD_REG; i++) lr_gap_bitmaps[i] = NULL; + HTAB_CREATE (lr_gap_t, lr_gap_tab, 1024, lr_gap_hash, lr_gap_eq, NULL); +} + +static void finish_lr_gap_tab (gen_ctx_t gen_ctx) { + for (int i = 0; i <= MAX_HARD_REG; i++) + if (lr_gap_bitmaps[i] != NULL) bitmap_destroy (lr_gap_bitmaps[i]); + HTAB_DESTROY (lr_gap_t, lr_gap_tab); +} + +static int allocno_info_compare_func (const void *a1, const void *a2) { + const allocno_info_t *allocno_info1 = (const allocno_info_t *) a1, + *allocno_info2 = (const allocno_info_t *) a2; + MIR_reg_t reg1 = allocno_info1->reg, reg2 = allocno_info2->reg; + reg_info_t *reg_infos = allocno_info1->reg_infos; + long diff; + + gen_assert (reg_infos == allocno_info2->reg_infos); + if (allocno_info1->tied_reg_p) { + if (allocno_info2->tied_reg_p) return -1; + } else if (allocno_info2->tied_reg_p) { + return 1; } - used_locs_addr = VARR_ADDR (bitmap_t, used_locs); - for (i = 0; i <= MAX_HARD_REG; i++) - FOREACH_BITMAP_BIT (bi, VARR_GET (bitmap_t, var_bbs, i), nel) { - bitmap_set_bit_p (used_locs_addr[nel], i); + if ((diff = reg_infos[reg2].freq - reg_infos[reg1].freq) != 0) return diff; + if (reg_infos[reg2].live_length < reg_infos[reg1].live_length) return -1; + if (reg_infos[reg1].live_length < reg_infos[reg2].live_length) return 1; + return reg1 < reg2 ? -1 : 1; /* make sort stable */ +} + +static int hreg_in_bitmap_p (int hreg, MIR_type_t type, int nregs, bitmap_t bm) { + for (int i = 0; i < nregs; i++) + if (bitmap_bit_p (bm, target_nth_loc (hreg, type, i))) return TRUE; + return FALSE; +} + +static MIR_reg_t get_hard_reg (gen_ctx_t gen_ctx, MIR_reg_t type, bitmap_t conflict_locs) { + MIR_reg_t hreg, curr_hreg, best_hreg = MIR_NON_VAR; + int n, k, nregs, best_saved_p = FALSE; + for (n = 0; n <= MAX_HARD_REG; n++) { +#ifdef TARGET_HARD_REG_ALLOC_ORDER + hreg = TARGET_HARD_REG_ALLOC_ORDER (n); +#else + hreg = n; +#endif + if (bitmap_bit_p (conflict_locs, hreg)) continue; + if (!target_hard_reg_type_ok_p (hreg, type) || target_fixed_hard_reg_p (hreg)) continue; + if ((nregs = target_locs_num (hreg, type)) > 1) { + if (target_nth_loc (hreg, type, nregs - 1) > MAX_HARD_REG) break; + for (k = nregs - 1; k > 0; k--) { + curr_hreg = target_nth_loc (hreg, type, k); + if (target_fixed_hard_reg_p (curr_hreg) || bitmap_bit_p (conflict_locs, curr_hreg)) break; + } + if (k > 0) continue; } - bitmap_clear (func_used_hard_regs); - for (i = 0; i < nregs; i++) { /* hard reg and stack slot assignment */ - breg = i; - reg = breg2reg (gen_ctx, breg); - var = reg2var (gen_ctx, reg); - bitmap_clear (conflict_locs); - FOREACH_BITMAP_BIT (bi, VARR_GET (bitmap_t, var_bbs, var), nel) { - bitmap_ior (conflict_locs, conflict_locs, used_locs_addr[nel]); - } - type = MIR_reg_type (gen_ctx->ctx, reg, curr_func_item->u.func); - /* Call used hard regs are already taken into account above for call crossed regs. */ - best_loc = MIR_NON_HARD_REG; - for (loc = 0; loc <= MAX_HARD_REG; loc++) { - if (bitmap_bit_p (conflict_locs, loc)) continue; - if (!target_hard_reg_type_ok_p (loc, type) || target_fixed_hard_reg_p (loc)) continue; - if ((slots_num = target_locs_num (loc, type)) > 1) { - if (target_nth_loc (loc, type, slots_num - 1) > MAX_HARD_REG) break; - for (k = slots_num - 1; k > 0; k--) { - curr_loc = target_nth_loc (loc, type, k); - if (target_fixed_hard_reg_p (curr_loc) || bitmap_bit_p (conflict_locs, curr_loc)) break; - } - if (k > 0) continue; + if (best_hreg == MIR_NON_VAR + || (best_saved_p && bitmap_bit_p (call_used_hard_regs[MIR_T_UNDEF], hreg))) { + best_hreg = hreg; + best_saved_p = !bitmap_bit_p (call_used_hard_regs[MIR_T_UNDEF], hreg); + } + } + return best_hreg; +} + +static int available_hreg_p (int hreg, MIR_reg_t type, int nregs, bitmap_t *conflict_locs, + live_range_t lr) { + for (int j = lr->start; j <= lr->finish; j++) { + if (bitmap_bit_p (conflict_locs[j], hreg)) return FALSE; + if (nregs > 1) { + if (target_nth_loc (hreg, type, nregs - 1) > MAX_HARD_REG) return FALSE; + for (int k = nregs - 1; k > 0; k--) { + MIR_reg_t curr_hreg = target_nth_loc (hreg, type, k); + if (bitmap_bit_p (conflict_locs[j], curr_hreg)) return FALSE; } - best_loc = loc; - break; } - if (best_loc != MIR_NON_HARD_REG) { - setup_used_hard_regs (gen_ctx, type, best_loc); + } + return TRUE; +} + +/* Return cost spill of given lr */ +static int gap_lr_spill_cost (gen_ctx_t gen_ctx, live_range_t lr) { + int cost_factor = LOOP_COST_FACTOR / 2; + bitmap_clear (temp_bitmap); + for (lr_bb_t lr_bb = lr->lr_bb; lr_bb != NULL; lr_bb = lr_bb->next) + bitmap_set_bit_p (temp_bitmap, lr_bb->bb->index); + int cost = 0; + for (lr_bb_t lr_bb = lr->lr_bb; lr_bb != NULL; lr_bb = lr_bb->next) { + bb_t bb = lr_bb->bb; + int level, max_level, bb_level = bb_loop_level (bb) + 1; + edge_t e; + max_level = -1; + for (e = DLIST_HEAD (out_edge_t, bb->out_edges); e != NULL; e = DLIST_NEXT (out_edge_t, e)) + if (!bitmap_bit_p (temp_bitmap, e->dst->index) + && (level = bb_loop_level (e->dst) + 1) > max_level) + max_level = level; + if (max_level >= 0) cost += (max_level < bb_level ? max_level : bb_level) * cost_factor; + max_level = -1; + for (e = DLIST_HEAD (in_edge_t, bb->in_edges); e != NULL; e = DLIST_NEXT (in_edge_t, e)) + if (!bitmap_bit_p (temp_bitmap, e->src->index) + && (level = bb_loop_level (e->src) + 1) > max_level) + max_level = level; + if (max_level >= 0) cost += (max_level < bb_level ? max_level : bb_level) * cost_factor; + } + return cost; +} + +static void find_lr_gaps (gen_ctx_t gen_ctx, live_range_t for_lr, MIR_reg_t hreg, MIR_type_t type, + int nregs MIR_UNUSED, int *spill_cost, VARR (lr_gap_t) * lr_gaps) { + MIR_reg_t curr_hreg; + int i, j, cont, slots_num = target_locs_num (hreg, type); + int found_p MIR_UNUSED; + lr_gap_t lr_gap, last_lr_gap; + for (i = 0; i < slots_num; i++) { + curr_hreg = target_nth_loc (hreg, type, i); + gen_assert (curr_hreg <= MAX_HARD_REG); + if (VARR_LENGTH (lr_gap_t, lr_gaps) == 0) { + last_lr_gap.lr = NULL; } else { - for (loc = MAX_HARD_REG + 1; loc <= func_stack_slots_num + MAX_HARD_REG; loc++) { - slots_num = target_locs_num (loc, type); - if (target_nth_loc (loc, type, slots_num - 1) > func_stack_slots_num + MAX_HARD_REG) break; - for (k = 0; k < slots_num; k++) { - curr_loc = target_nth_loc (loc, type, k); - if (bitmap_bit_p (conflict_locs, curr_loc)) break; - } - if (k < slots_num) continue; - if ((loc - MAX_HARD_REG - 1) % slots_num != 0) - continue; /* we align stack slots according to the type size */ - best_loc = loc; - break; - } - if (best_loc == MIR_NON_HARD_REG) { /* Add stack slot ??? */ - slots_num = 1; - for (k = 0; k < slots_num; k++) { - if (k == 0) { - best_loc = func_stack_slots_num + MAX_HARD_REG + 1; - slots_num = target_locs_num (best_loc, type); - } - func_stack_slots_num++; - if (k == 0 && (best_loc - MAX_HARD_REG - 1) % slots_num != 0) k--; /* align */ - } + last_lr_gap = VARR_LAST (lr_gap_t, lr_gaps); + } + *spill_cost = 0; + for (j = for_lr->start; j >= 0; j--) + if (find_lr_gap (gen_ctx, curr_hreg, j, &lr_gap)) break; + cont = for_lr->start + 1; + if (j >= 0 && lr_gap.lr->finish >= for_lr->start) { /* found the leftmost interesecting */ + cont = lr_gap.lr->finish + 1; + if (last_lr_gap.lr != lr_gap.lr) { + VARR_PUSH (lr_gap_t, lr_gaps, lr_gap); + *spill_cost = gap_lr_spill_cost (gen_ctx, lr_gap.lr); + last_lr_gap = lr_gap; } } - DEBUG (2, { - fprintf (debug_file, " Assigning to %s:var=%3u, breg=%3u -- %lu\n", - MIR_reg_name (gen_ctx->ctx, reg, curr_func_item->u.func), reg2var (gen_ctx, reg), - breg, (unsigned long) best_loc); - }); - VARR_SET (MIR_reg_t, breg_renumber, breg, best_loc); - slots_num = target_locs_num (best_loc, type); - FOREACH_BITMAP_BIT (bi, VARR_GET (bitmap_t, var_bbs, var), nel) { - for (k = 0; k < slots_num; k++) - bitmap_set_bit_p (used_locs_addr[nel], target_nth_loc (best_loc, type, k)); + for (j = cont; j <= for_lr->finish; j++) { + if (!find_lr_gap (gen_ctx, curr_hreg, j, &lr_gap)) continue; + if (last_lr_gap.lr != lr_gap.lr) { + VARR_PUSH (lr_gap_t, lr_gaps, lr_gap); + *spill_cost += gap_lr_spill_cost (gen_ctx, lr_gap.lr); + last_lr_gap = lr_gap; + } + j = lr_gap.lr->finish; } } } -#undef live_in -#undef live_out -#undef live_kill -#undef live_gen - -static void process_move_to_form_thread (gen_ctx_t gen_ctx, mv_t mv) { - MIR_op_t op1 = mv->bb_insn->insn->ops[0], op2 = mv->bb_insn->insn->ops[1]; - MIR_reg_t breg1, breg2, breg1_first, breg2_first, last; - - if (op1.mode != MIR_OP_REG || op2.mode != MIR_OP_REG) return; - breg1 = reg2breg (gen_ctx, op1.u.reg); - breg2 = reg2breg (gen_ctx, op2.u.reg); - breg1_first = curr_breg_infos[breg1].thread_first; - breg2_first = curr_breg_infos[breg2].thread_first; - if (breg1_first != breg2_first) { - for (last = breg2_first; curr_breg_infos[last].thread_next != MIR_MAX_REG_NUM; - last = curr_breg_infos[last].thread_next) - curr_breg_infos[last].thread_first = breg1_first; - curr_breg_infos[last].thread_first = breg1_first; - curr_breg_infos[last].thread_next = curr_breg_infos[breg1_first].thread_next; - curr_breg_infos[breg1_first].thread_next = breg2_first; - if (curr_breg_infos[breg1_first].thread_freq - < LONG_MAX - curr_breg_infos[breg2_first].thread_freq) - curr_breg_infos[breg1_first].thread_freq += curr_breg_infos[breg2_first].thread_freq; - else - curr_breg_infos[breg1_first].thread_freq = LONG_MAX; +/* If we find a hard reg then info about spilled lrs will in spill_gaps */ +static MIR_reg_t get_hard_reg_with_split (gen_ctx_t gen_ctx, MIR_reg_t reg, MIR_reg_t type, + live_range_t start_lr) { + MIR_reg_t hreg, curr_hreg, best_hreg = MIR_NON_VAR; + int n, k, nregs, profit, cost, best_profit = 0, gap_size, best_gap_size = 0; + int clobbered_p, best_saved_p = FALSE; + live_range_t lr; + bitmap_t *all_locs = VARR_ADDR (bitmap_t, used_locs); + bitmap_t *busy_locs = VARR_ADDR (bitmap_t, busy_used_locs); + VARR (lr_gap_t) * temp_gaps; + for (n = 0; n <= MAX_HARD_REG; n++) { +#ifdef TARGET_HARD_REG_ALLOC_ORDER + hreg = TARGET_HARD_REG_ALLOC_ORDER (n); +#else + hreg = n; +#endif + if (!target_hard_reg_type_ok_p (hreg, type) || target_fixed_hard_reg_p (hreg)) continue; + if ((nregs = target_locs_num (hreg, type)) > 1) { + if (target_nth_loc (hreg, type, nregs - 1) > MAX_HARD_REG) break; + for (k = nregs - 1; k > 0; k--) { + curr_hreg = target_nth_loc (hreg, type, k); + if (target_fixed_hard_reg_p (curr_hreg)) break; + } + if (k > 0) continue; + } + VARR_TRUNC (lr_gap_t, curr_gaps, 0); + profit = curr_reg_infos[reg].freq; + gap_size = 0; + for (lr = start_lr; lr != NULL; lr = lr->next) { + if (available_hreg_p (hreg, type, nregs, all_locs, lr)) { + } else if (available_hreg_p (hreg, type, nregs, busy_locs, lr)) { + /* spill other pseudo regs in their gap */ + find_lr_gaps (gen_ctx, lr, hreg, type, nregs, &cost, curr_gaps); + profit -= cost; + gap_size += (lr->finish - lr->start + 1); + } else if (lr->lr_bb == NULL) { /* not a gap */ + break; + } else { /* spill this pseudo reg gap */ + lr_gap_t lr_gap = {hreg, type, reg, lr}; + cost = gap_lr_spill_cost (gen_ctx, lr_gap.lr); + profit -= cost; + VARR_PUSH (lr_gap_t, curr_gaps, lr_gap); + } + } + if (lr != NULL || profit < 0) continue; + clobbered_p = bitmap_bit_p (call_used_hard_regs[MIR_T_UNDEF], hreg); + if (best_hreg == MIR_NON_VAR || profit > best_profit + || (profit == best_profit && best_saved_p && clobbered_p) + || (profit == best_profit && best_saved_p == !clobbered_p && gap_size > best_gap_size)) { + best_hreg = hreg; + best_profit = profit; + best_saved_p = !clobbered_p; + best_gap_size = gap_size; + SWAP (spill_gaps, curr_gaps, temp_gaps); + } } - if (curr_breg_infos[breg1_first].thread_freq < 2 * mv->freq) - curr_breg_infos[breg1_first].thread_freq = 0; - else - curr_breg_infos[breg1_first].thread_freq -= 2 * mv->freq; - gen_assert (curr_breg_infos[breg1_first].thread_freq >= 0); -} - -static int breg_info_compare_func (const void *a1, const void *a2) { - const breg_info_t *breg_info1 = (const breg_info_t *) a1, *breg_info2 = (const breg_info_t *) a2; - MIR_reg_t breg1 = breg_info1->breg, breg2 = breg_info2->breg; - reg_info_t *breg_infos = breg_info1->breg_infos; - MIR_reg_t t1 = breg_infos[breg1].thread_first, t2 = breg_infos[breg2].thread_first; - long diff; - - gen_assert (breg_infos == breg_info2->breg_infos); - if ((diff = breg_infos[t2].thread_freq - breg_infos[t1].thread_freq) != 0) return diff; - if (t1 < t2) return -1; - if (t2 < t1) return 1; - if (breg_infos[breg2].live_length < breg_infos[breg1].live_length) return -1; - if (breg_infos[breg1].live_length < breg_infos[breg2].live_length) return 1; - return breg1 < breg2 ? -1 : 1; /* make sort stable */ + return best_hreg; } -static void setup_loc_profit_from_op (gen_ctx_t gen_ctx, MIR_op_t op, size_t freq) { - MIR_reg_t loc; - size_t *curr_loc_profits = VARR_ADDR (size_t, loc_profits); - size_t *curr_loc_profit_ages = VARR_ADDR (size_t, loc_profit_ages); +static MIR_reg_t get_new_stack_slot (gen_ctx_t gen_ctx, MIR_reg_t type, int *slots_num_ref) { + MIR_reg_t best_loc; + int k, slots_num = 1; - if (op.mode == MIR_OP_HARD_REG) - loc = op.u.hard_reg; - else if ((loc = VARR_GET (MIR_reg_t, breg_renumber, reg2breg (gen_ctx, op.u.reg))) - == MIR_NON_HARD_REG) - return; - if (curr_loc_profit_ages[loc] == curr_age) { - if (curr_loc_profits[loc] < SIZE_MAX - freq) - curr_loc_profits[loc] += freq; - else - curr_loc_profits[loc] = SIZE_MAX; - } else { - curr_loc_profit_ages[loc] = curr_age; - curr_loc_profits[loc] = freq; + for (k = 0; k < slots_num; k++) { + if (k == 0) { + best_loc = (MIR_reg_t) func_stack_slots_num + MAX_HARD_REG + 1; + slots_num = target_locs_num (best_loc, type); + } + func_stack_slots_num++; + if (k == 0 && (best_loc - MAX_HARD_REG - 1) % slots_num != 0) k--; /* align */ + } + *slots_num_ref = slots_num; + return best_loc; +} + +static MIR_reg_t get_stack_loc (gen_ctx_t gen_ctx, MIR_reg_t start_loc, MIR_type_t type, + bitmap_t conflict_locs, int *slots_num_ref) { + MIR_reg_t loc, curr_loc, best_loc = MIR_NON_VAR; + int k, slots_num = 1; + for (loc = start_loc; loc <= func_stack_slots_num + MAX_HARD_REG; loc++) { + slots_num = target_locs_num (loc, type); + if (target_nth_loc (loc, type, slots_num - 1) > func_stack_slots_num + MAX_HARD_REG) break; + for (k = 0; k < slots_num; k++) { + curr_loc = target_nth_loc (loc, type, k); + if (bitmap_bit_p (conflict_locs, curr_loc)) break; + } + if (k < slots_num) continue; + if ((loc - MAX_HARD_REG - 1) % slots_num != 0) + continue; /* we align stack slots according to the type size */ + if (best_loc == MIR_NON_VAR) best_loc = loc; } + if (best_loc == MIR_NON_VAR) best_loc = get_new_stack_slot (gen_ctx, type, &slots_num); + *slots_num_ref = slots_num; + return best_loc; } -static void setup_loc_profits (gen_ctx_t gen_ctx, MIR_reg_t breg) { - mv_t mv; - reg_info_t *info = &curr_breg_infos[breg]; - - for (mv = DLIST_HEAD (dst_mv_t, info->dst_moves); mv != NULL; mv = DLIST_NEXT (dst_mv_t, mv)) - setup_loc_profit_from_op (gen_ctx, mv->bb_insn->insn->ops[0], mv->freq); - for (mv = DLIST_HEAD (src_mv_t, info->src_moves); mv != NULL; mv = DLIST_NEXT (src_mv_t, mv)) - setup_loc_profit_from_op (gen_ctx, mv->bb_insn->insn->ops[1], mv->freq); -} +#define ONLY_SIMPLIFIED_RA FALSE -static void quality_assign (gen_ctx_t gen_ctx) { - MIR_reg_t loc, curr_loc, best_loc, i, reg, breg, var, nregs = get_nregs (gen_ctx); +static void assign (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_reg_t best_loc, i, reg, var, max_var = get_max_var (gen_ctx); MIR_type_t type; - int n, slots_num; - int j, k; - live_range_t lr; + int slots_num; + int j, k, simple_loc_update_p, reserve_p, nregs; + live_range_t start_lr, lr; bitmap_t bm; - size_t length, profit, best_profit; - bitmap_t *used_locs_addr; - breg_info_t breg_info; + size_t length; + bitmap_t *used_locs_addr, *busy_used_locs_addr; + allocno_info_t allocno_info; + MIR_func_t func = curr_func_item->u.func; + bitmap_t global_hard_regs = _MIR_get_module_global_var_hard_regs (ctx, curr_func_item->module); + const char *msg; + const int simplified_p = ONLY_SIMPLIFIED_RA || optimize_level < 2; + bitmap_t conflict_locs = conflict_locs1, spill_lr_starts = temp_bitmap2; func_stack_slots_num = 0; - if (nregs == 0) return; - curr_breg_infos = VARR_ADDR (reg_info_t, curr_cfg->breg_info); - VARR_TRUNC (MIR_reg_t, breg_renumber, 0); - for (i = 0; i < nregs; i++) { - VARR_PUSH (MIR_reg_t, breg_renumber, MIR_NON_HARD_REG); - curr_breg_infos[i].thread_freq = curr_breg_infos[i].freq; - } - for (mv_t mv = DLIST_HEAD (mv_t, curr_cfg->used_moves); mv != NULL; mv = DLIST_NEXT (mv_t, mv)) - process_move_to_form_thread (gen_ctx, mv); - /* min_reg, max_reg for func */ - VARR_TRUNC (breg_info_t, sorted_bregs, 0); - breg_info.breg_infos = curr_breg_infos; - for (i = 0; i < nregs; i++) { - breg_info.breg = i; - VARR_PUSH (breg_info_t, sorted_bregs, breg_info); - var = reg2var (gen_ctx, breg2reg (gen_ctx, i)); - for (length = 0, lr = VARR_GET (live_range_t, var_live_ranges, var); lr != NULL; lr = lr->next) + curr_reg_infos = VARR_ADDR (reg_info_t, curr_cfg->reg_info); + VARR_TRUNC (MIR_reg_t, reg_renumber, 0); + for (i = 0; i <= max_var; i++) { + VARR_PUSH (MIR_reg_t, reg_renumber, MIR_NON_VAR); + } + /* max_var for func */ + VARR_TRUNC (allocno_info_t, sorted_regs, 0); + allocno_info.reg_infos = curr_reg_infos; + start_mem_loc = MAX_HARD_REG + 1; + for (reg = MAX_HARD_REG + 1; reg <= max_var; reg++) { + allocno_info.reg = reg; + allocno_info.tied_reg_p = bitmap_bit_p (tied_regs, reg); + if (bitmap_bit_p (addr_regs, reg)) { + type = MIR_reg_type (gen_ctx->ctx, reg - MAX_HARD_REG, func); + best_loc = get_new_stack_slot (gen_ctx, type, &slots_num); + VARR_SET (MIR_reg_t, reg_renumber, reg, best_loc); + start_mem_loc = best_loc + slots_num; + DEBUG (2, { + fprintf (debug_file, " Assigning to addressable %s:reg=%3u (freq %-3ld) -- %lu\n", + MIR_reg_name (gen_ctx->ctx, reg - MAX_HARD_REG, func), reg, + curr_reg_infos[reg].freq, (unsigned long) best_loc); + }); + continue; + } + VARR_PUSH (allocno_info_t, sorted_regs, allocno_info); + for (length = 0, lr = VARR_GET (live_range_t, var_live_ranges, reg); lr != NULL; lr = lr->next) length += lr->finish - lr->start + 1; - curr_breg_infos[i].live_length = length; + curr_reg_infos[reg].live_length = length; } - VARR_TRUNC (size_t, loc_profits, 0); - VARR_TRUNC (size_t, loc_profit_ages, 0); - for (i = 0; i <= MAX_HARD_REG; i++) { - VARR_PUSH (size_t, loc_profits, 0); - VARR_PUSH (size_t, loc_profit_ages, 0); - } - for (size_t n = 0; n <= curr_point && n < VARR_LENGTH (bitmap_t, used_locs); n++) - bitmap_clear (VARR_GET (bitmap_t, used_locs, n)); - while (VARR_LENGTH (bitmap_t, used_locs) <= curr_point) { + for (int n = 0; n <= curr_point && n < (int) VARR_LENGTH (bitmap_t, used_locs); n++) + if (global_hard_regs == NULL) { + bitmap_clear (VARR_GET (bitmap_t, used_locs, n)); + if (!simplified_p) bitmap_clear (VARR_GET (bitmap_t, busy_used_locs, n)); + } else { + bitmap_copy (VARR_GET (bitmap_t, used_locs, n), global_hard_regs); + if (!simplified_p) bitmap_copy (VARR_GET (bitmap_t, busy_used_locs, n), global_hard_regs); + } + while ((int) VARR_LENGTH (bitmap_t, used_locs) <= curr_point) { bm = bitmap_create2 (MAX_HARD_REG + 1); + if (global_hard_regs != NULL) bitmap_copy (bm, global_hard_regs); VARR_PUSH (bitmap_t, used_locs, bm); + if (!simplified_p) { + bm = bitmap_create2 (MAX_HARD_REG + 1); + if (global_hard_regs != NULL) bitmap_copy (bm, global_hard_regs); + VARR_PUSH (bitmap_t, busy_used_locs, bm); + } } - qsort (VARR_ADDR (breg_info_t, sorted_bregs), nregs, sizeof (breg_info_t), - breg_info_compare_func); - curr_age = 0; + nregs = (int) VARR_LENGTH (allocno_info_t, sorted_regs); + qsort (VARR_ADDR (allocno_info_t, sorted_regs), nregs, sizeof (allocno_info_t), + allocno_info_compare_func); used_locs_addr = VARR_ADDR (bitmap_t, used_locs); + busy_used_locs_addr = VARR_ADDR (bitmap_t, busy_used_locs); + /* Mark ranges used by hard regs for pseudo reg splitting: */ for (i = 0; i <= MAX_HARD_REG; i++) { for (lr = VARR_GET (live_range_t, var_live_ranges, i); lr != NULL; lr = lr->next) - for (j = lr->start; j <= lr->finish; j++) bitmap_set_bit_p (used_locs_addr[j], i); + for (j = lr->start; j <= lr->finish; j++) { + bitmap_set_bit_p (used_locs_addr[j], i); + if (!simplified_p) bitmap_set_bit_p (busy_used_locs_addr[j], i); + } } bitmap_clear (func_used_hard_regs); - for (i = 0; i < nregs; i++) { /* hard reg and stack slot assignment */ - breg = VARR_GET (breg_info_t, sorted_bregs, i).breg; - if (VARR_GET (MIR_reg_t, breg_renumber, breg) != MIR_NON_HARD_REG) continue; - reg = breg2reg (gen_ctx, breg); - var = reg2var (gen_ctx, reg); + if (!simplified_p) HTAB_CLEAR (lr_gap_t, lr_gap_tab); + for (int n = 0; n < nregs; n++) { /* hard reg and stack slot assignment */ + reg = VARR_GET (allocno_info_t, sorted_regs, n).reg; + if (VARR_GET (MIR_reg_t, reg_renumber, reg) != MIR_NON_VAR) continue; + type = MIR_reg_type (gen_ctx->ctx, reg - MAX_HARD_REG, func); + if (VARR_GET (allocno_info_t, sorted_regs, n).tied_reg_p) { + const char *hard_reg_name = MIR_reg_hard_reg_name (ctx, reg - MAX_HARD_REG, func); + int hard_reg = _MIR_get_hard_reg (ctx, hard_reg_name); + gen_assert (hard_reg >= 0 && hard_reg <= MAX_HARD_REG + && target_locs_num (hard_reg, type) == 1); + VARR_SET (MIR_reg_t, reg_renumber, reg, hard_reg); +#ifndef NDEBUG + for (lr = VARR_GET (live_range_t, var_live_ranges, reg); lr != NULL; lr = lr->next) + for (j = lr->start; j <= lr->finish; j++) + gen_assert (bitmap_bit_p (used_locs_addr[j], hard_reg)); +#endif + if (hard_reg_name == NULL) setup_used_hard_regs (gen_ctx, type, hard_reg); + DEBUG (2, { + fprintf (debug_file, " Assigning to global %s:reg=%3u (freq %-3ld) -- %lu\n", + MIR_reg_name (gen_ctx->ctx, reg - MAX_HARD_REG, func), reg, + curr_reg_infos[reg].freq, (unsigned long) hard_reg); + }); + continue; + } + var = reg; + if ((start_lr = VARR_GET (live_range_t, var_live_ranges, var)) == NULL) continue; bitmap_clear (conflict_locs); - for (lr = VARR_GET (live_range_t, var_live_ranges, var); lr != NULL; lr = lr->next) + for (lr = start_lr; lr != NULL; lr = lr->next) { for (j = lr->start; j <= lr->finish; j++) bitmap_ior (conflict_locs, conflict_locs, used_locs_addr[j]); - curr_age++; - setup_loc_profits (gen_ctx, breg); - best_loc = MIR_NON_HARD_REG; - best_profit = 0; - type = MIR_reg_type (gen_ctx->ctx, reg, curr_func_item->u.func); - if (bitmap_bit_p (curr_cfg->call_crossed_bregs, breg)) - bitmap_ior (conflict_locs, conflict_locs, call_used_hard_regs[type]); - for (n = 0; n <= MAX_HARD_REG; n++) { -#ifdef TARGET_HARD_REG_ALLOC_ORDER - loc = TARGET_HARD_REG_ALLOC_ORDER (n); -#else - loc = n; -#endif - if (bitmap_bit_p (conflict_locs, loc)) continue; - if (!target_hard_reg_type_ok_p (loc, type) || target_fixed_hard_reg_p (loc)) continue; - if ((slots_num = target_locs_num (loc, type)) > 1) { - if (target_nth_loc (loc, type, slots_num - 1) > MAX_HARD_REG) break; - for (k = slots_num - 1; k > 0; k--) { - curr_loc = target_nth_loc (loc, type, k); - if (target_fixed_hard_reg_p (curr_loc) || bitmap_bit_p (conflict_locs, curr_loc)) break; - } - if (k > 0) continue; - } - profit = (VARR_GET (size_t, loc_profit_ages, loc) != curr_age - ? 0 - : VARR_GET (size_t, loc_profits, loc)); - if (best_loc == MIR_NON_HARD_REG || best_profit < profit) { - best_loc = loc; - best_profit = profit; - } } - if (best_loc != MIR_NON_HARD_REG) { + msg = ""; + VARR_TRUNC (lr_gap_t, spill_gaps, 0); + if ((best_loc = get_hard_reg (gen_ctx, type, conflict_locs)) != MIR_NON_VAR) { setup_used_hard_regs (gen_ctx, type, best_loc); + } else if (!simplified_p + && ((best_loc = get_hard_reg_with_split (gen_ctx, reg, type, start_lr)) + != MIR_NON_VAR)) { + /* try to use gaps in already allocated pseudos or given pseudo: */ + setup_used_hard_regs (gen_ctx, type, best_loc); + msg = "(with splitting live ranges)"; } else { - for (loc = MAX_HARD_REG + 1; loc <= func_stack_slots_num + MAX_HARD_REG; loc++) { - slots_num = target_locs_num (loc, type); - if (target_nth_loc (loc, type, slots_num - 1) > func_stack_slots_num + MAX_HARD_REG) break; - for (k = 0; k < slots_num; k++) { - curr_loc = target_nth_loc (loc, type, k); - if (bitmap_bit_p (conflict_locs, curr_loc)) break; - } - if (k < slots_num) continue; - if ((loc - MAX_HARD_REG - 1) % slots_num != 0) - continue; /* we align stack slots according to the type size */ - profit = (VARR_GET (size_t, loc_profit_ages, loc) != curr_age - ? 0 - : VARR_GET (size_t, loc_profits, loc)); - if (best_loc == MIR_NON_HARD_REG || best_profit < profit) { - best_loc = loc; - best_profit = profit; - } + best_loc = get_stack_loc (gen_ctx, start_mem_loc, type, conflict_locs, &slots_num); + } + DEBUG (2, { + fprintf (debug_file, " Assigning %s to %s:reg=%3u (freq %-3ld) -- %lu\n", msg, + MIR_reg_name (gen_ctx->ctx, reg - MAX_HARD_REG, func), reg, curr_reg_infos[reg].freq, + (unsigned long) best_loc); + fprintf (debug_file, " live range: "); + print_live_ranges (gen_ctx, start_lr); + }); + bitmap_clear (spill_lr_starts); + while (VARR_LENGTH (lr_gap_t, spill_gaps) != 0) { + lr_gap_t lr_gap = VARR_POP (lr_gap_t, spill_gaps); + DEBUG (4, { + fprintf (debug_file, " Splitting lr gap: r%d%s, h%d [%d..%d]\n", lr_gap.reg, + lr_gap.reg == reg ? "*" : "", lr_gap.hreg, lr_gap.lr->start, lr_gap.lr->finish); + }); + for (lr_bb_t lr_bb = lr_gap.lr->lr_bb; lr_bb != NULL; lr_bb = lr_bb->next) + bitmap_set_bit_p (lr_bb->bb->spill_gen, lr_gap.reg); + if (lr_gap.reg == reg) { + bitmap_set_bit_p (spill_lr_starts, lr_gap.lr->start); + continue; /* spilled gap of the current reg */ + } + slots_num = target_locs_num (lr_gap.hreg, lr_gap.type); + for (k = 0; k < slots_num; k++) { + MIR_reg_t curr_hr = target_nth_loc (lr_gap.hreg, lr_gap.type, k); + delete_lr_gap (gen_ctx, curr_hr, lr_gap.lr); + for (j = lr_gap.lr->start; j <= lr_gap.lr->finish; j++) + bitmap_clear_bit_p (used_locs_addr[j], curr_hr); + } + } + VARR_SET (MIR_reg_t, reg_renumber, reg, best_loc); + slots_num = target_locs_num (best_loc, type); + simple_loc_update_p = simplified_p || best_loc > MAX_HARD_REG; + for (lr = VARR_GET (live_range_t, var_live_ranges, var); lr != NULL; lr = lr->next) { + if ((reserve_p = simple_loc_update_p || !bitmap_bit_p (spill_lr_starts, lr->start))) { + for (j = lr->start; j <= lr->finish; j++) + for (k = 0; k < slots_num; k++) + bitmap_set_bit_p (used_locs_addr[j], target_nth_loc (best_loc, type, k)); } - if (best_loc == MIR_NON_HARD_REG) { /* Add stack slot ??? */ - slots_num = 1; + if (simple_loc_update_p) continue; + if (lr->lr_bb == NULL) { + for (j = lr->start; j <= lr->finish; j++) + for (k = 0; k < slots_num; k++) + bitmap_set_bit_p (busy_used_locs_addr[j], target_nth_loc (best_loc, type, k)); + } else if (reserve_p) { for (k = 0; k < slots_num; k++) { - if (k == 0) { - best_loc = VARR_LENGTH (size_t, loc_profits); - slots_num = target_locs_num (best_loc, type); - } - VARR_PUSH (size_t, loc_profits, 0); - VARR_PUSH (size_t, loc_profit_ages, 0); - if (k == 0 && (best_loc - MAX_HARD_REG - 1) % slots_num != 0) k--; /* align */ + MIR_reg_t hr = target_nth_loc (best_loc, type, k); + DEBUG (4, { + fprintf (debug_file, " Adding lr gap: r%d, h%d [%d..%d]\n", reg, hr, lr->start, + lr->finish); + }); + insert_lr_gap (gen_ctx, hr, type, reg, lr); } - func_stack_slots_num = VARR_LENGTH (size_t, loc_profits) - MAX_HARD_REG - 1; } } - DEBUG (2, { - MIR_reg_t thread_breg = curr_breg_infos[breg].thread_first; - - fprintf (debug_file, - " Assigning to %s:var=%3u, breg=%3u (freq %-3ld), thread breg=%3u (freq %-3ld) " - "-- " - "%lu\n", - MIR_reg_name (gen_ctx->ctx, reg, curr_func_item->u.func), reg2var (gen_ctx, reg), - breg, curr_breg_infos[breg].freq, thread_breg, - curr_breg_infos[thread_breg].thread_freq, (unsigned long) best_loc); - }); - VARR_SET (MIR_reg_t, breg_renumber, breg, best_loc); - slots_num = target_locs_num (best_loc, type); - for (lr = VARR_GET (live_range_t, var_live_ranges, var); lr != NULL; lr = lr->next) - for (j = lr->start; j <= lr->finish; j++) - for (k = 0; k < slots_num; k++) - bitmap_set_bit_p (used_locs_addr[j], target_nth_loc (best_loc, type, k)); } } -static void assign (gen_ctx_t gen_ctx) { - MIR_reg_t i, reg, nregs = get_nregs (gen_ctx); - - if (optimize_level == 0) - fast_assign (gen_ctx); - else - quality_assign (gen_ctx); - DEBUG (2, { - fprintf (debug_file, "+++++++++++++Disposition after assignment:"); - for (i = 0; i < nregs; i++) { - if (i % 8 == 0) fprintf (debug_file, "\n"); - reg = breg2reg (gen_ctx, i); - fprintf (debug_file, " %3u=>%-2u", reg2var (gen_ctx, reg), - VARR_GET (MIR_reg_t, breg_renumber, i)); - } - fprintf (debug_file, "\n"); - }); -} - -static MIR_reg_t change_reg (gen_ctx_t gen_ctx, MIR_op_t *mem_op, MIR_reg_t reg, - MIR_op_mode_t data_mode, int first_p, MIR_insn_t insn, int out_p) { +/* Add store (st_p) or load of hard_reg with data mode to/from memory + loc using temp_hard_reg for addressing an put it before after anchor. */ +static MIR_reg_t add_ld_st (gen_ctx_t gen_ctx, MIR_op_t *mem_op, MIR_reg_t loc, MIR_reg_t base_reg, + MIR_op_mode_t data_mode, MIR_reg_t hard_reg, int st_p, + MIR_reg_t temp_hard_reg, MIR_insn_t anchor, int after_p) { MIR_context_t ctx = gen_ctx->ctx; - MIR_reg_t loc = VARR_GET (MIR_reg_t, breg_renumber, reg2breg (gen_ctx, reg)); - MIR_reg_t hard_reg; MIR_disp_t offset; MIR_insn_code_t code; MIR_insn_t new_insn, new_insns[3]; @@ -4209,46 +7695,49 @@ static MIR_reg_t change_reg (gen_ctx_t gen_ctx, MIR_op_t *mem_op, MIR_reg_t reg, MIR_op_t hard_reg_op; size_t n; - gen_assert (loc != MIR_NON_HARD_REG); - if (loc <= MAX_HARD_REG) return loc; - gen_assert (data_mode == MIR_OP_INT || data_mode == MIR_OP_FLOAT || data_mode == MIR_OP_DOUBLE - || data_mode == MIR_OP_LDOUBLE); - if (data_mode == MIR_OP_INT) { - type = MIR_T_I64; - code = MIR_MOV; - } else if (data_mode == MIR_OP_FLOAT) { - type = MIR_T_F; - code = MIR_FMOV; - } else if (data_mode == MIR_OP_DOUBLE) { - type = MIR_T_D; - code = MIR_DMOV; - } else { - type = MIR_T_LD; - code = MIR_LDMOV; - } - hard_reg = get_temp_hard_reg (type, first_p); - setup_used_hard_regs (gen_ctx, type, hard_reg); + gen_assert (temp_hard_reg != MIR_NON_VAR); + type = mode2type (data_mode); + code = (type == MIR_T_I64 ? MIR_MOV + : type == MIR_T_F ? MIR_FMOV + : type == MIR_T_D ? MIR_DMOV + : MIR_LDMOV); + if (hard_reg != MIR_NON_VAR) setup_used_hard_regs (gen_ctx, type, hard_reg); offset = target_get_stack_slot_offset (gen_ctx, type, loc - MAX_HARD_REG - 1); n = 0; if (target_valid_mem_offset_p (gen_ctx, type, offset)) { - *mem_op = _MIR_new_hard_reg_mem_op (ctx, type, offset, FP_HARD_REG, MIR_NON_HARD_REG, 0); + *mem_op = _MIR_new_var_mem_op (ctx, type, offset, base_reg, MIR_NON_VAR, 0); } else { - MIR_reg_t temp_hard_reg - = (first_p && !out_p) || (out_p && !first_p) ? TEMP_INT_HARD_REG1 : TEMP_INT_HARD_REG2; - new_insns[0] = MIR_new_insn (ctx, MIR_MOV, _MIR_new_hard_reg_op (ctx, temp_hard_reg), + new_insns[0] = MIR_new_insn (ctx, MIR_MOV, _MIR_new_var_op (ctx, temp_hard_reg), MIR_new_int_op (ctx, offset)); - new_insns[1] = MIR_new_insn (ctx, MIR_ADD, _MIR_new_hard_reg_op (ctx, temp_hard_reg), - _MIR_new_hard_reg_op (ctx, temp_hard_reg), - _MIR_new_hard_reg_op (ctx, FP_HARD_REG)); + new_insns[1] + = MIR_new_insn (ctx, MIR_ADD, _MIR_new_var_op (ctx, temp_hard_reg), + _MIR_new_var_op (ctx, temp_hard_reg), _MIR_new_var_op (ctx, base_reg)); n = 2; - *mem_op = _MIR_new_hard_reg_mem_op (ctx, type, 0, temp_hard_reg, MIR_NON_HARD_REG, 0); + *mem_op = _MIR_new_var_mem_op (ctx, type, 0, temp_hard_reg, MIR_NON_VAR, 0); } - if (hard_reg == MIR_NON_HARD_REG) return hard_reg; - hard_reg_op = _MIR_new_hard_reg_op (ctx, hard_reg); - if (!out_p) { + if (hard_reg == MIR_NON_VAR) return hard_reg; /* LD vars can be always kept in memory */ + hard_reg_op = _MIR_new_var_op (ctx, hard_reg); + if (!st_p) { new_insns[n++] = MIR_new_insn (ctx, code, hard_reg_op, *mem_op); } else { new_insns[n++] = MIR_new_insn (ctx, code, *mem_op, hard_reg_op); + } + DEBUG (2, { + bb_t bb = get_insn_bb (gen_ctx, anchor); + fprintf (debug_file, " Adding %s insn ", after_p ? "after" : "before"); + fprintf (debug_file, " (in BB %lu", (unsigned long) bb->index); + if (optimize_level == 0 || bb->loop_node == NULL) + fprintf (debug_file, ") "); + else + fprintf (debug_file, ", level %d) ", bb_loop_level (bb)); + MIR_output_insn (ctx, debug_file, anchor, curr_func_item->u.func, FALSE); + fprintf (debug_file, ":\n"); + for (size_t i = 0; i < n; i++) { + fprintf (debug_file, " "); + MIR_output_insn (ctx, debug_file, new_insns[i], curr_func_item->u.func, TRUE); + } + }); + if (after_p) { for (size_t i = 0, j = n - 1; i < j; i++, j--) { /* reverse for subsequent correct insertion: */ new_insn = new_insns[i]; new_insns[i] = new_insns[j]; @@ -4257,183 +7746,838 @@ static MIR_reg_t change_reg (gen_ctx_t gen_ctx, MIR_op_t *mem_op, MIR_reg_t reg, } for (size_t i = 0; i < n; i++) { new_insn = new_insns[i]; - if (out_p) - MIR_insert_insn_after (ctx, curr_func_item, insn, new_insn); + if (after_p) + MIR_insert_insn_after (ctx, curr_func_item, anchor, new_insn); else - MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + MIR_insert_insn_before (ctx, curr_func_item, anchor, new_insn); if (optimize_level == 0) { - new_insn->data = get_insn_data_bb (insn); + new_insn->data = get_insn_data_bb (anchor); } else { - bb_insn = insn->data; + bb_insn = anchor->data; new_bb_insn = create_bb_insn (gen_ctx, new_insn, bb_insn->bb); - if (out_p) + if (after_p) DLIST_INSERT_AFTER (bb_insn_t, bb_insn->bb->bb_insns, bb_insn, new_bb_insn); else DLIST_INSERT_BEFORE (bb_insn_t, bb_insn->bb->bb_insns, bb_insn, new_bb_insn); } } - return hard_reg; + return hard_reg; +} + +static MIR_reg_t get_reload_hreg (gen_ctx_t gen_ctx, MIR_reg_t var, MIR_reg_t type, int out_p) { + MIR_reg_t hr; + insn_reload_t *reloads = out_p ? out_reloads : in_reloads; + int rld_num = 0, *reloads_num = out_p ? &out_reloads_num : &in_reloads_num; + + for (int i = 0; i < *reloads_num; i++) { + if (var != MIR_NON_VAR && reloads[i].var == var) return reloads[i].hreg; + if (rld_num == 0 && reloads[i].hreg == get_temp_hard_reg (type, TRUE)) + rld_num = 1; + else if (reloads[i].hreg == get_temp_hard_reg (type, FALSE)) + rld_num = 2; + } + gen_assert (rld_num <= 1); + hr = get_temp_hard_reg (type, rld_num == 0); + rld_num = (*reloads_num)++; + gen_assert (rld_num < MAX_INSN_RELOADS); + reloads[rld_num].var = var; + reloads[rld_num].type = type; + reloads[rld_num].hreg = hr; + return hr; +} + +/* Return hard reg to use in insn instead of pseudo (reg) with given data_mode. + If reg got a stack slot (will be in *mem_op after the call), add load or store insn + from this slot depending on out_p using base_reg and possibly a temp hard reg + depending on out_p. */ +static MIR_reg_t change_reg (gen_ctx_t gen_ctx, MIR_op_t *mem_op, MIR_reg_t reg, MIR_reg_t base_reg, + MIR_op_mode_t data_mode, MIR_insn_t insn, int out_p) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_reg_t loc = VARR_GET (MIR_reg_t, reg_renumber, reg); + if (loc <= MAX_HARD_REG) return loc; + MIR_type_t type = MIR_reg_type (ctx, reg - MAX_HARD_REG, curr_func_item->u.func); + MIR_reg_t reload_hreg = get_reload_hreg (gen_ctx, reg, type, out_p); + MIR_reg_t temp_addr_hreg + = (out_p || type != MIR_T_I64 ? get_reload_hreg (gen_ctx, MIR_NON_VAR, MIR_T_I64, out_p) + : reload_hreg); + gen_assert (!MIR_addr_code_p (insn->code)); + return add_ld_st (gen_ctx, mem_op, loc, base_reg, data_mode, reload_hreg, out_p, temp_addr_hreg, + insn, out_p); +} + +static void update_live (MIR_reg_t var, int out_p, bitmap_t live) { + if (out_p) { + bitmap_clear_bit_p (live, var); + } else { + bitmap_set_bit_p (live, var); + } +} + +static int get_spill_mem_loc (gen_ctx_t gen_ctx, MIR_reg_t reg) { + MIR_context_t ctx = gen_ctx->ctx; + int slots_num; + MIR_type_t type; + MIR_reg_t var, slot; + live_range_t lr; + bitmap_t conflict_locs = conflict_locs1; + bitmap_t *used_locs_addr = VARR_ADDR (bitmap_t, used_locs); + spill_cache_el_t *spill_cache_addr = VARR_ADDR (spill_cache_el_t, spill_cache); + gen_assert (reg != MIR_NON_VAR && reg < VARR_LENGTH (spill_cache_el_t, spill_cache)); + if (spill_cache_addr[reg].age == spill_cache_age) return spill_cache_addr[reg].slot; + bitmap_clear (conflict_locs); + var = reg; + for (lr = VARR_GET (live_range_t, var_live_ranges, var); lr != NULL; lr = lr->next) + for (int j = lr->start; j <= lr->finish; j++) + bitmap_ior (conflict_locs, conflict_locs, used_locs_addr[j]); + type = MIR_reg_type (ctx, reg - MAX_HARD_REG, curr_func_item->u.func); + spill_cache_addr[reg].slot = slot + = get_stack_loc (gen_ctx, start_mem_loc, type, conflict_locs, &slots_num); + spill_cache_addr[reg].age = spill_cache_age; + for (lr = VARR_GET (live_range_t, var_live_ranges, var); lr != NULL; lr = lr->next) + for (int j = lr->start; j <= lr->finish; j++) + for (int k = 0; k < slots_num; k++) + bitmap_set_bit_p (used_locs_addr[j], target_nth_loc (slot, type, k)); + return slot; +} + +/* Add spill or restore (restore_p) of pseudo (reg) assigned to hard reg and + put the insns after anchor. Use base_reg to address the stack lot. */ +static void spill_restore_reg (gen_ctx_t gen_ctx, MIR_reg_t reg, MIR_reg_t base_reg, + MIR_insn_t anchor, int after_p, int restore_p) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_reg_t mem_loc; + MIR_op_t mem_op; + MIR_type_t type = MIR_reg_type (ctx, reg - MAX_HARD_REG, curr_func_item->u.func); + MIR_op_mode_t data_mode = type2mode (type); + MIR_reg_t hard_reg = VARR_GET (MIR_reg_t, reg_renumber, reg); + gen_assert (hard_reg <= MAX_HARD_REG); + mem_loc = get_spill_mem_loc (gen_ctx, reg); + DEBUG (2, { fprintf (debug_file, " %s r%d:", restore_p ? "Restore" : "Spill", reg); }); + add_ld_st (gen_ctx, &mem_op, mem_loc, base_reg, data_mode, hard_reg, !restore_p, + TEMP_INT_HARD_REG1, anchor, after_p); +} + +static void reload_addr (gen_ctx_t gen_ctx, MIR_insn_t insn, int in_mem_op_num, int out_mem_op_num, + MIR_reg_t base_reg) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_op_t mem_op, temp_op1, temp_op2, *ops = insn->ops; + gen_assert (in_mem_op_num >= 0 || out_mem_op_num >= 0); + int op_num = out_mem_op_num >= 0 ? out_mem_op_num : in_mem_op_num; + MIR_reg_t base = ops[op_num].u.var_mem.base, index = ops[op_num].u.var_mem.index; + MIR_insn_t new_insn; + gen_assert (in_mem_op_num < 0 || out_mem_op_num < 0 + || MIR_op_eq_p (ctx, ops[in_mem_op_num], ops[out_mem_op_num])); + add_ld_st (gen_ctx, &mem_op, VARR_GET (MIR_reg_t, reg_renumber, base), base_reg, MIR_OP_INT, + TEMP_INT_HARD_REG1, FALSE, TEMP_INT_HARD_REG1, insn, FALSE); + add_ld_st (gen_ctx, &mem_op, VARR_GET (MIR_reg_t, reg_renumber, index), base_reg, MIR_OP_INT, + TEMP_INT_HARD_REG2, FALSE, TEMP_INT_HARD_REG2, insn, FALSE); + temp_op1 = _MIR_new_var_op (ctx, TEMP_INT_HARD_REG1); + temp_op2 = _MIR_new_var_op (ctx, TEMP_INT_HARD_REG2); + if (ops[op_num].u.var_mem.scale != 1) { + new_insn = MIR_new_insn (ctx, MIR_LSH, temp_op2, temp_op2, + MIR_new_int_op (ctx, gen_int_log2 (ops[op_num].u.var_mem.scale))); + gen_add_insn_before (gen_ctx, insn, new_insn); + /* continuation of debug output in add_ld_st: */ + DEBUG (2, { + fprintf (debug_file, " "); + MIR_output_insn (ctx, debug_file, new_insn, curr_func_item->u.func, TRUE); + }); + } + new_insn = MIR_new_insn (ctx, MIR_ADD, temp_op1, temp_op1, temp_op2); + gen_add_insn_before (gen_ctx, insn, new_insn); + DEBUG (2, { + fprintf (debug_file, " "); + MIR_output_insn (ctx, debug_file, new_insn, curr_func_item->u.func, TRUE); + }); + if (out_mem_op_num >= 0) { + ops[out_mem_op_num].u.var_mem.base = TEMP_INT_HARD_REG1; + ops[out_mem_op_num].u.var_mem.index = MIR_NON_VAR; + } + if (in_mem_op_num >= 0) { + ops[in_mem_op_num].u.var_mem.base = TEMP_INT_HARD_REG1; + ops[in_mem_op_num].u.var_mem.index = MIR_NON_VAR; + } +} + +struct rewrite_data { + bb_t bb; + bitmap_t live, regs_to_save; +}; + +#define MAX_INSN_RELOAD_MEM_OPS 2 +static int try_spilled_reg_mem (gen_ctx_t gen_ctx, MIR_insn_t insn, int nop, MIR_reg_t loc, + MIR_reg_t base_reg) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_op_t *op = &insn->ops[nop]; + MIR_type_t type = MIR_reg_type (ctx, op->u.var - MAX_HARD_REG, curr_func_item->u.func); + MIR_disp_t offset = target_get_stack_slot_offset (gen_ctx, type, loc - MAX_HARD_REG - 1); + if (!target_valid_mem_offset_p (gen_ctx, type, offset)) return FALSE; + MIR_reg_t reg = op->u.var; + MIR_op_t saved_op = *op; + MIR_op_t mem_op = _MIR_new_var_mem_op (ctx, type, offset, base_reg, MIR_NON_VAR, 0); + int n = 0, op_nums[MAX_INSN_RELOAD_MEM_OPS]; + for (int i = nop; i < (int) insn->nops; i++) + if (insn->ops[i].mode == MIR_OP_VAR && insn->ops[i].u.var == reg) { + insn->ops[i] = mem_op; + gen_assert (n < MAX_INSN_RELOAD_MEM_OPS); + op_nums[n++] = i; + } + if (target_insn_ok_p (gen_ctx, insn)) return TRUE; + for (int i = 0; i < n; i++) insn->ops[op_nums[i]] = saved_op; + return FALSE; +} + +static void transform_addr (gen_ctx_t gen_ctx, MIR_insn_t insn, MIR_reg_t base_reg) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_reg_t reg, temp_hard_reg, loc; + MIR_type_t type; + MIR_disp_t offset; + MIR_insn_t new_insn1, new_insn2; + gen_assert (MIR_addr_code_p (insn->code)); + gen_assert (insn->ops[1].mode == MIR_OP_VAR); + reg = insn->ops[1].u.reg; + gen_assert (reg > MAX_HARD_REG && reg != MIR_NON_VAR); + loc = VARR_GET (MIR_reg_t, reg_renumber, reg); + type = MIR_reg_type (ctx, reg - MAX_HARD_REG, curr_func_item->u.func); + gen_assert (loc > MAX_HARD_REG); + offset = target_get_stack_slot_offset (gen_ctx, type, loc - MAX_HARD_REG - 1); + temp_hard_reg = get_reload_hreg (gen_ctx, MIR_NON_VAR, MIR_T_I64, FALSE); + new_insn1 = MIR_new_insn (ctx, MIR_MOV, _MIR_new_var_op (ctx, temp_hard_reg), + MIR_new_int_op (ctx, offset + _MIR_addr_offset (ctx, insn->code))); + new_insn2 = MIR_new_insn (ctx, MIR_ADD, _MIR_new_var_op (ctx, temp_hard_reg), + _MIR_new_var_op (ctx, temp_hard_reg), _MIR_new_var_op (ctx, base_reg)); + DEBUG (2, { + bb_t bb = get_insn_bb (gen_ctx, insn); + fprintf (debug_file, " Adding before insn (in BB %lu) ", (unsigned long) bb->index); + MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, FALSE); + fprintf (debug_file, ":\n "); + MIR_output_insn (ctx, debug_file, new_insn1, curr_func_item->u.func, TRUE); + fprintf (debug_file, " "); + MIR_output_insn (ctx, debug_file, new_insn2, curr_func_item->u.func, TRUE); + }); + gen_add_insn_before (gen_ctx, insn, new_insn1); + gen_add_insn_before (gen_ctx, insn, new_insn2); + DEBUG (2, { + fprintf (debug_file, "Changing "); + MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, FALSE); + }); + insn->code = MIR_MOV; + insn->ops[1] = _MIR_new_var_op (ctx, temp_hard_reg); + DEBUG (2, { + fprintf (debug_file, " to "); + MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, TRUE); + }); +} + +static int rewrite_insn (gen_ctx_t gen_ctx, MIR_insn_t insn, MIR_reg_t base_reg, + struct rewrite_data *data) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_type_t type; + size_t nops, i; + MIR_op_t *op, mem_op; +#if !MIR_NO_GEN_DEBUG + MIR_op_t in_op = MIR_new_int_op (ctx, 0), + out_op = MIR_new_int_op (ctx, 0); /* To remove unitilized warning */ +#endif + MIR_op_mode_t data_mode; + MIR_reg_t hard_reg, reg; + int out_p, call_p, addr_reload_p, rld_in_num; + int out_mem_op_num, in_mem_op_num, other_mem_op_num; + nops = MIR_insn_nops (ctx, insn); + out_mem_op_num = in_mem_op_num = -1; + rld_in_num = 0; + /* Update live info, save/restore regs living across calls, and check do we need addr reload: + */ + for (int niter = 0; niter <= 1; niter++) { + for (i = 0; i < nops; i++) { + op = &insn->ops[i]; + data_mode = MIR_insn_op_mode (ctx, insn, i, &out_p); + if (niter == 0 && (!out_p || op->mode == MIR_OP_VAR_MEM)) continue; + if (niter == 1 && (out_p && op->mode != MIR_OP_VAR_MEM)) continue; + switch (op->mode) { + case MIR_OP_VAR: + if (op->u.var <= MAX_HARD_REG) { + bitmap_set_bit_p (func_used_hard_regs, op->u.var); + if (data != NULL) update_live (op->u.var, out_p, data->live); + } else { + if (!out_p && VARR_GET (MIR_reg_t, reg_renumber, op->u.var) > MAX_HARD_REG + && mode2type (data_mode) == MIR_T_I64) + rld_in_num++; + if (data != NULL) { + update_live (op->u.var, out_p, data->live); + if (bitmap_clear_bit_p (data->regs_to_save, op->u.var)) /* put (slot<-hr) after insn */ + spill_restore_reg (gen_ctx, op->u.var, base_reg, insn, TRUE, FALSE); + } + } + break; + case MIR_OP_VAR_MEM: + if (op->u.var_mem.base <= MAX_HARD_REG) + bitmap_set_bit_p (func_used_hard_regs, op->u.var_mem.base); + if (op->u.var_mem.index <= MAX_HARD_REG) + bitmap_set_bit_p (func_used_hard_regs, op->u.var_mem.index); + if (op->u.var_mem.base != MIR_NON_VAR && op->u.var_mem.index != MIR_NON_VAR + && op->u.var_mem.base > MAX_HARD_REG && op->u.var_mem.index > MAX_HARD_REG + && VARR_GET (MIR_reg_t, reg_renumber, op->u.var_mem.base) > MAX_HARD_REG + && VARR_GET (MIR_reg_t, reg_renumber, op->u.var_mem.index) > MAX_HARD_REG) { + other_mem_op_num = -1; + if (out_p) { + gen_assert (out_mem_op_num < 0); + out_mem_op_num = (int) i; + if (in_mem_op_num >= 0) other_mem_op_num = in_mem_op_num; + } else { + gen_assert (in_mem_op_num < 0); + in_mem_op_num = (int) i; + if (out_mem_op_num >= 0) other_mem_op_num = out_mem_op_num; + } + if (other_mem_op_num < 0 + || op->u.var_mem.base != insn->ops[other_mem_op_num].u.var_mem.base + || op->u.var_mem.index != insn->ops[other_mem_op_num].u.var_mem.index) + rld_in_num += 2; + } + if (data != NULL) { + if (op->u.var_mem.base != MIR_NON_VAR) { + update_live (op->u.var_mem.base, FALSE, data->live); + if (op->u.var_mem.base > MAX_HARD_REG) { + if (bitmap_clear_bit_p (data->regs_to_save, + op->u.var_mem.base)) /* put slot<-hr after */ + spill_restore_reg (gen_ctx, op->u.var_mem.base, base_reg, insn, TRUE, FALSE); + } + } + if (op->u.var_mem.index != MIR_NON_VAR) { + update_live (op->u.var_mem.index, FALSE, data->live); + if (op->u.var_mem.index > MAX_HARD_REG) { + if (bitmap_clear_bit_p (data->regs_to_save, + op->u.var_mem.index)) /* put slot<-hr after */ + spill_restore_reg (gen_ctx, op->u.var_mem.index, base_reg, insn, TRUE, FALSE); + } + } + } + break; + default: /* do nothing */ break; + } + } + if (data != NULL && niter == 0) { /* right after processing outputs */ + MIR_reg_t early_clobbered_hard_reg1, early_clobbered_hard_reg2; + target_get_early_clobbered_hard_regs (insn, &early_clobbered_hard_reg1, + &early_clobbered_hard_reg2); + if (early_clobbered_hard_reg1 != MIR_NON_VAR) + update_live (early_clobbered_hard_reg1, TRUE, data->live); + if (early_clobbered_hard_reg2 != MIR_NON_VAR) + update_live (early_clobbered_hard_reg2, TRUE, data->live); + if (MIR_call_code_p (insn->code)) { + size_t nel; + bb_insn_t bb_insn = insn->data; + bitmap_iterator_t bi; + FOREACH_BITMAP_BIT (bi, call_used_hard_regs[MIR_T_UNDEF], nel) { + update_live ((MIR_reg_t) nel, TRUE, data->live); + } + FOREACH_BITMAP_BIT (bi, bb_insn->call_hard_reg_args, nel) { + update_live ((MIR_reg_t) nel, FALSE, data->live); + } + FOREACH_BITMAP_BIT (bi, data->live, nel) { + if (nel <= MAX_HARD_REG) continue; + reg = (MIR_reg_t) nel; + if (bitmap_bit_p (data->bb->spill_gen, reg)) continue; /* will be spilled in split */ + MIR_reg_t loc = VARR_GET (MIR_reg_t, reg_renumber, reg); + if (loc > MAX_HARD_REG) continue; + type = MIR_reg_type (gen_ctx->ctx, reg - MAX_HARD_REG, curr_func_item->u.func); + int nregs = target_locs_num (loc, type); + if (hreg_in_bitmap_p (loc, type, nregs, call_used_hard_regs[MIR_T_UNDEF]) + && bitmap_set_bit_p (data->regs_to_save, reg)) /* put (hr<-slot) after call */ + spill_restore_reg (gen_ctx, reg, base_reg, insn, TRUE, TRUE); + } + } + } + } + addr_reload_p = rld_in_num > 2; + out_reloads_num = in_reloads_num = 0; + if (addr_reload_p) { /* not enough 2 temp int hard regs: address reload: */ + reload_addr (gen_ctx, insn, in_mem_op_num, out_mem_op_num, base_reg); + get_reload_hreg (gen_ctx, MIR_NON_VAR, MIR_T_I64, + FALSE); /* reserve the 1st int temp hard reg */ + } + if (MIR_addr_code_p (insn->code)) transform_addr (gen_ctx, insn, base_reg); + call_p = MIR_call_code_p (insn->code); + for (i = 0; i < nops; i++) { + op = &insn->ops[i]; + data_mode = MIR_insn_op_mode (ctx, insn, i, &out_p); + DEBUG (2, { + if (out_p) + out_op = *op; /* we don't care about multiple call outputs here */ + else + in_op = *op; + }); + switch (op->mode) { + case MIR_OP_VAR: + if (op->u.var <= MAX_HARD_REG) break; + if (data_mode == MIR_OP_VAR) { + gen_assert (insn->code == MIR_USE || (MIR_addr_code_p (insn->code) && i == 1)); + type = MIR_reg_type (ctx, op->u.var - MAX_HARD_REG, curr_func_item->u.func); + data_mode = type == MIR_T_F ? MIR_OP_FLOAT + : type == MIR_T_D ? MIR_OP_DOUBLE + : type == MIR_T_LD ? MIR_OP_LDOUBLE + : MIR_OP_INT; + } + MIR_reg_t loc = VARR_GET (MIR_reg_t, reg_renumber, op->u.var); + if (!MIR_addr_code_p (insn->code) && i == 0 && loc > MAX_HARD_REG + && try_spilled_reg_mem (gen_ctx, insn, (int) i, loc, base_reg)) + break; + hard_reg = change_reg (gen_ctx, &mem_op, op->u.var, base_reg, data_mode, insn, out_p); + if (hard_reg == MIR_NON_VAR) { /* we don't use hard regs for this type reg (e.g. ldouble) */ + *op = mem_op; + } else { + op->u.var = hard_reg; + } + break; + case MIR_OP_VAR_MEM: + if (call_p && MIR_blk_type_p (op->u.var_mem.type)) break; + if (op->u.var_mem.base > MAX_HARD_REG && op->u.var_mem.base != MIR_NON_VAR) { + op->u.var_mem.base + = change_reg (gen_ctx, &mem_op, op->u.var_mem.base, base_reg, MIR_OP_INT, insn, FALSE); + gen_assert (op->u.var_mem.base != MIR_NON_VAR); /* we can always use GP regs */ + } + if (op->u.var_mem.index > MAX_HARD_REG && op->u.var_mem.index != MIR_NON_VAR) { + op->u.var_mem.index + = change_reg (gen_ctx, &mem_op, op->u.var_mem.index, base_reg, MIR_OP_INT, insn, FALSE); + gen_assert (op->u.var_mem.index != MIR_NON_VAR); /* we can always use GP regs */ + } + break; + default: /* do nothing */ break; + } + } + if (move_code_p (insn->code)) { + if (MIR_op_eq_p (ctx, insn->ops[0], insn->ops[1])) { + DEBUG (2, { + fprintf (debug_file, "Deleting noop move "); + MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, FALSE); + fprintf (debug_file, " which was "); + insn->ops[0] = out_op; + insn->ops[1] = in_op; + MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, TRUE); + }); + bb_insn_t bb_insn; + if (optimize_level > 0 && (bb_insn = insn->data) != NULL + && DLIST_HEAD (bb_insn_t, bb_insn->bb->bb_insns) == bb_insn + && DLIST_TAIL (bb_insn_t, bb_insn->bb->bb_insns) == bb_insn) { /* avoid empty bb */ + MIR_insn_t nop = MIR_new_insn_arr (gen_ctx->ctx, MIR_USE, 0, NULL); + MIR_insert_insn_before (gen_ctx->ctx, curr_func_item, bb_insn->insn, nop); + add_new_bb_insn (gen_ctx, nop, bb_insn->bb, FALSE); + } + gen_delete_insn (gen_ctx, insn); + return 1; + } + } + return 0; +} + +static void rewrite (gen_ctx_t gen_ctx) { + MIR_insn_t insn, next_insn, head_insn; + MIR_reg_t base_reg = target_get_stack_slot_base_reg (gen_ctx); + size_t insns_num = 0, movs_num = 0, deleted_movs_num = 0; + bitmap_t global_hard_regs + = _MIR_get_module_global_var_hard_regs (gen_ctx->ctx, curr_func_item->module); + const int simplified_p = ONLY_SIMPLIFIED_RA || optimize_level < 2; + + if (simplified_p) { + for (insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); insn != NULL; + insn = next_insn) { + next_insn = DLIST_NEXT (MIR_insn_t, insn); + if (move_code_p (insn->code)) movs_num++; + deleted_movs_num += rewrite_insn (gen_ctx, insn, base_reg, NULL); + insns_num++; + } + } else { + MIR_reg_t reg; + bb_insn_t prev_bb_insn; + bitmap_t live = temp_bitmap, regs_to_save = temp_bitmap2; + bitmap_iterator_t bi; + size_t nel; + spill_cache_el_t spill_cache_el = {0, 0}; + spill_cache_age++; + VARR_TRUNC (spill_cache_el_t, spill_cache, 0); + while (VARR_LENGTH (spill_cache_el_t, spill_cache) <= curr_cfg->max_var) + VARR_PUSH (spill_cache_el_t, spill_cache, spill_cache_el); + for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + struct rewrite_data data = {bb, live, regs_to_save}; + bitmap_copy (live, bb->live_out); + bitmap_clear (regs_to_save); + for (bb_insn_t bb_insn = DLIST_TAIL (bb_insn_t, bb->bb_insns); bb_insn != NULL; + bb_insn = prev_bb_insn) { + prev_bb_insn = DLIST_PREV (bb_insn_t, bb_insn); + insn = bb_insn->insn; + if (move_code_p (insn->code)) movs_num++; + deleted_movs_num += rewrite_insn (gen_ctx, insn, base_reg, &data); + insns_num++; + } + gen_assert (bitmap_equal_p (live, bb->live_in)); + FOREACH_BITMAP_BIT (bi, regs_to_save, nel) { + gen_assert (nel > MAX_HARD_REG); + reg = (MIR_reg_t) nel; + gen_assert (bitmap_bit_p (bb->spill_kill, reg)); + head_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns)->insn; + spill_restore_reg (gen_ctx, reg, base_reg, head_insn, head_insn->code == MIR_LABEL, FALSE); + } + } + } + DEBUG (1, { + fprintf (debug_file, + "%5lu deleted RA noop moves out of %lu all moves (%.1f%%), out of %lu all insns " + "(%.1f%%)\n", + (unsigned long) deleted_movs_num, (unsigned long) movs_num, + deleted_movs_num * 100.0 / movs_num, (unsigned long) insns_num, + deleted_movs_num * 100.0 / insns_num); + }); + if (global_hard_regs != NULL) /* we should not save/restore hard regs used by globals */ + bitmap_and_compl (func_used_hard_regs, func_used_hard_regs, global_hard_regs); } -static void rewrite (gen_ctx_t gen_ctx) { - MIR_context_t ctx = gen_ctx->ctx; - MIR_insn_t insn, next_insn; - size_t nops, i; - MIR_op_t *op, mem_op; #if !MIR_NO_GEN_DEBUG - MIR_op_t in_op = MIR_new_int_op (ctx, 0), - out_op = MIR_new_int_op (ctx, 0); /* To remove unitilized warning */ +static void output_bb_spill_info (gen_ctx_t gen_ctx, bb_t bb) { + output_bitmap (gen_ctx, " live_in:", bb->live_in, TRUE, NULL); + output_bitmap (gen_ctx, " live_out:", bb->live_out, TRUE, NULL); + output_bitmap (gen_ctx, " spill_gen:", bb->spill_gen, TRUE, NULL); + output_bitmap (gen_ctx, " spill_kill:", bb->spill_kill, TRUE, NULL); +} #endif - MIR_mem_t mem; - MIR_op_mode_t data_mode; - MIR_reg_t hard_reg; - int out_p, first_in_p; - size_t insns_num = 0, movs_num = 0, deleted_movs_num = 0; - for (insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); insn != NULL; - insn = next_insn) { - next_insn = DLIST_NEXT (MIR_insn_t, insn); - nops = MIR_insn_nops (ctx, insn); - first_in_p = TRUE; - for (i = 0; i < nops; i++) { - op = &insn->ops[i]; - data_mode = MIR_insn_op_mode (ctx, insn, i, &out_p); - DEBUG (2, { - if (out_p) - out_op = *op; /* we don't care about multiple call outputs here */ - else - in_op = *op; - }); - switch (op->mode) { - case MIR_OP_HARD_REG: bitmap_set_bit_p (func_used_hard_regs, op->u.hard_reg); break; - case MIR_OP_HARD_REG_MEM: - if (op->u.hard_reg_mem.base != MIR_NON_HARD_REG) - bitmap_set_bit_p (func_used_hard_regs, op->u.hard_reg_mem.base); - if (op->u.hard_reg_mem.index != MIR_NON_HARD_REG) - bitmap_set_bit_p (func_used_hard_regs, op->u.hard_reg_mem.index); - break; - case MIR_OP_REG: - hard_reg - = change_reg (gen_ctx, &mem_op, op->u.reg, data_mode, out_p || first_in_p, insn, out_p); - if (!out_p) first_in_p = FALSE; - if (hard_reg == MIR_NON_HARD_REG) { - *op = mem_op; - } else { - op->mode = MIR_OP_HARD_REG; - op->u.hard_reg = hard_reg; - } - break; - case MIR_OP_MEM: - mem = op->u.mem; - /* Always second for mov MEM[R2], R1 or mov R1, MEM[R2]. */ - if (op->u.mem.base == 0) { - mem.base = MIR_NON_HARD_REG; - } else { - mem.base = change_reg (gen_ctx, &mem_op, op->u.mem.base, MIR_OP_INT, FALSE, insn, FALSE); - gen_assert (mem.base != MIR_NON_HARD_REG); /* we can always use GP regs */ - } - gen_assert (op->u.mem.index == 0); - mem.index = MIR_NON_HARD_REG; - op->mode = MIR_OP_HARD_REG_MEM; - op->u.hard_reg_mem = mem; - break; - default: /* do nothing */ break; +static void collect_spill_els (gen_ctx_t gen_ctx) { + bb_t bb; + edge_t e; + size_t nel; + bitmap_iterator_t bi; + spill_el_t spill_el; + + VARR_TRUNC (spill_el_t, spill_els, 0); /* collect spill elements */ + for (bb = DLIST_EL (bb_t, curr_cfg->bbs, 2); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + /* we need non-empty BBs for splitting. we can not remove empty BB as a reg can be splitted + around the BB and we need to generate spills/restores in this BB. */ + gen_assert (DLIST_HEAD (bb_insn_t, bb->bb_insns) != NULL); + /* skip entry/exit bbs and split bbs */ + DEBUG (2, { + fprintf (debug_file, " Process BB%lu(level %d) for splitting\n", (unsigned long) bb->index, + bb_loop_level (bb)); + }); + /* Process out edges for spills: */ + spill_el.edge_p = spill_el.bb_end_p = TRUE; + for (e = DLIST_TAIL (out_edge_t, bb->out_edges); e != NULL; e = DLIST_PREV (out_edge_t, e)) { + bitmap_and_compl (temp_bitmap, e->dst->spill_gen, bb->spill_gen); + if (bitmap_empty_p (temp_bitmap)) continue; + FOREACH_BITMAP_BIT (bi, temp_bitmap, nel) { + gen_assert (nel > MAX_HARD_REG); + spill_el.reg = (MIR_reg_t) nel; + spill_el.spill_p = TRUE; + spill_el.u.e = e; + VARR_PUSH (spill_el_t, spill_els, spill_el); } } - insns_num++; - if (move_code_p (insn->code)) { - movs_num++; - if (MIR_op_eq_p (ctx, insn->ops[0], insn->ops[1])) { - DEBUG (2, { - fprintf (debug_file, "Deleting noop move "); - MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, FALSE); - fprintf (debug_file, " which was "); - insn->ops[0] = out_op; - insn->ops[1] = in_op; - MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, TRUE); - }); - gen_delete_insn (gen_ctx, insn); - deleted_movs_num++; + /* Process input edges for restores: */ + for (e = DLIST_TAIL (in_edge_t, bb->in_edges); e != NULL; e = DLIST_PREV (in_edge_t, e)) { + bitmap_clear (temp_bitmap); + FOREACH_BITMAP_BIT (bi, e->src->spill_gen, nel) { + if (bitmap_bit_p (bb->spill_gen, nel) || !bitmap_bit_p (bb->live_in, nel)) continue; + bitmap_set_bit_p (temp_bitmap, nel); + } + if (bitmap_empty_p (temp_bitmap)) continue; + FOREACH_BITMAP_BIT (bi, temp_bitmap, nel) { + gen_assert (nel > MAX_HARD_REG); + spill_el.reg = (MIR_reg_t) nel; + spill_el.spill_p = FALSE; + spill_el.u.e = e; + VARR_PUSH (spill_el_t, spill_els, spill_el); } } } - DEBUG (1, { - fprintf (debug_file, - "%5lu deleted RA noop moves out of %lu non-conflicting moves " - "(%.1f%%), " - "out of %lu all moves (%.1f%%), out of %lu all insns (%.1f%%)\n", - (unsigned long) deleted_movs_num, (unsigned long) curr_cfg->non_conflicting_moves, - curr_cfg->non_conflicting_moves == 0 - ? 100.0 - : deleted_movs_num * 100.0 / curr_cfg->non_conflicting_moves, - (unsigned long) movs_num, deleted_movs_num * 100.0 / movs_num, - (unsigned long) insns_num, deleted_movs_num * 100.0 / insns_num); +} + +#undef live_in +#undef live_out +#undef spill_gen +#undef spill_kill + +static int spill_el_cmp (const spill_el_t *e1, const spill_el_t *e2) { + if (e1->edge_p != e2->edge_p) return e1->edge_p - e2->edge_p; /* put bb first */ + if (e1->edge_p && e1->u.e != e2->u.e) return e1->u.e < e2->u.e ? -1 : 1; + if (!e1->edge_p && e1->u.bb != e2->u.bb) return e1->u.bb->index < e2->u.bb->index ? -1 : 1; + if (!e1->edge_p && e1->bb_end_p != e2->bb_end_p) + return e1->bb_end_p - e2->bb_end_p; /* start first */ + if (e1->spill_p != e2->spill_p) /* load first for bb start, store first otherwise: */ + return !e1->edge_p && !e1->bb_end_p ? e1->spill_p - e2->spill_p : e2->spill_p - e1->spill_p; + return e1->reg == e2->reg ? 0 : e1->reg < e2->reg ? -1 : 1; /* smaller reg first */ +} + +static int spill_el_sort_cmp (const void *e1, const void *e2) { return spill_el_cmp (e1, e2); } + +static void make_uniq_spill_els (gen_ctx_t gen_ctx) { + size_t i, last, len = VARR_LENGTH (spill_el_t, spill_els); + spill_el_t *els = VARR_ADDR (spill_el_t, spill_els); + if (len == 0) return; + for (last = 0, i = 1; i < len; i++) { + if (spill_el_cmp (&els[last], &els[i]) == 0) continue; + els[++last] = els[i]; + } + VARR_TRUNC (spill_el_t, spill_els, last + 1); +} + +#define at_start gen +#define at_end kill +#define at_src_p flag1 +#define at_dst_p flag2 +static void transform_edge_to_bb_placement (gen_ctx_t gen_ctx) { + size_t i, j; + MIR_insn_t insn; + bb_t bb; + edge_t e, e2, first_e; + bitmap_t edge_regs = temp_bitmap; + spill_el_t *spill_els_addr = VARR_ADDR (spill_el_t, spill_els); + + if (VARR_LENGTH (spill_el_t, spill_els) == 0) return; + /* Initialize: */ + for (bb = DLIST_EL (bb_t, curr_cfg->bbs, 2); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { + bitmap_clear (bb->at_end); + bitmap_clear (bb->at_start); + for (e = DLIST_HEAD (out_edge_t, bb->out_edges); e != NULL; e = DLIST_NEXT (out_edge_t, e)) + e->at_src_p = e->at_dst_p = FALSE; + } + /* Setup common at_{start,end} and initial at_{src,dst}_p: */ + for (i = 0; i < VARR_LENGTH (spill_el_t, spill_els); i++) { + gen_assert (spill_els_addr[i].edge_p); + e = spill_els_addr[i].u.e; + insn = DLIST_TAIL (bb_insn_t, e->src->bb_insns)->insn; + /* remember restores sorted after spills: */ + e->at_src_p + = spill_els_addr[i].spill_p || !MIR_any_branch_code_p (insn->code) || insn->code == MIR_JMP; + e->at_dst_p = TRUE; + bitmap_set_bit_p (e->src->at_end, spill_els_addr[i].reg); + bitmap_set_bit_p (e->dst->at_start, spill_els_addr[i].reg); + } + /* Check edge spills/restores and with common one at src end and dst start and refine + at_{src,dst}_p: */ + for (i = 0; i < VARR_LENGTH (spill_el_t, spill_els); i = j) { + e = spill_els_addr[i].u.e; + bitmap_clear (edge_regs); + for (j = i; j < VARR_LENGTH (spill_el_t, spill_els) && e == spill_els_addr[j].u.e; j++) + bitmap_set_bit_p (edge_regs, spill_els_addr[j].reg); + if (e->at_src_p) { + first_e = e2 = DLIST_HEAD (out_edge_t, e->src->out_edges); + while (e2 != NULL && e2->at_src_p) e2 = DLIST_NEXT (out_edge_t, e2); + if (e2 != NULL || !bitmap_equal_p (edge_regs, e->src->at_end)) + for (e2 = first_e; e2 != NULL; e2 = DLIST_NEXT (out_edge_t, e2)) e2->at_src_p = FALSE; + } + if (e->at_dst_p) { + first_e = e2 = DLIST_HEAD (in_edge_t, e->dst->in_edges); + while (e2 != NULL && e2->at_dst_p) e2 = DLIST_NEXT (in_edge_t, e2); + if (e2 != NULL || !bitmap_equal_p (edge_regs, e->dst->at_start)) + for (e2 = first_e; e2 != NULL; e2 = DLIST_NEXT (in_edge_t, e2)) e2->at_dst_p = FALSE; + } + } + for (size_t n = 0; n < VARR_LENGTH (spill_el_t, spill_els); n++) { + e = spill_els_addr[n].u.e; + if (!e->at_src_p || !e->at_dst_p) continue; + if (DLIST_HEAD (out_edge_t, e->src->out_edges) == DLIST_TAIL (out_edge_t, e->src->out_edges)) + e->at_src_p = FALSE; + else if (DLIST_HEAD (in_edge_t, e->dst->in_edges) == DLIST_TAIL (in_edge_t, e->dst->in_edges)) + e->at_dst_p = FALSE; + } + uint32_t start_split_bb_index = curr_bb_index; + /* Changing to BB placement with splitting edges if necessary */ + for (size_t n = 0; n < VARR_LENGTH (spill_el_t, spill_els); n++) { + gen_assert (spill_els_addr[n].edge_p); + e = spill_els_addr[n].u.e; + spill_els_addr[n].edge_p = FALSE; + spill_els_addr[n].bb_end_p = FALSE; + if (e->at_src_p) { + spill_els_addr[n].u.bb = e->src; + spill_els_addr[n].bb_end_p = TRUE; + } else if (e->at_dst_p) { + spill_els_addr[n].u.bb = e->dst; + } else if (e->src->index >= start_split_bb_index) { // ??? split_bb + gen_assert (DLIST_LENGTH (out_edge_t, e->src->out_edges) == 1 + && DLIST_LENGTH (in_edge_t, e->src->in_edges) == 1); + spill_els_addr[n].u.bb = e->src; + } else if (e->dst->index >= start_split_bb_index) { // ?? split_bb + gen_assert (DLIST_LENGTH (out_edge_t, e->dst->out_edges) == 1 + && DLIST_LENGTH (in_edge_t, e->dst->in_edges) == 1); + spill_els_addr[n].u.bb = e->dst; + } else { + /* put at split bb start, as we reuse existing edge to connect split bb, we will put + next spill at the same split bb -- see processing order above */ + // ??? check reuse existing edge (start,end) in split_edge_if_necessary + bb = split_edge_if_necessary (gen_ctx, e); + spill_els_addr[n].u.bb = bb; + } + } +} + +#undef at_start +#undef at_end +#undef at_src_p +#undef at_dst_p + +static void split (gen_ctx_t gen_ctx) { /* split by putting spill/restore insns */ + int restore_p, after_p; + bb_t bb; + bb_insn_t bb_insn = NULL; + MIR_reg_t reg, base_hreg = target_get_stack_slot_base_reg (gen_ctx); + spill_el_t *spill_els_addr; + + collect_spill_els (gen_ctx); + spill_els_addr = VARR_ADDR (spill_el_t, spill_els); + qsort (spill_els_addr, VARR_LENGTH (spill_el_t, spill_els), sizeof (spill_el_t), + spill_el_sort_cmp); + DEBUG (2, { + fprintf (debug_file, " Spills on edges:\n"); + for (size_t i = 0; i < VARR_LENGTH (spill_el_t, spill_els); i++) { + gen_assert (spill_els_addr[i].edge_p); + fprintf (debug_file, " %s r%d on %s of edge bb%lu->bb%lu\n", + spill_els_addr[i].spill_p ? "spill" : "restore", spill_els_addr[i].reg, + spill_els_addr[i].bb_end_p ? "end" : "start", + (unsigned long) spill_els_addr[i].u.e->src->index, + (unsigned long) spill_els_addr[i].u.e->dst->index); + } + }); + transform_edge_to_bb_placement (gen_ctx); + qsort (spill_els_addr, VARR_LENGTH (spill_el_t, spill_els), sizeof (spill_el_t), + spill_el_sort_cmp); + make_uniq_spill_els (gen_ctx); + spill_els_addr = VARR_ADDR (spill_el_t, spill_els); + DEBUG (2, { + fprintf (debug_file, "+++++++++++++MIR after splitting edges:\n"); + print_CFG (gen_ctx, TRUE, FALSE, TRUE, FALSE, NULL); + fprintf (debug_file, " Spills on BBs:\n"); + for (size_t i = 0; i < VARR_LENGTH (spill_el_t, spill_els); i++) { + gen_assert (!spill_els_addr[i].edge_p); + fprintf (debug_file, " %s r%d on %s of bb%lu\n", + spill_els_addr[i].spill_p ? "spill" : "restore", spill_els_addr[i].reg, + spill_els_addr[i].bb_end_p ? "end" : "start", + (unsigned long) spill_els_addr[i].u.bb->index); + } + }); + /* place spills and restores: */ + for (size_t i = 0; i < VARR_LENGTH (spill_el_t, spill_els); i++) { /* ??? debug info */ + bb = spill_els_addr[i].u.bb; + reg = spill_els_addr[i].reg; + gen_assert (reg > MAX_HARD_REG); + restore_p = !spill_els_addr[i].spill_p; + after_p = FALSE; + if (spill_els_addr[i].bb_end_p) { + bb_insn = DLIST_TAIL (bb_insn_t, bb->bb_insns); + if (!MIR_any_branch_code_p (bb_insn->insn->code)) after_p = TRUE; + } else { + bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); + if (bb_insn->insn->code == MIR_LABEL) after_p = TRUE; + } + spill_restore_reg (gen_ctx, reg, base_hreg, bb_insn->insn, after_p, restore_p); + } +} + +static void reg_alloc (gen_ctx_t gen_ctx) { + MIR_reg_t reg, max_var = get_max_var (gen_ctx); + const int simplified_p = ONLY_SIMPLIFIED_RA || optimize_level < 2; + + build_live_ranges (gen_ctx); + assign (gen_ctx); + DEBUG (2, { + fprintf (debug_file, "+++++++++++++Disposition after assignment:"); + for (reg = MAX_HARD_REG + 1; reg <= max_var; reg++) { + if ((reg - MAX_HARD_REG + 1) % 8 == 0) fprintf (debug_file, "\n"); + fprintf (debug_file, " %3u=>", reg); + if (VARR_LENGTH (live_range_t, var_live_ranges) <= reg + || VARR_GET (live_range_t, var_live_ranges, reg) == NULL) + fprintf (debug_file, "UA"); + else + fprintf (debug_file, "%-2u", VARR_GET (MIR_reg_t, reg_renumber, reg)); + } + fprintf (debug_file, "\n"); }); + rewrite (gen_ctx); /* After rewrite the BB live info is invalid as it is used for spill info */ + if (!simplified_p) { + DEBUG (2, { + fprintf (debug_file, "+++++++++++++Spill info:\n"); + print_CFG (gen_ctx, TRUE, FALSE, FALSE, FALSE, output_bb_spill_info); + }); + split (gen_ctx); + } + free_func_live_ranges (gen_ctx); } static void init_ra (gen_ctx_t gen_ctx) { gen_ctx->ra_ctx = gen_malloc (gen_ctx, sizeof (struct ra_ctx)); - VARR_CREATE (MIR_reg_t, breg_renumber, 0); - VARR_CREATE (breg_info_t, sorted_bregs, 0); + VARR_CREATE (MIR_reg_t, reg_renumber, 0); + VARR_CREATE (allocno_info_t, sorted_regs, 0); VARR_CREATE (bitmap_t, used_locs, 0); + VARR_CREATE (bitmap_t, busy_used_locs, 0); VARR_CREATE (bitmap_t, var_bbs, 0); - VARR_CREATE (size_t, loc_profits, 0); - VARR_CREATE (size_t, loc_profit_ages, 0); - conflict_locs = bitmap_create2 (3 * MAX_HARD_REG / 2); + VARR_CREATE (lr_gap_t, spill_gaps, 0); + VARR_CREATE (lr_gap_t, curr_gaps, 0); + VARR_CREATE (spill_el_t, spill_els, 0); + init_lr_gap_tab (gen_ctx); + VARR_CREATE (spill_cache_el_t, spill_cache, 0); + spill_cache_age = 0; + conflict_locs1 = bitmap_create2 (3 * MAX_HARD_REG / 2); } static void finish_ra (gen_ctx_t gen_ctx) { - VARR_DESTROY (MIR_reg_t, breg_renumber); - VARR_DESTROY (breg_info_t, sorted_bregs); + VARR_DESTROY (MIR_reg_t, reg_renumber); + VARR_DESTROY (allocno_info_t, sorted_regs); while (VARR_LENGTH (bitmap_t, used_locs) != 0) bitmap_destroy (VARR_POP (bitmap_t, used_locs)); VARR_DESTROY (bitmap_t, used_locs); + while (VARR_LENGTH (bitmap_t, busy_used_locs) != 0) + bitmap_destroy (VARR_POP (bitmap_t, busy_used_locs)); + VARR_DESTROY (bitmap_t, busy_used_locs); while (VARR_LENGTH (bitmap_t, var_bbs) != 0) bitmap_destroy (VARR_POP (bitmap_t, var_bbs)); VARR_DESTROY (bitmap_t, var_bbs); - VARR_DESTROY (size_t, loc_profits); - VARR_DESTROY (size_t, loc_profit_ages); - bitmap_destroy (conflict_locs); + VARR_DESTROY (lr_gap_t, spill_gaps); + VARR_DESTROY (lr_gap_t, curr_gaps); + VARR_DESTROY (spill_el_t, spill_els); + finish_lr_gap_tab (gen_ctx); + VARR_DESTROY (spill_cache_el_t, spill_cache); + bitmap_destroy (conflict_locs1); free (gen_ctx->ra_ctx); gen_ctx->ra_ctx = NULL; } /* New Page */ -/* Code selection */ +/* Insn combining after RA requires dead notes and is done in forward insn processing. + It is done for the following cases: + o splitting insns: lr restore (r = mem) ; bcmp r => bcmp mem + o meeting 2-op constraints after RA (when p2 and p0 got hr0): + p1=mem; add p2,p0,p1(dead p1) => hr1=mem; add hr0,hr0,hr1 => add hr0,mem */ -struct hreg_ref { /* We keep track of the last hard reg ref in this struct. */ +struct var_ref { /* We keep track of the last reg ref in this struct. */ MIR_insn_t insn; size_t insn_num; size_t nop; char def_p, del_p; /* def/use and deleted */ }; -typedef struct hreg_ref hreg_ref_t; +typedef struct var_ref var_ref_t; -DEF_VARR (hreg_ref_t); +DEF_VARR (var_ref_t); -struct selection_ctx { - VARR (size_t) * hreg_ref_ages; - VARR (hreg_ref_t) * hreg_refs; - hreg_ref_t *hreg_refs_addr; - size_t *hreg_ref_ages_addr; - size_t curr_bb_hreg_ref_age; +struct combine_ctx { + VARR (size_t) * var_ref_ages; + VARR (var_ref_t) * var_refs; + var_ref_t *var_refs_addr; + size_t *var_ref_ages_addr; + size_t curr_bb_var_ref_age; size_t last_mem_ref_insn_num; - VARR (MIR_reg_t) * insn_hard_regs; /* registers considered for substitution */ + VARR (MIR_reg_t) * insn_vars; /* registers considered for substitution */ VARR (size_t) * changed_op_numbers; VARR (MIR_op_t) * last_right_ops; - bitmap_t hard_regs_bitmap; + bitmap_t vars_bitmap; }; -#define hreg_ref_ages gen_ctx->selection_ctx->hreg_ref_ages -#define hreg_refs gen_ctx->selection_ctx->hreg_refs -#define hreg_refs_addr gen_ctx->selection_ctx->hreg_refs_addr -#define hreg_ref_ages_addr gen_ctx->selection_ctx->hreg_ref_ages_addr -#define curr_bb_hreg_ref_age gen_ctx->selection_ctx->curr_bb_hreg_ref_age -#define last_mem_ref_insn_num gen_ctx->selection_ctx->last_mem_ref_insn_num -#define insn_hard_regs gen_ctx->selection_ctx->insn_hard_regs -#define changed_op_numbers gen_ctx->selection_ctx->changed_op_numbers -#define last_right_ops gen_ctx->selection_ctx->last_right_ops -#define hard_regs_bitmap gen_ctx->selection_ctx->hard_regs_bitmap +#define var_ref_ages gen_ctx->combine_ctx->var_ref_ages +#define var_refs gen_ctx->combine_ctx->var_refs +#define var_refs_addr gen_ctx->combine_ctx->var_refs_addr +#define var_ref_ages_addr gen_ctx->combine_ctx->var_ref_ages_addr +#define curr_bb_var_ref_age gen_ctx->combine_ctx->curr_bb_var_ref_age +#define last_mem_ref_insn_num gen_ctx->combine_ctx->last_mem_ref_insn_num +#define insn_vars gen_ctx->combine_ctx->insn_vars +#define changed_op_numbers gen_ctx->combine_ctx->changed_op_numbers +#define last_right_ops gen_ctx->combine_ctx->last_right_ops +#define vars_bitmap gen_ctx->combine_ctx->vars_bitmap static MIR_insn_code_t commutative_insn_code (MIR_insn_code_t insn_code) { switch (insn_code) { @@ -4445,6 +8589,10 @@ static MIR_insn_code_t commutative_insn_code (MIR_insn_code_t insn_code) { case MIR_LDADD: case MIR_MUL: case MIR_MULS: + case MIR_MULO: + case MIR_MULOS: + case MIR_UMULO: + case MIR_UMULOS: case MIR_FMUL: case MIR_DMUL: case MIR_LDMUL: @@ -4510,74 +8658,84 @@ static MIR_insn_code_t commutative_insn_code (MIR_insn_code_t insn_code) { } } -static int obsolete_hard_reg_p (gen_ctx_t gen_ctx, MIR_reg_t hard_reg, size_t def_insn_num) { - return (hreg_ref_ages_addr[hard_reg] == curr_bb_hreg_ref_age - && hreg_refs_addr[hard_reg].insn_num > def_insn_num); +static int obsolete_var_p (gen_ctx_t gen_ctx, MIR_reg_t var, size_t def_insn_num) { + return (var < VARR_LENGTH (size_t, var_ref_ages) && var_ref_ages_addr[var] == curr_bb_var_ref_age + && var_refs_addr[var].insn_num > def_insn_num); } -static int obsolete_hard_reg_op_p (gen_ctx_t gen_ctx, MIR_op_t op, size_t def_insn_num) { - return op.mode == MIR_OP_HARD_REG && obsolete_hard_reg_p (gen_ctx, op.u.hard_reg, def_insn_num); +static int obsolete_var_op_p (gen_ctx_t gen_ctx, MIR_op_t op, size_t def_insn_num) { + return op.mode == MIR_OP_VAR && obsolete_var_p (gen_ctx, op.u.var, def_insn_num); } static int obsolete_op_p (gen_ctx_t gen_ctx, MIR_op_t op, size_t def_insn_num) { - if (obsolete_hard_reg_op_p (gen_ctx, op, def_insn_num)) return TRUE; - if (op.mode != MIR_OP_HARD_REG_MEM) return FALSE; - if (op.u.hard_reg_mem.base != MIR_NON_HARD_REG - && obsolete_hard_reg_p (gen_ctx, op.u.hard_reg_mem.base, def_insn_num)) + if (obsolete_var_op_p (gen_ctx, op, def_insn_num)) return TRUE; + if (op.mode != MIR_OP_VAR_MEM) return FALSE; + if (op.u.var_mem.base != MIR_NON_VAR && obsolete_var_p (gen_ctx, op.u.var_mem.base, def_insn_num)) return TRUE; - if (op.u.hard_reg_mem.index != MIR_NON_HARD_REG - && obsolete_hard_reg_p (gen_ctx, op.u.hard_reg_mem.index, def_insn_num)) + if (op.u.var_mem.index != MIR_NON_VAR + && obsolete_var_p (gen_ctx, op.u.var_mem.index, def_insn_num)) return TRUE; return last_mem_ref_insn_num > def_insn_num; } -static int safe_hreg_substitution_p (gen_ctx_t gen_ctx, MIR_reg_t hr, bb_insn_t bb_insn) { - return (hr != MIR_NON_HARD_REG - && hreg_ref_ages_addr[hr] == curr_bb_hreg_ref_age +static int safe_var_substitution_p (gen_ctx_t gen_ctx, MIR_reg_t var, bb_insn_t bb_insn) { + return (var != MIR_NON_VAR && var < VARR_LENGTH (size_t, var_ref_ages) + && var_ref_ages_addr[var] == curr_bb_var_ref_age + && var_refs_addr[var].def_p /* It is not safe to substitute if there is another use after def insn before the current insn as we delete def insn after the substitution. */ - && hreg_refs_addr[hr].def_p && find_bb_insn_dead_var (bb_insn, hr) != NULL); + && find_bb_insn_dead_var (bb_insn, var) != NULL); } -static void combine_process_hard_reg (gen_ctx_t gen_ctx, MIR_reg_t hr, bb_insn_t bb_insn) { - if (!safe_hreg_substitution_p (gen_ctx, hr, bb_insn) || !bitmap_set_bit_p (hard_regs_bitmap, hr)) +static void combine_process_var (gen_ctx_t gen_ctx, MIR_reg_t var, bb_insn_t bb_insn) { + if (!safe_var_substitution_p (gen_ctx, var, bb_insn) || !bitmap_set_bit_p (vars_bitmap, var)) return; - VARR_PUSH (MIR_reg_t, insn_hard_regs, hr); + VARR_PUSH (MIR_reg_t, insn_vars, var); } static void combine_process_op (gen_ctx_t gen_ctx, const MIR_op_t *op_ref, bb_insn_t bb_insn) { - if (op_ref->mode == MIR_OP_HARD_REG) { - combine_process_hard_reg (gen_ctx, op_ref->u.hard_reg, bb_insn); - } else if (op_ref->mode == MIR_OP_HARD_REG_MEM) { - if (op_ref->u.hard_reg_mem.base != MIR_NON_HARD_REG) - combine_process_hard_reg (gen_ctx, op_ref->u.hard_reg_mem.base, bb_insn); - if (op_ref->u.hard_reg_mem.index != MIR_NON_HARD_REG) - combine_process_hard_reg (gen_ctx, op_ref->u.hard_reg_mem.index, bb_insn); + if (op_ref->mode == MIR_OP_VAR) { + combine_process_var (gen_ctx, op_ref->u.var, bb_insn); + } else if (op_ref->mode == MIR_OP_VAR_MEM) { + if (op_ref->u.var_mem.base != MIR_NON_VAR) + combine_process_var (gen_ctx, op_ref->u.var_mem.base, bb_insn); + if (op_ref->u.var_mem.index != MIR_NON_VAR) + combine_process_var (gen_ctx, op_ref->u.var_mem.index, bb_insn); + } +} + +static int hard_reg_used_in_bb_insn_p (gen_ctx_t gen_ctx, bb_insn_t bb_insn, MIR_reg_t var) { + int op_num; + MIR_reg_t v; + insn_var_iterator_t iter; + + FOREACH_IN_INSN_VAR (gen_ctx, iter, bb_insn->insn, v, op_num) { + if (v == var) return TRUE; } + return FALSE; } static int combine_delete_insn (gen_ctx_t gen_ctx, MIR_insn_t def_insn, bb_insn_t bb_insn) { - MIR_reg_t hr; + MIR_reg_t var; - gen_assert (def_insn->ops[0].mode == MIR_OP_HARD_REG); - hr = def_insn->ops[0].u.hard_reg; - if (hreg_ref_ages_addr[hr] != curr_bb_hreg_ref_age || hreg_refs_addr[hr].del_p) return FALSE; + gen_assert (def_insn->ops[0].mode == MIR_OP_VAR); + var = def_insn->ops[0].u.var; + if (var_ref_ages_addr[var] != curr_bb_var_ref_age || var_refs_addr[var].del_p) return FALSE; DEBUG (2, { fprintf (debug_file, " deleting now dead insn "); print_bb_insn (gen_ctx, def_insn->data, TRUE); }); - remove_bb_insn_dead_var (gen_ctx, bb_insn, hr); - move_bb_insn_dead_vars (bb_insn, def_insn->data); - /* We should delete the def insn here because of possible - substitution of the def insn 'r0 = ... r0 ...'. We still - need valid entry for def here to find obsolete definiton, - e.g. "hr1 = hr0; hr0 = ...; hr0 = ... (deleted); ...= ...hr1..." */ + remove_bb_insn_dead_var (gen_ctx, bb_insn, var); + move_bb_insn_dead_vars (gen_ctx, bb_insn, def_insn->data, hard_reg_used_in_bb_insn_p); + /* We should delete the def insn here because of possible substitution of the def + insn 'r0 = ... r0 ...'. We still need valid entry for def here to find obsolete + definiton, e.g. "r1 = r0; r0 = ...; r0 = ... (deleted); ...= ...r1..." */ gen_delete_insn (gen_ctx, def_insn); - hreg_refs_addr[hr].del_p = TRUE; /* to exclude repetitive deletion */ + var_refs_addr[var].del_p = TRUE; /* to exclude repetitive deletion */ return TRUE; } -static int64_t power2 (int64_t p) { +static MIR_UNUSED int64_t power2 (int64_t p) { int64_t n = 1; if (p < 0) return 0; @@ -4585,25 +8743,16 @@ static int64_t power2 (int64_t p) { return n; } -static int64_t int_log2 (int64_t i) { - int64_t n; - - if (i <= 0) return -1; - for (n = 0; (i & 1) == 0; n++, i >>= 1) - ; - return i == 1 ? n : -1; -} - -static MIR_insn_t get_uptodate_def_insn (gen_ctx_t gen_ctx, int hr) { +static MIR_insn_t get_uptodate_def_insn (gen_ctx_t gen_ctx, int var) { MIR_insn_t def_insn; - if (!hreg_refs_addr[hr].def_p) return NULL; - gen_assert (!hreg_refs_addr[hr].del_p); - def_insn = hreg_refs_addr[hr].insn; - /* Checking hr0 = ... hr1 ...; ...; hr1 = ...; ...; insn */ - if ((def_insn->nops > 1 && obsolete_op_p (gen_ctx, def_insn->ops[1], hreg_refs_addr[hr].insn_num)) + if (!var_refs_addr[var].def_p) return NULL; + gen_assert (!var_refs_addr[var].del_p); + def_insn = var_refs_addr[var].insn; + /* Checking r0 = ... r1 ...; ...; r1 = ...; ...; insn */ + if ((def_insn->nops > 1 && obsolete_op_p (gen_ctx, def_insn->ops[1], var_refs_addr[var].insn_num)) || (def_insn->nops > 2 - && obsolete_op_p (gen_ctx, def_insn->ops[2], hreg_refs_addr[hr].insn_num))) + && obsolete_op_p (gen_ctx, def_insn->ops[2], var_refs_addr[var].insn_num))) return NULL; return def_insn; } @@ -4613,37 +8762,38 @@ static int combine_substitute (gen_ctx_t gen_ctx, bb_insn_t *bb_insn_ref, long * bb_insn_t bb_insn = *bb_insn_ref; MIR_insn_t insn = bb_insn->insn, def_insn; size_t i, nops = insn->nops; - int out_p, insn_change_p, insn_hr_change_p, op_change_p, mem_reg_change_p, success_p; - MIR_op_t *op_ref, *src_op_ref, *src_op2_ref, saved_op; - MIR_reg_t hr, early_clobbered_hard_reg1, early_clobbered_hard_reg2; - int64_t scale; + int out_p, insn_change_p, insn_var_change_p, op_change_p, success_p; + MIR_op_t *op_ref, saved_op; + MIR_reg_t var, early_clobbered_hard_reg1, early_clobbered_hard_reg2; if (nops == 0) return FALSE; VARR_TRUNC (MIR_op_t, last_right_ops, 0); for (i = 0; i < nops; i++) VARR_PUSH (MIR_op_t, last_right_ops, insn->ops[i]); - VARR_TRUNC (MIR_reg_t, insn_hard_regs, 0); - bitmap_clear (hard_regs_bitmap); + VARR_TRUNC (MIR_reg_t, insn_vars, 0); + bitmap_clear (vars_bitmap); for (i = 0; i < nops; i++) { MIR_insn_op_mode (ctx, insn, i, &out_p); - if (out_p && insn->ops[i].mode != MIR_OP_HARD_REG_MEM) continue; + if (out_p || insn->ops[i].mode == MIR_OP_VAR_MEM) continue; combine_process_op (gen_ctx, &insn->ops[i], bb_insn); } - if (move_code_p (insn->code) && VARR_LENGTH (MIR_reg_t, insn_hard_regs) == 1) { - /* We processed all other regs already. Try to change insn the following way: - hr0 = hr2 op hr3; ...; ... = hr0 => ...; ... = hr2 op hr3 */ - hr = VARR_GET (MIR_reg_t, insn_hard_regs, 0); - if ((def_insn = get_uptodate_def_insn (gen_ctx, hr)) == NULL + + if (move_code_p (insn->code) && insn->ops[1].mode == MIR_OP_VAR + && VARR_LENGTH (MIR_reg_t, insn_vars) != 0 + && VARR_LAST (MIR_reg_t, insn_vars) == insn->ops[1].u.var) { + /* We can change move src. Try to change insn the following way: + r0 = r2 op r3; ...; ... = r0 => ...; ... = r2 op r3 */ + var = insn->ops[1].u.var; + if ((def_insn = get_uptodate_def_insn (gen_ctx, var)) == NULL || MIR_call_code_p (def_insn->code)) return FALSE; target_get_early_clobbered_hard_regs (def_insn, &early_clobbered_hard_reg1, &early_clobbered_hard_reg2); - if (!move_code_p (def_insn->code) && early_clobbered_hard_reg1 == MIR_NON_HARD_REG - && early_clobbered_hard_reg2 == MIR_NON_HARD_REG && insn->ops[1].mode == MIR_OP_HARD_REG - && insn->ops[1].u.hard_reg == hr + if (!move_code_p (def_insn->code) && early_clobbered_hard_reg1 == MIR_NON_VAR + && early_clobbered_hard_reg2 == MIR_NON_VAR && insn->ops[1].mode == MIR_OP_VAR + && insn->ops[1].u.var == var /* Check that insn->ops[0] is not mem[...hr0...]: */ - && (insn->ops[0].mode != MIR_OP_HARD_REG_MEM - || (insn->ops[0].u.hard_reg_mem.base != hr - && insn->ops[0].u.hard_reg_mem.index != hr))) { + && (insn->ops[0].mode != MIR_OP_VAR_MEM + || (insn->ops[0].u.var_mem.base != var && insn->ops[0].u.var_mem.index != var))) { saved_op = def_insn->ops[0]; def_insn->ops[0] = insn->ops[0]; success_p = target_insn_ok_p (gen_ctx, def_insn); @@ -4670,103 +8820,27 @@ static int combine_substitute (gen_ctx_t gen_ctx, bb_insn_t *bb_insn_ref, long * } } insn_change_p = FALSE; - while (VARR_LENGTH (MIR_reg_t, insn_hard_regs) != 0) { - hr = VARR_POP (MIR_reg_t, insn_hard_regs); - if ((def_insn = get_uptodate_def_insn (gen_ctx, hr)) == NULL) continue; - insn_hr_change_p = FALSE; - for (i = 0; i < nops; i++) { /* Change all hr occurences: */ + while (VARR_LENGTH (MIR_reg_t, insn_vars) != 0) { + var = VARR_POP (MIR_reg_t, insn_vars); + if ((def_insn = get_uptodate_def_insn (gen_ctx, var)) == NULL) continue; + if (!move_code_p (def_insn->code)) continue; + insn_var_change_p = FALSE; + for (i = 0; i < nops; i++) { /* Change all var occurences: */ op_ref = &insn->ops[i]; op_change_p = FALSE; MIR_insn_op_mode (ctx, insn, i, &out_p); - if (!out_p && op_ref->mode == MIR_OP_HARD_REG && op_ref->u.hard_reg == hr) { - if (!move_code_p (def_insn->code)) break; + if (!out_p && op_ref->mode == MIR_OP_VAR && op_ref->u.var == var) { /* It is not safe to substitute if there is another use after def insn before the current as we delete def insn after substitution. */ insn->ops[i] = def_insn->ops[1]; - insn_hr_change_p = op_change_p = TRUE; - } else if (op_ref->mode == MIR_OP_HARD_REG_MEM) { - src_op_ref = &def_insn->ops[1]; - if (op_ref->u.hard_reg_mem.index == hr) { - mem_reg_change_p = FALSE; - if (src_op_ref->mode != MIR_OP_HARD_REG) { - } else if (def_insn->code == MIR_MOV) { /* index = r */ - insn->ops[i].u.hard_reg_mem.index = src_op_ref->u.hard_reg; - mem_reg_change_p = op_change_p = insn_hr_change_p = TRUE; - } else if (def_insn->code == MIR_ADD) { /* index = r + const */ - gen_assert (src_op_ref->u.hard_reg != MIR_NON_HARD_REG); - if ((src_op2_ref = &def_insn->ops[2])->mode == MIR_OP_INT) { - insn->ops[i].u.hard_reg_mem.index = src_op_ref->u.hard_reg; - insn->ops[i].u.hard_reg_mem.disp - += src_op2_ref->u.i * insn->ops[i].u.hard_reg_mem.scale; - mem_reg_change_p = op_change_p = insn_hr_change_p = TRUE; - } - } else if ((def_insn->code == MIR_MUL || def_insn->code == MIR_LSH) - && op_ref->u.mem.scale >= 1 && op_ref->u.mem.scale <= MIR_MAX_SCALE - && (src_op2_ref = &def_insn->ops[2])->mode == MIR_OP_INT) { - scale = def_insn->code == MIR_MUL ? src_op2_ref->u.i : power2 (src_op2_ref->u.i); - if (scale >= 1 && scale <= MIR_MAX_SCALE - && insn->ops[i].u.hard_reg_mem.scale * scale <= MIR_MAX_SCALE) { - /* index = r * const */ - gen_assert (src_op_ref->u.hard_reg != MIR_NON_HARD_REG); - insn->ops[i].u.hard_reg_mem.index = src_op_ref->u.hard_reg; - insn->ops[i].u.hard_reg_mem.scale *= scale; - mem_reg_change_p = op_change_p = insn_hr_change_p = TRUE; - } - } - if (!mem_reg_change_p) break; - } - if (op_ref->u.hard_reg_mem.base == hr) { - mem_reg_change_p = FALSE; - op_ref = &insn->ops[i]; - if (def_insn->code == MIR_MOV) { - if (src_op_ref->mode == MIR_OP_HARD_REG) { /* base = r */ - insn->ops[i].u.hard_reg_mem.base = src_op_ref->u.hard_reg; - mem_reg_change_p = op_change_p = insn_hr_change_p = TRUE; - } else if (src_op_ref->mode == MIR_OP_INT) { /* base = const */ - if (insn->ops[i].u.hard_reg_mem.scale != 1) { - insn->ops[i].u.hard_reg_mem.base = MIR_NON_HARD_REG; - } else { - insn->ops[i].u.hard_reg_mem.base = insn->ops[i].u.hard_reg_mem.index; - insn->ops[i].u.hard_reg_mem.index = MIR_NON_HARD_REG; - } - insn->ops[i].u.hard_reg_mem.disp += src_op_ref->u.i; - mem_reg_change_p = op_change_p = insn_hr_change_p = TRUE; - } - } else if (src_op_ref->mode != MIR_OP_HARD_REG) { /* Can do nothing */ - ; - } else if (def_insn->code == MIR_ADD) { - gen_assert (src_op_ref->u.hard_reg != MIR_NON_HARD_REG); - src_op2_ref = &def_insn->ops[2]; - if (src_op2_ref->mode == MIR_OP_HARD_REG - && op_ref->u.hard_reg_mem.index == MIR_NON_HARD_REG) { /* base = r1 + r2 */ - insn->ops[i].u.hard_reg_mem.base = src_op_ref->u.hard_reg; - insn->ops[i].u.hard_reg_mem.index = src_op2_ref->u.hard_reg; - insn->ops[i].u.hard_reg_mem.scale = 1; - mem_reg_change_p = op_change_p = insn_hr_change_p = TRUE; - } else if (src_op2_ref->mode == MIR_OP_INT) { /* base = r + const */ - insn->ops[i].u.hard_reg_mem.base = src_op_ref->u.hard_reg; - insn->ops[i].u.hard_reg_mem.disp += src_op2_ref->u.i; - mem_reg_change_p = op_change_p = insn_hr_change_p = TRUE; - } - } else if (def_insn->code == MIR_MUL && op_ref->u.hard_reg_mem.index == MIR_NON_HARD_REG - && (src_op2_ref = &def_insn->ops[2])->mode == MIR_OP_INT - && src_op2_ref->u.i >= 1 - && src_op2_ref->u.i <= MIR_MAX_SCALE) { /* base = r * const */ - gen_assert (src_op_ref->u.hard_reg != MIR_NON_HARD_REG && src_op2_ref->u.i != 1); - insn->ops[i].u.hard_reg_mem.base = MIR_NON_HARD_REG; - insn->ops[i].u.hard_reg_mem.index = src_op_ref->u.hard_reg; - insn->ops[i].u.hard_reg_mem.scale = src_op2_ref->u.i; - mem_reg_change_p = op_change_p = insn_hr_change_p = TRUE; - } - if (!mem_reg_change_p) { - if (op_change_p) VARR_PUSH (size_t, changed_op_numbers, i); /* index was changed */ - break; - } - } + insn_var_change_p = op_change_p = TRUE; + } else { + gen_assert (op_ref->mode != MIR_OP_VAR_MEM + || (op_ref->u.var_mem.base != var && op_ref->u.var_mem.index != var)); } if (op_change_p) VARR_PUSH (size_t, changed_op_numbers, i); } - if (insn_hr_change_p) { + if (insn_var_change_p) { if ((success_p = i >= nops && target_insn_ok_p (gen_ctx, insn))) insn_change_p = TRUE; while (VARR_LENGTH (size_t, changed_op_numbers)) { i = VARR_POP (size_t, changed_op_numbers); @@ -4788,120 +8862,20 @@ static int combine_substitute (gen_ctx_t gen_ctx, bb_insn_t *bb_insn_ref, long * return insn_change_p; } -static MIR_insn_code_t get_combined_br_code (int true_p, MIR_insn_code_t cmp_code) { - switch (cmp_code) { - case MIR_EQ: return true_p ? MIR_BEQ : MIR_BNE; - case MIR_EQS: return true_p ? MIR_BEQS : MIR_BNES; - case MIR_NE: return true_p ? MIR_BNE : MIR_BEQ; - case MIR_NES: return true_p ? MIR_BNES : MIR_BEQS; - case MIR_LT: return true_p ? MIR_BLT : MIR_BGE; - case MIR_LTS: return true_p ? MIR_BLTS : MIR_BGES; - case MIR_ULT: return true_p ? MIR_UBLT : MIR_UBGE; - case MIR_ULTS: return true_p ? MIR_UBLTS : MIR_UBGES; - case MIR_LE: return true_p ? MIR_BLE : MIR_BGT; - case MIR_LES: return true_p ? MIR_BLES : MIR_BGTS; - case MIR_ULE: return true_p ? MIR_UBLE : MIR_UBGT; - case MIR_ULES: return true_p ? MIR_UBLES : MIR_UBGTS; - case MIR_GT: return true_p ? MIR_BGT : MIR_BLE; - case MIR_GTS: return true_p ? MIR_BGTS : MIR_BLES; - case MIR_UGT: return true_p ? MIR_UBGT : MIR_UBLE; - case MIR_UGTS: return true_p ? MIR_UBGTS : MIR_UBLES; - case MIR_GE: return true_p ? MIR_BGE : MIR_BLT; - case MIR_GES: return true_p ? MIR_BGES : MIR_BLTS; - case MIR_UGE: return true_p ? MIR_UBGE : MIR_UBLT; - case MIR_UGES: - return true_p ? MIR_UBGES : MIR_UBLTS; - /* Cannot revert in the false case for IEEE754: */ - case MIR_FEQ: return true_p ? MIR_FBEQ : MIR_INSN_BOUND; - case MIR_DEQ: return true_p ? MIR_DBEQ : MIR_INSN_BOUND; - case MIR_LDEQ: return true_p ? MIR_LDBEQ : MIR_INSN_BOUND; - case MIR_FNE: return true_p ? MIR_FBNE : MIR_INSN_BOUND; - case MIR_DNE: return true_p ? MIR_DBNE : MIR_INSN_BOUND; - case MIR_LDNE: return true_p ? MIR_LDBNE : MIR_INSN_BOUND; - case MIR_FLT: return true_p ? MIR_FBLT : MIR_INSN_BOUND; - case MIR_DLT: return true_p ? MIR_DBLT : MIR_INSN_BOUND; - case MIR_LDLT: return true_p ? MIR_LDBLT : MIR_INSN_BOUND; - case MIR_FLE: return true_p ? MIR_FBLE : MIR_INSN_BOUND; - case MIR_DLE: return true_p ? MIR_DBLE : MIR_INSN_BOUND; - case MIR_LDLE: return true_p ? MIR_LDBLE : MIR_INSN_BOUND; - case MIR_FGT: return true_p ? MIR_FBGT : MIR_INSN_BOUND; - case MIR_DGT: return true_p ? MIR_DBGT : MIR_INSN_BOUND; - case MIR_LDGT: return true_p ? MIR_LDBGT : MIR_INSN_BOUND; - case MIR_FGE: return true_p ? MIR_FBGE : MIR_INSN_BOUND; - case MIR_DGE: return true_p ? MIR_DBGE : MIR_INSN_BOUND; - case MIR_LDGE: return true_p ? MIR_LDBGE : MIR_INSN_BOUND; - default: return MIR_INSN_BOUND; - } -} - -static MIR_insn_t combine_branch_and_cmp (gen_ctx_t gen_ctx, bb_insn_t bb_insn, - long *deleted_insns_num) { - MIR_context_t ctx = gen_ctx->ctx; - MIR_insn_t def_insn, new_insn, insn = bb_insn->insn; - MIR_insn_code_t code = insn->code; - MIR_op_t op; - - if (code != MIR_BT && code != MIR_BF && code != MIR_BTS && code != MIR_BFS) return NULL; - op = insn->ops[1]; - if (op.mode != MIR_OP_HARD_REG || !safe_hreg_substitution_p (gen_ctx, op.u.hard_reg, bb_insn)) - return NULL; - def_insn = hreg_refs_addr[op.u.hard_reg].insn; - if ((code = get_combined_br_code (code == MIR_BT || code == MIR_BTS, def_insn->code)) - == MIR_INSN_BOUND) - return NULL; - if (obsolete_op_p (gen_ctx, def_insn->ops[1], hreg_refs_addr[op.u.hard_reg].insn_num) - || obsolete_op_p (gen_ctx, def_insn->ops[2], hreg_refs_addr[op.u.hard_reg].insn_num)) - return NULL; - new_insn = MIR_new_insn (ctx, code, insn->ops[0], def_insn->ops[1], def_insn->ops[2]); - MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); - if (!target_insn_ok_p (gen_ctx, new_insn)) { - MIR_remove_insn (ctx, curr_func_item, new_insn); - return NULL; - } else { - MIR_remove_insn (ctx, curr_func_item, insn); - new_insn->data = bb_insn; - bb_insn->insn = new_insn; - DEBUG (2, { - fprintf (debug_file, " changing to "); - print_bb_insn (gen_ctx, bb_insn, TRUE); - }); - if (combine_delete_insn (gen_ctx, def_insn, bb_insn)) (*deleted_insns_num)++; - return new_insn; - } -} - static MIR_insn_t combine_exts (gen_ctx_t gen_ctx, bb_insn_t bb_insn, long *deleted_insns_num) { MIR_insn_t def_insn, insn = bb_insn->insn; MIR_insn_code_t code = insn->code; - MIR_op_t op, saved_op; - int size, size2, sign_p = FALSE, sign2_p = FALSE, ok_p; - - switch (code) { - case MIR_EXT8: sign2_p = TRUE; - case MIR_UEXT8: size2 = 1; break; - case MIR_EXT16: sign2_p = TRUE; - case MIR_UEXT16: size2 = 2; break; - case MIR_EXT32: sign2_p = TRUE; - case MIR_UEXT32: size2 = 3; break; - default: return NULL; - } - op = insn->ops[1]; - if (op.mode != MIR_OP_HARD_REG || !safe_hreg_substitution_p (gen_ctx, op.u.hard_reg, bb_insn)) - return NULL; - def_insn = hreg_refs_addr[op.u.hard_reg].insn; - switch (def_insn->code) { - case MIR_EXT8: sign_p = TRUE; - case MIR_UEXT8: size = 1; break; - case MIR_EXT16: sign_p = TRUE; - case MIR_UEXT16: size = 2; break; - case MIR_EXT32: sign_p = TRUE; - case MIR_UEXT32: size = 3; break; - default: return NULL; - } - if (obsolete_op_p (gen_ctx, def_insn->ops[1], hreg_refs_addr[op.u.hard_reg].insn_num)) - return NULL; - if (size2 <= size) { - /* [u]ext b,a; ... [u]ext c,b -> [u]ext c,a when <= : */ + MIR_op_t *op, saved_op; + int w, w2, sign_p = FALSE, sign2_p = FALSE, ok_p; + + if ((w = get_ext_params (code, &sign_p)) == 0) return NULL; + op = &insn->ops[1]; + if (op->mode != MIR_OP_VAR || !safe_var_substitution_p (gen_ctx, op->u.var, bb_insn)) return NULL; + def_insn = var_refs_addr[op->u.var].insn; + if ((w2 = get_ext_params (def_insn->code, &sign2_p)) == 0) return NULL; + if (obsolete_op_p (gen_ctx, def_insn->ops[1], var_refs_addr[op->u.var].insn_num)) return NULL; + if (w <= w2) { + /* [u]ext b,a; ... [u]ext c,b -> [u]ext c,a when <= : */ saved_op = insn->ops[1]; insn->ops[1] = def_insn->ops[1]; if (!target_insn_ok_p (gen_ctx, insn)) { @@ -4914,203 +8888,152 @@ static MIR_insn_t combine_exts (gen_ctx_t gen_ctx, bb_insn_t bb_insn, long *dele }); if (combine_delete_insn (gen_ctx, def_insn, bb_insn)) (*deleted_insns_num)++; return insn; - } else if (sign_p == sign2_p && size < size2) { - saved_op = def_insn->ops[0]; - def_insn->ops[0] = insn->ops[0]; - ok_p = target_insn_ok_p (gen_ctx, def_insn); - def_insn->ops[0] = saved_op; + } else if (w2 < w && (sign_p || !sign2_p)) { /* exclude ext, uext pair */ + /* [u]ext b,a; .. [u]ext c,b -> [[u]ext b,a;] .. [u]ext c,a */ + saved_op = insn->ops[1]; + insn->ops[1] = def_insn->ops[1]; + insn->code = def_insn->code; + ok_p = target_insn_ok_p (gen_ctx, insn); + insn->ops[1] = saved_op; + insn->code = code; if (!ok_p) return NULL; - gen_move_insn_before (gen_ctx, insn, def_insn); DEBUG (2, { - fprintf (debug_file, " moving insn "); - print_bb_insn (gen_ctx, def_insn->data, FALSE); - fprintf (debug_file, " before insn "); - print_bb_insn (gen_ctx, bb_insn, TRUE); + fprintf (debug_file, " changing "); + print_bb_insn (gen_ctx, bb_insn, FALSE); }); - def_insn->ops[0] = insn->ops[0]; + insn->ops[1] = def_insn->ops[1]; + insn->code = def_insn->code; DEBUG (2, { - fprintf (debug_file, " changing it to "); - print_bb_insn (gen_ctx, def_insn->data, TRUE); - fprintf (debug_file, " deleting insn "); + fprintf (debug_file, " to "); print_bb_insn (gen_ctx, bb_insn, TRUE); }); - gen_delete_insn (gen_ctx, insn); - (*deleted_insns_num)++; - return def_insn; + if (combine_delete_insn (gen_ctx, def_insn, bb_insn)) (*deleted_insns_num)++; + return insn; } return NULL; } -static MIR_insn_t combine_mul_div_substitute (gen_ctx_t gen_ctx, bb_insn_t bb_insn, - long *deleted_insns_num) { - MIR_context_t ctx = gen_ctx->ctx; - MIR_insn_t def_insn = NULL, new_insns[6], insn = bb_insn->insn; - MIR_insn_code_t new_code, code = insn->code; - int n, sh, ok_p; - MIR_op_t op, temp; +static void setup_var_ref (gen_ctx_t gen_ctx, MIR_reg_t var, MIR_insn_t insn, size_t nop, + size_t insn_num, int def_p) { + static const var_ref_t var_ref = {NULL, 0, 0, FALSE, FALSE}; - switch (code) { - case MIR_MUL: new_code = MIR_LSH; break; - case MIR_MULS: new_code = MIR_LSHS; break; - case MIR_UDIV: new_code = MIR_URSH; break; - case MIR_UDIVS: new_code = MIR_URSHS; break; - case MIR_DIV: new_code = MIR_RSH; break; - case MIR_DIVS: new_code = MIR_RSHS; break; - default: return NULL; - } - op = insn->ops[2]; - if (op.mode == MIR_OP_HARD_REG && safe_hreg_substitution_p (gen_ctx, op.u.hard_reg, bb_insn)) { - def_insn = hreg_refs_addr[op.u.hard_reg].insn; - if (def_insn->code != MIR_MOV) return NULL; - op = def_insn->ops[1]; - } - if (op.mode != MIR_OP_INT && op.mode != MIR_OP_UINT) return NULL; - if ((sh = int_log2 (op.u.i)) < 0) return NULL; - if (sh == 0) { - new_insns[0] = MIR_new_insn (ctx, MIR_MOV, insn->ops[0], insn->ops[1]); - gen_add_insn_before (gen_ctx, insn, new_insns[0]); - move_bb_insn_dead_vars (new_insns[0]->data, bb_insn); + if (var == MIR_NON_VAR) return; + gen_assert (VARR_LENGTH (var_ref_t, var_refs) == VARR_LENGTH (size_t, var_ref_ages)); + if (VARR_LENGTH (var_ref_t, var_refs) <= var) { + do { + VARR_PUSH (size_t, var_ref_ages, 0); + VARR_PUSH (var_ref_t, var_refs, var_ref); + } while (VARR_LENGTH (var_ref_t, var_refs) <= var); + var_refs_addr = VARR_ADDR (var_ref_t, var_refs); + var_ref_ages_addr = VARR_ADDR (size_t, var_ref_ages); + } + var_ref_ages_addr[var] = curr_bb_var_ref_age; + var_refs_addr[var].insn = insn; + var_refs_addr[var].nop = nop; + var_refs_addr[var].insn_num = insn_num; + var_refs_addr[var].def_p = def_p; + var_refs_addr[var].del_p = FALSE; +} + +static void remove_property_insn (gen_ctx_t gen_ctx, MIR_insn_t insn) { + gen_assert (insn->code == MIR_PRSET || insn->code == MIR_PRBEQ || insn->code == MIR_PRBNE); + if (insn->code == MIR_PRSET + || (insn->code == MIR_PRBEQ && (insn->ops[2].mode != MIR_OP_INT || insn->ops[2].u.i != 0)) + || (insn->code == MIR_PRBNE && (insn->ops[2].mode != MIR_OP_INT || insn->ops[2].u.i == 0))) { DEBUG (2, { - fprintf (debug_file, " changing to "); - print_bb_insn (gen_ctx, new_insns[0]->data, TRUE); + fprintf (debug_file, " removing "); + print_insn (gen_ctx, insn, TRUE); }); gen_delete_insn (gen_ctx, insn); - if (def_insn != NULL) { - DEBUG (2, { - fprintf (debug_file, " deleting now dead insn "); - print_bb_insn (gen_ctx, def_insn->data, TRUE); - }); - gen_delete_insn (gen_ctx, def_insn); - (*deleted_insns_num)++; - } - return new_insns[0]; - } else if (code == MIR_MUL || code == MIR_MULS || code == MIR_UDIV || code == MIR_UDIVS) { - new_insns[0] - = MIR_new_insn (ctx, new_code, insn->ops[0], insn->ops[1], MIR_new_int_op (ctx, sh)); - MIR_insert_insn_after (ctx, curr_func_item, insn, new_insns[0]); - if ((ok_p = target_insn_ok_p (gen_ctx, new_insns[0]))) { - insn->code = new_insns[0]->code; - insn->ops[2] = new_insns[0]->ops[2]; - DEBUG (2, { - fprintf (debug_file, " changing to "); - print_bb_insn (gen_ctx, bb_insn, TRUE); - }); - } - MIR_remove_insn (ctx, curr_func_item, new_insns[0]); - return ok_p ? insn : NULL; - } else if (insn->ops[1].mode == MIR_OP_HARD_REG - && insn->ops[1].u.hard_reg == TEMP_INT_HARD_REG2) { - } else if (insn->ops[1].mode == MIR_OP_HARD_REG_MEM - && (insn->ops[1].u.hard_reg_mem.base == TEMP_INT_HARD_REG2 - || insn->ops[1].u.hard_reg_mem.index == TEMP_INT_HARD_REG2)) { - } else { - temp = _MIR_new_hard_reg_op (ctx, TEMP_INT_HARD_REG2); - gen_assert (code == MIR_DIV || code == MIR_DIVS); - new_insns[0] = MIR_new_insn (ctx, MIR_MOV, temp, insn->ops[1]); - if (code == MIR_DIV) { - new_insns[1] = MIR_new_insn (ctx, MIR_RSH, temp, temp, MIR_new_int_op (ctx, 63)); - new_insns[2] = MIR_new_insn (ctx, MIR_AND, temp, temp, MIR_new_int_op (ctx, op.u.i - 1)); - new_insns[3] = MIR_new_insn (ctx, MIR_ADD, temp, temp, insn->ops[1]); - } else { - new_insns[1] = MIR_new_insn (ctx, MIR_RSHS, temp, temp, MIR_new_int_op (ctx, 31)); - new_insns[2] = MIR_new_insn (ctx, MIR_ANDS, temp, temp, MIR_new_int_op (ctx, op.u.i - 1)); - new_insns[3] = MIR_new_insn (ctx, MIR_ADDS, temp, temp, insn->ops[1]); - } - new_insns[4] = MIR_new_insn (ctx, new_code, temp, temp, MIR_new_int_op (ctx, sh)); - new_insns[5] = MIR_new_insn (ctx, MIR_MOV, insn->ops[0], temp); - for (n = 0; n < 6; n++) gen_add_insn_before (gen_ctx, insn, new_insns[n]); - for (n = 0; n < 6; n++) - if (!target_insn_ok_p (gen_ctx, new_insns[n])) break; - if (n < 6) { - for (n = 0; n < 6; n++) gen_delete_insn (gen_ctx, new_insns[n]); - } else { - move_bb_insn_dead_vars (new_insns[3]->data, bb_insn); - add_bb_insn_dead_var (gen_ctx, new_insns[5]->data, TEMP_INT_HARD_REG2); - DEBUG (2, { - fprintf (debug_file, " changing to "); - for (n = 0; n < 6; n++) { - if (n != 0) fprintf (debug_file, " "); - print_bb_insn (gen_ctx, new_insns[n]->data, TRUE); - } - }); - gen_delete_insn (gen_ctx, insn); - *deleted_insns_num -= 5; - if (def_insn != NULL) { - DEBUG (2, { - fprintf (debug_file, " deleting now dead insn "); - print_bb_insn (gen_ctx, def_insn->data, TRUE); - }); - gen_delete_insn (gen_ctx, def_insn); - (*deleted_insns_num)++; - } - return new_insns[0]; + } else { /* make unconditional jump */ + MIR_context_t ctx = gen_ctx->ctx; + MIR_insn_t new_insn = MIR_new_insn (ctx, MIR_JMP, insn->ops[0]); + MIR_insert_insn_before (ctx, curr_func_item, insn, new_insn); + DEBUG (2, { + fprintf (debug_file, " changing "); + print_insn (gen_ctx, insn, FALSE); + }); + new_insn->data = insn->data; + if (optimize_level > 0) { + bb_insn_t bb_insn = insn->data; + bb_insn->insn = new_insn; } + MIR_remove_insn (ctx, curr_func_item, insn); + DEBUG (2, { + fprintf (debug_file, " to "); + print_insn (gen_ctx, new_insn, TRUE); + }); } - return NULL; } -static void setup_hreg_ref (gen_ctx_t gen_ctx, MIR_reg_t hr, MIR_insn_t insn, size_t nop, - size_t insn_num, int def_p) { - if (hr == MIR_NON_HARD_REG) return; - hreg_ref_ages_addr[hr] = curr_bb_hreg_ref_age; - hreg_refs_addr[hr].insn = insn; - hreg_refs_addr[hr].nop = nop; - hreg_refs_addr[hr].insn_num = insn_num; - hreg_refs_addr[hr].def_p = def_p; - hreg_refs_addr[hr].del_p = FALSE; -} - -static void combine (gen_ctx_t gen_ctx) { +static void combine (gen_ctx_t gen_ctx, int no_property_p) { MIR_context_t ctx = gen_ctx->ctx; MIR_insn_code_t code, new_code; MIR_insn_t insn, new_insn; - bb_insn_t bb_insn; + bb_insn_t bb_insn, next_bb_insn; size_t iter, nops, i, curr_insn_num; MIR_op_t temp_op, *op_ref; MIR_reg_t early_clobbered_hard_reg1, early_clobbered_hard_reg2; - int out_p, change_p, block_change_p; + int out_p, change_p, block_change_p, label_only_p; long insns_num = 0, deleted_insns_num = 0; - hreg_refs_addr = VARR_ADDR (hreg_ref_t, hreg_refs); - hreg_ref_ages_addr = VARR_ADDR (size_t, hreg_ref_ages); + gen_assert (optimize_level > 0); for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { do { DEBUG (2, { fprintf (debug_file, "Processing bb%lu\n", (unsigned long) bb->index); }); block_change_p = FALSE; - curr_bb_hreg_ref_age++; + curr_bb_var_ref_age++; last_mem_ref_insn_num = 0; /* means undef */ + label_only_p = TRUE; for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns), curr_insn_num = 1; bb_insn != NULL; - bb_insn = DLIST_NEXT (bb_insn_t, bb_insn), curr_insn_num++) { + bb_insn = next_bb_insn, curr_insn_num++) { + next_bb_insn = DLIST_NEXT (bb_insn_t, bb_insn); insn = bb_insn->insn; nops = MIR_insn_nops (ctx, insn); - if (insn->code != MIR_LABEL) insns_num++; + if (insn->code == MIR_LABEL) { + if (!label_only_p) { + /* We can move insns with temp hard regs inside BB. It + is important to remove labels inside BB as we use labels to find BBs for lazy BB + generation and temp regs can be used between BBs in this generation mode. */ + DEBUG (2, { + fprintf (debug_file, " Remove label inside BB "); + print_bb_insn (gen_ctx, bb_insn, TRUE); + }); + gen_delete_insn (gen_ctx, insn); + } + continue; + } + label_only_p = FALSE; + insns_num++; DEBUG (2, { fprintf (debug_file, " Processing "); print_bb_insn (gen_ctx, bb_insn, TRUE); }); + if (insn->code == MIR_PRSET || insn->code == MIR_PRBEQ || insn->code == MIR_PRBNE) { + if (no_property_p) remove_property_insn (gen_ctx, insn); + continue; + } target_get_early_clobbered_hard_regs (insn, &early_clobbered_hard_reg1, &early_clobbered_hard_reg2); - if (early_clobbered_hard_reg1 != MIR_NON_HARD_REG) - setup_hreg_ref (gen_ctx, early_clobbered_hard_reg1, insn, 0 /* whatever */, curr_insn_num, - TRUE); - if (early_clobbered_hard_reg2 != MIR_NON_HARD_REG) - setup_hreg_ref (gen_ctx, early_clobbered_hard_reg2, insn, 0 /* whatever */, curr_insn_num, - TRUE); + if (early_clobbered_hard_reg1 != MIR_NON_VAR) + setup_var_ref (gen_ctx, early_clobbered_hard_reg1, insn, 0 /* whatever */, curr_insn_num, + TRUE); + if (early_clobbered_hard_reg2 != MIR_NON_VAR) + setup_var_ref (gen_ctx, early_clobbered_hard_reg2, insn, 0 /* whatever */, curr_insn_num, + TRUE); if (MIR_call_code_p (code = insn->code)) { for (size_t hr = 0; hr <= MAX_HARD_REG; hr++) if (bitmap_bit_p (call_used_hard_regs[MIR_T_UNDEF], hr)) { - setup_hreg_ref (gen_ctx, hr, insn, 0 /* whatever */, curr_insn_num, TRUE); + setup_var_ref (gen_ctx, (MIR_reg_t) hr, insn, 0 /* whatever */, curr_insn_num, TRUE); } last_mem_ref_insn_num = curr_insn_num; /* Potentially call can change memory */ } else if (code == MIR_VA_BLOCK_ARG) { last_mem_ref_insn_num = curr_insn_num; /* Change memory */ } else if (code == MIR_RET) { /* ret is transformed in machinize and should be not modified after that */ - } else if ((new_insn = combine_branch_and_cmp (gen_ctx, bb_insn, &deleted_insns_num)) - != NULL - || (new_insn = combine_exts (gen_ctx, bb_insn, &deleted_insns_num)) != NULL - || (new_insn = combine_mul_div_substitute (gen_ctx, bb_insn, &deleted_insns_num)) - != NULL) { + } else if ((new_insn = combine_exts (gen_ctx, bb_insn, &deleted_insns_num)) != NULL) { + /* ssa ext removal is not enough as we can add ext insn in machinize for args and rets + */ bb_insn = new_insn->data; insn = new_insn; nops = MIR_insn_nops (ctx, insn); @@ -5140,21 +9063,19 @@ static void combine (gen_ctx_t gen_ctx) { if (code == MIR_BSTART || code == MIR_BEND) last_mem_ref_insn_num = curr_insn_num; } - for (iter = 0; iter < 2; iter++) { /* update hreg ref info: */ + for (iter = 0; iter < 2; iter++) { /* update var ref info: */ for (i = 0; i < nops; i++) { op_ref = &insn->ops[i]; MIR_insn_op_mode (ctx, insn, i, &out_p); - if (op_ref->mode == MIR_OP_HARD_REG && !iter == !out_p) { + if (op_ref->mode == MIR_OP_VAR && !iter == !out_p) { /* process in ops on 0th iteration and out ops on 1th iteration */ - setup_hreg_ref (gen_ctx, op_ref->u.hard_reg, insn, i, curr_insn_num, iter == 1); - } else if (op_ref->mode == MIR_OP_HARD_REG_MEM) { + setup_var_ref (gen_ctx, op_ref->u.var, insn, i, curr_insn_num, iter == 1); + } else if (op_ref->mode == MIR_OP_VAR_MEM) { if (out_p && iter == 1) last_mem_ref_insn_num = curr_insn_num; else if (iter == 0) { - setup_hreg_ref (gen_ctx, op_ref->u.hard_reg_mem.base, insn, i, curr_insn_num, - FALSE); - setup_hreg_ref (gen_ctx, op_ref->u.hard_reg_mem.index, insn, i, curr_insn_num, - FALSE); + setup_var_ref (gen_ctx, op_ref->u.var_mem.base, insn, i, curr_insn_num, FALSE); + setup_var_ref (gen_ctx, op_ref->u.var_mem.index, insn, i, curr_insn_num, FALSE); } } } @@ -5168,32 +9089,36 @@ static void combine (gen_ctx_t gen_ctx) { }); } -static void init_selection (gen_ctx_t gen_ctx) { - hreg_ref_t hreg_ref = {NULL, 0, 0}; - - gen_ctx->selection_ctx = gen_malloc (gen_ctx, sizeof (struct selection_ctx)); - curr_bb_hreg_ref_age = 0; - VARR_CREATE (size_t, hreg_ref_ages, MAX_HARD_REG + 1); - VARR_CREATE (hreg_ref_t, hreg_refs, MAX_HARD_REG + 1); - VARR_CREATE (MIR_reg_t, insn_hard_regs, 0); +static void init_combine (gen_ctx_t gen_ctx) { + gen_ctx->combine_ctx = gen_malloc (gen_ctx, sizeof (struct combine_ctx)); + curr_bb_var_ref_age = 0; + VARR_CREATE (size_t, var_ref_ages, 0); + VARR_CREATE (var_ref_t, var_refs, 0); + VARR_CREATE (MIR_reg_t, insn_vars, 0); VARR_CREATE (size_t, changed_op_numbers, 16); VARR_CREATE (MIR_op_t, last_right_ops, 16); - hard_regs_bitmap = bitmap_create2 (MAX_HARD_REG + 1); - for (MIR_reg_t i = 0; i <= MAX_HARD_REG; i++) { - VARR_PUSH (hreg_ref_t, hreg_refs, hreg_ref); - VARR_PUSH (size_t, hreg_ref_ages, 0); - } + vars_bitmap = bitmap_create (); } -static void finish_selection (gen_ctx_t gen_ctx) { - VARR_DESTROY (size_t, hreg_ref_ages); - VARR_DESTROY (hreg_ref_t, hreg_refs); - VARR_DESTROY (MIR_reg_t, insn_hard_regs); +static void finish_combine (gen_ctx_t gen_ctx) { + VARR_DESTROY (size_t, var_ref_ages); + VARR_DESTROY (var_ref_t, var_refs); + VARR_DESTROY (MIR_reg_t, insn_vars); VARR_DESTROY (size_t, changed_op_numbers); VARR_DESTROY (MIR_op_t, last_right_ops); - bitmap_destroy (hard_regs_bitmap); - free (gen_ctx->selection_ctx); - gen_ctx->selection_ctx = NULL; + bitmap_destroy (vars_bitmap); + free (gen_ctx->combine_ctx); + gen_ctx->combine_ctx = NULL; +} + +static void remove_property_insns (gen_ctx_t gen_ctx) { + MIR_insn_t insn, next_insn; + for (insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); insn != NULL; + insn = next_insn) { + next_insn = DLIST_NEXT (MIR_insn_t, insn); + if (insn->code == MIR_PRSET || insn->code == MIR_PRBEQ || insn->code == MIR_PRBNE) + remove_property_insn (gen_ctx, insn); + } } /* New Page */ @@ -5203,15 +9128,17 @@ static void finish_selection (gen_ctx_t gen_ctx) { #define live_out out static void dead_code_elimination (gen_ctx_t gen_ctx) { - MIR_insn_t insn; + MIR_insn_t insn, nop_insn; bb_insn_t bb_insn, prev_bb_insn; - size_t passed_mem_num; MIR_reg_t var, early_clobbered_hard_reg1, early_clobbered_hard_reg2; - int op_num, out_p, reg_def_p, dead_p, mem_p; + int op_num, reg_def_p, dead_p; bitmap_t live; insn_var_iterator_t insn_var_iter; long dead_insns_num = 0; + bitmap_t global_hard_regs + = _MIR_get_module_global_var_hard_regs (gen_ctx->ctx, curr_func_item->module); + gen_assert (optimize_level > 0); DEBUG (2, { fprintf (debug_file, "+++++++++++++Dead code elimination:\n"); }); live = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) { @@ -5221,36 +9148,48 @@ static void dead_code_elimination (gen_ctx_t gen_ctx) { insn = bb_insn->insn; reg_def_p = FALSE; dead_p = TRUE; - FOREACH_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (!out_p) continue; + FOREACH_OUT_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num) { reg_def_p = TRUE; - if (bitmap_clear_bit_p (live, var)) dead_p = FALSE; + if (bitmap_clear_bit_p (live, var) || bitmap_bit_p (addr_regs, var)) dead_p = FALSE; } if (!reg_def_p) dead_p = FALSE; - if (dead_p && !MIR_call_code_p (insn->code) && insn->code != MIR_RET + if (dead_p && !MIR_call_code_p (insn->code) && insn->code != MIR_RET && insn->code != MIR_JRET && insn->code != MIR_ALLOCA && insn->code != MIR_BSTART && insn->code != MIR_BEND && insn->code != MIR_VA_START && insn->code != MIR_VA_ARG && insn->code != MIR_VA_END - && !(insn->ops[0].mode == MIR_OP_HARD_REG - && (insn->ops[0].u.hard_reg == FP_HARD_REG - || insn->ops[0].u.hard_reg == SP_HARD_REG))) { + && !(MIR_overflow_insn_code_p (insn->code) + && reachable_bo_exists_p (DLIST_NEXT (bb_insn_t, bb_insn))) + && !(insn->ops[0].mode == MIR_OP_VAR + && (insn->ops[0].u.var == FP_HARD_REG || insn->ops[0].u.var == SP_HARD_REG))) { DEBUG (2, { fprintf (debug_file, " Removing dead insn %-5lu", (unsigned long) bb_insn->index); MIR_output_insn (gen_ctx->ctx, debug_file, insn, curr_func_item->u.func, TRUE); }); + if (DLIST_HEAD (bb_insn_t, bb->bb_insns) == DLIST_TAIL (bb_insn_t, bb->bb_insns)) { + gen_assert (bb_insn == DLIST_HEAD (bb_insn_t, bb->bb_insns)); + nop_insn = MIR_new_insn_arr (gen_ctx->ctx, MIR_USE, 0, NULL); + DEBUG (2, { + fprintf (debug_file, + " Adding nop to keep bb%lu non-empty: ", (unsigned long) bb->index); + MIR_output_insn (gen_ctx->ctx, debug_file, nop_insn, curr_func_item->u.func, TRUE); + }); + gen_add_insn_after (gen_ctx, insn, nop_insn); + } gen_delete_insn (gen_ctx, insn); dead_insns_num++; continue; } - if (MIR_call_code_p (insn->code)) + if (MIR_call_code_p (insn->code)) { bitmap_and_compl (live, live, call_used_hard_regs[MIR_T_UNDEF]); - FOREACH_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (!out_p) bitmap_set_bit_p (live, var); + if (global_hard_regs != NULL) bitmap_ior (live, live, global_hard_regs); + } + FOREACH_IN_INSN_VAR (gen_ctx, insn_var_iter, insn, var, op_num) { + bitmap_set_bit_p (live, var); } target_get_early_clobbered_hard_regs (insn, &early_clobbered_hard_reg1, &early_clobbered_hard_reg2); - if (early_clobbered_hard_reg1 != MIR_NON_HARD_REG) + if (early_clobbered_hard_reg1 != MIR_NON_VAR) bitmap_clear_bit_p (live, early_clobbered_hard_reg1); - if (early_clobbered_hard_reg2 != MIR_NON_HARD_REG) + if (early_clobbered_hard_reg2 != MIR_NON_VAR) bitmap_clear_bit_p (live, early_clobbered_hard_reg2); if (MIR_call_code_p (insn->code)) bitmap_ior (live, live, bb_insn->call_hard_reg_args); } @@ -5263,70 +9202,6 @@ static void dead_code_elimination (gen_ctx_t gen_ctx) { /* New Page */ -/* SSA dead code elimination */ - -static int dead_insn_p (gen_ctx_t gen_ctx, bb_insn_t bb_insn) { - MIR_insn_t insn = bb_insn->insn; - int op_num, out_p, mem_p, output_exists_p = FALSE; - size_t passed_mem_num; - MIR_reg_t var; - insn_var_iterator_t iter; - ssa_edge_t ssa_edge; - - /* check control insns with possible output: */ - if (MIR_call_code_p (insn->code) || insn->code == MIR_ALLOCA || insn->code == MIR_BSTART - || insn->code == MIR_VA_START || insn->code == MIR_VA_ARG - || (insn->nops > 0 && insn->ops[0].mode == MIR_OP_HARD_REG - && (insn->ops[0].u.hard_reg == FP_HARD_REG || insn->ops[0].u.hard_reg == SP_HARD_REG))) - return FALSE; - if (start_insn_p (gen_ctx, bb_insn)) return FALSE; - FOREACH_INSN_VAR (gen_ctx, iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (!out_p) continue; - output_exists_p = TRUE; - if (mem_p || (ssa_edge = insn->ops[op_num].data) != NULL) return FALSE; - } - return output_exists_p; -} - -static void ssa_dead_code_elimination (gen_ctx_t gen_ctx) { - MIR_insn_t insn; - bb_insn_t bb_insn, def; - int op_num, out_p, mem_p; - size_t passed_mem_num; - MIR_reg_t var; - insn_var_iterator_t iter; - ssa_edge_t ssa_edge; - long dead_insns_num = 0; - - DEBUG (2, { fprintf (debug_file, "+++++++++++++Dead code elimination:\n"); }); - gen_assert (def_use_repr_p); - VARR_TRUNC (bb_insn_t, dead_bb_insns, 0); - for (bb_t bb = DLIST_HEAD (bb_t, curr_cfg->bbs); bb != NULL; bb = DLIST_NEXT (bb_t, bb)) - for (bb_insn = DLIST_HEAD (bb_insn_t, bb->bb_insns); bb_insn != NULL; - bb_insn = DLIST_NEXT (bb_insn_t, bb_insn)) - if (dead_insn_p (gen_ctx, bb_insn)) VARR_PUSH (bb_insn_t, dead_bb_insns, bb_insn); - while (VARR_LENGTH (bb_insn_t, dead_bb_insns) != 0) { - bb_insn = VARR_POP (bb_insn_t, dead_bb_insns); - insn = bb_insn->insn; - DEBUG (2, { - fprintf (debug_file, " Removing dead insn %-5lu", (unsigned long) bb_insn->index); - MIR_output_insn (gen_ctx->ctx, debug_file, insn, curr_func_item->u.func, TRUE); - }); - FOREACH_INSN_VAR (gen_ctx, iter, insn, var, op_num, out_p, mem_p, passed_mem_num) { - if (out_p && !mem_p) continue; - if ((ssa_edge = insn->ops[op_num].data) == NULL) continue; - def = ssa_edge->def; - remove_ssa_edge (gen_ctx, ssa_edge); - if (dead_insn_p (gen_ctx, def)) VARR_PUSH (bb_insn_t, dead_bb_insns, def); - } - gen_delete_insn (gen_ctx, insn); - dead_insns_num++; - } - DEBUG (1, { fprintf (debug_file, "%5ld removed SSA dead insns\n", dead_insns_num); }); -} - -/* New Page */ - #if !MIR_NO_GEN_DEBUG #include "real-time.h" #endif @@ -5339,23 +9214,16 @@ static void *print_and_execute_wrapper (gen_ctx_t gen_ctx, MIR_item_t called_fun } #endif -static void parallel_error (MIR_context_t ctx, const char *err_message) { - MIR_get_error_func (ctx) (MIR_parallel_error, err_message); -} +static const int collect_bb_stat_p = FALSE; -void *MIR_gen (MIR_context_t ctx, int gen_num, MIR_item_t func_item) { - struct all_gen_ctx *all_gen_ctx = *all_gen_ctx_loc (ctx); - gen_ctx_t gen_ctx; +static void *generate_func_code (MIR_context_t ctx, MIR_item_t func_item, int machine_code_p) { + gen_ctx_t gen_ctx = *gen_ctx_loc (ctx); uint8_t *code; - void *machine_code; - size_t code_len; + void *machine_code = NULL; + size_t code_len = 0; double start_time = real_usec_time (); + uint32_t bbs_num; -#if !MIR_PARALLEL_GEN - gen_num = 0; -#endif - gen_assert (gen_num >= 0 && gen_num < all_gen_ctx->gens_num); - gen_ctx = &all_gen_ctx->gen_ctx[gen_num]; gen_assert (func_item->item_type == MIR_func_item && func_item->data == NULL); if (func_item->u.func->machine_code != NULL) { gen_assert (func_item->u.func->call_addr != NULL); @@ -5377,31 +9245,43 @@ void *MIR_gen (MIR_context_t ctx, int gen_num, MIR_item_t func_item) { _MIR_duplicate_func_insns (ctx, func_item); curr_cfg = func_item->data = gen_malloc (gen_ctx, sizeof (struct func_cfg)); build_func_cfg (gen_ctx); + bbs_num = curr_bb_index; DEBUG (2, { fprintf (debug_file, "+++++++++++++MIR after building CFG:\n"); print_CFG (gen_ctx, TRUE, FALSE, TRUE, FALSE, NULL); }); + if (optimize_level >= 2 && !addr_insn_p && clone_bbs (gen_ctx)) { + /* do not clone bbs before addr transformation: it can prevent addr transformations */ + DEBUG (2, { + fprintf (debug_file, "+++++++++++++MIR after cloning BBs:\n"); + print_CFG (gen_ctx, TRUE, FALSE, TRUE, FALSE, NULL); + }); + } if (optimize_level >= 2) { - build_ssa (gen_ctx); + build_ssa (gen_ctx, !addr_insn_p); DEBUG (2, { - fprintf (debug_file, "+++++++++++++MIR after building SSA:\n"); + fprintf (debug_file, "+++++++++++++MIR after building SSA%s:\n", + addr_insn_p ? " for address transformation" : ""); print_varr_insns (gen_ctx, "undef init", undef_insns); print_varr_insns (gen_ctx, "arg init", arg_bb_insns); fprintf (debug_file, "\n"); print_CFG (gen_ctx, TRUE, FALSE, TRUE, TRUE, NULL); }); } -#ifndef NO_COPY_PROP - if (optimize_level >= 2) { - DEBUG (2, { fprintf (debug_file, "+++++++++++++Copy Propagation:\n"); }); - copy_prop (gen_ctx); + if (optimize_level >= 2 && addr_insn_p) { + DEBUG (2, { fprintf (debug_file, "+++++++++++++Transform Addr Insns and cloning BBs:\n"); }); + transform_addrs (gen_ctx); + undo_build_ssa (gen_ctx); + clone_bbs (gen_ctx); + build_ssa (gen_ctx, TRUE); DEBUG (2, { - fprintf (debug_file, "+++++++++++++MIR after Copy Propagation:\n"); + fprintf (debug_file, "+++++++++++++MIR after Addr Insns Transformation and cloning BBs:\n"); + print_varr_insns (gen_ctx, "undef init", undef_insns); + print_varr_insns (gen_ctx, "arg init", arg_bb_insns); + fprintf (debug_file, "\n"); print_CFG (gen_ctx, TRUE, FALSE, TRUE, TRUE, NULL); }); } -#endif /* #ifndef NO_COPY_PROP */ -#ifndef NO_GVN if (optimize_level >= 2) { DEBUG (2, { fprintf (debug_file, "+++++++++++++GVN:\n"); }); gvn (gen_ctx); @@ -5411,93 +9291,145 @@ void *MIR_gen (MIR_context_t ctx, int gen_num, MIR_item_t func_item) { }); gvn_clear (gen_ctx); } -#endif /* #ifndef NO_GVN */ -#ifndef NO_GVN + if (optimize_level >= 2) { + DEBUG (2, { fprintf (debug_file, "+++++++++++++Copy Propagation:\n"); }); + copy_prop (gen_ctx); + DEBUG (2, { + fprintf (debug_file, "+++++++++++++MIR after Copy Propagation:\n"); + print_CFG (gen_ctx, TRUE, FALSE, TRUE, TRUE, NULL); + }); + } + if (optimize_level >= 2) { + DEBUG (2, { fprintf (debug_file, "+++++++++++++DSE:\n"); }); + dse (gen_ctx); + DEBUG (2, { + fprintf (debug_file, "+++++++++++++MIR after DSE:\n"); + print_CFG (gen_ctx, TRUE, FALSE, TRUE, TRUE, NULL); + }); + } if (optimize_level >= 2) { ssa_dead_code_elimination (gen_ctx); DEBUG (2, { - fprintf (debug_file, "+++++++++++++MIR after dead code elimination after GVN:\n"); + fprintf (debug_file, "+++++++++++++MIR after dead code elimination:\n"); print_CFG (gen_ctx, TRUE, TRUE, TRUE, TRUE, NULL); }); } -#endif /* #ifndef NO_GVN */ -#ifndef NO_CCP if (optimize_level >= 2) { - DEBUG (2, { fprintf (debug_file, "+++++++++++++CCP:\n"); }); - if (ccp (gen_ctx)) { - DEBUG (2, { - fprintf (debug_file, "+++++++++++++MIR after CCP:\n"); - print_CFG (gen_ctx, TRUE, FALSE, TRUE, TRUE, NULL); - }); - ssa_dead_code_elimination (gen_ctx); + build_loop_tree (gen_ctx); + DEBUG (2, { print_loop_tree (gen_ctx, TRUE); }); + if (licm (gen_ctx)) { DEBUG (2, { - fprintf (debug_file, "+++++++++++++MIR after dead code elimination after CCP:\n"); + fprintf (debug_file, "+++++++++++++MIR after loop invariant motion:\n"); print_CFG (gen_ctx, TRUE, TRUE, TRUE, TRUE, NULL); }); } + destroy_loop_tree (gen_ctx, curr_cfg->root_loop_node); + curr_cfg->root_loop_node = NULL; + } + if (optimize_level >= 2 && pressure_relief (gen_ctx)) { + DEBUG (2, { + fprintf (debug_file, "+++++++++++++MIR after pressure relief:\n"); + print_CFG (gen_ctx, TRUE, TRUE, TRUE, TRUE, NULL); + }); + } + if (optimize_level >= 2) { + make_conventional_ssa (gen_ctx); + DEBUG (2, { + fprintf (debug_file, "+++++++++++++MIR after transformation to conventional SSA:\n"); + print_CFG (gen_ctx, TRUE, TRUE, TRUE, TRUE, NULL); + }); + ssa_combine (gen_ctx); + DEBUG (2, { + fprintf (debug_file, "+++++++++++++MIR after ssa combine:\n"); + print_CFG (gen_ctx, TRUE, TRUE, TRUE, TRUE, NULL); + }); + undo_build_ssa (gen_ctx); + DEBUG (2, { + fprintf (debug_file, "+++++++++++++MIR after destroying ssa:\n"); + print_varr_insns (gen_ctx, "undef init", undef_insns); + print_varr_insns (gen_ctx, "arg init", arg_bb_insns); + fprintf (debug_file, "\n"); + print_CFG (gen_ctx, TRUE, FALSE, TRUE, TRUE, NULL); + }); + } + if (optimize_level >= 2) { + DEBUG (2, { fprintf (debug_file, "+++++++++++++Jump optimization:\n"); }); + jump_opt (gen_ctx); + DEBUG (2, { + fprintf (debug_file, "+++++++++++++MIR after Jump optimization:\n"); + print_CFG (gen_ctx, TRUE, FALSE, TRUE, TRUE, NULL); + }); } -#endif /* #ifndef NO_CCP */ - if (optimize_level >= 2) undo_build_ssa (gen_ctx); - make_io_dup_op_insns (gen_ctx); target_machinize (gen_ctx); + make_io_dup_op_insns (gen_ctx); DEBUG (2, { fprintf (debug_file, "+++++++++++++MIR after machinize:\n"); print_CFG (gen_ctx, FALSE, FALSE, TRUE, TRUE, NULL); }); - if (optimize_level != 0) build_loop_tree (gen_ctx); - calculate_func_cfg_live_info (gen_ctx, optimize_level != 0); - DEBUG (2, { - add_bb_insn_dead_vars (gen_ctx); - fprintf (debug_file, "+++++++++++++MIR after building live_info:\n"); - print_loop_tree (gen_ctx, TRUE); - print_CFG (gen_ctx, TRUE, TRUE, FALSE, FALSE, output_bb_live_info); - }); - if (optimize_level != 0) build_live_ranges (gen_ctx); - assign (gen_ctx); - rewrite (gen_ctx); /* After rewrite the BB live info is still valid */ + if (optimize_level >= 1) { + build_loop_tree (gen_ctx); + DEBUG (2, { print_loop_tree (gen_ctx, TRUE); }); + } + if (optimize_level >= 2) { + collect_moves (gen_ctx); + if (consider_move_vars_only (gen_ctx)) { + calculate_func_cfg_live_info (gen_ctx, FALSE); + print_live_info (gen_ctx, "Live info before coalesce", TRUE, FALSE); + coalesce (gen_ctx); + DEBUG (2, { + fprintf (debug_file, "+++++++++++++MIR after coalescing:\n"); + print_CFG (gen_ctx, TRUE, TRUE, TRUE, TRUE, output_bb_border_live_info); + }); + } + } + consider_all_live_vars (gen_ctx); + calculate_func_cfg_live_info (gen_ctx, TRUE); + print_live_info (gen_ctx, "Live info before RA", optimize_level > 0, TRUE); + reg_alloc (gen_ctx); DEBUG (2, { - fprintf (debug_file, "+++++++++++++MIR after rewrite:\n"); - print_CFG (gen_ctx, FALSE, FALSE, TRUE, FALSE, NULL); + fprintf (debug_file, "+++++++++++++MIR after RA:\n"); + print_CFG (gen_ctx, TRUE, FALSE, TRUE, FALSE, NULL); }); -#ifndef NO_COMBINE + if (optimize_level < 2 && machine_code_p) remove_property_insns (gen_ctx); if (optimize_level >= 1) { + consider_all_live_vars (gen_ctx); calculate_func_cfg_live_info (gen_ctx, FALSE); add_bb_insn_dead_vars (gen_ctx); - DEBUG (2, { - fprintf (debug_file, "+++++++++++++MIR before combine:\n"); - print_CFG (gen_ctx, FALSE, FALSE, TRUE, FALSE, NULL); - }); - combine (gen_ctx); /* After combine the BB live info is still valid */ + print_live_info (gen_ctx, "Live info before combine", FALSE, FALSE); + combine (gen_ctx, machine_code_p); /* After combine the BB live info is still valid */ DEBUG (2, { fprintf (debug_file, "+++++++++++++MIR after combine:\n"); print_CFG (gen_ctx, FALSE, FALSE, TRUE, FALSE, NULL); }); dead_code_elimination (gen_ctx); DEBUG (2, { - fprintf (debug_file, "+++++++++++++MIR after dead code elimination after combine:\n"); + fprintf (debug_file, "+++++++++++++MIR after dead code elimination after 2nd combine:\n"); print_CFG (gen_ctx, TRUE, TRUE, TRUE, FALSE, output_bb_live_info); }); } -#endif /* #ifndef NO_COMBINE */ target_make_prolog_epilog (gen_ctx, func_used_hard_regs, func_stack_slots_num); + target_split_insns (gen_ctx); DEBUG (2, { - fprintf (debug_file, "+++++++++++++MIR after forming prolog/epilog:\n"); - print_CFG (gen_ctx, FALSE, FALSE, TRUE, FALSE, NULL); + fprintf (debug_file, "+++++++++++++MIR after forming prolog/epilog and insn splitting:\n"); + print_CFG (gen_ctx, FALSE, FALSE, TRUE, TRUE, NULL); }); - code = target_translate (gen_ctx, &code_len); - machine_code = func_item->u.func->call_addr = _MIR_publish_code (ctx, code, code_len); - target_rebase (gen_ctx, func_item->u.func->call_addr); + if (machine_code_p) { + code = target_translate (gen_ctx, &code_len); + machine_code = func_item->u.func->call_addr = _MIR_publish_code (ctx, code, code_len); + target_rebase (gen_ctx, func_item->u.func->call_addr); #if MIR_GEN_CALL_TRACE - func_item->u.func->call_addr = _MIR_get_wrapper (ctx, func_item, print_and_execute_wrapper); + func_item->u.func->call_addr = _MIR_get_wrapper (ctx, func_item, print_and_execute_wrapper); #endif - DEBUG (2, { - _MIR_dump_code (NULL, gen_ctx->gen_num, machine_code, code_len); - fprintf (debug_file, "code size = %lu:\n", (unsigned long) code_len); - }); - _MIR_redirect_thunk (ctx, func_item->addr, func_item->u.func->call_addr); - destroy_func_live_ranges (gen_ctx); + DEBUG (2, { + _MIR_dump_code (NULL, machine_code, code_len); + fprintf (debug_file, "code size = %lu:\n", (unsigned long) code_len); + }); + _MIR_redirect_thunk (ctx, func_item->addr, func_item->u.func->call_addr); + } if (optimize_level != 0) destroy_loop_tree (gen_ctx, curr_cfg->root_loop_node); destroy_func_cfg (gen_ctx); + if (collect_bb_stat_p) overall_bbs_num += bbs_num; + if (!machine_code_p) return NULL; DEBUG (0, { fprintf (debug_file, " Code generation for %s: %lu MIR insns (addr=%llx, len=%lu) -- time %.2f ms\n", @@ -5508,299 +9440,278 @@ void *MIR_gen (MIR_context_t ctx, int gen_num, MIR_item_t func_item) { }); _MIR_restore_func_insns (ctx, func_item); /* ??? We should use atomic here but c2mir does not implement them yet. */ -#if MIR_PARALLEL_GEN - if (mir_mutex_lock (&queue_mutex)) parallel_error (ctx, "error in mutex lock"); -#endif func_item->u.func->machine_code = machine_code; -#if MIR_PARALLEL_GEN - if (mir_cond_broadcast (&done_signal)) parallel_error (ctx, "error in cond broadcast"); - if (mir_mutex_unlock (&queue_mutex)) parallel_error (ctx, "error in mutex lock"); -#endif return func_item->addr; } -void MIR_gen_set_debug_file (MIR_context_t ctx, int gen_num, FILE *f) { -#if !MIR_NO_GEN_DEBUG - struct all_gen_ctx *all_gen_ctx = *all_gen_ctx_loc (ctx); - gen_ctx_t gen_ctx; +void *MIR_gen (MIR_context_t ctx, MIR_item_t func_item) { + return generate_func_code (ctx, func_item, TRUE); +} - if (all_gen_ctx == NULL) { +void MIR_gen_set_debug_file (MIR_context_t ctx, FILE *f) { +#if !MIR_NO_GEN_DEBUG + gen_ctx_t gen_ctx = *gen_ctx_loc (ctx); + if (gen_ctx == NULL) { fprintf (stderr, "Calling MIR_gen_set_debug_file before MIR_gen_init -- good bye\n"); exit (1); } -#if !MIR_PARALLEL_GEN - gen_num = 0; -#endif - gen_assert (gen_num >= 0 && gen_num < all_gen_ctx->gens_num); - gen_ctx = &all_gen_ctx->gen_ctx[gen_num]; debug_file = f; #endif } -void MIR_gen_set_debug_level (MIR_context_t ctx, int gen_num, int level) { +void MIR_gen_set_debug_level (MIR_context_t ctx, int level) { #if !MIR_NO_GEN_DEBUG - struct all_gen_ctx *all_gen_ctx = *all_gen_ctx_loc (ctx); - gen_ctx_t gen_ctx; - - if (all_gen_ctx == NULL) { + gen_ctx_t gen_ctx = *gen_ctx_loc (ctx); + if (gen_ctx == NULL) { fprintf (stderr, "Calling MIR_gen_set_debug_level before MIR_gen_init -- good bye\n"); exit (1); } -#if !MIR_PARALLEL_GEN - gen_num = 0; -#endif - gen_assert (gen_num >= 0 && gen_num < all_gen_ctx->gens_num); - gen_ctx = &all_gen_ctx->gen_ctx[gen_num]; debug_level = level; #endif } -void MIR_gen_set_optimize_level (MIR_context_t ctx, int gen_num, unsigned int level) { - struct all_gen_ctx *all_gen_ctx = *all_gen_ctx_loc (ctx); - gen_ctx_t gen_ctx; - -#if !MIR_PARALLEL_GEN - gen_num = 0; -#endif - gen_assert (gen_num >= 0 && gen_num < all_gen_ctx->gens_num); - gen_ctx = &all_gen_ctx->gen_ctx[gen_num]; +void MIR_gen_set_optimize_level (MIR_context_t ctx, unsigned int level) { + gen_ctx_t gen_ctx = *gen_ctx_loc (ctx); + if (gen_ctx == NULL) { + fprintf (stderr, "Calling MIR_gen_set_optimize_level before MIR_gen_init -- good bye\n"); + exit (1); + } optimize_level = level; } -#if MIR_PARALLEL_GEN -static void *gen (void *arg) { - MIR_item_t func_item; - gen_ctx_t gen_ctx = arg; - struct all_gen_ctx *all_gen_ctx = gen_ctx->all_gen_ctx; - MIR_context_t ctx = all_gen_ctx->ctx; - size_t len; +static void generate_bb_version_machine_code (gen_ctx_t gen_ctx, bb_version_t bb_version); +static void *bb_version_generator (gen_ctx_t gen_ctx, bb_version_t bb_version); - for (;;) { - if (mir_mutex_lock (&queue_mutex)) parallel_error (ctx, "error in mutex lock"); - while (VARR_LENGTH (MIR_item_t, funcs_to_generate) <= funcs_start) - if (mir_cond_wait (&generate_signal, &queue_mutex)) - parallel_error (ctx, "error in cond wait"); - if ((func_item = VARR_GET (MIR_item_t, funcs_to_generate, funcs_start)) == NULL) { - if (mir_mutex_unlock (&queue_mutex)) parallel_error (ctx, "error in mutex unlock"); - break; +static bb_version_t get_bb_version (gen_ctx_t gen_ctx, bb_stub_t bb_stub, uint32_t n_attrs, + spot_attr_t *attrs, int call_p, void **addr) { + MIR_context_t ctx = gen_ctx->ctx; + bb_version_t bb_version; + + if ((bb_version = DLIST_HEAD (bb_version_t, bb_stub->bb_versions)) != NULL) { + VARR_PUSH (target_bb_version_t, target_succ_bb_versions, NULL); + *addr = bb_version->addr; + return bb_version; + } + bb_version = gen_malloc_and_mark_to_free (gen_ctx, sizeof (struct bb_version) + + (n_attrs <= 1 ? 0 : n_attrs) + * sizeof (spot_attr_t)); + target_init_bb_version_data (&bb_version->target_data); + VARR_PUSH (target_bb_version_t, target_succ_bb_versions, + call_p ? NULL : &bb_version->target_data); + bb_version->bb_stub = bb_stub; + bb_version->n_attrs = n_attrs; + if (n_attrs != 0) memcpy (bb_version->attrs, attrs, n_attrs * sizeof (spot_attr_t)); + bb_version->call_p = call_p; + DLIST_APPEND (bb_version_t, bb_stub->bb_versions, bb_version); + bb_version->machine_code = NULL; + *addr = bb_version->addr = _MIR_get_bb_thunk (ctx, bb_version, bb_wrapper); + return bb_version; +} + +/* create bb stubs and set up label data to the corresponding bb stub */ +/* todo finish bb on calls ??? */ +static void create_bb_stubs (gen_ctx_t gen_ctx) { + MIR_context_t ctx = gen_ctx->ctx; + MIR_insn_t insn, last_lab_insn; + size_t n_bbs; + int new_bb_p = TRUE; + bb_stub_t bb_stubs; + + n_bbs = 0; + for (insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); insn != NULL; + insn = DLIST_NEXT (MIR_insn_t, insn)) { + if (insn->code == MIR_LABEL || new_bb_p) { + last_lab_insn = insn; + if (insn->code == MIR_LABEL) + for (insn = DLIST_NEXT (MIR_insn_t, insn); insn != NULL && insn->code == MIR_LABEL; + last_lab_insn = insn, insn = DLIST_NEXT (MIR_insn_t, insn)) + ; + insn = last_lab_insn; + n_bbs++; } - funcs_start++; - if (funcs_start > 64 && VARR_LENGTH (MIR_item_t, funcs_to_generate) < 2 * funcs_start) { - len = VARR_LENGTH (MIR_item_t, funcs_to_generate) - funcs_start; - memmove (VARR_ADDR (MIR_item_t, funcs_to_generate), /* compact */ - VARR_ADDR (MIR_item_t, funcs_to_generate) + funcs_start, len * sizeof (MIR_item_t)); - VARR_TRUNC (MIR_item_t, funcs_to_generate, len); - funcs_start = 0; - } - gen_ctx->busy_p = TRUE; - if (mir_mutex_unlock (&queue_mutex)) parallel_error (ctx, "error in mutex unlock"); - MIR_gen (gen_ctx->ctx, gen_ctx->gen_num, func_item); - if (mir_mutex_lock (&queue_mutex)) parallel_error (ctx, "error in mutex lock"); - gen_ctx->busy_p = FALSE; - if (mir_cond_signal (&done_signal)) parallel_error (ctx, "error in cond signal"); - if (mir_mutex_unlock (&queue_mutex)) parallel_error (ctx, "error in mutex unlock"); + new_bb_p = MIR_any_branch_code_p (insn->code) || insn->code == MIR_RET || insn->code == MIR_JRET + || insn->code == MIR_PRBEQ || insn->code == MIR_PRBNE; } - return NULL; -} - -static void signal_threads_to_finish (struct all_gen_ctx *all_gen_ctx) { - MIR_context_t ctx = all_gen_ctx->ctx; - - if (mir_mutex_lock (&queue_mutex)) parallel_error (ctx, "error in mutex lock"); - funcs_start = 0; - VARR_TRUNC (MIR_item_t, funcs_to_generate, 0); - VARR_PUSH (MIR_item_t, funcs_to_generate, NULL); /* flag to finish threads */ - if (mir_cond_broadcast (&generate_signal)) parallel_error (ctx, "error in cond broadcast"); - if (mir_mutex_unlock (&queue_mutex)) parallel_error (ctx, "error in mutex unlock"); -} -#endif - -void MIR_gen_init (MIR_context_t ctx, int gens_num) { - struct all_gen_ctx **all_gen_ctx_ptr = all_gen_ctx_loc (ctx), *all_gen_ctx; - gen_ctx_t gen_ctx; - -#if !MIR_PARALLEL_GEN - gens_num = 1; -#else - if (gens_num < 1) gens_num = 1; -#endif - *all_gen_ctx_ptr = all_gen_ctx - = gen_malloc (NULL, sizeof (struct all_gen_ctx) + sizeof (struct gen_ctx) * (gens_num - 1)); - all_gen_ctx->ctx = ctx; - all_gen_ctx->gens_num = gens_num; -#if MIR_PARALLEL_GEN - funcs_start = 0; - VARR_CREATE (MIR_item_t, funcs_to_generate, 0); - if (mir_mutex_init (&queue_mutex, NULL) != 0) { - (*MIR_get_error_func (ctx)) (MIR_parallel_error, "can not create a generator thread lock"); - } else if (mir_cond_init (&generate_signal, NULL) != 0) { - mir_mutex_destroy (&queue_mutex); - (*MIR_get_error_func (ctx)) (MIR_parallel_error, "can not create a generator thread signal"); - } else if (mir_cond_init (&done_signal, NULL) != 0) { - mir_cond_destroy (&generate_signal); - mir_mutex_destroy (&queue_mutex); - (*MIR_get_error_func (ctx)) (MIR_parallel_error, "can not create a generator thread signal"); - } else { - for (int i = 0; i < gens_num; i++) { - gen_ctx = &all_gen_ctx->gen_ctx[i]; - gen_ctx->busy_p = FALSE; - gen_ctx->gen_num = i; - gen_ctx->all_gen_ctx = all_gen_ctx; - if (mir_thread_create (&gen_ctx->gen_thread, NULL, gen, gen_ctx) != 0) { - signal_threads_to_finish (all_gen_ctx); - for (int j = 0; j < i; j++) mir_thread_join (all_gen_ctx->gen_ctx[j].gen_thread, NULL); - mir_cond_destroy (&done_signal); - mir_cond_destroy (&generate_signal); - mir_mutex_destroy (&queue_mutex); - (*MIR_get_error_func (ctx)) (MIR_parallel_error, "can not create a generator thread"); + curr_func_item->data = bb_stubs = gen_malloc (gen_ctx, sizeof (struct bb_stub) * n_bbs); + n_bbs = 0; + new_bb_p = TRUE; + for (insn = DLIST_HEAD (MIR_insn_t, curr_func_item->u.func->insns); insn != NULL; + insn = DLIST_NEXT (MIR_insn_t, insn)) { + if (insn->code == MIR_LABEL || new_bb_p) { + if (n_bbs != 0) bb_stubs[n_bbs - 1].last_insn = DLIST_PREV (MIR_insn_t, insn); + bb_stubs[n_bbs].func_item = curr_func_item; + bb_stubs[n_bbs].first_insn = insn; + DLIST_INIT (bb_version_t, bb_stubs[n_bbs].bb_versions); + last_lab_insn = insn; + if (insn->code == MIR_LABEL) { + insn->data = &bb_stubs[n_bbs]; + for (insn = DLIST_NEXT (MIR_insn_t, insn); insn != NULL && insn->code == MIR_LABEL; + last_lab_insn = insn, insn = DLIST_NEXT (MIR_insn_t, insn)) + insn->data = &bb_stubs[n_bbs]; + } + insn = last_lab_insn; + n_bbs++; + } + new_bb_p = MIR_any_branch_code_p (insn->code) || insn->code == MIR_RET || insn->code == MIR_JRET + || insn->code == MIR_PRBEQ || insn->code == MIR_PRBNE; + } + bb_stubs[n_bbs - 1].last_insn = DLIST_TAIL (MIR_insn_t, curr_func_item->u.func->insns); + if (debug_file != NULL) { + fprintf (debug_file, "BBs for lazy code generation:\n"); + for (size_t i = 0; i < n_bbs; i++) { + fprintf (debug_file, " BBStub%llx:\n", (long long unsigned) &bb_stubs[i]); + for (insn = bb_stubs[i].first_insn;; insn = DLIST_NEXT (MIR_insn_t, insn)) { + MIR_output_insn (ctx, debug_file, insn, curr_func_item->u.func, TRUE); + if (insn == bb_stubs[i].last_insn) break; } } } -#endif - for (int n = 0; n < gens_num; n++) { - gen_ctx = &all_gen_ctx->gen_ctx[n]; -#if !MIR_PARALLEL_GEN - gen_ctx->all_gen_ctx = all_gen_ctx; - gen_ctx->gen_num = n; -#endif - gen_ctx->ctx = ctx; - optimize_level = 2; - gen_ctx->target_ctx = NULL; - gen_ctx->data_flow_ctx = NULL; - gen_ctx->gvn_ctx = NULL; - gen_ctx->ccp_ctx = NULL; - gen_ctx->lr_ctx = NULL; - gen_ctx->ra_ctx = NULL; - gen_ctx->selection_ctx = NULL; + for (MIR_lref_data_t lref = curr_func_item->u.func->first_lref; lref != NULL; lref = lref->next) { + bb_stub_t lab_bb_stub = lref->label->data; + void *addr, *addr2; + (void) get_bb_version (gen_ctx, lab_bb_stub, 0, NULL, FALSE, &addr); + if (lref->label2 != NULL) { + lab_bb_stub = lref->label2->data; + (void) get_bb_version (gen_ctx, lab_bb_stub, 0, NULL, FALSE, &addr2); + addr = (void *) ((char *) addr - (char *) addr2); + } + addr = (void *) ((char *) addr + lref->disp); + *(void **) lref->load_addr = addr; + } +} + +void MIR_gen_init (MIR_context_t ctx) { + gen_ctx_t gen_ctx, *gen_ctx_ptr = gen_ctx_loc (ctx); + + *gen_ctx_ptr = gen_ctx = gen_malloc (NULL, sizeof (struct gen_ctx)); + gen_ctx->ctx = ctx; + optimize_level = 2; + gen_ctx->target_ctx = NULL; + gen_ctx->data_flow_ctx = NULL; + gen_ctx->gvn_ctx = NULL; + gen_ctx->lr_ctx = NULL; + gen_ctx->ra_ctx = NULL; + gen_ctx->combine_ctx = NULL; #if !MIR_NO_GEN_DEBUG - debug_file = NULL; - debug_level = 100; + debug_file = NULL; + debug_level = 100; #endif - VARR_CREATE (bb_insn_t, dead_bb_insns, 16); - VARR_CREATE (loop_node_t, loop_nodes, 32); - VARR_CREATE (loop_node_t, queue_nodes, 32); - VARR_CREATE (loop_node_t, loop_entries, 16); - init_dead_vars (gen_ctx); - init_data_flow (gen_ctx); - init_ssa (gen_ctx); - init_gvn (gen_ctx); - init_ccp (gen_ctx); - temp_bitmap = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); - temp_bitmap2 = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); - init_live_ranges (gen_ctx); - init_ra (gen_ctx); - init_selection (gen_ctx); - target_init (gen_ctx); - max_int_hard_regs = max_fp_hard_regs = 0; + VARR_CREATE (void_ptr_t, to_free, 0); + addr_insn_p = FALSE; + VARR_CREATE (MIR_op_t, temp_ops, 16); + VARR_CREATE (MIR_insn_t, temp_insns, 16); + VARR_CREATE (MIR_insn_t, temp_insns2, 16); + VARR_CREATE (bb_insn_t, temp_bb_insns, 16); + VARR_CREATE (bb_insn_t, temp_bb_insns2, 16); + VARR_CREATE (loop_node_t, loop_nodes, 32); + VARR_CREATE (loop_node_t, queue_nodes, 32); + VARR_CREATE (loop_node_t, loop_entries, 16); + VARR_CREATE (mem_attr_t, mem_attrs, 32); + VARR_CREATE (target_bb_version_t, target_succ_bb_versions, 16); + VARR_CREATE (void_ptr_t, succ_bb_addrs, 16); + VARR_CREATE (spot_attr_t, spot_attrs, 32); + VARR_CREATE (spot_attr_t, spot2attr, 32); + temp_bitmap = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); + temp_bitmap2 = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); + temp_bitmap3 = bitmap_create2 (DEFAULT_INIT_BITMAP_BITS_NUM); + init_dead_vars (gen_ctx); + init_data_flow (gen_ctx); + init_ssa (gen_ctx); + init_gvn (gen_ctx); + init_live_ranges (gen_ctx); + init_coalesce (gen_ctx); + init_ra (gen_ctx); + init_combine (gen_ctx); + target_init (gen_ctx); + max_int_hard_regs = max_fp_hard_regs = 0; + for (int i = 0; i <= MAX_HARD_REG; i++) { + if (target_fixed_hard_reg_p (i)) continue; + target_hard_reg_type_ok_p (i, MIR_T_I32) ? max_int_hard_regs++ : max_fp_hard_regs++; + } + for (MIR_type_t type = MIR_T_I8; type < MIR_T_BOUND; type++) { + call_used_hard_regs[type] = bitmap_create2 (MAX_HARD_REG + 1); for (int i = 0; i <= MAX_HARD_REG; i++) { - if (target_fixed_hard_reg_p (i)) continue; - target_hard_reg_type_ok_p (i, MIR_T_I32) ? max_int_hard_regs++ : max_fp_hard_regs++; - } - for (MIR_type_t type = MIR_T_I8; type < MIR_T_BOUND; type++) { - call_used_hard_regs[type] = bitmap_create2 (MAX_HARD_REG + 1); - for (int i = 0; i <= MAX_HARD_REG; i++) { - /* We need call_used_hard_regs even for fixed regs in combine. */ - if (target_call_used_hard_reg_p (i, type)) bitmap_set_bit_p (call_used_hard_regs[type], i); - } + /* We need call_used_hard_regs even for fixed regs in combine. */ + if (target_call_used_hard_reg_p (i, type)) bitmap_set_bit_p (call_used_hard_regs[type], i); } - insn_to_consider = bitmap_create2 (1024); - func_used_hard_regs = bitmap_create2 (MAX_HARD_REG + 1); } + tied_regs = bitmap_create2 (256); + addr_regs = bitmap_create2 (256); + insn_to_consider = bitmap_create2 (1024); + func_used_hard_regs = bitmap_create2 (MAX_HARD_REG + 1); + bb_wrapper = _MIR_get_bb_wrapper (ctx, gen_ctx, bb_version_generator); + overall_bbs_num = overall_gen_bbs_num = 0; } void MIR_gen_finish (MIR_context_t ctx) { - struct all_gen_ctx **all_gen_ctx_ptr = all_gen_ctx_loc (ctx), *all_gen_ctx = *all_gen_ctx_ptr; - gen_ctx_t gen_ctx; - -#if MIR_PARALLEL_GEN - signal_threads_to_finish (all_gen_ctx); - for (int i = 0; i < all_gen_ctx->gens_num; i++) - mir_thread_join (all_gen_ctx->gen_ctx[i].gen_thread, NULL); - if (mir_mutex_destroy (&queue_mutex) != 0 || mir_cond_destroy (&generate_signal) != 0 - || mir_cond_destroy (&done_signal) != 0) { // ??? - (*MIR_get_error_func (all_gen_ctx->ctx)) (MIR_parallel_error, - "can not destroy generator mutex or signals"); - } - VARR_DESTROY (MIR_item_t, funcs_to_generate); -#endif - for (int i = 0; i < all_gen_ctx->gens_num; i++) { - gen_ctx = &all_gen_ctx->gen_ctx[i]; - finish_data_flow (gen_ctx); - finish_ssa (gen_ctx); - finish_gvn (gen_ctx); - finish_ccp (gen_ctx); - bitmap_destroy (temp_bitmap); - bitmap_destroy (temp_bitmap2); - finish_live_ranges (gen_ctx); - finish_ra (gen_ctx); - finish_selection (gen_ctx); - for (MIR_type_t type = MIR_T_I8; type < MIR_T_BOUND; type++) - bitmap_destroy (call_used_hard_regs[type]); - bitmap_destroy (insn_to_consider); - bitmap_destroy (func_used_hard_regs); - target_finish (gen_ctx); - finish_dead_vars (gen_ctx); - free (gen_ctx->data_flow_ctx); - VARR_DESTROY (bb_insn_t, dead_bb_insns); - VARR_DESTROY (loop_node_t, loop_nodes); - VARR_DESTROY (loop_node_t, queue_nodes); - VARR_DESTROY (loop_node_t, loop_entries); - } - free (all_gen_ctx); - *all_gen_ctx_ptr = NULL; -} + gen_ctx_t *gen_ctx_ptr = gen_ctx_loc (ctx), gen_ctx = *gen_ctx_ptr; -void MIR_set_gen_interface (MIR_context_t ctx, MIR_item_t func_item) { - if (func_item == NULL) return; /* finish setting interfaces */ - MIR_gen (ctx, 0, func_item); + if (gen_ctx == NULL) { + fprintf (stderr, "Calling MIR_gen_finish before MIR_gen_init -- good bye\n"); + exit (1); + } + finish_data_flow (gen_ctx); + finish_ssa (gen_ctx); + finish_gvn (gen_ctx); + finish_live_ranges (gen_ctx); + finish_coalesce (gen_ctx); + finish_ra (gen_ctx); + finish_combine (gen_ctx); + for (MIR_type_t type = MIR_T_I8; type < MIR_T_BOUND; type++) + bitmap_destroy (call_used_hard_regs[type]); + bitmap_destroy (tied_regs); + bitmap_destroy (addr_regs); + bitmap_destroy (insn_to_consider); + bitmap_destroy (func_used_hard_regs); + target_finish (gen_ctx); + finish_dead_vars (gen_ctx); + free (gen_ctx->data_flow_ctx); + bitmap_destroy (temp_bitmap); + bitmap_destroy (temp_bitmap2); + bitmap_destroy (temp_bitmap3); + VARR_DESTROY (MIR_op_t, temp_ops); + VARR_DESTROY (MIR_insn_t, temp_insns); + VARR_DESTROY (MIR_insn_t, temp_insns2); + VARR_DESTROY (bb_insn_t, temp_bb_insns); + VARR_DESTROY (bb_insn_t, temp_bb_insns2); + VARR_DESTROY (loop_node_t, loop_nodes); + VARR_DESTROY (loop_node_t, queue_nodes); + VARR_DESTROY (loop_node_t, loop_entries); + VARR_DESTROY (mem_attr_t, mem_attrs); + VARR_DESTROY (target_bb_version_t, target_succ_bb_versions); + VARR_DESTROY (void_ptr_t, succ_bb_addrs); + VARR_DESTROY (spot_attr_t, spot_attrs); + VARR_DESTROY (spot_attr_t, spot2attr); + while (VARR_LENGTH (void_ptr_t, to_free) != 0) free (VARR_POP (void_ptr_t, to_free)); + VARR_DESTROY (void_ptr_t, to_free); + if (collect_bb_stat_p) + fprintf (stderr, "Overall bbs num = %llu, generated bbs num = %llu\n", overall_bbs_num, + overall_gen_bbs_num); + free (gen_ctx); + *gen_ctx_ptr = NULL; } -void MIR_set_parallel_gen_interface (MIR_context_t ctx, MIR_item_t func_item) { - struct all_gen_ctx *all_gen_ctx = *all_gen_ctx_loc (ctx); - -#if !MIR_PARALLEL_GEN - if (func_item == NULL) return; /* finish setting interfaces */ - MIR_gen (ctx, 0, func_item); -#else - if (func_item == NULL) { - size_t i; - - if (mir_mutex_lock (&queue_mutex)) parallel_error (ctx, "error in mutex lock"); - for (;;) { - for (i = 0; i < all_gen_ctx->gens_num; i++) - if (all_gen_ctx->gen_ctx[i].busy_p) break; - if (VARR_LENGTH (MIR_item_t, funcs_to_generate) <= funcs_start && i >= all_gen_ctx->gens_num) - break; /* nothing to generate and nothing is being generated */ - if (mir_cond_wait (&done_signal, &queue_mutex)) parallel_error (ctx, "error in cond wait"); - } - if (mir_mutex_unlock (&queue_mutex)) parallel_error (ctx, "error in mutex unlock"); +void MIR_set_gen_interface (MIR_context_t ctx, MIR_item_t func_item) { + if (func_item == NULL) { /* finish setting interfaces */ + target_change_to_direct_calls (ctx); } else { - if (mir_mutex_lock (&queue_mutex)) parallel_error (ctx, "error in mutex lock"); - VARR_PUSH (MIR_item_t, funcs_to_generate, func_item); - if (mir_cond_broadcast (&generate_signal)) parallel_error (ctx, "error in cond broadcast"); - if (mir_mutex_unlock (&queue_mutex)) parallel_error (ctx, "error in mutex unlock"); + MIR_gen (ctx, func_item); } -#endif } -static void *gen_and_redirect (MIR_context_t ctx, MIR_item_t func_item) { -#if !MIR_PARALLEL_GEN - MIR_gen (ctx, 0, func_item); -#else - struct all_gen_ctx *all_gen_ctx = *all_gen_ctx_loc (ctx); - MIR_func_t func = func_item->u.func; - - if (mir_mutex_lock (&queue_mutex)) parallel_error (ctx, "error in mutex lock"); - VARR_PUSH (MIR_item_t, funcs_to_generate, func_item); - if (mir_cond_broadcast (&generate_signal)) parallel_error (ctx, "error in cond broadcast"); - if (mir_mutex_unlock (&queue_mutex)) parallel_error (ctx, "error in mutex unlock"); - if (mir_mutex_lock (&queue_mutex)) parallel_error (ctx, "error in mutex lock"); - for (;;) { - if (func->machine_code != NULL) break; - if (mir_cond_wait (&done_signal, &queue_mutex)) parallel_error (ctx, "error in cond wait"); - } - if (mir_mutex_unlock (&queue_mutex)) parallel_error (ctx, "error in mutex unlock"); -#endif +/* Lazy func generation is done right away. */ +static void generate_func_and_redirect (MIR_context_t ctx, MIR_item_t func_item, int full_p) { + generate_func_code (ctx, func_item, full_p); + if (full_p) return; + gen_ctx_t gen_ctx = *gen_ctx_loc (ctx); + void *addr; + create_bb_stubs (gen_ctx); + (void) get_bb_version (gen_ctx, &((struct bb_stub *) func_item->data)[0], 0, NULL, TRUE, &addr); + _MIR_redirect_thunk (ctx, func_item->addr, addr); +} + +static void *generate_func_and_redirect_to_func_code (MIR_context_t ctx, MIR_item_t func_item) { + generate_func_and_redirect (ctx, func_item, TRUE); return func_item->u.func->machine_code; } @@ -5808,7 +9719,235 @@ void MIR_set_lazy_gen_interface (MIR_context_t ctx, MIR_item_t func_item) { void *addr; if (func_item == NULL) return; - addr = _MIR_get_wrapper (ctx, func_item, gen_and_redirect); + addr = _MIR_get_wrapper (ctx, func_item, generate_func_and_redirect_to_func_code); + _MIR_redirect_thunk (ctx, func_item->addr, addr); +} + +static void set_spot2attr (gen_ctx_t gen_ctx, const spot_attr_t *attr) { + gen_assert (attr->spot != 0 && attr->prop != 0); + while (VARR_LENGTH (spot_attr_t, spot2attr) <= attr->spot) + VARR_PUSH (spot_attr_t, spot2attr, *attr); + VARR_SET (spot_attr_t, spot2attr, attr->spot, *attr); +} + +#define FIRST_MEM_SPOT (MAX_HARD_REG + 2) +static int mem_spot_p (uint32_t spot) { return spot >= FIRST_MEM_SPOT; } + +static uint32_t mem_nloc2spot (uint32_t nloc) { + return nloc == 0 ? 0 : nloc + 1 + MAX_HARD_REG + 1; +} + +static uint32_t op2spot (MIR_op_t *op_ref) { + if (op_ref->mode == MIR_OP_VAR) return op_ref->u.var + 1; + if (op_ref->mode == MIR_OP_VAR_MEM) return mem_nloc2spot (op_ref->u.var_mem.nloc); + return 0; +} + +static void generate_bb_version_machine_code (gen_ctx_t gen_ctx, bb_version_t bb_version) { + MIR_context_t ctx = gen_ctx->ctx; + int skip_p; + bb_stub_t branch_bb_stub, bb_stub = bb_version->bb_stub; + MIR_insn_t curr_insn, new_insn, next_insn; + void *addr; + uint8_t *code; + size_t code_len, nel; + uint32_t prop, spot, dest_spot, src_spot, max_spot = 0; + bitmap_t nonzero_property_spots = temp_bitmap; + bitmap_iterator_t bi; + spot_attr_t spot_attr; + + bitmap_clear (nonzero_property_spots); + DEBUG (2, { + fprintf (debug_file, " IN BBStub%llx nonzero properties: ", (long long unsigned) bb_stub); + }); + for (size_t i = 0; i < bb_version->n_attrs; i++) { + bitmap_set_bit_p (nonzero_property_spots, bb_version->attrs[i].spot); + set_spot2attr (gen_ctx, &bb_version->attrs[i]); + DEBUG (2, { + if (i != 0) fprintf (debug_file, ", "); + fprintf (debug_file, "(spot=%u,prop=%u)", (unsigned) bb_version->attrs[i].spot, + (unsigned) bb_version->attrs[i].prop); + }); + } + DEBUG (2, { fprintf (debug_file, "\n"); }); + if (bb_version->n_attrs != 0) max_spot = bb_version->attrs[bb_version->n_attrs - 1].spot; + VARR_TRUNC (target_bb_version_t, target_succ_bb_versions, 0); + target_bb_translate_start (gen_ctx); + for (curr_insn = bb_stub->first_insn;; curr_insn = next_insn) { + next_insn = DLIST_NEXT (MIR_insn_t, curr_insn); + if (MIR_any_branch_code_p (curr_insn->code)) break; + skip_p = FALSE; + switch (curr_insn->code) { + case MIR_USE: skip_p = TRUE; break; + case MIR_PRSET: + gen_assert (curr_insn->ops[1].mode == MIR_OP_INT || curr_insn->ops[1].mode == MIR_OP_UINT); + dest_spot = op2spot (&curr_insn->ops[0]); + if (dest_spot == 0) { + } else if (curr_insn->ops[1].u.i == 0) { /* ??? aliased */ + bitmap_clear_bit_p (nonzero_property_spots, dest_spot); + } else { + bitmap_set_bit_p (nonzero_property_spots, dest_spot); + spot_attr.spot = dest_spot; + spot_attr.prop = (uint32_t) curr_insn->ops[1].u.i; + spot_attr.mem_ref = mem_spot_p (dest_spot) ? &curr_insn->ops[0] : NULL; + set_spot2attr (gen_ctx, &spot_attr); + } + skip_p = TRUE; + break; + case MIR_PRBEQ: + case MIR_PRBNE: + gen_assert (curr_insn->ops[2].mode == MIR_OP_INT || curr_insn->ops[2].mode == MIR_OP_UINT); + spot = op2spot (&curr_insn->ops[1]); + prop = 0; + if (bitmap_bit_p (nonzero_property_spots, spot)) { + spot_attr = VARR_GET (spot_attr_t, spot2attr, spot); + prop = spot_attr.prop; + } + if ((curr_insn->code == MIR_PRBEQ && curr_insn->ops[2].u.i != prop) + || (curr_insn->code == MIR_PRBNE && curr_insn->ops[2].u.i == prop)) { + DEBUG (2, { + fprintf (debug_file, " Remove property insn "); + MIR_output_insn (ctx, debug_file, curr_insn, curr_func_item->u.func, TRUE); + }); + MIR_remove_insn (ctx, curr_func_item, curr_insn); + skip_p = TRUE; + break; + } else { /* make unconditional jump */ + new_insn = MIR_new_insn (ctx, MIR_JMP, curr_insn->ops[0]); + MIR_insert_insn_before (ctx, curr_func_item, curr_insn, new_insn); + DEBUG (2, { + fprintf (debug_file, " Change "); + MIR_output_insn (ctx, debug_file, curr_insn, curr_func_item->u.func, FALSE); + fprintf (debug_file, " to "); + MIR_output_insn (ctx, debug_file, new_insn, curr_func_item->u.func, TRUE); + }); + MIR_remove_insn (ctx, curr_func_item, curr_insn); + next_insn = new_insn; + continue; + } + case MIR_MOV: + case MIR_FMOV: + case MIR_DMOV: + case MIR_LDMOV: + dest_spot = op2spot (&curr_insn->ops[0]); + src_spot = op2spot (&curr_insn->ops[1]); + if (src_spot == 0) { + bitmap_clear_bit_p (nonzero_property_spots, dest_spot); + } else if (dest_spot == 0) { /* clear attrs of all memory locations */ + if (max_spot >= FIRST_MEM_SPOT) + bitmap_clear_bit_range_p (nonzero_property_spots, FIRST_MEM_SPOT, + max_spot - FIRST_MEM_SPOT + 1); + } else if (bitmap_bit_p (nonzero_property_spots, src_spot)) { + spot_attr = VARR_GET (spot_attr_t, spot2attr, src_spot); + spot_attr.mem_ref = NULL; + if (mem_spot_p (dest_spot)) { + spot_attr_t *spot_attr_addr = VARR_ADDR (spot_attr_t, spot_attrs); + for (spot = FIRST_MEM_SPOT; spot <= max_spot; spot++) + if (may_mem_alias_p (spot_attr_addr[dest_spot].mem_ref, spot_attr_addr[spot].mem_ref)) + bitmap_clear_bit_p (nonzero_property_spots, spot); + spot_attr.mem_ref = &curr_insn->ops[0]; + } + bitmap_set_bit_p (nonzero_property_spots, dest_spot); + spot_attr.spot = dest_spot; + set_spot2attr (gen_ctx, &spot_attr); + } + break; + default: break; + } + if (!skip_p) { + if (curr_insn->code != MIR_LADDR) { + target_bb_insn_translate (gen_ctx, curr_insn, NULL); + } else { + VARR_TRUNC (spot_attr_t, spot_attrs, 0); + FOREACH_BITMAP_BIT (bi, nonzero_property_spots, nel) { + VARR_PUSH (spot_attr_t, spot_attrs, VARR_GET (spot_attr_t, spot2attr, nel)); + } + VARR_TRUNC (void_ptr_t, succ_bb_addrs, 0); + branch_bb_stub = curr_insn->ops[1].u.label->data; + (void) get_bb_version (gen_ctx, branch_bb_stub, + (uint32_t) VARR_LENGTH (spot_attr_t, spot_attrs), + VARR_ADDR (spot_attr_t, spot_attrs), FALSE, &addr); + VARR_PUSH (void_ptr_t, succ_bb_addrs, addr); + target_bb_insn_translate (gen_ctx, curr_insn, VARR_ADDR (void_ptr_t, succ_bb_addrs)); + } + } + if (curr_insn == bb_stub->last_insn) break; + } + VARR_TRUNC (spot_attr_t, spot_attrs, 0); + DEBUG (2, { + fprintf (debug_file, " OUT BBStub%llx nonzero properties: ", (long long unsigned) bb_stub); + }); + FOREACH_BITMAP_BIT (bi, nonzero_property_spots, nel) { + if (MIR_call_code_p (curr_insn->code) && mem_spot_p ((uint32_t) nel)) break; + spot_attr = VARR_GET (spot_attr_t, spot2attr, nel); + DEBUG (2, { + if (VARR_LENGTH (spot_attr_t, spot_attrs) != 0) fprintf (debug_file, ", "); + fprintf (debug_file, "(spot=%u,prop=%u)", (unsigned) spot_attr.spot, + (unsigned) spot_attr.prop); + }); + VARR_PUSH (spot_attr_t, spot_attrs, spot_attr); + } + DEBUG (2, { fprintf (debug_file, "\n"); }); + VARR_TRUNC (void_ptr_t, succ_bb_addrs, 0); + if (curr_insn->code == MIR_JMPI) { + target_bb_insn_translate (gen_ctx, curr_insn, NULL); + } else if (curr_insn->code == MIR_SWITCH) { + for (size_t i = 1; i < curr_insn->nops; i++) { + branch_bb_stub = curr_insn->ops[i].u.label->data; + (void) get_bb_version (gen_ctx, branch_bb_stub, + (uint32_t) VARR_LENGTH (spot_attr_t, spot_attrs), + VARR_ADDR (spot_attr_t, spot_attrs), FALSE, &addr); + VARR_PUSH (void_ptr_t, succ_bb_addrs, addr); + } + target_bb_insn_translate (gen_ctx, curr_insn, VARR_ADDR (void_ptr_t, succ_bb_addrs)); + } else if (MIR_branch_code_p (curr_insn->code)) { // ??? generate branch + branch_bb_stub = curr_insn->ops[0].u.label->data; + (void) get_bb_version (gen_ctx, branch_bb_stub, + (uint32_t) VARR_LENGTH (spot_attr_t, spot_attrs), + VARR_ADDR (spot_attr_t, spot_attrs), FALSE, &addr); + VARR_PUSH (void_ptr_t, succ_bb_addrs, addr); + target_bb_insn_translate (gen_ctx, curr_insn, VARR_ADDR (void_ptr_t, succ_bb_addrs)); + } + if (curr_insn->code != MIR_JMP && curr_insn->code != MIR_JMPI && curr_insn->code != MIR_SWITCH + && curr_insn->code != MIR_RET && curr_insn->code != MIR_JRET) { + VARR_TRUNC (void_ptr_t, succ_bb_addrs, 0); + (void) get_bb_version (gen_ctx, bb_stub + 1, (uint32_t) VARR_LENGTH (spot_attr_t, spot_attrs), + VARR_ADDR (spot_attr_t, spot_attrs), FALSE, &addr); + VARR_PUSH (void_ptr_t, succ_bb_addrs, addr); + target_output_jump (gen_ctx, VARR_ADDR (void_ptr_t, succ_bb_addrs)); + } + code = target_bb_translate_finish (gen_ctx, &code_len); + addr = _MIR_publish_code (ctx, code, code_len); + target_bb_rebase (gen_ctx, addr); + target_setup_succ_bb_version_data (gen_ctx, addr); + DEBUG (1, { + _MIR_dump_code (NULL, addr, code_len); + fprintf (debug_file, "BBStub%llx code size = %lu:\n", (unsigned long long) bb_stub, + (unsigned long) code_len); + }); + target_redirect_bb_origin_branch (gen_ctx, &bb_version->target_data, addr); + _MIR_replace_bb_thunk (ctx, bb_version->addr, addr); + bb_version->addr = addr; + overall_gen_bbs_num++; + bb_version->machine_code = addr; +} + +static void *bb_version_generator (gen_ctx_t gen_ctx, bb_version_t bb_version) { + generate_bb_version_machine_code (gen_ctx, bb_version); + return bb_version->machine_code; +} + +/* attrs ignored ??? implement versions */ +static void *generate_func_and_redirect_to_bb_gen (MIR_context_t ctx, MIR_item_t func_item) { + generate_func_and_redirect (ctx, func_item, FALSE); + return func_item->addr; +} + +void MIR_set_lazy_bb_gen_interface (MIR_context_t ctx, MIR_item_t func_item) { + void *addr; + + if (func_item == NULL) return; /* finish setting interfaces */ + addr = _MIR_get_wrapper (ctx, func_item, generate_func_and_redirect_to_bb_gen); _MIR_redirect_thunk (ctx, func_item->addr, addr); } diff --git a/mir/mir-gen.h b/mir/mir-gen.h index a1fca94b..5ec08680 100644 --- a/mir/mir-gen.h +++ b/mir/mir-gen.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ #ifndef MIR_GEN_H @@ -16,14 +16,14 @@ extern "C" { #endif -extern void MIR_gen_init (MIR_context_t ctx, int gens_num); -extern void MIR_gen_set_debug_file (MIR_context_t ctx, int gen_num, FILE *f); -extern void MIR_gen_set_debug_level (MIR_context_t ctx, int gen_num, int debug_level); -extern void MIR_gen_set_optimize_level (MIR_context_t ctx, int gen_num, unsigned int level); -extern void *MIR_gen (MIR_context_t ctx, int gen_num, MIR_item_t func_item); +extern void MIR_gen_init (MIR_context_t ctx); +extern void MIR_gen_set_debug_file (MIR_context_t ctx, FILE *f); +extern void MIR_gen_set_debug_level (MIR_context_t ctx, int debug_level); +extern void MIR_gen_set_optimize_level (MIR_context_t ctx, unsigned int level); +extern void *MIR_gen (MIR_context_t ctx, MIR_item_t func_item); extern void MIR_set_gen_interface (MIR_context_t ctx, MIR_item_t func_item); -extern void MIR_set_parallel_gen_interface (MIR_context_t ctx, MIR_item_t func_item); extern void MIR_set_lazy_gen_interface (MIR_context_t ctx, MIR_item_t func_item); +extern void MIR_set_lazy_bb_gen_interface (MIR_context_t ctx, MIR_item_t func_item); extern void MIR_gen_finish (MIR_context_t ctx); #ifdef __cplusplus diff --git a/mir/mir-hash.h b/mir/mir-hash.h index 6ce7d97f..1b5ae668 100644 --- a/mir/mir-hash.h +++ b/mir/mir-hash.h @@ -1,6 +1,6 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ /* Simple high-quality multiplicative hash passing demerphq-smhasher, @@ -45,7 +45,7 @@ static inline uint64_t mir_get_key_part (const uint8_t *v, size_t len, int relax return tail; } -static const uint64_t p1 = 0X65862b62bdf5ef4d, p2 = 0X288eea216831e6a7; +static const uint64_t mir_hash_p1 = 0X65862b62bdf5ef4d, mir_hash_p2 = 0X288eea216831e6a7; static inline uint64_t mir_mum (uint64_t v, uint64_t c, int relax_p) { if (relax_p) { #if defined(__SIZEOF_INT128__) @@ -58,8 +58,8 @@ static inline uint64_t mir_mum (uint64_t v, uint64_t c, int relax_p) { } static inline uint64_t mir_round (uint64_t state, uint64_t v, int relax_p) { - state ^= mir_mum (v, p1, relax_p); - return state ^ mir_mum (state, p2, relax_p); + state ^= mir_mum (v, mir_hash_p1, relax_p); + return state ^ mir_mum (state, mir_hash_p2, relax_p); } static inline uint64_t mir_hash_1 (const void *key, size_t len, uint64_t seed, int relax_p) { @@ -67,15 +67,15 @@ static inline uint64_t mir_hash_1 (const void *key, size_t len, uint64_t seed, i uint64_t r = seed + len; for (; len >= 16; len -= 16, v += 16) { - r ^= mir_mum (mir_get_key_part (v, 8, relax_p), p1, relax_p); - r ^= mir_mum (mir_get_key_part (v + 8, 8, relax_p), p2, relax_p); - r ^= mir_mum (r, p1, relax_p); + r ^= mir_mum (mir_get_key_part (v, 8, relax_p), mir_hash_p1, relax_p); + r ^= mir_mum (mir_get_key_part (v + 8, 8, relax_p), mir_hash_p2, relax_p); + r ^= mir_mum (r, mir_hash_p1, relax_p); } if (len >= 8) { - r ^= mir_mum (mir_get_key_part (v, 8, relax_p), p1, relax_p); + r ^= mir_mum (mir_get_key_part (v, 8, relax_p), mir_hash_p1, relax_p); len -= 8, v += 8; } - if (len != 0) r ^= mir_mum (mir_get_key_part (v, len, relax_p), p2, relax_p); + if (len != 0) r ^= mir_mum (mir_get_key_part (v, len, relax_p), mir_hash_p2, relax_p); return mir_round (r, r, relax_p); } diff --git a/mir/mir-htab.h b/mir/mir-htab.h index abd914a1..af7593a1 100644 --- a/mir/mir-htab.h +++ b/mir/mir-htab.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ #ifndef MIR_HTAB_H diff --git a/mir/mir-interp.c b/mir/mir-interp.c index 7b5a38e4..64d3c34f 100644 --- a/mir/mir-interp.c +++ b/mir/mir-interp.c @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . File contains MIR interpreter which is an obligatory part of MIR API. */ @@ -42,7 +42,7 @@ typedef struct func_desc { MIR_reg_t nregs; MIR_item_t func_item; MIR_val_t code[1]; -} * func_desc_t; +} *func_desc_t; static void update_max_nreg (MIR_reg_t reg, MIR_reg_t *max_nreg) { if (*max_nreg < reg) *max_nreg = reg; @@ -62,8 +62,9 @@ typedef enum { REP6 (IC_EL, LDU8, LDI16, LDU16, LDI32, LDU32, LDI64), REP3 (IC_EL, LDF, LDD, LDLD), REP7 (IC_EL, STI8, STU8, STI16, STU16, STI32, STU32, STI64), - REP3 (IC_EL, STF, STD, STLD), - REP7 (IC_EL, MOVI, MOVP, MOVF, MOVD, MOVLD, IMM_CALL, INSN_BOUND), + REP8 (IC_EL, STF, STD, STLD, MOVI, MOVP, MOVF, MOVD, MOVLD), + REP6 (IC_EL, IMM_CALL, IMM_JCALL, MOVFG, FMOVFG, DMOVFG, LDMOVFG), + REP5 (IC_EL, MOVTG, FMOVTG, DMOVTG, LDMOVTG, INSN_BOUND), } MIR_full_insn_code_t; #undef REP_SEP @@ -85,6 +86,7 @@ struct interp_ctx { #if DIRECT_THREADED_DISPATCH void *dispatch_label_tab[IC_INSN_BOUND]; #endif + MIR_val_t global_regs[MAX_HARD_REG + 1]; VARR (MIR_val_t) * code_varr; VARR (MIR_insn_t) * branches; VARR (MIR_val_t) * arg_vals_varr; @@ -94,6 +96,7 @@ struct interp_ctx { #endif void *(*bstart_builtin) (void); void (*bend_builtin) (void *); + void *jret_addr; VARR (MIR_val_t) * call_res_args_varr; MIR_val_t *call_res_args; VARR (_MIR_arg_desc_t) * call_arg_descs_varr; @@ -102,8 +105,10 @@ struct interp_ctx { }; #define dispatch_label_tab interp_ctx->dispatch_label_tab +#define global_regs interp_ctx->global_regs #define code_varr interp_ctx->code_varr #define branches interp_ctx->branches +#define jret_addr interp_ctx->jret_addr #define arg_vals_varr interp_ctx->arg_vals_varr #define arg_vals interp_ctx->arg_vals #define trace_insn_ident interp_ctx->trace_insn_ident @@ -124,7 +129,8 @@ static void get_icode (struct interp_ctx *interp_ctx, MIR_val_t *v, int code) { #endif } -static void push_insn_start (struct interp_ctx *interp_ctx, int code, MIR_insn_t original_insn) { +static void push_insn_start (struct interp_ctx *interp_ctx, int code, + MIR_insn_t original_insn MIR_UNUSED) { MIR_val_t v; get_icode (interp_ctx, &v, code); @@ -170,6 +176,7 @@ static void generate_icode (MIR_context_t ctx, MIR_item_t func_item) { int imm_call_p; MIR_func_t func = func_item->u.func; MIR_insn_t insn, label; + MIR_type_t type; MIR_val_t v; size_t i; MIR_reg_t max_nreg = 0; @@ -216,12 +223,44 @@ static void generate_icode (MIR_context_t ctx, MIR_item_t func_item) { v.a = item->addr; VARR_PUSH (MIR_val_t, code_varr, v); } else { - mir_assert (ops[1].mode == MIR_OP_REG); - push_insn_start (interp_ctx, code, insn); - v.i = get_reg (ops[0], &max_nreg); - VARR_PUSH (MIR_val_t, code_varr, v); - v.i = ops[1].u.reg; - VARR_PUSH (MIR_val_t, code_varr, v); + const char *hard_reg_name; + regreg: + mir_assert (ops[0].mode == MIR_OP_REG && ops[1].mode == MIR_OP_REG); + type = MIR_reg_type (ctx, ops[0].u.reg, func); + mir_assert (type == MIR_reg_type (ctx, ops[1].u.reg, func)); + if ((hard_reg_name = MIR_reg_hard_reg_name (ctx, ops[0].u.reg, func)) != NULL) { + mir_assert (MIR_reg_hard_reg_name (ctx, ops[1].u.reg, func) == NULL); + push_insn_start (interp_ctx, + type == MIR_T_F ? IC_FMOVTG + : type == MIR_T_D ? IC_DMOVTG + : type == MIR_T_LD ? IC_LDMOVTG + : IC_MOVTG, + insn); + v.i = _MIR_get_hard_reg (ctx, hard_reg_name); + mir_assert (v.i <= MAX_HARD_REG); + VARR_PUSH (MIR_val_t, code_varr, v); + v.i = get_reg (ops[1], &max_nreg); + VARR_PUSH (MIR_val_t, code_varr, v); + } else if ((hard_reg_name = MIR_reg_hard_reg_name (ctx, ops[1].u.reg, func)) != NULL) { + mir_assert (MIR_reg_hard_reg_name (ctx, ops[0].u.reg, func) == NULL); + push_insn_start (interp_ctx, + type == MIR_T_F ? IC_FMOVFG + : type == MIR_T_D ? IC_DMOVFG + : type == MIR_T_LD ? IC_LDMOVFG + : IC_MOVFG, + insn); + v.i = get_reg (ops[0], &max_nreg); + VARR_PUSH (MIR_val_t, code_varr, v); + v.i = _MIR_get_hard_reg (ctx, hard_reg_name); + mir_assert (v.i <= MAX_HARD_REG); + VARR_PUSH (MIR_val_t, code_varr, v); + } else { + push_insn_start (interp_ctx, code, insn); + v.i = get_reg (ops[0], &max_nreg); + VARR_PUSH (MIR_val_t, code_varr, v); + v.i = get_reg (ops[1], &max_nreg); + VARR_PUSH (MIR_val_t, code_varr, v); + } } break; case MIR_FMOV: @@ -242,12 +281,7 @@ static void generate_icode (MIR_context_t ctx, MIR_item_t func_item) { v.f = ops[1].u.f; VARR_PUSH (MIR_val_t, code_varr, v); } else { - mir_assert (ops[1].mode == MIR_OP_REG); - push_insn_start (interp_ctx, code, insn); - v.i = get_reg (ops[0], &max_nreg); - VARR_PUSH (MIR_val_t, code_varr, v); - v.i = ops[1].u.reg; - VARR_PUSH (MIR_val_t, code_varr, v); + goto regreg; } break; case MIR_DMOV: @@ -268,12 +302,7 @@ static void generate_icode (MIR_context_t ctx, MIR_item_t func_item) { v.d = ops[1].u.d; VARR_PUSH (MIR_val_t, code_varr, v); } else { - mir_assert (ops[1].mode == MIR_OP_REG); - push_insn_start (interp_ctx, code, insn); - v.i = get_reg (ops[0], &max_nreg); - VARR_PUSH (MIR_val_t, code_varr, v); - v.i = ops[1].u.reg; - VARR_PUSH (MIR_val_t, code_varr, v); + goto regreg; } break; case MIR_LDMOV: @@ -294,12 +323,7 @@ static void generate_icode (MIR_context_t ctx, MIR_item_t func_item) { v.ld = ops[1].u.ld; VARR_PUSH (MIR_val_t, code_varr, v); } else { - mir_assert (ops[1].mode == MIR_OP_REG); - push_insn_start (interp_ctx, code, insn); - v.i = get_reg (ops[0], &max_nreg); - VARR_PUSH (MIR_val_t, code_varr, v); - v.i = ops[1].u.reg; - VARR_PUSH (MIR_val_t, code_varr, v); + goto regreg; } break; case MIR_LABEL: break; @@ -312,6 +336,14 @@ static void generate_icode (MIR_context_t ctx, MIR_item_t func_item) { v.i = 0; VARR_PUSH (MIR_val_t, code_varr, v); break; + case MIR_LADDR: + VARR_PUSH (MIR_insn_t, branches, insn); + push_insn_start (interp_ctx, code, insn); + v.i = get_reg (ops[0], &max_nreg); + VARR_PUSH (MIR_val_t, code_varr, v); + v.i = 0; + VARR_PUSH (MIR_val_t, code_varr, v); + break; case MIR_BT: case MIR_BTS: case MIR_BF: @@ -370,6 +402,27 @@ static void generate_icode (MIR_context_t ctx, MIR_item_t func_item) { v.i = get_reg (ops[2], &max_nreg); VARR_PUSH (MIR_val_t, code_varr, v); break; + case MIR_BO: + case MIR_UBO: + case MIR_BNO: + case MIR_UBNO: + VARR_PUSH (MIR_insn_t, branches, insn); + push_insn_start (interp_ctx, code, insn); + v.i = 0; + VARR_PUSH (MIR_val_t, code_varr, v); + break; + case MIR_PRSET: break; /* just ignore */ + case MIR_PRBEQ: /* make jump if property is zero or ignore otherwise */ + if (ops[2].mode == MIR_OP_INT && ops[2].u.i == 0) goto jump; + break; + case MIR_PRBNE: /* make jump if property is nonzero or ignore otherwise */ + if (ops[2].mode != MIR_OP_INT || ops[2].u.i == 0) break; + jump: + VARR_PUSH (MIR_insn_t, branches, insn); + push_insn_start (interp_ctx, MIR_JMP, insn); + v.i = 0; + VARR_PUSH (MIR_val_t, code_varr, v); /* place for label */ + break; default: imm_call_p = FALSE; if (MIR_call_code_p (code)) @@ -379,7 +432,7 @@ static void generate_icode (MIR_context_t ctx, MIR_item_t func_item) { || ops[1].u.ref->item_type == MIR_forward_item || ops[1].u.ref->item_type == MIR_func_item)); push_insn_start (interp_ctx, - imm_call_p ? IC_IMM_CALL + imm_call_p ? (code == MIR_JCALL ? IC_IMM_JCALL : IC_IMM_CALL) : code == MIR_INLINE ? MIR_CALL : code, insn); @@ -417,7 +470,7 @@ static void generate_icode (MIR_context_t ctx, MIR_item_t func_item) { } else if (MIR_call_code_p (code) && ops[i].mode == MIR_OP_MEM) { mir_assert (MIR_all_blk_type_p (ops[i].u.mem.type)); v.i = ops[i].u.mem.base; - update_max_nreg (v.i, &max_nreg); + update_max_nreg ((MIR_reg_t) v.i, &max_nreg); } else { mir_assert (ops[i].mode == MIR_OP_REG); v.i = get_reg (ops[i], &max_nreg); @@ -430,7 +483,10 @@ static void generate_icode (MIR_context_t ctx, MIR_item_t func_item) { size_t start_label_nop = 0, bound_label_nop = 1, start_label_loc = 1, n; insn = VARR_GET (MIR_insn_t, branches, i); - if (insn->code == MIR_SWITCH) { + if (insn->code == MIR_LADDR) { + start_label_nop = 1; + bound_label_nop = 2; + } else if (insn->code == MIR_SWITCH) { start_label_nop = 1; bound_label_nop = start_label_nop + insn->nops - 1; start_label_loc++; /* we put nops for MIR_SWITCH */ @@ -451,6 +507,14 @@ static void generate_icode (MIR_context_t ctx, MIR_item_t func_item) { (*MIR_get_error_func (ctx)) (MIR_alloc_error, "no memory for interpreter code"); memmove (func_desc->code, VARR_ADDR (MIR_val_t, code_varr), VARR_LENGTH (MIR_val_t, code_varr) * sizeof (MIR_val_t)); + for (MIR_lref_data_t lref = func->first_lref; lref != NULL; lref = lref->next) { + if (lref->label2 == NULL) + *(void **) lref->load_addr + = (char *) (func_desc->code + (int64_t) lref->label->data) + lref->disp; + else + *(int64_t *) lref->load_addr + = (int64_t) lref->label->data - (int64_t) lref->label2->data + lref->disp; + } mir_assert (max_nreg < MIR_MAX_REG_NUM); func_desc->nregs = max_nreg + 1; func_desc->func_item = func_item; @@ -485,7 +549,7 @@ static ALWAYS_INLINE int64_t *get_2iops (MIR_val_t *bp, code_t c, int64_t *p) { } static ALWAYS_INLINE int64_t *get_2isops (MIR_val_t *bp, code_t c, int32_t *p) { - *p = *get_iop (bp, c + 1); + *p = (int32_t) *get_iop (bp, c + 1); return get_iop (bp, c); } @@ -496,8 +560,8 @@ static ALWAYS_INLINE int64_t *get_3iops (MIR_val_t *bp, code_t c, int64_t *p1, i } static ALWAYS_INLINE int64_t *get_3isops (MIR_val_t *bp, code_t c, int32_t *p1, int32_t *p2) { - *p1 = *get_iop (bp, c + 1); - *p2 = *get_iop (bp, c + 2); + *p1 = (int32_t) *get_iop (bp, c + 1); + *p2 = (int32_t) *get_iop (bp, c + 2); return get_iop (bp, c); } @@ -508,8 +572,8 @@ static ALWAYS_INLINE uint64_t *get_3uops (MIR_val_t *bp, code_t c, uint64_t *p1, } static ALWAYS_INLINE uint64_t *get_3usops (MIR_val_t *bp, code_t c, uint32_t *p1, uint32_t *p2) { - *p1 = *get_uop (bp, c + 1); - *p2 = *get_uop (bp, c + 2); + *p1 = (uint32_t) *get_uop (bp, c + 1); + *p2 = (uint32_t) *get_uop (bp, c + 2); return get_uop (bp, c); } @@ -568,11 +632,11 @@ static ALWAYS_INLINE int64_t *get_ldcmp_ops (MIR_val_t *bp, code_t c, long doubl static ALWAYS_INLINE int64_t get_mem_addr (MIR_val_t *bp, code_t c) { return bp[get_i (c)].i; } -#define EXT(tp) \ - do { \ - int64_t *r = get_iop (bp, ops); \ - tp s = *get_iop (bp, ops + 1); \ - *r = s; \ +#define EXT(tp) \ + do { \ + int64_t *r = get_iop (bp, ops); \ + tp s = (tp) * get_iop (bp, ops + 1); \ + *r = (int64_t) s; \ } while (0) #define IOP2(op) \ do { \ @@ -618,10 +682,10 @@ static ALWAYS_INLINE int64_t get_mem_addr (MIR_val_t *bp, code_t c) { return bp[ int64_t op1 = *get_iop (bp, ops + 1), op2 = *get_iop (bp, ops + 2); \ if (op1 op op2) pc = code + get_i (ops); \ } while (0) -#define BICMPS(op) \ - do { \ - int32_t op1 = *get_iop (bp, ops + 1), op2 = *get_iop (bp, ops + 2); \ - if (op1 op op2) pc = code + get_i (ops); \ +#define BICMPS(op) \ + do { \ + int32_t op1 = (int32_t) * get_iop (bp, ops + 1), op2 = (int32_t) * get_iop (bp, ops + 2); \ + if (op1 op op2) pc = code + get_i (ops); \ } while (0) #define UOP3(op) \ do { \ @@ -667,10 +731,10 @@ static ALWAYS_INLINE int64_t get_mem_addr (MIR_val_t *bp, code_t c) { return bp[ uint64_t op1 = *get_uop (bp, ops + 1), op2 = *get_uop (bp, ops + 2); \ if (op1 op op2) pc = code + get_i (ops); \ } while (0) -#define BUCMPS(op) \ - do { \ - uint32_t op1 = *get_uop (bp, ops + 1), op2 = *get_uop (bp, ops + 2); \ - if (op1 op op2) pc = code + get_i (ops); \ +#define BUCMPS(op) \ + do { \ + uint32_t op1 = (uint32_t) * get_uop (bp, ops + 1), op2 = (uint32_t) * get_uop (bp, ops + 2); \ + if (op1 op op2) pc = code + get_i (ops); \ } while (0) #define FOP2(op) \ @@ -754,11 +818,11 @@ static ALWAYS_INLINE int64_t get_mem_addr (MIR_val_t *bp, code_t c) { return bp[ int64_t a = get_mem_addr (bp, ops + 1); \ *r = *((mem_type *) a); \ } while (0) -#define ST(op, val_type, mem_type) \ - do { \ - val_type v = *get_##op (bp, ops); \ - int64_t a = get_mem_addr (bp, ops + 1); \ - *((mem_type *) a) = v; \ +#define ST(op, val_type, mem_type) \ + do { \ + val_type v = (val_type) * get_##op (bp, ops); \ + int64_t a = get_mem_addr (bp, ops + 1); \ + *((mem_type *) a) = (mem_type) v; \ } while (0) #if !MIR_INTERP_TRACE && defined(__GNUC__) && !defined(__clang__) @@ -790,8 +854,10 @@ static void start_insn_trace (MIR_context_t ctx, const char *name, func_desc_t f static void finish_insn_trace (MIR_context_t ctx, MIR_full_insn_code_t code, code_t ops, MIR_val_t *bp) { + struct interp_ctx *interp_ctx = ctx->interp_ctx; int out_p; MIR_op_mode_t op_mode = MIR_OP_UNDEF; + MIR_val_t *res = bp; switch (code) { case IC_LDI8: @@ -802,12 +868,28 @@ static void finish_insn_trace (MIR_context_t ctx, MIR_full_insn_code_t code, cod case IC_LDU32: case IC_LDI64: case IC_MOVI: + case IC_MOVTG: + res = global_regs; + /* falls through */ + case IC_MOVFG: case IC_MOVP: op_mode = MIR_OP_INT; break; case IC_LDF: + case IC_FMOVTG: + res = global_regs; + /* falls through */ + case IC_FMOVFG: case IC_MOVF: op_mode = MIR_OP_FLOAT; break; case IC_LDD: + case IC_DMOVTG: + res = global_regs; + /* falls through */ + case IC_DMOVFG: case IC_MOVD: op_mode = MIR_OP_DOUBLE; break; case IC_LDLD: + case IC_LDMOVTG: + res = global_regs; + /* falls through */ + case IC_LDMOVFG: case IC_MOVLD: op_mode = MIR_OP_LDOUBLE; break; case IC_STI8: case IC_STU8: @@ -820,6 +902,7 @@ static void finish_insn_trace (MIR_context_t ctx, MIR_full_insn_code_t code, cod case IC_STD:; case IC_STLD: break; case IC_IMM_CALL: break; + case IC_IMM_JCALL: break; default: op_mode = _MIR_insn_code_op_mode (ctx, (MIR_insn_code_t) code, 0, &out_p); if (op_mode == MIR_OP_BOUND || !out_p) op_mode = MIR_OP_UNDEF; @@ -828,16 +911,16 @@ static void finish_insn_trace (MIR_context_t ctx, MIR_full_insn_code_t code, cod switch (op_mode) { case MIR_OP_INT: case MIR_OP_UINT: - fprintf (stderr, "\t# res = %" PRId64 " (%" PRIu64 "u, 0x%" PRIx64 ")", bp[ops[0].i].i, - bp[ops[0].i].u, bp[ops[0].i].u); + fprintf (stderr, "\t# res = %" PRId64 " (%" PRIu64 "u, 0x%" PRIx64 ")", res[ops[0].i].i, + res[ops[0].i].u, res[ops[0].i].u); break; - case MIR_OP_FLOAT: fprintf (stderr, "\t# res = %.*ef", FLT_DECIMAL_DIG, bp[ops[0].i].f); break; + case MIR_OP_FLOAT: fprintf (stderr, "\t# res = %.*ef", FLT_DECIMAL_DIG, res[ops[0].i].f); break; case MIR_OP_LDOUBLE: #ifndef _WIN32 - fprintf (stderr, "\t# res = %.*Le", LDBL_DECIMAL_DIG, bp[ops[0].i].ld); + fprintf (stderr, "\t# res = %.*Le", LDBL_DECIMAL_DIG, res[ops[0].i].ld); break; #endif - case MIR_OP_DOUBLE: fprintf (stderr, "\t# res = %.*e", DBL_DECIMAL_DIG, bp[ops[0].i].d); break; + case MIR_OP_DOUBLE: fprintf (stderr, "\t# res = %.*e", DBL_DECIMAL_DIG, res[ops[0].i].d); break; default: assert (op_mode == MIR_OP_UNDEF); } fprintf (stderr, "\n"); @@ -855,7 +938,7 @@ static code_t call_insn_execute (MIR_context_t ctx, code_t pc, MIR_val_t *bp, co if (VARR_EXPAND (MIR_val_t, arg_vals_varr, nops)) arg_vals = VARR_ADDR (MIR_val_t, arg_vals_varr); - for (size_t i = start; i < nops + 3; i++) arg_vals[i - start] = bp[get_i (ops + i)]; + for (size_t i = start; i < (size_t) nops + 3; i++) arg_vals[i - start] = bp[get_i (ops + i)]; #if MIR_INTERP_TRACE trace_insn_ident += 2; @@ -870,10 +953,16 @@ static code_t call_insn_execute (MIR_context_t ctx, code_t pc, MIR_val_t *bp, co return pc; } +static int64_t addr_offset8, addr_offset16, addr_offset32; + static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t *bp, MIR_val_t *results) { struct interp_ctx *interp_ctx = ctx->interp_ctx; + MIR_val_t *globals = global_regs; code_t pc, ops, code; + void *jmpi_val; /* where label thunk execution result will be: */ + int64_t offset; + int signed_overflow_p = FALSE, unsigned_overflow_p = FALSE; /* to avoid uninitialized warnings */ #if MIR_INTERP_TRACE MIR_full_insn_code_t trace_insn_code; @@ -902,7 +991,8 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * REP6 (LAB_EL, MIR_EXT8, MIR_EXT16, MIR_EXT32, MIR_UEXT8, MIR_UEXT16, MIR_UEXT32); REP6 (LAB_EL, MIR_I2F, MIR_I2D, MIR_I2LD, MIR_UI2F, MIR_UI2D, MIR_UI2LD); REP8 (LAB_EL, MIR_F2I, MIR_D2I, MIR_LD2I, MIR_F2D, MIR_F2LD, MIR_D2F, MIR_D2LD, MIR_LD2F); - REP8 (LAB_EL, MIR_LD2D, MIR_NEG, MIR_NEGS, MIR_FNEG, MIR_DNEG, MIR_LDNEG, MIR_ADD, MIR_ADDS); + REP6 (LAB_EL, MIR_LD2D, MIR_NEG, MIR_NEGS, MIR_FNEG, MIR_DNEG, MIR_LDNEG); + REP6 (LAB_EL, MIR_ADDR, MIR_ADDR8, MIR_ADDR16, MIR_ADDR32, MIR_ADD, MIR_ADDS); REP8 (LAB_EL, MIR_FADD, MIR_DADD, MIR_LDADD, MIR_SUB, MIR_SUBS, MIR_FSUB, MIR_DSUB, MIR_LDSUB); REP8 (LAB_EL, MIR_MUL, MIR_MULS, MIR_FMUL, MIR_DMUL, MIR_LDMUL, MIR_DIV, MIR_DIVS, MIR_UDIV); REP8 (LAB_EL, MIR_UDIVS, MIR_FDIV, MIR_DDIV, MIR_LDDIV, MIR_MOD, MIR_MODS, MIR_UMOD, MIR_UMODS); @@ -912,19 +1002,24 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * REP8 (LAB_EL, MIR_ULT, MIR_ULTS, MIR_FLT, MIR_DLT, MIR_LDLT, MIR_LE, MIR_LES, MIR_ULE); REP8 (LAB_EL, MIR_ULES, MIR_FLE, MIR_DLE, MIR_LDLE, MIR_GT, MIR_GTS, MIR_UGT, MIR_UGTS); REP8 (LAB_EL, MIR_FGT, MIR_DGT, MIR_LDGT, MIR_GE, MIR_GES, MIR_UGE, MIR_UGES, MIR_FGE); - REP8 (LAB_EL, MIR_DGE, MIR_LDGE, MIR_JMP, MIR_BT, MIR_BTS, MIR_BF, MIR_BFS, MIR_BEQ); + REP6 (LAB_EL, MIR_DGE, MIR_LDGE, MIR_ADDO, MIR_ADDOS, MIR_SUBO, MIR_SUBOS); + REP4 (LAB_EL, MIR_MULO, MIR_MULOS, MIR_UMULO, MIR_UMULOS); + REP6 (LAB_EL, MIR_JMP, MIR_BT, MIR_BTS, MIR_BF, MIR_BFS, MIR_BEQ); REP8 (LAB_EL, MIR_BEQS, MIR_FBEQ, MIR_DBEQ, MIR_LDBEQ, MIR_BNE, MIR_BNES, MIR_FBNE, MIR_DBNE); REP8 (LAB_EL, MIR_LDBNE, MIR_BLT, MIR_BLTS, MIR_UBLT, MIR_UBLTS, MIR_FBLT, MIR_DBLT, MIR_LDBLT); REP8 (LAB_EL, MIR_BLE, MIR_BLES, MIR_UBLE, MIR_UBLES, MIR_FBLE, MIR_DBLE, MIR_LDBLE, MIR_BGT); REP8 (LAB_EL, MIR_BGTS, MIR_UBGT, MIR_UBGTS, MIR_FBGT, MIR_DBGT, MIR_LDBGT, MIR_BGE, MIR_BGES); REP5 (LAB_EL, MIR_UBGE, MIR_UBGES, MIR_FBGE, MIR_DBGE, MIR_LDBGE); - REP4 (LAB_EL, MIR_CALL, MIR_INLINE, MIR_SWITCH, MIR_RET); + REP6 (LAB_EL, MIR_BO, MIR_UBO, MIR_BNO, MIR_UBNO, MIR_LADDR, MIR_JMPI); + REP6 (LAB_EL, MIR_CALL, MIR_INLINE, MIR_JCALL, MIR_SWITCH, MIR_RET, MIR_JRET); REP3 (LAB_EL, MIR_ALLOCA, MIR_BSTART, MIR_BEND); REP4 (LAB_EL, MIR_VA_ARG, MIR_VA_BLOCK_ARG, MIR_VA_START, MIR_VA_END); REP8 (LAB_EL, IC_LDI8, IC_LDU8, IC_LDI16, IC_LDU16, IC_LDI32, IC_LDU32, IC_LDI64, IC_LDF); REP8 (LAB_EL, IC_LDD, IC_LDLD, IC_STI8, IC_STU8, IC_STI16, IC_STU16, IC_STI32, IC_STU32); REP8 (LAB_EL, IC_STI64, IC_STF, IC_STD, IC_STLD, IC_MOVI, IC_MOVP, IC_MOVF, IC_MOVD); - REP2 (LAB_EL, IC_MOVLD, IC_IMM_CALL); + REP3 (LAB_EL, IC_MOVLD, IC_IMM_CALL, IC_IMM_JCALL); + REP4 (LAB_EL, IC_MOVFG, IC_FMOVFG, IC_DMOVFG, IC_LDMOVFG); + REP4 (LAB_EL, IC_MOVTG, IC_FMOVTG, IC_DMOVTG, IC_LDMOVTG); return; } #undef REP_SEP @@ -971,6 +1066,14 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * switch (insn_code) { #endif +#if 0 + L_jmpi_finish : +#endif + { /* jmpi thunk return */ + pc = jmpi_val; + END_INSN; + } + CASE (MIR_MOV, 2) { int64_t p, *r = get_2iops (bp, ops, &p); *r = p; @@ -991,6 +1094,49 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * *r = p; END_INSN; } + + CASE (IC_MOVFG, 2) { + int64_t l = get_i (ops), r = get_i (ops + 1); + bp[l].i = globals[r].i; + END_INSN; + } + CASE (IC_FMOVFG, 2) { + int64_t l = get_i (ops), r = get_i (ops + 1); + bp[l].f = globals[r].f; + END_INSN; + } + CASE (IC_DMOVFG, 2) { + int64_t l = get_i (ops), r = get_i (ops + 1); + bp[l].d = globals[r].d; + END_INSN; + } + CASE (IC_LDMOVFG, 2) { + int64_t l = get_i (ops), r = get_i (ops + 1); + bp[l].ld = globals[r].ld; + END_INSN; + } + + CASE (IC_MOVTG, 2) { + int64_t l = get_i (ops), r = get_i (ops + 1); + globals[l].i = bp[r].i; + END_INSN; + } + CASE (IC_FMOVTG, 2) { + int64_t l = get_i (ops), r = get_i (ops + 1); + globals[l].f = bp[r].f; + END_INSN; + } + CASE (IC_DMOVTG, 2) { + int64_t l = get_i (ops), r = get_i (ops + 1); + globals[l].d = bp[r].d; + END_INSN; + } + CASE (IC_LDMOVTG, 2) { + int64_t l = get_i (ops), r = get_i (ops + 1); + globals[l].ld = bp[r].ld; + END_INSN; + } + SCASE (MIR_EXT8, 2, EXT (int8_t)); SCASE (MIR_EXT16, 2, EXT (int16_t)); SCASE (MIR_EXT32, 2, EXT (int32_t)); @@ -1001,21 +1147,21 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * float *r = get_fop (bp, ops); int64_t i = *get_iop (bp, ops + 1); - *r = i; + *r = (float) i; END_INSN; } CASE (MIR_I2D, 2) { double *r = get_dop (bp, ops); int64_t i = *get_iop (bp, ops + 1); - *r = i; + *r = (double) i; END_INSN; } CASE (MIR_I2LD, 2) { long double *r = get_ldop (bp, ops); int64_t i = *get_iop (bp, ops + 1); - *r = i; + *r = (long double) i; END_INSN; } @@ -1023,21 +1169,21 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * float *r = get_fop (bp, ops); uint64_t i = *get_iop (bp, ops + 1); - *r = i; + *r = (float) i; END_INSN; } CASE (MIR_UI2D, 2) { double *r = get_dop (bp, ops); uint64_t i = *get_iop (bp, ops + 1); - *r = i; + *r = (double) i; END_INSN; } CASE (MIR_UI2LD, 2) { long double *r = get_ldop (bp, ops); uint64_t i = *get_iop (bp, ops + 1); - *r = i; + *r = (long double) i; END_INSN; } @@ -1045,21 +1191,21 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * int64_t *r = get_iop (bp, ops); float f = *get_fop (bp, ops + 1); - *r = f; + *r = (int64_t) f; END_INSN; } CASE (MIR_D2I, 2) { int64_t *r = get_iop (bp, ops); double d = *get_dop (bp, ops + 1); - *r = d; + *r = (int64_t) d; END_INSN; } CASE (MIR_LD2I, 2) { int64_t *r = get_iop (bp, ops); long double ld = *get_ldop (bp, ops + 1); - *r = ld; + *r = (int64_t) ld; END_INSN; } @@ -1080,7 +1226,7 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * float *r = get_fop (bp, ops); double d = *get_dop (bp, ops + 1); - *r = d; + *r = (float) d; END_INSN; } CASE (MIR_D2LD, 2) { @@ -1094,7 +1240,7 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * float *r = get_fop (bp, ops); long double ld = *get_ldop (bp, ops + 1); - *r = ld; + *r = (float) ld; END_INSN; } @@ -1112,6 +1258,29 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * SCASE (MIR_DNEG, 2, DOP2 (-)); SCASE (MIR_LDNEG, 2, LDOP2 (-)); + CASE (MIR_ADDR8, 2) { + offset = addr_offset8; + goto common_addr; + } + CASE (MIR_ADDR16, 2) { + offset = addr_offset16; + goto common_addr; + } + CASE (MIR_ADDR32, 2) { + offset = addr_offset32; + goto common_addr; + } + CASE (MIR_ADDR, 2) + offset = 0; +common_addr:; + { + int64_t *r = get_iop (bp, ops); + void **p = get_aop (bp, ops + 1); + + *r = (int64_t) p + offset; + END_INSN; + } + SCASE (MIR_ADD, 3, IOP3 (+)); SCASE (MIR_ADDS, 3, IOP3S (+)); SCASE (MIR_FADD, 3, FOP3 (+)); @@ -1201,6 +1370,80 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * SCASE (MIR_DGE, 3, DCMP (>=)); SCASE (MIR_LDGE, 3, LDCMP (>=)); + CASE (MIR_ADDO, 3) { + int64_t *r = get_iop (bp, ops); + int64_t op1 = *get_iop (bp, ops + 1), op2 = *get_iop (bp, ops + 2); + unsigned_overflow_p = (uint64_t) op1 > UINT64_MAX - (uint64_t) op2; + signed_overflow_p = op2 >= 0 ? op1 > INT64_MAX - op2 : op1 < INT64_MIN - op2; + *r = op1 + op2; + END_INSN; + } + + CASE (MIR_ADDOS, 3) { + int64_t *r = get_iop (bp, ops); + int32_t op1 = (int32_t) *get_iop (bp, ops + 1), op2 = (int32_t) *get_iop (bp, ops + 2); + unsigned_overflow_p = (uint32_t) op1 > UINT32_MAX - (uint32_t) op2; + signed_overflow_p = op2 >= 0 ? op1 > INT32_MAX - op2 : op1 < INT32_MIN - op2; + *r = op1 + op2; + END_INSN; + } + + CASE (MIR_SUBO, 3) { + int64_t *r = get_iop (bp, ops); + int64_t op1 = *get_iop (bp, ops + 1), op2 = *get_iop (bp, ops + 2); + unsigned_overflow_p = (uint64_t) op1 < (uint64_t) op2; + signed_overflow_p = op2 < 0 ? op1 > INT64_MAX + op2 : op1 < INT64_MIN + op2; + *r = op1 - op2; + END_INSN; + } + + CASE (MIR_SUBOS, 3) { + int64_t *r = get_iop (bp, ops); + int32_t op1 = (int32_t) *get_iop (bp, ops + 1), op2 = (int32_t) *get_iop (bp, ops + 2); + unsigned_overflow_p = (uint32_t) op1 < (uint32_t) op2; + signed_overflow_p = op2 < 0 ? op1 > INT32_MAX + op2 : op1 < INT32_MIN + op2; + *r = op1 - op2; + END_INSN; + } + + CASE (MIR_MULO, 3) { + int64_t *r = get_iop (bp, ops); + int64_t op1 = *get_iop (bp, ops + 1), op2 = *get_iop (bp, ops + 2); + signed_overflow_p = (op1 == 0 ? FALSE + : op1 == -1 ? op2 < -INT64_MAX + : op1 > 0 ? (op2 > 0 ? INT64_MAX / op1 < op2 : INT64_MIN / op1 > op2) + : (op2 > 0 ? INT64_MIN / op1 < op2 : INT64_MAX / op1 > op2)); + *r = op1 * op2; + END_INSN; + } + + CASE (MIR_MULOS, 3) { + int64_t *r = get_iop (bp, ops); + int32_t op1 = (int32_t) *get_iop (bp, ops + 1), op2 = (int32_t) *get_iop (bp, ops + 2); + signed_overflow_p = (op1 == 0 ? FALSE + : op1 == -1 ? op2 < -INT32_MAX + : op1 > 0 ? (op2 > 0 ? INT32_MAX / op1 < op2 : INT32_MIN / op1 > op2) + : (op2 > 0 ? INT32_MIN / op1 < op2 : INT32_MAX / op1 > op2)); + *r = op1 * op2; + END_INSN; + } + + CASE (MIR_UMULO, 3) { + uint64_t *r = get_uop (bp, ops); + uint64_t op1 = *get_uop (bp, ops + 1), op2 = *get_uop (bp, ops + 2); + unsigned_overflow_p = op1 == 0 ? FALSE : UINT64_MAX / op1 < op2; + *r = op1 * op2; + END_INSN; + } + + CASE (MIR_UMULOS, 3) { + uint64_t *r = get_uop (bp, ops); + uint32_t op1 = (uint32_t) *get_uop (bp, ops + 1), op2 = (uint32_t) *get_uop (bp, ops + 2); + unsigned_overflow_p = op1 == 0 ? FALSE : UINT32_MAX / op1 < op2; + *r = op1 * op2; + END_INSN; + } + SCASE (MIR_JMP, 1, pc = code + get_i (ops)); CASE (MIR_BT, 2) { int64_t cond = *get_iop (bp, ops + 1); @@ -1215,13 +1458,13 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * END_INSN; } CASE (MIR_BTS, 2) { - int32_t cond = *get_iop (bp, ops + 1); + int32_t cond = (int32_t) *get_iop (bp, ops + 1); if (cond) pc = code + get_i (ops); END_INSN; } CASE (MIR_BFS, 2) { - int32_t cond = *get_iop (bp, ops + 1); + int32_t cond = (int32_t) *get_iop (bp, ops + 1); if (!cond) pc = code + get_i (ops); END_INSN; @@ -1265,6 +1508,34 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * SCASE (MIR_DBGE, 3, BDCMP (>=)); SCASE (MIR_LDBGE, 3, BLDCMP (>=)); + CASE (MIR_BO, 1) { + if (signed_overflow_p) pc = code + get_i (ops); + END_INSN; + } + CASE (MIR_UBO, 1) { + if (unsigned_overflow_p) pc = code + get_i (ops); + END_INSN; + } + CASE (MIR_BNO, 1) { + if (!signed_overflow_p) pc = code + get_i (ops); + END_INSN; + } + CASE (MIR_UBNO, 1) { + if (!unsigned_overflow_p) pc = code + get_i (ops); + END_INSN; + } + CASE (MIR_LADDR, 2) { + void **r = get_aop (bp, ops); + *r = code + get_i (ops + 1); + END_INSN; + } + + CASE (MIR_JMPI, 1) { /* jmpi thunk */ + void **r = get_aop (bp, ops); + pc = *r; + END_INSN; + } + CASE (MIR_CALL, 0) { int (*func_addr) (void *buf) = *get_aop (bp, ops + 4); @@ -1306,6 +1577,23 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * SCASE (MIR_INLINE, 0, mir_assert (FALSE)); + CASE (MIR_JCALL, 0) { + int (*func_addr) (void *buf) = *get_aop (bp, ops + 4); + if (func_addr == setjmp_addr) + (*MIR_get_error_func (ctx)) (MIR_invalid_insn_error, "jcall of setjmp"); + call_insn_execute (ctx, pc, bp, ops, FALSE); + pc = jret_addr; + END_INSN; + } + CASE (IC_IMM_JCALL, 0) { + int (*func_addr) (void *buf) = get_a (ops + 4); + if (func_addr == setjmp_addr) + (*MIR_get_error_func (ctx)) (MIR_invalid_insn_error, "jcall of setjmp"); + call_insn_execute (ctx, pc, bp, ops, TRUE); + pc = jret_addr; + END_INSN; + } + CASE (MIR_SWITCH, 0) { int64_t nops = get_i (ops); /* #ops */ int64_t index = *get_iop (bp, ops + 1); @@ -1317,13 +1605,18 @@ static void OPTIMIZE eval (MIR_context_t ctx, func_desc_t func_desc, MIR_val_t * CASE (MIR_RET, 0) { int64_t nops = get_i (ops); /* #ops */ - for (int64_t i = 0; i < nops; i++) results[i] = bp[get_i (ops + i + 1)]; pc += nops + 1; return; END_INSN; } + CASE (MIR_JRET, 0) { + jret_addr = bp[get_i (ops)].a; /* pc for continuation */ + return; + END_INSN; + } + CASE (MIR_ALLOCA, 2) { int64_t *r, s; @@ -1413,19 +1706,20 @@ static inline func_desc_t get_func_desc (MIR_item_t func_item) { return func_item->data; } -static htab_hash_t ff_interface_hash (ff_interface_t i, void *arg) { - htab_hash_t h = mir_hash_step (mir_hash_init (0), i->nres); - h = mir_hash_step (h, i->nargs); - h = mir_hash_step (h, i->arg_vars_num); - h = mir_hash (i->res_types, sizeof (MIR_type_t) * i->nres, h); +static htab_hash_t ff_interface_hash (ff_interface_t i, void *arg MIR_UNUSED) { + htab_hash_t h = (htab_hash_t) mir_hash_step (mir_hash_init (0), i->nres); + h = (htab_hash_t) mir_hash_step (h, i->nargs); + h = (htab_hash_t) mir_hash_step (h, i->arg_vars_num); + h = (htab_hash_t) mir_hash (i->res_types, sizeof (MIR_type_t) * i->nres, h); for (size_t n = 0; n < i->nargs; n++) { - h = mir_hash_step (h, i->arg_descs[n].type); - if (MIR_all_blk_type_p (i->arg_descs[n].type)) h = mir_hash_step (h, i->arg_descs[n].size); + h = (htab_hash_t) mir_hash_step (h, i->arg_descs[n].type); + if (MIR_all_blk_type_p (i->arg_descs[n].type)) + h = (htab_hash_t) mir_hash_step (h, i->arg_descs[n].size); } - return mir_hash_finish (h); + return (htab_hash_t) mir_hash_finish (h); } -static int ff_interface_eq (ff_interface_t i1, ff_interface_t i2, void *arg) { +static int ff_interface_eq (ff_interface_t i1, ff_interface_t i2, void *arg MIR_UNUSED) { if (i1->nres != i2->nres || i1->nargs != i2->nargs || i1->arg_vars_num != i2->arg_vars_num) return FALSE; if (memcmp (i1->res_types, i2->res_types, sizeof (MIR_type_t) * i1->nres) != 0) return FALSE; @@ -1438,11 +1732,11 @@ static int ff_interface_eq (ff_interface_t i1, ff_interface_t i2, void *arg) { return TRUE; } -static void ff_interface_clear (ff_interface_t ffi, void *arg) { free (ffi); } +static void ff_interface_clear (ff_interface_t ffi, void *arg MIR_UNUSED) { free (ffi); } static void *get_ff_interface (MIR_context_t ctx, size_t arg_vars_num, size_t nres, MIR_type_t *res_types, size_t nargs, _MIR_arg_desc_t *arg_descs, - int vararg_p) { + int vararg_p MIR_UNUSED) { struct interp_ctx *interp_ctx = ctx->interp_ctx; struct ff_interface ffi_s; ff_interface_t tab_ffi, ffi; @@ -1574,6 +1868,9 @@ static void call (MIR_context_t ctx, MIR_val_t *bp, MIR_op_t *insn_arg_ops, code static void interp_init (MIR_context_t ctx) { struct interp_ctx *interp_ctx; + addr_offset8 = _MIR_addr_offset (ctx, MIR_ADDR8); + addr_offset16 = _MIR_addr_offset (ctx, MIR_ADDR16); + addr_offset32 = _MIR_addr_offset (ctx, MIR_ADDR32); if ((interp_ctx = ctx->interp_ctx = malloc (sizeof (struct interp_ctx))) == NULL) MIR_get_error_func (ctx) (MIR_alloc_error, "Not enough memory for ctx"); #if DIRECT_THREADED_DISPATCH diff --git a/mir/mir-ppc64.c b/mir/mir-ppc64.c index b601686d..1453311c 100644 --- a/mir/mir-ppc64.c +++ b/mir/mir-ppc64.c @@ -1,22 +1,14 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ +#include "mir-ppc64.h" + /* All BLK type values is passed in int regs, and if the regs are not enough, the rest is passed on the stack. RBLK is always passed by address. */ #define VA_LIST_IS_ARRAY_P 1 /* one element which is a pointer to args */ -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define PPC64_STACK_HEADER_SIZE 32 -#define PPC64_TOC_OFFSET 24 -#define PPC64_FUNC_DESC_LEN 0 -#else -#define PPC64_STACK_HEADER_SIZE 48 -#define PPC64_TOC_OFFSET 40 -#define PPC64_FUNC_DESC_LEN 24 -#endif - static void ppc64_push_func_desc (VARR (uint8_t) * *insn_varr); void (*ppc64_func_desc) (VARR (uint8_t) * *insn_varr) = ppc64_push_func_desc; @@ -26,28 +18,13 @@ static void ppc64_push_func_desc (VARR (uint8_t) * *insn_varr) { VARR_PUSH (uint8_t, *insn_varr, ((uint8_t *) ppc64_func_desc)[i]); } -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -static void ppc64_redirect_func_desc (MIR_context_t ctx, void *desc, void *to) { - mir_assert (((uint64_t) desc & 0x3) == 0 && ((uint64_t) to & 0x3) == 0); /* alignment */ - _MIR_change_code (ctx, desc, (uint8_t *) &to, sizeof (to)); -} -#endif - static void *ppc64_publish_func_and_redirect (MIR_context_t ctx, VARR (uint8_t) * insn_varr) { void *res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, insn_varr), VARR_LENGTH (uint8_t, insn_varr)); -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - ppc64_redirect_func_desc (ctx, res, (uint8_t *) res + PPC64_FUNC_DESC_LEN); -#endif VARR_DESTROY (uint8_t, insn_varr); return res; } -static void push_insn (VARR (uint8_t) * insn_varr, uint32_t insn) { - uint8_t *p = (uint8_t *) &insn; - for (size_t i = 0; i < 4; i++) VARR_PUSH (uint8_t, insn_varr, p[i]); -} - static void push_insns (VARR (uint8_t) * insn_varr, const uint32_t *pat, size_t pat_len) { uint8_t *p = (uint8_t *) pat; for (size_t i = 0; i < pat_len; i++) VARR_PUSH (uint8_t, insn_varr, p[i]); @@ -99,36 +76,15 @@ static void ppc64_gen_stdu (VARR (uint8_t) * insn_varr, int disp) { push_insn (insn_varr, 0xf8210001 | (disp & 0xfffc)); /* stdu 1, disp (1) */ } -static void ppc64_gen_address (VARR (uint8_t) * insn_varr, unsigned int reg, void *p) { - uint64_t a = (uint64_t) p; - if ((a >> 32) == 0) { - if (((a >> 31) & 1) == 0) { /* lis r,0,Z2 */ - push_insn (insn_varr, (15 << 26) | (reg << 21) | (0 << 16) | ((a >> 16) & 0xffff)); - } else { /* xor r,r,r; oris r,r,Z2 */ - push_insn (insn_varr, (31 << 26) | (316 << 1) | (reg << 21) | (reg << 16) | (reg << 11)); - push_insn (insn_varr, (25 << 26) | (reg << 21) | (reg << 16) | ((a >> 16) & 0xffff)); - } - } else { - /* lis r,0,Z0; ori r,r,Z1; rldicr r,r,32,31; oris r,r,Z2; ori r,r,Z3: */ - push_insn (insn_varr, (15 << 26) | (reg << 21) | (0 << 16) | (a >> 48)); - push_insn (insn_varr, (24 << 26) | (reg << 21) | (reg << 16) | ((a >> 32) & 0xffff)); - push_insn (insn_varr, (30 << 26) | (reg << 21) | (reg << 16) | 0x07c6); - push_insn (insn_varr, (25 << 26) | (reg << 21) | (reg << 16) | ((a >> 16) & 0xffff)); - } - push_insn (insn_varr, (24 << 26) | (reg << 21) | (reg << 16) | (a & 0xffff)); +static void ppc64_gen_jump (VARR (uint8_t) * insn_varr, unsigned int reg) { + push_insn (insn_varr, (31 << 26) | (467 << 1) | (reg << 21) | (9 << 16)); /* mctr reg */ + push_insn (insn_varr, (19 << 26) | (528 << 1) | (20 << 21)); /* bcctr */ } -static void ppc64_gen_jump (VARR (uint8_t) * insn_varr, unsigned int reg, int call_p) { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - assert (reg != 0); - ppc64_gen_ld (insn_varr, 0, reg, 0, MIR_T_I64); /* 0 = func addr */ - ppc64_gen_ld (insn_varr, 2, reg, 8, MIR_T_I64); /* r2 = TOC */ - push_insn (insn_varr, (31 << 26) | (467 << 1) | (0 << 21) | (9 << 16)); /* mctr 0 */ -#else +static void ppc64_gen_call (VARR (uint8_t) * insn_varr, unsigned int reg) { if (reg != 12) ppc64_gen_mov (insn_varr, 12, reg); /* 12 = func addr */ push_insn (insn_varr, (31 << 26) | (467 << 1) | (12 << 21) | (9 << 16)); /* mctr 12 */ -#endif - push_insn (insn_varr, (19 << 26) | (528 << 1) | (20 << 21) | (call_p ? 1 : 0)); /* bcctr[l] */ + push_insn (insn_varr, (19 << 26) | (528 << 1) | (20 << 21) | 1); /* bcctrl */ } /* r11=addr_reg+addr_disp; r15=r1(sp)+sp_offset; r0=qwords-1; @@ -183,41 +139,46 @@ void *_MIR_get_bend_builtin (MIR_context_t ctx) { return ppc64_publish_func_and_redirect (ctx, code); } +static const int max_thunk_len = (7 * 4 + 8); /* 5 for r=addr, 2 for goto r, addr itself */ + void *_MIR_get_thunk (MIR_context_t ctx) { /* emit 3 doublewords for func descriptor: */ VARR (uint8_t) * code; - -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - ppc64_push_func_desc (&code); - return ppc64_publish_func_and_redirect (ctx, code); -#else - const uint32_t nop_insn = 24 << (32 - 6); /* ori 0,0,0 */ - const int max_thunk_len = (7 * 8); void *res; VARR_CREATE (uint8_t, code, 128); - for (int i = 0; i < max_thunk_len; i++) push_insn (code, nop_insn); + for (int i = 0; i < max_thunk_len / 4; i++) push_insn (code, TARGET_NOP); res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); VARR_DESTROY (uint8_t, code); return res; -#endif } +static const uint32_t thunk_code_end[] = { + 0x7d8903a6, /* mtctr r12 */ + 0x4e800420, /* bctr */ +}; + void _MIR_redirect_thunk (MIR_context_t ctx, void *thunk, void *to) { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - ppc64_redirect_func_desc (ctx, thunk, to); -#else - static const uint32_t global_entry_end[] = { - 0x7d8903a6, /* mtctr r12 */ - 0x4e800420, /* bctr */ - }; VARR (uint8_t) * code; - VARR_CREATE (uint8_t, code, 256); ppc64_gen_address (code, 12, to); - push_insns (code, global_entry_end, sizeof (global_entry_end)); + push_insns (code, thunk_code_end, sizeof (thunk_code_end)); + mir_assert ((VARR_LENGTH (uint8_t, code) & 0x3) == 0 + && VARR_LENGTH (uint8_t, code) <= (size_t) max_thunk_len); + push_insns (code, (uint32_t *) &to, sizeof (to)); _MIR_change_code (ctx, thunk, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); VARR_DESTROY (uint8_t, code); -#endif +} + +static void *get_jump_addr (uint32_t *insns) { + int i; + for (i = 0; i < 8; i++) + if (insns[i] == 0x4e800420) break; /* bctr */ + mir_assert (i < 8); + return (void *) (insns[i + 1] | ((uint64_t) insns[i + 2] << 32)); +} + +void *_MIR_get_thunk_addr (MIR_context_t ctx MIR_UNUSED, void *thunk) { + return get_jump_addr (thunk); } struct ppc64_va_list { @@ -234,20 +195,17 @@ void *va_arg_builtin (void *p, uint64_t t) { } else { va->arg_area++; } -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - if (type == MIR_T_F || type == MIR_T_I32) a = (char *) a + 4; /* 2nd word of doubleword */ -#endif return a; } -void va_block_arg_builtin (void *res, void *p, size_t s, uint64_t ncase) { +void va_block_arg_builtin (void *res, void *p, size_t s, uint64_t ncase MIR_UNUSED) { struct ppc64_va_list *va = p; void *a = va->arg_area; - memcpy (res, a, s); + if (res != NULL) memcpy (res, a, s); va->arg_area += (s + sizeof (uint64_t) - 1) / sizeof (uint64_t); } -void va_start_interp_builtin (MIR_context_t ctx, void *p, void *a) { +void va_start_interp_builtin (MIR_context_t ctx MIR_UNUSED, void *p, void *a) { struct ppc64_va_list **va = p; va_list *vap = a; @@ -255,7 +213,7 @@ void va_start_interp_builtin (MIR_context_t ctx, void *p, void *a) { *va = (struct ppc64_va_list *) vap; } -void va_end_interp_builtin (MIR_context_t ctx, void *p) {} +void va_end_interp_builtin (MIR_context_t ctx MIR_UNUSED, void *p MIR_UNUSED) {} /* Generation: fun (fun_addr, res_arg_addresses): save lr (r1 + 16); allocate and form minimal stack frame (with necessary param area); save @@ -359,7 +317,7 @@ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, s param_offset += 16; n_gpregs += type == MIR_T_LD ? 2 : 1; } - ppc64_gen_jump (code, 12, TRUE); /* call func_addr */ + ppc64_gen_call (code, 12); /* call func_addr */ n_gpregs = n_fpregs = 0; disp = 0; for (uint32_t i = 0; i < nres; i++) { @@ -495,7 +453,7 @@ void *_MIR_get_interp_shim (MIR_context_t ctx, MIR_item_t func_item, void *handl ppc64_gen_mov (code, 5, va_reg); ppc64_gen_mov (code, 6, res_reg); ppc64_gen_address (code, 12, handler); - ppc64_gen_jump (code, 12, TRUE); + ppc64_gen_call (code, 12); disp = n_gpregs = n_fpregs = 0; for (uint32_t i = 0; i < nres; i++) { type = res_types[i]; @@ -527,36 +485,115 @@ void *_MIR_get_interp_shim (MIR_context_t ctx, MIR_item_t func_item, void *handl return res; } -/* Brief: save lr (r1+16); update r1, save all param regs (r1+header+64); - allocate and form minimal wrapper stack frame (param area = 8*8); - r3 = call hook_address (ctx, called_func); r12=r3 - restore params regs (r1+header+64), r1, lr (r1+16); ctr=r12; b *ctr */ +static void redirect_bb_thunk (MIR_context_t ctx, VARR (uint8_t) * code, void *start, void *to) { + int64_t offset = (uint8_t *) to - (uint8_t *) start; + mir_assert ((offset & 0x3) == 0); + VARR_TRUNC (uint8_t, code, 0); + if (((offset < 0 ? -offset : offset) & ~(int64_t) 0x1ffffff) == 0) { /* just jump */ + uint32_t insn + = (PPC_JUMP_OPCODE << (32 - 6)) /* jump opcode */ | (((offset / 4) & 0xffffff) << 2); + push_insn (code, insn); + } else { + ppc64_gen_address (code, 12, to); /* r12 = to */ + push_insns (code, thunk_code_end, sizeof (thunk_code_end)); + mir_assert ((VARR_LENGTH (uint8_t, code) & 0x3) == 0 + && VARR_LENGTH (uint8_t, code) <= (size_t) max_thunk_len); + } + _MIR_change_code (ctx, start, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); +} + +/* r11=; jump handler ??? mutex free */ +void *_MIR_get_bb_thunk (MIR_context_t ctx, void *bb_version, void *handler) { + void *res; + size_t offset; + VARR (uint8_t) * code; + + VARR_CREATE (uint8_t, code, 64); + ppc64_gen_address (code, 11, bb_version); /* x11 = bb_version */ + offset = VARR_LENGTH (uint8_t, code); + for (int i = 0; i < max_thunk_len / 4; i++) push_insn (code, TARGET_NOP); + res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); + redirect_bb_thunk (ctx, code, (uint8_t *) res + offset, handler); +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("bb thunk:", res, offset + VARR_LENGTH (uint8_t, code)); +#endif + VARR_DESTROY (uint8_t, code); + return res; +} + +/* change to jump to */ +void _MIR_replace_bb_thunk (MIR_context_t ctx, void *thunk, void *to) { + size_t i, offset; + VARR (uint8_t) * code; + uint32_t opcode, *insns = (uint32_t *) thunk; + + /* find jump code offset (see ppc64_gen_address): */ + for (i = 0; i <= 5; i++) { + if ((opcode = insns[i] >> 26) == (uint32_t) PPC_JUMP_OPCODE) break; /* uncond branch */ + if ((opcode == LI_OPCODE || opcode == LIS_OPCODE + || opcode == XOR_OPCODE) /* (li|lis|xor) r12, ... */ + && ((insns[i] >> 21) & 0x1f) == 12) + break; + } + mir_assert (i <= 5); + offset = i * 4; + VARR_CREATE (uint8_t, code, 64); + redirect_bb_thunk (ctx, code, (char *) thunk + offset, to); + VARR_DESTROY (uint8_t, code); +} + +static const int wrapper_frame_size = PPC64_STACK_HEADER_SIZE + 8 * 8 + 13 * 8 + 8 * 8; + +/* save lr(r1+16);update r1,save r3,r4 regs;r3=ctx;r4=called_func;r12=hook_address;jmp wrap_end */ void *_MIR_get_wrapper (MIR_context_t ctx, MIR_item_t called_func, void *hook_address) { - static uint32_t prologue[] = { + static const uint32_t prologue[] = { 0x7c0802a6, /* mflr r0 */ 0xf8010010, /* std r0,16(r1) */ }; - static uint32_t epilogue[] = { - 0xe8010010, /* ld r0,16(r1) */ - 0x7c0803a6, /* mtlr r0 */ - }; - int frame_size = PPC64_STACK_HEADER_SIZE + 8 * 8 + 13 * 8 + 8 * 8; VARR (uint8_t) * code; void *res; + int frame_size = wrapper_frame_size; VARR_CREATE (uint8_t, code, 256); push_insns (code, prologue, sizeof (prologue)); /* stdu r1,n(r1): header + 8(gp args) + 13(fp args) + 8(param area): */ if (frame_size % 16 != 0) frame_size += 8; ppc64_gen_stdu (code, -frame_size); - for (unsigned reg = 3; reg <= 10; reg++) /* std rn,dispn(r1) : */ + for (unsigned reg = 3; reg <= 4; reg++) /* std rn,dispn(r1) : */ ppc64_gen_st (code, reg, 1, PPC64_STACK_HEADER_SIZE + (reg - 3) * 8 + 64, MIR_T_I64); - for (unsigned reg = 1; reg <= 13; reg++) /* stfd fn,dispn(r1) : */ - ppc64_gen_st (code, reg, 1, PPC64_STACK_HEADER_SIZE + (reg - 1 + 8) * 8 + 64, MIR_T_D); ppc64_gen_address (code, 3, ctx); ppc64_gen_address (code, 4, called_func); ppc64_gen_address (code, 12, hook_address); - ppc64_gen_jump (code, 12, TRUE); + ppc64_gen_address (code, 11, wrapper_end_addr); + ppc64_gen_jump (code, 11); + res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("wapper:", res, VARR_LENGTH (uint8_t, code)); +#endif + VARR_DESTROY (uint8_t, code); + return res; +} + +/* save all param regs but r3, r4; allocate and form minimal wrapper stack frame (param area = 8*8); + r3 = call r12 (r3, r4); r12=r3; restore all params regs,r1,lr (r1+16);ctr=r12; b *ctr */ +void *_MIR_get_wrapper_end (MIR_context_t ctx) { + static const uint32_t epilogue[] = { + 0xe8010010, /* ld r0,16(r1) */ + 0x7c0803a6, /* mtlr r0 */ + }; + VARR (uint8_t) * code; + void *res; + int frame_size = wrapper_frame_size; + + if (frame_size % 16 != 0) frame_size += 8; + VARR_CREATE (uint8_t, code, 256); + for (unsigned reg = 5; reg <= 10; reg++) /* std rn,dispn(r1) : */ + ppc64_gen_st (code, reg, 1, PPC64_STACK_HEADER_SIZE + (reg - 3) * 8 + 64, MIR_T_I64); + for (unsigned reg = 1; reg <= 13; reg++) /* stfd fn,dispn(r1) : */ + ppc64_gen_st (code, reg, 1, PPC64_STACK_HEADER_SIZE + (reg - 1 + 8) * 8 + 64, MIR_T_D); + ppc64_gen_call (code, 12); ppc64_gen_mov (code, 12, 3); for (unsigned reg = 3; reg <= 10; reg++) /* ld rn,dispn(r1) : */ ppc64_gen_ld (code, reg, 1, PPC64_STACK_HEADER_SIZE + (reg - 3) * 8 + 64, MIR_T_I64); @@ -567,6 +604,69 @@ void *_MIR_get_wrapper (MIR_context_t ctx, MIR_item_t called_func, void *hook_ad push_insn (code, (31 << 26) | (467 << 1) | (12 << 21) | (9 << 16)); /* mctr 12 */ push_insn (code, (19 << 26) | (528 << 1) | (20 << 21)); /* bcctr */ res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("wapper end:", res, VARR_LENGTH (uint8_t, code)); +#endif + VARR_DESTROY (uint8_t, code); + return res; +} + +/* save all clobbered regs but r11 and r12; r11 = call hook_address (data, r11); restore regs; br + r11 r11 is a generator temp reg which is not used across bb borders. */ +void *_MIR_get_bb_wrapper (MIR_context_t ctx, void *data, void *hook_address) { + static const uint32_t prologue[] = { + 0x7d800026, /* mfcr r12 */ + 0xf9810008, /* std r12,8(r1) */ + 0x7d8802a6, /* mflr r12 */ + 0xf9810010, /* std r12,16(r1) */ + }; + static const uint32_t epilogue[] = { + 0xe9810010, /* ld r12,16(r1) */ + 0x7d8803a6, /* mtlr r12 */ + 0xe9810008, /* ld r12,8(r1) */ + 0x7d8ff120, /* mtcr r12 */ + }; + int frame_size = PPC64_STACK_HEADER_SIZE + 14 * 8 + 14 * 8 + 8 * 8; + void *res; + VARR (uint8_t) * code; + + VARR_CREATE (uint8_t, code, 256); + push_insns (code, prologue, sizeof (prologue)); + /* stdu r1,n(r1): header + 14(gp regs, r{1,2,11} space alloc is not used) + 14(fp args) + 8(param + * area): */ + if (frame_size % 16 != 0) frame_size += 8; + ppc64_gen_stdu (code, -frame_size); + ppc64_gen_st (code, R0_HARD_REG, SP_HARD_REG, PPC64_STACK_HEADER_SIZE + R0_HARD_REG * 8 + 64, + MIR_T_I64); + for (unsigned reg = R2_HARD_REG; reg <= R10_HARD_REG; reg++) /* ld rn,dispn(r1) : */ + ppc64_gen_st (code, reg, SP_HARD_REG, PPC64_STACK_HEADER_SIZE + reg * 8 + 64, MIR_T_I64); + ppc64_gen_st (code, R13_HARD_REG, SP_HARD_REG, PPC64_STACK_HEADER_SIZE + R13_HARD_REG * 8 + 64, + MIR_T_I64); + for (unsigned reg = 0; reg <= F13_HARD_REG - F0_HARD_REG; reg++) /* lfd fn,dispn(r1) : */ + ppc64_gen_st (code, reg, SP_HARD_REG, PPC64_STACK_HEADER_SIZE + (reg + 14) * 8 + 64, MIR_T_D); + ppc64_gen_address (code, 3, data); /* r3 = data */ + ppc64_gen_mov (code, 4, 11); /* r4 = r11 */ + ppc64_gen_address (code, 12, hook_address); /* r12 = hook addres */ + ppc64_gen_call (code, 12); /* call r12 */ + ppc64_gen_mov (code, 11, 3); /* r11 = r3 */ + ppc64_gen_ld (code, R0_HARD_REG, SP_HARD_REG, PPC64_STACK_HEADER_SIZE + R0_HARD_REG * 8 + 64, + MIR_T_I64); + for (unsigned reg = R2_HARD_REG; reg <= R10_HARD_REG; reg++) /* ld rn,dispn(r1) : */ + ppc64_gen_ld (code, reg, SP_HARD_REG, PPC64_STACK_HEADER_SIZE + reg * 8 + 64, MIR_T_I64); + ppc64_gen_ld (code, R13_HARD_REG, SP_HARD_REG, PPC64_STACK_HEADER_SIZE + R13_HARD_REG * 8 + 64, + MIR_T_I64); + for (unsigned reg = 0; reg <= F13_HARD_REG - F0_HARD_REG; reg++) /* lfd fn,dispn(r1) : */ + ppc64_gen_ld (code, reg, SP_HARD_REG, PPC64_STACK_HEADER_SIZE + (reg + 14) * 8 + 64, MIR_T_D); + ppc64_gen_addi (code, 1, 1, frame_size); + push_insns (code, epilogue, sizeof (epilogue)); + push_insn (code, (31 << 26) | (467 << 1) | (11 << 21) | (9 << 16)); /* mctr 11 */ + push_insn (code, (19 << 26) | (528 << 1) | (20 << 21)); /* bcctr */ + res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("bb wrapper:", res, VARR_LENGTH (uint8_t, code)); +#endif VARR_DESTROY (uint8_t, code); return res; } diff --git a/mir/mir-ppc64.h b/mir/mir-ppc64.h new file mode 100644 index 00000000..dae5345c --- /dev/null +++ b/mir/mir-ppc64.h @@ -0,0 +1,114 @@ +/* This file is a part of MIR project. + Copyright (C) 2018-2024 Vladimir Makarov . + A common include file for mir-ppc64.c and mir-gen-ppc64.c +*/ + +#include "mir.h" + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#error "pp64 big endian is not supported anymore" +#endif + +#define PPC64_STACK_HEADER_SIZE 32 +#define PPC64_TOC_OFFSET 24 +#define PPC64_FUNC_DESC_LEN 0 + +#define TARGET_NOP (24 << (32 - 6)) /* ori 0,0,0 */ + +#define HREG_EL(h) h##_HARD_REG +#define REP_SEP , +enum { + REP8 (HREG_EL, R0, R1, R2, R3, R4, R5, R6, R7), + REP8 (HREG_EL, R8, R9, R10, R11, R12, R13, R14, R15), + REP8 (HREG_EL, R16, R17, R18, R19, R20, R21, R22, R23), + REP8 (HREG_EL, R24, R25, R26, R27, R28, R29, R30, R31), + REP8 (HREG_EL, F0, F1, F2, F3, F4, F5, F6, F7), + REP8 (HREG_EL, F8, F9, F10, F11, F12, F13, F14, F15), + REP8 (HREG_EL, F16, F17, F18, F19, F20, F21, F22, F23), + REP8 (HREG_EL, F24, F25, F26, F27, F28, F29, F30, F31), + HREG_EL (LR), +}; +#undef REP_SEP + +static const char *const target_hard_reg_names[] = { + "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12", + "r13", "r14", "r15", "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23", "r24", "r25", + "r26", "r27", "r28", "r29", "r30", "r31", "f0", "f1", "f2", "f3", "f4", "f5", "f6", + "f7", "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", "f19", + "f20", "f21", "f22", "f23", "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", "lr", +}; + +#define MAX_HARD_REG LR_HARD_REG +static const MIR_reg_t SP_HARD_REG = R1_HARD_REG; +static const MIR_reg_t FP_HARD_REG = R31_HARD_REG; + +/* Hard regs not used in machinized code, preferably call used ones. */ +static const MIR_reg_t TEMP_INT_HARD_REG1 = R11_HARD_REG, TEMP_INT_HARD_REG2 = R12_HARD_REG; +static const MIR_reg_t TEMP_FLOAT_HARD_REG1 = F11_HARD_REG, TEMP_FLOAT_HARD_REG2 = F12_HARD_REG; +static const MIR_reg_t TEMP_DOUBLE_HARD_REG1 = F11_HARD_REG, TEMP_DOUBLE_HARD_REG2 = F12_HARD_REG; +static const MIR_reg_t TEMP_LDOUBLE_HARD_REG1 = F11_HARD_REG; //??? +static const MIR_reg_t TEMP_LDOUBLE_HARD_REG2 = F12_HARD_REG; + +static inline int target_hard_reg_type_ok_p (MIR_reg_t hard_reg, MIR_type_t type) { + assert (hard_reg <= MAX_HARD_REG); + if (type == MIR_T_LD) return FALSE; + return MIR_fp_type_p (type) ? F0_HARD_REG <= hard_reg && hard_reg <= F31_HARD_REG + : hard_reg < F0_HARD_REG; +} + +static inline int target_fixed_hard_reg_p (MIR_reg_t hard_reg) { + assert (hard_reg <= MAX_HARD_REG); + return (hard_reg == FP_HARD_REG || hard_reg == SP_HARD_REG + || hard_reg == LR_HARD_REG + /* don't bother to allocate R0 as it has special meaning for base reg and of addi: */ + || hard_reg == R0_HARD_REG || hard_reg == R2_HARD_REG || hard_reg == R13_HARD_REG + || hard_reg == TEMP_INT_HARD_REG1 || hard_reg == TEMP_INT_HARD_REG2 + || hard_reg == TEMP_FLOAT_HARD_REG1 || hard_reg == TEMP_FLOAT_HARD_REG2 + || hard_reg == TEMP_DOUBLE_HARD_REG1 || hard_reg == TEMP_DOUBLE_HARD_REG2 + || hard_reg == TEMP_LDOUBLE_HARD_REG1 || hard_reg == TEMP_LDOUBLE_HARD_REG2); +} + +static int target_locs_num (MIR_reg_t loc MIR_UNUSED, MIR_type_t type) { + return /*loc > MAX_HARD_REG && */ type == MIR_T_LD ? 2 : 1; +} + +static inline void push_insn (VARR (uint8_t) * insn_varr, uint32_t insn) { + uint8_t *p = (uint8_t *) &insn; + for (size_t i = 0; i < 4; i++) VARR_PUSH (uint8_t, insn_varr, p[i]); +} + +static const int PPC_JUMP_OPCODE = 18; + +#define LI_OPCODE 14 +#define LIS_OPCODE 15 +#define ORI_OPCODE 24 +#define ORIS_OPCODE 25 +#define XOR_OPCODE 31 +static inline void ppc64_gen_address (VARR (uint8_t) * insn_varr, unsigned int reg, void *p) { + uint64_t a = (uint64_t) p; + if ((a >> 32) == 0) { + if (((a >> 31) & 1) == 0) { /* lis r,0,Z2 */ + push_insn (insn_varr, (LIS_OPCODE << 26) | (reg << 21) | (0 << 16) | ((a >> 16) & 0xffff)); + } else { /* xor r,r,r; oris r,r,Z2 */ + push_insn (insn_varr, + (XOR_OPCODE << 26) | (316 << 1) | (reg << 21) | (reg << 16) | (reg << 11)); + push_insn (insn_varr, (ORIS_OPCODE << 26) | (reg << 21) | (reg << 16) | ((a >> 16) & 0xffff)); + } + } else { + if ((a >> 47) != 0) { + /* lis r,0,Z0; [ori r,r,Z1]; rldicr r,r,32,31; [oris r,r,Z2]; [ori r,r,Z3]: */ + push_insn (insn_varr, (LIS_OPCODE << 26) | (reg << 21) | (0 << 16) | (a >> 48)); + if (((a >> 32) & 0xffff) != 0) + push_insn (insn_varr, + (ORI_OPCODE << 26) | (reg << 21) | (reg << 16) | ((a >> 32) & 0xffff)); + } else { + /* li r,0,Z1; rldicr r,r,32,31; [oris r,r,Z2]; [ori r,r,Z3]: */ + push_insn (insn_varr, (LI_OPCODE << 26) | (reg << 21) | (0 << 16) | ((a >> 32) & 0xffff)); + } + push_insn (insn_varr, (30 << 26) | (reg << 21) | (reg << 16) | 0x07c6); + if (((a >> 16) & 0xffff) != 0) + push_insn (insn_varr, (ORIS_OPCODE << 26) | (reg << 21) | (reg << 16) | ((a >> 16) & 0xffff)); + } + if ((a & 0xffff) != 0) + push_insn (insn_varr, (ORI_OPCODE << 26) | (reg << 21) | (reg << 16) | (a & 0xffff)); +} diff --git a/mir/mir-reduce.h b/mir/mir-reduce.h index fe7daa03..89bd50d3 100644 --- a/mir/mir-reduce.h +++ b/mir/mir-reduce.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ #ifndef MIR_REDUCE_H @@ -121,7 +121,7 @@ static inline void _reduce_uint_write (struct reduce_data *data, uint32_t u) { int n; assert (u < (1 << 7 * 4)); - for (n = 1; n <= 4 && u >= (1 << 7 * n); n++) + for (n = 1; n <= 4 && u >= (1u << 7 * n); n++) ; _reduce_put (data, (1 << (8 - n)) | ((u >> (n - 1) * 8) & 0xff)); /* tag */ for (int i = 2; i <= n; i++) _reduce_put (data, (u >> (n - i) * 8) & 0xff); @@ -145,13 +145,13 @@ static inline int64_t _reduce_uint_read (reduce_reader_t reader, void *aux_data) static inline void _reduce_hash_write (struct reduce_data *data, uint64_t h) { _reduce_put (data, 0); /* 0 tag */ - for (int i = 0; i < sizeof (uint64_t); i++) _reduce_put (data, (h >> i * 8) & 0xff); + for (size_t i = 0; i < sizeof (uint64_t); i++) _reduce_put (data, (h >> i * 8) & 0xff); } static inline uint64_t _reduce_str2hash (const uint8_t *s) { uint64_t h = 0; - for (int i = 0; i < sizeof (uint64_t); i++) h |= (uint64_t) s[i] << i * 8; + for (size_t i = 0; i < sizeof (uint64_t); i++) h |= (uint64_t) s[i] << i * 8; return h; } @@ -209,6 +209,7 @@ static inline uint32_t _reduce_dict_find_longest (struct reduce_data *data, uint compression speed by 10%. */ hash = mir_hash_strict (&data->buf[pos], _REDUCE_START_LEN, _REDUCE_HASH_SEED) % _REDUCE_TABLE_SIZE; + best_len = 0; /* to remove a warning */ for (curr = encode_data->table[hash].head; curr != UINT32_MAX; curr = next) { next = encode_data->table[curr].next; el = &encode_data->table[curr]; @@ -385,7 +386,7 @@ static inline int reduce_decode_get (struct reduce_data *data) { if (sym_len != 0) { if (sym_len == _REDUCE_SYMB_TAG_LONG) { if ((r = _reduce_uint_read (reader, data->aux_data)) < 0) break; - sym_len = r; + sym_len = (uint32_t) r; } if (sym_len > _REDUCE_MAX_SYMB_LEN || pos + sym_len > _REDUCE_BUF_LEN) break; if (reader (&data->buf[pos], sym_len, data->aux_data) != sym_len) break; @@ -396,11 +397,11 @@ static inline int reduce_decode_get (struct reduce_data *data) { if (ref_len != 0) { if (ref_len == _REDUCE_REF_TAG_LONG) { if ((r = _reduce_uint_read (reader, data->aux_data)) < 0) break; - ref_len = r; + ref_len = (uint32_t) r; } ref_len += _REDUCE_START_LEN - 1; if ((r = _reduce_uint_read (reader, data->aux_data)) < 0) break; - ref_ind = r; + ref_ind = (uint32_t) r; if (curr_ind < ref_ind) break; sym_pos = decode_data->ind2pos[curr_ind - ref_ind]; if (sym_pos + ref_len > _REDUCE_BUF_LEN) break; diff --git a/mir/mir-riscv64.c b/mir/mir-riscv64.c index df328bd8..31991c0f 100644 --- a/mir/mir-riscv64.c +++ b/mir/mir-riscv64.c @@ -1,7 +1,9 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ +#include "mir-riscv64.h" + /* x0 (zero) - always zero; x1 (ra) - link reg; x2 (sp) - sp, x3 (gp) - global pointer, x4 (tp) - thread pointer; x8 (s0/fp) - fp; x10-x11 (a0-a1), f10-f11 (fa0-fa1) - ret values, x10-x17 (a0-a7), f10-f17 (fa0-fa7) - arg regs; x8-x9 (s0-s1), x18-x27 (s2-s11) - callee-saved; x1 (ra), @@ -101,10 +103,10 @@ void va_block_arg_builtin (void *res, void *p, size_t s, uint64_t ncase) { a = *(void **) a; va->arg_area++; } - memcpy (res, a, s); + if (res != NULL) memcpy (res, a, s); } -void va_start_interp_builtin (MIR_context_t ctx, void *p, void *a) { +void va_start_interp_builtin (MIR_context_t ctx MIR_UNUSED, void *p, void *a) { struct riscv64_va_list *va = p; va_list *vap = a; @@ -112,7 +114,7 @@ void va_start_interp_builtin (MIR_context_t ctx, void *p, void *a) { *va = *(struct riscv64_va_list *) vap; } -void va_end_interp_builtin (MIR_context_t ctx, void *p) {} +void va_end_interp_builtin (MIR_context_t ctx MIR_UNUSED, void *p MIR_UNUSED) {} static uint8_t *push_insns (VARR (uint8_t) * insn_varr, const void *pat, size_t pat_len) { const uint8_t *p = pat; @@ -120,20 +122,73 @@ static uint8_t *push_insns (VARR (uint8_t) * insn_varr, const void *pat, size_t return VARR_ADDR (uint8_t, insn_varr) + VARR_LENGTH (uint8_t, insn_varr) - pat_len; } +#define MAX_JUMP_CODE 6 /* in insns */ +/* Possible combinations: jal (20-bits); auipc+jalr (32-bits); auipc+ld+jalr+64-bit abs address: */ void *_MIR_get_thunk (MIR_context_t ctx) { - static const uint32_t call_pat[] = { - 0x00000297, /* 0:auipc t0,0x0 */ - 0x0102b283, /* 4:ld t0,16(t0) */ - 0x00028067, /* 8:jr t0 */ - 0x00000013, /* nop:addi zero,zero,0 */ - 0x00000013, /* nop:addi zero,zero,0 */ - 0x00000013, /* nop:addi zero,zero,0 */ + static const uint32_t call_pat[MAX_JUMP_CODE] = { + /* max 3-insns and aligned abs addr */ + TARGET_NOP, TARGET_NOP, TARGET_NOP, TARGET_NOP, TARGET_NOP, TARGET_NOP, }; return _MIR_publish_code (ctx, (uint8_t *) call_pat, sizeof (call_pat)); } +static int get_jump_code (uint32_t *insns, void *to, int64_t offset, int temp_hard_reg) { + assert ((offset & 1) == 0 && 0 <= temp_hard_reg && temp_hard_reg < 32); + if (-(1 << 20) <= offset && offset < (1 << 20)) { + insns[0] = 0x6f | get_j_format_imm (offset); /* jal */ + insns[1] = TARGET_NOP; /* size should be aligned to 8 */ + return 8; + } else if (-(1l << 31) <= offset && offset < (1 << 31)) { + int hi = offset >> 12, low = offset & 0xfff; + if ((low & 0x800) != 0) hi++; + insns[0] = 0x17 | (temp_hard_reg << 7) | (hi << 12); /* auipc t */ + insns[1] = 0x67 | (temp_hard_reg << 15) | (low << 20); /* jalr t */ + return 8; + } else { + insns[0] = 0x17 | (temp_hard_reg << 7); /* auipc t,0x0 */ + insns[1] + = 0x0003003 | (16 << 20) | (temp_hard_reg << 7) | (temp_hard_reg << 15); /* ld t,16(t) */ + insns[2] = 0x67 | (temp_hard_reg << 15); /* jalr t */ + *(void **) &insns[4] = to; + return 24; + } +} + +static void *get_jump_addr (uint32_t *insns) { /* see get_jump_code */ + int32_t offset; + if ((insns[0] & 0x7f) == 0x6f) { /* jal */ + offset = (((int32_t) insns[0] >> 30) << 20) | (insns[0] & 0xff000) | ((insns[0] >> 9) & 0x800) + | ((insns[0] >> 20) & 0x7fe); + return (int8_t *) insns + offset; + } else if ((insns[0] & 0x7f) == 0x17 && (insns[1] & 0x7f) == 0x67) { + int32_t hi = (int32_t) insns[0] & 0xfffff000, low = (int32_t) insns[1] >> 20; + return (int8_t *) insns + hi + low; + } else { + assert ((insns[0] & ~0xf80) == 0x17 && (insns[1] & ~0xf8f80) == (0x0003003 | (16 << 20)) + && (insns[2] & 0x7f) == 0x67); + return *(void **) &insns[4]; + } +} + +void *_MIR_get_thunk_addr (MIR_context_t ctx MIR_UNUSED, void *thunk) { + return get_jump_addr (thunk); +} + +static void redirect_thunk (MIR_context_t ctx, void *thunk, void *to, int temp_hard_reg) { + uint32_t insns[MAX_JUMP_CODE]; + uint64_t offset = (uint8_t *) to - (uint8_t *) thunk; + int len = get_jump_code (insns, to, offset, temp_hard_reg); + + assert (len <= MAX_JUMP_CODE * 4); + _MIR_change_code (ctx, (uint8_t *) thunk, (uint8_t *) insns, len); +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("thunk", thunk, len); +#endif +} + void _MIR_redirect_thunk (MIR_context_t ctx, void *thunk, void *to) { - _MIR_change_code (ctx, (uint8_t *) thunk + 16, (uint8_t *) &to, sizeof (to)); + redirect_thunk (ctx, thunk, to, T0_HARD_REG); } static const uint32_t add_sp_pat = 0x00010113; /* addi sp,sp,0 */ @@ -166,12 +221,6 @@ static uint32_t get_u_format_imm (int offset) { return offset << 12; } -static uint32_t get_j_format_imm (int offset) { - int d = offset >> 1; /* scale */ - assert (-(1 << 19) <= d && d < (1 << 19)); - return ((d & 0x80000) | ((d & 0x3ff) << 9) | (((d >> 10) & 0x1) << 8) | ((d >> 11) & 0xff)) << 12; -} - static uint32_t get_opfp_format_rd (int reg) { assert (0 <= reg && reg < 32); return reg << 7; @@ -350,7 +399,7 @@ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, s pat = ld_arg_pat | get_i_format_imm (offset) | get_i_format_rd (7); /* ld t2,offset(s1) */ push_insns (code, &pat, sizeof (pat)); if (type == MIR_T_BLK + 1) n_xregs = (n_xregs + 1) / 2 * 2; /* Make even */ - for (int n = 0; n < parts; n++) { + for (uint32_t n = 0; n < parts; n++) { if (n_xregs < 8) { pat = ld_word_pat | get_i_format_imm (n * 8) | get_i_format_rd (n_xregs + a0_num); push_insns (code, &pat, sizeof (pat)); @@ -482,7 +531,7 @@ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, s res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); #if 0 if (getenv ("MIR_code_dump") != NULL) - _MIR_dump_code ("ffi:", 0, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code), res); + _MIR_dump_code ("ffi:", VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); #endif VARR_DESTROY (uint8_t, code); return res; @@ -554,7 +603,7 @@ void *_MIR_get_interp_shim (MIR_context_t ctx, MIR_item_t func_item, void *handl sp_offset += 8; n_xregs = (n_xregs + 1) / 2 * 2; } - for (int n = 0; n < parts; n++) { + for (uint32_t n = 0; n < parts; n++) { if (n_xregs < 8) { n_xregs++; } @@ -599,7 +648,7 @@ void *_MIR_get_interp_shim (MIR_context_t ctx, MIR_item_t func_item, void *handl sp_offset += 8; n_xregs = (n_xregs + 1) / 2 * 2; } - for (int n = 0; n < parts; n++) { + for (uint32_t n = 0; n < parts; n++) { if (n_xregs < 8) { pat = sd_arg_pat | get_s_format_imm (sp_offset) | get_s_format_rs2 (n_xregs + a0_num); push_insns (code2, &pat, sizeof (pat)); @@ -712,7 +761,7 @@ void *_MIR_get_interp_shim (MIR_context_t ctx, MIR_item_t func_item, void *handl res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); #if 0 if (getenv ("MIR_code_dump") != NULL) - _MIR_dump_code (func->name, 0, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code), res); + _MIR_dump_code (func->name, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); #endif VARR_DESTROY (uint8_t, code); VARR_DESTROY (uint8_t, code2); @@ -800,9 +849,48 @@ static const uint32_t restore_insns[] = { }; #endif +/* set t0 to [ctx,called_func,hook_address]; jump wrapp_end */ +void *_MIR_get_wrapper (MIR_context_t ctx, MIR_item_t called_func, void *hook_address) { + static const uint32_t set_pat[] = { + 0x00000297, /* auipc t0,0x0 */ + 0x00028293, /* addi t0,t0,0 */ + }; + uint8_t *base_addr, *res_code; + size_t offset = 0; + VARR (uint8_t) * code; + uint32_t insns[MAX_JUMP_CODE]; + int len = 64; /* initial len */ + + VARR_CREATE (uint8_t, code, 128); + for (;;) { /* dealing with moving code to another page as the immediate call is pc relative */ + base_addr = _MIR_get_new_code_addr (ctx, len); + if (base_addr == NULL) break; + VARR_TRUNC (uint8_t, code, 0); + push_insns (code, set_pat, sizeof (set_pat)); + len = get_jump_code (insns, wrapper_end_addr, + (uint8_t *) wrapper_end_addr - base_addr - sizeof (set_pat), T1_HARD_REG); + push_insns (code, insns, len); + offset = VARR_LENGTH (uint8_t, code); + assert (offset % 8 == 0); + push_insns (code, &ctx, sizeof (ctx)); + push_insns (code, &called_func, sizeof (called_func)); + push_insns (code, &hook_address, sizeof (hook_address)); + ((uint32_t *) (VARR_ADDR (uint8_t, code)))[1] |= get_i_format_imm (offset); + len = VARR_LENGTH (uint8_t, code); + res_code = _MIR_publish_code_by_addr (ctx, base_addr, VARR_ADDR (uint8_t, code), len); + if (res_code != NULL) break; + } +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("wrapper:", res_code, VARR_LENGTH (uint8_t, code)); +#endif + VARR_DESTROY (uint8_t, code); + return res_code; +} + /* Save regs ra,a0-a7,fa0-fa7; a0 = call hook_address (ctx, called_func); t0=a0; restore regs; br t0 */ -void *_MIR_get_wrapper (MIR_context_t ctx, MIR_item_t called_func, void *hook_address) { +void *_MIR_get_wrapper_end (MIR_context_t ctx) { static const uint32_t jmp_insn = 0x00028067; /* jalr zero,0(t0) */ #if __riscv_compressed static const uint16_t sub_sp = 0x7175; /* c.addi16sp sp,-144 */ @@ -816,19 +904,17 @@ void *_MIR_get_wrapper (MIR_context_t ctx, MIR_item_t called_func, void *hook_ad static const uint32_t restore_ra = 0x00013083; /* ld ra,0(sp) */ #endif static const uint32_t call_pat[] = { - 0x00000297, /* auipc t0,0x0 */ 0x0002b503, /* ld a0,0(t0) */ 0x0002b583, /* ld a1,0(t0) */ 0x0002b603, /* ld a2,0(t0) */ 0x000600e7, /* jalr ra,0(a2) */ 0x00050293, /* mv t0,a0 */ }; + size_t args_start; uint8_t *res_code; - size_t args_start, offset; VARR (uint8_t) * code; VARR_CREATE (uint8_t, code, 128); - VARR_TRUNC (uint8_t, code, 0); push_insns (code, &sub_sp, sizeof (sub_sp)); push_insns (code, &save_ra, sizeof (save_ra)); push_insns (code, save_insns, sizeof (save_insns)); @@ -838,15 +924,254 @@ void *_MIR_get_wrapper (MIR_context_t ctx, MIR_item_t called_func, void *hook_ad push_insns (code, restore_insns, sizeof (restore_insns)); push_insns (code, &add_sp, sizeof (add_sp)); push_insns (code, &jmp_insn, sizeof (jmp_insn)); + ((uint32_t *) (VARR_ADDR (uint8_t, code) + args_start))[0] |= get_i_format_imm (0); + ((uint32_t *) (VARR_ADDR (uint8_t, code) + args_start))[1] |= get_i_format_imm (8); + ((uint32_t *) (VARR_ADDR (uint8_t, code) + args_start))[2] |= get_i_format_imm (16); + res_code = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("wrapper end:", res_code, VARR_LENGTH (uint8_t, code)); +#endif + VARR_DESTROY (uint8_t, code); + return res_code; +} + +/* save x5-x7,x10-x17,x28-x29,x31;f0-f7,f10-f17,f28-31: */ +#if __riscv_compressed +static const uint16_t bb_save_insns[] = { + 0xe816, /* sd t0,16(sp)*/ + 0xec1a, /* sd t1,24(sp)*/ + 0xf01e, /* sd t2,32(sp)*/ + 0xf42a, /* sd a0,40(sp)*/ + 0xf82e, /* sd a1,48(sp)*/ + 0xfc32, /* sd a2,56(sp)*/ + 0xe0b6, /* sd a3,64(sp)*/ + 0xe4ba, /* sd a4,72(sp)*/ + 0xe8be, /* sd a5,80(sp)*/ + 0xecc2, /* sd a6,88(sp)*/ + 0xf0c6, /* sd a7,96(sp)*/ + 0xf4f2, /* sd t3,104(sp)*/ + 0xf8f6, /* sd t4,112(sp)*/ + 0xfcfe, /* sd t6,120(sp)*/ + 0xa102, /* fsd ft0,128(sp)*/ + 0xa506, /* fsd ft1,136(sp)*/ + 0xa90a, /* fsd ft2,144(sp)*/ + 0xad0e, /* fsd ft3,152(sp)*/ + 0xb112, /* fsd ft4,160(sp)*/ + 0xb516, /* fsd ft5,168(sp)*/ + 0xb91a, /* fsd ft6,176(sp)*/ + 0xbd1e, /* fsd ft7,184(sp)*/ + 0xa1aa, /* fsd fa0,192(sp)*/ + 0xa5ae, /* fsd fa1,200(sp)*/ + 0xa9b2, /* fsd fa2,208(sp)*/ + 0xadb6, /* fsd fa3,216(sp)*/ + 0xb1ba, /* fsd fa4,224(sp)*/ + 0xb5be, /* fsd fa5,232(sp)*/ + 0xb9c2, /* fsd fa6,240(sp)*/ + 0xbdc6, /* fsd fa7,248(sp)*/ + 0xa272, /* fsd ft8,256(sp)*/ + 0xa676, /* fsd ft9,264(sp)*/ + 0xaa7a, /* fsd ft10,272(sp)*/ + 0xae7e, /* fsd ft11,280(sp)*/ +}; +#else +static const uint32_t bb_save_insns[] = { + 0x00513823, /* sd t0,16(sp)*/ + 0x00613c23, /* sd t1,24(sp)*/ + 0x02713023, /* sd t2,32(sp)*/ + 0x02a13423, /* sd a0,40(sp)*/ + 0x02b13823, /* sd a1,48(sp)*/ + 0x02c13c23, /* sd a2,56(sp)*/ + 0x04d13023, /* sd a3,64(sp)*/ + 0x04e13423, /* sd a4,72(sp)*/ + 0x04f13823, /* sd a5,80(sp)*/ + 0x05013c23, /* sd a6,88(sp)*/ + 0x07113023, /* sd a7,96(sp)*/ + 0x07c13423, /* sd t3,104(sp)*/ + 0x07d13823, /* sd t4,112(sp)*/ + 0x07f13c23, /* sd t6,120(sp)*/ + 0x08013027, /* fsd ft0,128(sp)*/ + 0x08113427, /* fsd ft1,136(sp)*/ + 0x08213827, /* fsd ft2,144(sp)*/ + 0x08313c27, /* fsd ft3,152(sp)*/ + 0x0a413027, /* fsd ft4,160(sp)*/ + 0x0a513427, /* fsd ft5,168(sp)*/ + 0x0a613827, /* fsd ft6,176(sp)*/ + 0x0a713c27, /* fsd ft7,184(sp)*/ + 0x0ca13027, /* fsd fa0,192(sp)*/ + 0x0cb13427, /* fsd fa1,200(sp)*/ + 0x0cc13827, /* fsd fa2,208(sp)*/ + 0x0cd13c27, /* fsd fa3,216(sp)*/ + 0x0ee13027, /* fsd fa4,224(sp)*/ + 0x0ef13427, /* fsd fa5,232(sp)*/ + 0x0f013827, /* fsd fa6,240(sp)*/ + 0x0f113c27, /* fsd fa7,248(sp)*/ + 0x11c13027, /* fsd ft8,256(sp)*/ + 0x11d13427, /* fsd ft9,264(sp)*/ + 0x11e13827, /* fsd ft10,272(sp)*/ + 0x11f13c27, /* fsd ft11,280(sp) */ +}; +#endif +/* restore x5-x7,x10-x17,x28-x29,x31;f0-f7,f10-f17,f28-31: */ +#if __riscv_compressed +static const uint16_t bb_restore_insns[] = { + 0x62c2, /* ld t0,16(sp)*/ + 0x6362, /* ld t1,24(sp)*/ + 0x7382, /* ld t2,32(sp)*/ + 0x7522, /* ld a0,40(sp)*/ + 0x75c2, /* ld a1,48(sp)*/ + 0x7662, /* ld a2,56(sp)*/ + 0x6686, /* ld a3,64(sp)*/ + 0x6726, /* ld a4,72(sp)*/ + 0x67c6, /* ld a5,80(sp)*/ + 0x6866, /* ld a6,88(sp)*/ + 0x7886, /* ld a7,96(sp)*/ + 0x7e26, /* ld t3,104(sp)*/ + 0x7ec6, /* ld t4,112(sp)*/ + 0x7fe6, /* ld t6,120(sp)*/ + 0x200a, /* fld ft0,128(sp)*/ + 0x20aa, /* fld ft1,136(sp)*/ + 0x214a, /* fld ft2,144(sp)*/ + 0x21ea, /* fld ft3,152(sp)*/ + 0x320a, /* fld ft4,160(sp)*/ + 0x32aa, /* fld ft5,168(sp)*/ + 0x334a, /* fld ft6,176(sp)*/ + 0x33ea, /* fld ft7,184(sp)*/ + 0x250e, /* fld fa0,192(sp)*/ + 0x25ae, /* fld fa1,200(sp)*/ + 0x264e, /* fld fa2,208(sp)*/ + 0x26ee, /* fld fa3,216(sp)*/ + 0x370e, /* fld fa4,224(sp)*/ + 0x37ae, /* fld fa5,232(sp)*/ + 0x384e, /* fld fa6,240(sp)*/ + 0x38ee, /* fld fa7,248(sp)*/ + 0x2e12, /* fld ft8,256(sp)*/ + 0x2eb2, /* fld ft9,264(sp)*/ + 0x2f52, /* fld ft10,272(sp)*/ + 0x2ff2, /* fld ft11,280(sp)*/ +}; +#else +static const uint32_t bb_restore_insns[] = { + 0x01013283, /* ld t0,16(sp)*/ + 0x01813303, /* ld t1,24(sp)*/ + 0x02013383, /* ld t2,32(sp)*/ + 0x02813503, /* ld a0,40(sp)*/ + 0x03013583, /* ld a1,48(sp)*/ + 0x03813603, /* ld a2,56(sp)*/ + 0x04013683, /* ld a3,64(sp)*/ + 0x04813703, /* ld a4,72(sp)*/ + 0x05013783, /* ld a5,80(sp)*/ + 0x05813803, /* ld a6,88(sp)*/ + 0x06013883, /* ld a7,96(sp)*/ + 0x06813e03, /* ld t3,104(sp)*/ + 0x07013e83, /* ld t4,112(sp)*/ + 0x07813f83, /* ld t6,120(sp)*/ + 0x08013007, /* fld ft0,128(sp)*/ + 0x08813087, /* fld ft1,136(sp)*/ + 0x09013107, /* fld ft2,144(sp)*/ + 0x09813187, /* fld ft3,152(sp)*/ + 0x0a013207, /* fld ft4,160(sp)*/ + 0x0a813287, /* fld ft5,168(sp)*/ + 0x0b013307, /* fld ft6,176(sp)*/ + 0x0b813387, /* fld ft7,184(sp)*/ + 0x0c013507, /* fld fa0,192(sp)*/ + 0x0c813587, /* fld fa1,200(sp)*/ + 0x0d013607, /* fld fa2,208(sp)*/ + 0x0d813687, /* fld fa3,216(sp)*/ + 0x0e013707, /* fld fa4,224(sp)*/ + 0x0e813787, /* fld fa5,232(sp)*/ + 0x0f013807, /* fld fa6,240(sp)*/ + 0x0f813887, /* fld fa7,248(sp)*/ + 0x10013e07, /* fld ft8,256(sp)*/ + 0x10813e87, /* fld ft9,264(sp)*/ + 0x11013f07, /* fld ft10,272(sp)*/ + 0x11813f87, /* fld ft11,280(sp)*/ +}; +#endif + +/* t5(x30)=; (b|br) handler ??? mutex free */ +void *_MIR_get_bb_thunk (MIR_context_t ctx, void *bb_version, void *handler) { + /* maximal size thunk -- see _MIR_redirect_thunk */ + uint32_t pat[2] = { + 0x00000f17, /* auipc t5,0 */ + 0x020f3f03, /* ld t5,32(t5) */ + }; + uint32_t nop = TARGET_NOP; + void *res; + VARR (uint8_t) * code; + + VARR_CREATE (uint8_t, code, 64); + assert (MAX_JUMP_CODE == 6); + push_insns (code, pat, sizeof (pat)); + for (int i = 0; i < MAX_JUMP_CODE + 2; i++) + push_insns (code, &nop, 4); /* for max branch and 2 for bb_version */ + *(void **) (VARR_ADDR (uint8_t, code) + 32) = bb_version; + res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); + redirect_thunk (ctx, (uint8_t *) res + 8, handler, T6_HARD_REG); +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("bb thunk:", res, VARR_LENGTH (uint8_t, code)); +#endif + VARR_DESTROY (uint8_t, code); + return res; +} + +/* change to jump */ +void _MIR_replace_bb_thunk (MIR_context_t ctx, void *thunk, void *to) { + redirect_thunk (ctx, thunk, to, TEMP_INT_HARD_REG1); +} + +/* save all clobbered regs but x30(t5); x30 = call hook_address (data, x30); restore regs; br x30 + x30 is a generator temp reg which is not used across bb borders. */ +void *_MIR_get_bb_wrapper (MIR_context_t ctx, void *data, void *hook_address) { + static const uint32_t jmp_insn = 0x000f0067; /* jalr zero,0(t5) */ +#if __riscv_compressed + static const uint16_t sub_sp = 0x712d; /* c.addi16sp sp,-288 */ + static const uint16_t add_sp = 0x6115; /* c.addi16sp sp,288 */ + static const uint16_t save_ra = 0xe006; /* sd ra,0(sp) */ + static const uint16_t restore_ra = 0x6082; /* ld ra,0(sp) */ + static const uint16_t mva1t5 = 0x85fa; /* mv a1,t5 */ +#else + static const uint32_t sub_sp = 0xee010113; /* addi sp,sp,-288 */ + static const uint32_t add_sp = 0x12010113; /* addi sp,sp,288 */ + static const uint32_t save_ra = 0x00113023; /* sd ra,0(sp) */ + static const uint32_t restore_ra = 0x00013083; /* ld ra,0(sp) */ + static const uint32_t mva1t5 = 0x000f0593; /* mv a1,t5 */ +#endif + static const uint32_t call_pat[] = { + 0x00000297, /* auipc t0,0x0 */ + 0x0002b503, /* ld a0,0(t0) */ + 0x0002b603, /* ld a2,0(t0) */ + 0x000600e7, /* jalr ra,0(a2) */ + 0x00050f13, /* mv t5,a0 */ + }; + uint8_t *res_code; + size_t args_start, offset; + VARR (uint8_t) * code; + + VARR_CREATE (uint8_t, code, 128); + VARR_TRUNC (uint8_t, code, 0); + push_insns (code, &sub_sp, sizeof (sub_sp)); + push_insns (code, &save_ra, sizeof (save_ra)); + push_insns (code, bb_save_insns, sizeof (bb_save_insns)); + push_insns (code, &mva1t5, sizeof (mva1t5)); + args_start = VARR_LENGTH (uint8_t, code); + push_insns (code, call_pat, sizeof (call_pat)); + push_insns (code, &restore_ra, sizeof (restore_ra)); + push_insns (code, bb_restore_insns, sizeof (bb_restore_insns)); + push_insns (code, &add_sp, sizeof (add_sp)); + push_insns (code, &jmp_insn, sizeof (jmp_insn)); while (VARR_LENGTH (uint8_t, code) % 8 != 0) VARR_PUSH (uint8_t, code, 0); /* align */ offset = VARR_LENGTH (uint8_t, code) - args_start; - push_insns (code, &ctx, sizeof (ctx)); - push_insns (code, &called_func, sizeof (called_func)); + push_insns (code, &data, sizeof (data)); push_insns (code, &hook_address, sizeof (hook_address)); ((uint32_t *) (VARR_ADDR (uint8_t, code) + args_start))[1] |= get_i_format_imm (offset); ((uint32_t *) (VARR_ADDR (uint8_t, code) + args_start))[2] |= get_i_format_imm (offset + 8); - ((uint32_t *) (VARR_ADDR (uint8_t, code) + args_start))[3] |= get_i_format_imm (offset + 16); res_code = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("bb wrapper:", res_code, VARR_LENGTH (uint8_t, code)); +#endif VARR_DESTROY (uint8_t, code); return res_code; } diff --git a/mir/mir-riscv64.h b/mir/mir-riscv64.h new file mode 100644 index 00000000..0d833b9f --- /dev/null +++ b/mir/mir-riscv64.h @@ -0,0 +1,78 @@ +/* This file is a part of MIR project. + Copyright (C) 2020-2024 Vladimir Makarov . + A common include file for mir-riscv64.c and mir-gen-riscv64.c +*/ + +#include "mir.h" + +#define TARGET_NOP 0x00000013 /* nop:addi zero,zero,0 */ + +#define HREG_EL(h) h##_HARD_REG +#define REP_SEP , +enum { + REP8 (HREG_EL, R0, R1, R2, R3, R4, R5, R6, R7), + REP8 (HREG_EL, R8, R9, R10, R11, R12, R13, R14, R15), + REP8 (HREG_EL, R16, R17, R18, R19, R20, R21, R22, R23), + REP8 (HREG_EL, R24, R25, R26, R27, R28, R29, R30, R31), + /*Aliases: */ ZERO_HARD_REG = R0_HARD_REG, + REP7 (HREG_EL, RA, SP, GP, TP, T0, T1, T2), + REP8 (HREG_EL, FP, S1, A0, A1, A2, A3, A4, A5), + REP8 (HREG_EL, A6, A7, S2, S3, S4, S5, S6, S7), + REP8 (HREG_EL, S8, S9, S10, S11, T3, T4, T5, T6), + + REP8 (HREG_EL, F0, F1, F2, F3, F4, F5, F6, F7), + REP8 (HREG_EL, F8, F9, F10, F11, F12, F13, F14, F15), + REP8 (HREG_EL, F16, F17, F18, F19, F20, F21, F22, F23), + REP8 (HREG_EL, F24, F25, F26, F27, F28, F29, F30, F31), + /* Aliases: */ FT0_HARD_REG = F0_HARD_REG, + REP7 (HREG_EL, FT1, FT2, FT3, FT4, FT5, FT6, FT7), + REP8 (HREG_EL, FS0, FS1, FA0, FA1, FA2, FA3, FA4, FA5), + REP8 (HREG_EL, FA6, FA7, FS2, FS3, FS4, FS5, FS6, FS7), + REP8 (HREG_EL, FS8, FS9, FS10, FS11, FT8, FT9, FT10, FT11), +}; +#undef REP_SEP + +static const char *const target_hard_reg_names[] = { + "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12", + "r13", "r14", "r15", "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23", "r24", "r25", + "r26", "r27", "r28", "r29", "r30", "r31", "f0", "f1", "f2", "f3", "f4", "f5", "f6", + "f7", "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17", "f18", "f19", + "f20", "f21", "f22", "f23", "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", +}; + +#define MAX_HARD_REG F31_HARD_REG + +/* Hard regs not used in machinized code, preferably call used ones. */ +static const MIR_reg_t TEMP_INT_HARD_REG1 = T5_HARD_REG, TEMP_INT_HARD_REG2 = T6_HARD_REG; +static const MIR_reg_t TEMP_FLOAT_HARD_REG1 = FT10_HARD_REG, TEMP_FLOAT_HARD_REG2 = FT11_HARD_REG; +static const MIR_reg_t TEMP_DOUBLE_HARD_REG1 = FT10_HARD_REG, TEMP_DOUBLE_HARD_REG2 = FT11_HARD_REG; +/* we use only builtins for long double ops: */ +static const MIR_reg_t TEMP_LDOUBLE_HARD_REG1 = MIR_NON_VAR; +static const MIR_reg_t TEMP_LDOUBLE_HARD_REG2 = MIR_NON_VAR; + +static inline int target_hard_reg_type_ok_p (MIR_reg_t hard_reg, MIR_type_t type) { + assert (hard_reg <= MAX_HARD_REG); + if (type == MIR_T_LD) return FALSE; /* long double can be in hard regs only for arg passing */ + return MIR_fp_type_p (type) ? hard_reg >= F0_HARD_REG : hard_reg < F0_HARD_REG; +} + +static inline int target_fixed_hard_reg_p (MIR_reg_t hard_reg) { + assert (hard_reg <= MAX_HARD_REG); + return (hard_reg == ZERO_HARD_REG || hard_reg == FP_HARD_REG || hard_reg == SP_HARD_REG + || hard_reg == GP_HARD_REG || hard_reg == TP_HARD_REG // ??? + || hard_reg == TEMP_INT_HARD_REG1 || hard_reg == TEMP_INT_HARD_REG2 + || hard_reg == TEMP_FLOAT_HARD_REG1 || hard_reg == TEMP_FLOAT_HARD_REG2 + || hard_reg == TEMP_DOUBLE_HARD_REG1 || hard_reg == TEMP_DOUBLE_HARD_REG2 + || hard_reg == TEMP_LDOUBLE_HARD_REG1 || hard_reg == TEMP_LDOUBLE_HARD_REG2); +} + +static inline int target_locs_num (MIR_reg_t loc, MIR_type_t type) { + return loc > MAX_HARD_REG && type == MIR_T_LD ? 2 : 1; +} + +static const uint32_t j_imm_mask = 0xfffff000; +static inline uint32_t get_j_format_imm (int32_t offset) { + int d = offset >> 1; /* scale */ + assert (-(1 << 19) <= d && d < (1 << 19)); + return ((d & 0x80000) | ((d & 0x3ff) << 9) | (((d >> 10) & 0x1) << 8) | ((d >> 11) & 0xff)) << 12; +} diff --git a/mir/mir-s390x.c b/mir/mir-s390x.c index 6230cfaf..2e484350 100644 --- a/mir/mir-s390x.c +++ b/mir/mir-s390x.c @@ -1,12 +1,14 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ +#include "mir-s390x.h" + /* Long doubles (-mlong-double=128) are always passed by its address (for args and results) */ /* All BLK type values and RBLK args are always passed by address. */ -#if 0 && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #error "s390x works only in BE mode" #endif @@ -77,7 +79,7 @@ static void s390x_gen_ldstm (VARR (uint8_t) * insn_varr, unsigned from, unsigned static void s390x_gen_jump (VARR (uint8_t) * insn_varr, unsigned int reg, int call_p) { uint16_t bcr = (0x7 << 8) | (15 << 4) | reg; /* bcr 15,reg: */ uint16_t balr = (0x5 << 8) | (14 << 4) | reg; /* balr 14,reg: */ - assert (reg < 16); + assert (reg != 0 && reg < 16); push_insns (insn_varr, (uint8_t *) (call_p ? &balr : &bcr), 2); } @@ -90,22 +92,25 @@ static void s390x_gen_addi (VARR (uint8_t) * insn_varr, unsigned dst, unsigned s } static void s390x_gen_3addrs (VARR (uint8_t) * insn_varr, unsigned int r1, void *a1, - unsigned int r2, void *a2, unsigned int r3, void *a3) { - /* 6b:lalr r3,22+align;6b:lg r1,0(r3);6b:lg r2,8(r3);6b:lg r3,16(r3);4b:bc m15,28;align;a1-a3 */ - size_t rem = (VARR_LENGTH (uint8_t, insn_varr) + 28) % 8; + unsigned int r2, void *a2, int r3, void *a3) { + /* 6b:lalr r3,22+align;6b:lg r1,0(r3);6b:lg r2,8(r3);[6b:lg r3,16(r3);]4b:bc m15,s;align;a1-a3:s + */ + size_t off = (r3 < 0 ? 22 : 28); + size_t rem = (VARR_LENGTH (uint8_t, insn_varr) + off) % 8; size_t padding = rem == 0 ? 0 : 8 - rem; - uint64_t lalr = ((0xc0l << 40) | ((uint64_t) r1 << 36) | (28 + padding) / 2) << 16; - uint32_t brc = (0xa7 << 24) | (15 << 20) | (4 << 16) | (28 + padding) / 2; /* brc m15,28: */ + uint64_t lalr = ((0xc0l << 40) | ((uint64_t) r1 << 36) | (off + padding) / 2) << 16; + uint32_t brc + = (0xa7 << 24) | (15 << 20) | (4 << 16) | ((r3 < 0 ? 20 : 28) + padding) / 2; /* brc m15,28: */ assert (r1 != 0); push_insns (insn_varr, (uint8_t *) &lalr, 6); - s390x_gen_ld (insn_varr, r3, r1, 16, MIR_T_I64); /* lg r3,16(r1) */ - s390x_gen_ld (insn_varr, r2, r1, 8, MIR_T_I64); /* lg r2,8(r1) */ - s390x_gen_ld (insn_varr, r1, r1, 0, MIR_T_I64); /* lg r1,0(r1) */ + if (r3 >= 0) s390x_gen_ld (insn_varr, r3, r1, 16, MIR_T_I64); /* lg r3,16(r1) */ + s390x_gen_ld (insn_varr, r2, r1, 8, MIR_T_I64); /* lg r2,8(r1) */ + s390x_gen_ld (insn_varr, r1, r1, 0, MIR_T_I64); /* lg r1,0(r1) */ push_insns (insn_varr, (uint8_t *) &brc, 4); for (size_t i = 0; i < padding; i++) VARR_PUSH (uint8_t, insn_varr, 0); push_insns (insn_varr, (uint8_t *) &a1, 8); push_insns (insn_varr, (uint8_t *) &a2, 8); - push_insns (insn_varr, (uint8_t *) &a3, 8); + if (r3 >= 0) push_insns (insn_varr, (uint8_t *) &a3, 8); } static void s390x_gen_blk_mov (VARR (uint8_t) * insn_varr, uint32_t param_offset, @@ -154,8 +159,8 @@ void *_MIR_get_bend_builtin (MIR_context_t ctx) { return res; } +static const int max_thunk_len = (4 * 8); /* see _MIR_redirect_thunk */ void *_MIR_get_thunk (MIR_context_t ctx) { - const int max_thunk_len = (4 * 8); /* see _MIR_redirect_thunk */ VARR (uint8_t) * code; void *res; @@ -166,22 +171,23 @@ void *_MIR_get_thunk (MIR_context_t ctx) { return res; } -void _MIR_redirect_thunk (MIR_context_t ctx, void *thunk, void *to) { +static void redirect_thunk (MIR_context_t ctx, void *thunk, void *to, long temp_reg) { int64_t offset = (uint8_t *) to - (uint8_t *) thunk; VARR (uint8_t) * code; + assert (temp_reg != 0); VARR_CREATE (uint8_t, code, 128); assert (offset % 2 == 0); offset /= 2; if (-(1l << 31) < offset && offset < (1l << 31)) { /* brcl m15,offset: */ uint64_t brcl = ((0xc0l << 40) | (15l << 36) | (4l << 32) | (offset & 0xffffffff)) << 16; push_insns (code, (uint8_t *) &brcl, 6); - } else { /* 6b:lalr r1,8+padding; 6b:lg r1,0(r1); 2b:bcr m15,r1;padding; 64-bit address: */ + } else { /* 6b:lalr tr,14+padding; 6b:lg tr,0(tr); 2b:bcr m15,tr;padding; 64-bit address: */ size_t rem = (VARR_LENGTH (uint8_t, code) + 14) % 8; size_t padding = rem == 0 ? 0 : 8 - rem; - uint64_t lalr = ((0xc0l << 40) | (1l << 36) | (14 + padding) / 2) << 16; - uint64_t lg = ((0xe3l << 40) | (1l << 36) | (1l << 28) | 0x4) << 16; - uint16_t bcr = (0x7 << 8) | (15 << 4) | 1; /* bcr 15,r1: */ + uint64_t lalr = ((0xc0l << 40) | (temp_reg << 36) | (14 + padding) / 2) << 16; + uint64_t lg = ((0xe3l << 40) | (temp_reg << 36) | (temp_reg << 28) | 0x4) << 16; + uint16_t bcr = (0x7 << 8) | (15 << 4) | temp_reg; /* bcr 15,tr: */ push_insns (code, (uint8_t *) &lalr, 6); push_insns (code, (uint8_t *) &lg, 6); push_insns (code, (uint8_t *) &bcr, 2); @@ -192,6 +198,25 @@ void _MIR_redirect_thunk (MIR_context_t ctx, void *thunk, void *to) { VARR_DESTROY (uint8_t, code); } +void _MIR_redirect_thunk (MIR_context_t ctx, void *thunk, void *to) { + redirect_thunk (ctx, thunk, to, 1); +} + +static void *get_jump_addr (uint8_t *insns) { + assert (insns[0] == 0xc0); + if ((insns[1] >> 4) == 15) { /* bcrl m15,offset */ + int32_t offset = (insns[2] << 24) | (insns[3] << 16) | (insns[4] << 8) | insns[5]; + return insns + offset * 2; + } else { + size_t addr = ((size_t) insns + 14 + 7) / 8 * 8; + return *(void **) addr; + } +} + +void *_MIR_get_thunk_addr (MIR_context_t ctx MIR_UNUSED, void *thunk) { + return get_jump_addr (thunk); +} + struct s390x_va_list { long __gpr, __fpr; /* number of args read until now */ void *__overflow_arg_area; /* argument on the stack to read next */ @@ -226,11 +251,12 @@ void *va_arg_builtin (void *p, uint64_t t) { return a; } -void va_block_arg_builtin (void *res, void *p, size_t s, uint64_t ncase) { - memcpy (res, *(void **) va_arg_builtin (p, MIR_T_I64), s); +void va_block_arg_builtin (void *res, void *p, size_t s, uint64_t ncase MIR_UNUSED) { + void *a = *(void **) va_arg_builtin (p, MIR_T_I64); + if (res != NULL) memcpy (res, a, s); } -void va_start_interp_builtin (MIR_context_t ctx, void *p, void *a) { +void va_start_interp_builtin (MIR_context_t ctx MIR_UNUSED, void *p, void *a) { struct s390x_va_list *va = p; va_list *vap = a; @@ -238,7 +264,7 @@ void va_start_interp_builtin (MIR_context_t ctx, void *p, void *a) { *va = *(struct s390x_va_list *) vap; } -void va_end_interp_builtin (MIR_context_t ctx, void *p) {} +void va_end_interp_builtin (MIR_context_t ctx MIR_UNUSED, void *p MIR_UNUSED) {} /* Generation: fun (fun_addr, res_arg_addresses): save r6, r7, r14 (r15 + 48,112); @@ -250,7 +276,7 @@ void va_end_interp_builtin (MIR_context_t ctx, void *p) {} r0=mem[r7,]; res_reg=mem[r0]; ... restore r15; restore r6, r7, r14; return. */ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, size_t nargs, - _MIR_arg_desc_t *arg_descs, size_t arg_vars_num) { + _MIR_arg_desc_t *arg_descs, size_t arg_vars_num MIR_UNUSED) { MIR_type_t type; int n_gpregs = 0, n_fpregs = 0, res_reg = 7, frame_size, disp, param_offset, blk_offset; uint32_t qwords, addr_reg; @@ -352,7 +378,7 @@ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, s res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); #if 0 if (getenv ("MIR_code_dump") != NULL) - _MIR_dump_code ("ffi:", 0, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); + _MIR_dump_code ("ffi:", VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); #endif VARR_DESTROY (uint8_t, code); return res; @@ -429,7 +455,7 @@ void *_MIR_get_interp_shim (MIR_context_t ctx, MIR_item_t func_item, void *handl res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); #if 0 if (getenv ("MIR_code_dump") != NULL) - _MIR_dump_code (func->name, 0, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); + _MIR_dump_code (func->name, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); #endif VARR_DESTROY (uint8_t, code); return res; @@ -439,7 +465,35 @@ void *_MIR_get_interp_shim (MIR_context_t ctx, MIR_item_t func_item, void *handl update r15; allocate and form minimal wrapper stack frame (S390X_STACK_HEADER_SIZE); r2 = call hook_address (ctx, called_func); r1=r2; restore all params regs, r15, r14; bcr r1 */ void *_MIR_get_wrapper (MIR_context_t ctx, MIR_item_t called_func, void *hook_address) { - int frame_size = S390X_STACK_HEADER_SIZE; + VARR (uint8_t) * code; + void *res; + /* 16b offset -- 6b:lalr r1; 2b(align): lr r0,r0; 6b:lg r1,24(r1); 2b: balr r1,r1; */ + uint64_t lalr = ((0xc0l << 40) | ((uint64_t) 1 << 36) | (16 / 2)) << 16; + uint16_t lr = (0x18 << 8); /* lr r0,r0 */ + uint16_t balr = (0x5 << 8) | (1 << 4) | 1; /* balr r1,r1: */ + + VARR_CREATE (uint8_t, code, 128); + push_insns (code, (uint8_t *) &lalr, 6); + push_insns (code, (uint8_t *) &lr, 2); + s390x_gen_ld (code, 1, 1, 24, MIR_T_I64); /* lg r1,24(r1) */ + push_insns (code, (uint8_t *) &balr, 2); /* balr r1,r1 */ + push_insns (code, (uint8_t *) &ctx, 8); + push_insns (code, (uint8_t *) &called_func, 8); + push_insns (code, (uint8_t *) &hook_address, 8); + push_insns (code, (uint8_t *) &wrapper_end_addr, 8); + res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("func wrapper:", res, VARR_LENGTH (uint8_t, code)); +#endif + VARR_DESTROY (uint8_t, code); + return res; +} + +/* Brief: save r14 (r15+120); save all param regs r2-r6 (r15+16),f0,f2,f4,f6 (r15+128); + update r15; allocate and form minimal wrapper stack frame (S390X_STACK_HEADER_SIZE); + r2 = call hook_address (ctx, called_func); r1=r2; restore all params regs, r15, r14; bcr r1 */ +void *_MIR_get_wrapper_end (MIR_context_t ctx) { VARR (uint8_t) * code; void *res; @@ -449,17 +503,98 @@ void *_MIR_get_wrapper (MIR_context_t ctx, MIR_item_t called_func, void *hook_ad for (unsigned reg = 0; reg <= 6; reg += 2) /* stdy f0,f2,f4,f6,128(r15) : */ s390x_gen_st (code, reg, 15, reg * 4 + 128, MIR_T_D); /* r15 -= frame_size: */ - s390x_gen_addi (code, 15, 15, -frame_size); - s390x_gen_3addrs (code, 2, ctx, 3, called_func, 4, hook_address); + s390x_gen_addi (code, 15, 15, -S390X_STACK_HEADER_SIZE); + s390x_gen_ld (code, 2, 1, 0, MIR_T_I64); /* lg r2,0(r1) */ + s390x_gen_ld (code, 3, 1, 8, MIR_T_I64); /* lg r3,8(r1) */ + s390x_gen_ld (code, 4, 1, 16, MIR_T_I64); /* lg r4,16(r1) */ s390x_gen_jump (code, 4, TRUE); s390x_gen_mov (code, 1, 2); - s390x_gen_addi (code, 15, 15, frame_size); + s390x_gen_addi (code, 15, 15, S390X_STACK_HEADER_SIZE); for (unsigned reg = 0; reg <= 6; reg += 2) /* ldy fn,disp(r15) : */ s390x_gen_ld (code, reg, 15, reg * 4 + 128, MIR_T_D); s390x_gen_ldstm (code, 2, 6, 15, 16, TRUE); /* lmg 2,6,16(r15) : */ s390x_gen_ld (code, 14, 15, 112, MIR_T_I64); /* lg 14,112(r15) */ s390x_gen_jump (code, 1, FALSE); res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("func wrapper end:", res, VARR_LENGTH (uint8_t, code)); +#endif + VARR_DESTROY (uint8_t, code); + return res; +} + +/* r0=; jump handler ??? mutex free */ +void *_MIR_get_bb_thunk (MIR_context_t ctx, void *bb_version, void *handler) { + void *res; + size_t offset; + VARR (uint8_t) * code; + uint64_t lalr = ((0xc0l << 40) | (1l << 36) | (16 + max_thunk_len) / 2) << 16; + uint64_t lg = ((0xe3l << 40) | (0l << 36) | (1l << 28) | 0x4) << 16; + uint32_t nop = (0x47 << 24); + + VARR_CREATE (uint8_t, code, 64); + /* 6b:lalr r1,8; 6b:lg r0,0(r1); 4b: nop for padding; */ + push_insns (code, (uint8_t *) &lalr, 6); + push_insns (code, (uint8_t *) &lg, 6); + push_insns (code, (uint8_t *) &nop, 4); + offset = VARR_LENGTH (uint8_t, code); + for (int i = 0; i < max_thunk_len; i++) VARR_PUSH (uint8_t, code, 0); + assert (max_thunk_len % 8 == 0 && VARR_LENGTH (uint8_t, code) % 8 == 0); + push_insns (code, (uint8_t *) &bb_version, 8); + res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); + _MIR_redirect_thunk (ctx, (uint8_t *) res + offset, handler); +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("bb thunk:", res, VARR_LENGTH (uint8_t, code)); +#endif + VARR_DESTROY (uint8_t, code); + return res; +} + +/* change to (b|br) to */ +void _MIR_replace_bb_thunk (MIR_context_t ctx, void *thunk, void *to) { + redirect_thunk (ctx, thunk, to, 1); +} + +/* save clobbered regs (r2-r6, f0-f7); r1 = call hook_address (data, r0); restore regs; br + r1, r1 is a generator temp reg which is not used across bb borders. */ +void *_MIR_get_bb_wrapper (MIR_context_t ctx, void *data, void *hook_address) { + void *res; + VARR (uint8_t) * code; + + VARR_CREATE (uint8_t, code, 128); + /* saving regs which can be put reg save area: */ + s390x_gen_ldstm (code, 2, 6, 15, 16, FALSE); /* stmg 2,6,16(r15) : */ + s390x_gen_st (code, 14, 15, 112, MIR_T_I64); /* ???do we need this: stg 14,112(r15) */ + for (unsigned reg = 0; reg <= 6; reg += 2) /* stdy f0,f2,f4,f6,128(r15) : */ + s390x_gen_st (code, reg, 15, reg * 4 + 128, MIR_T_D); + s390x_gen_addi (code, 15, 15, -48); + /* saving f1,f3,f5,f5: */ + for (unsigned reg = 1; reg <= 7; reg += 2) /* stdy f1,f3,f5,f7,16(r15) : */ + s390x_gen_st (code, reg, 15, (reg - 1) * 4 + 16, MIR_T_D); + /* r15 -= 160: */ + s390x_gen_addi (code, 15, 15, -S390X_STACK_HEADER_SIZE); + s390x_gen_3addrs (code, 1, hook_address, 2, data, -1, NULL); + s390x_gen_mov (code, 3, 0); /* r3=r0 */ + s390x_gen_jump (code, 1, TRUE); /* call r1 */ + s390x_gen_mov (code, 1, 2); /* r1=r2 */ + s390x_gen_addi (code, 15, 15, S390X_STACK_HEADER_SIZE); + /* restoring f1,f3,f5,f5: */ + for (unsigned reg = 1; reg <= 7; reg += 2) /* stdy f1,f3,f5,f7,16(r15) : */ + s390x_gen_ld (code, reg, 15, (reg - 1) * 4 + 16, MIR_T_D); + s390x_gen_addi (code, 15, 15, 48); + /* restoring regs which can be put reg save area: */ + for (unsigned reg = 0; reg <= 6; reg += 2) /* ldy fn,disp(r15) : */ + s390x_gen_ld (code, reg, 15, reg * 4 + 128, MIR_T_D); + s390x_gen_ld (code, 14, 15, 112, MIR_T_I64); /* ??? do we need this: lg 14,112(r15) */ + s390x_gen_ldstm (code, 2, 6, 15, 16, TRUE); /* lmg 2,6,16(r15) : */ + s390x_gen_jump (code, 1, FALSE); /* bcr r1 */ + res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); +#if 0 + if (getenv ("MIR_code_dump") != NULL) + _MIR_dump_code ("bb wrapper:", res, VARR_LENGTH (uint8_t, code)); +#endif VARR_DESTROY (uint8_t, code); return res; } diff --git a/mir/mir-s390x.h b/mir/mir-s390x.h new file mode 100644 index 00000000..4866833a --- /dev/null +++ b/mir/mir-s390x.h @@ -0,0 +1,57 @@ +/* This file is a part of MIR project. + Copyright (C) 2018-2024 Vladimir Makarov . + A common include file for mir-s390x.c and mir-gen-s390x.c +*/ + +#include "mir.h" + +#define HREG_EL(h) h##_HARD_REG +#define REP_SEP , +enum { + REP8 (HREG_EL, R0, R1, R2, R3, R4, R5, R6, R7), + REP8 (HREG_EL, R8, R9, R10, R11, R12, R13, R14, R15), + REP8 (HREG_EL, F0, F1, F2, F3, F4, F5, F6, F7), + REP8 (HREG_EL, F8, F9, F10, F11, F12, F13, F14, F15), +}; +#undef REP_SEP + +static const char *const target_hard_reg_names[] = { + "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", + "r11", "r12", "r13", "r14", "r15", "f0", "f1", "f2", "f3", "f4", "f5", + "f6", "f7", "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", +}; + +#define MAX_HARD_REG F15_HARD_REG +static const MIR_reg_t SP_HARD_REG = R15_HARD_REG; +static const MIR_reg_t FP_HARD_REG = R11_HARD_REG; + +static int target_locs_num (MIR_reg_t loc MIR_UNUSED, MIR_type_t type) { + return type == MIR_T_LD ? 2 : 1; +} + +/* Hard regs not used in machinized code and for passing args, preferably call saved ones. */ +static const MIR_reg_t TEMP_INT_HARD_REG1 = R1_HARD_REG, TEMP_INT_HARD_REG2 = R9_HARD_REG; +static const MIR_reg_t TEMP_FLOAT_HARD_REG1 = F8_HARD_REG, TEMP_FLOAT_HARD_REG2 = F10_HARD_REG; +static const MIR_reg_t TEMP_DOUBLE_HARD_REG1 = F8_HARD_REG, TEMP_DOUBLE_HARD_REG2 = F10_HARD_REG; +static const MIR_reg_t TEMP_LDOUBLE_HARD_REG1 = F8_HARD_REG; //??? +static const MIR_reg_t TEMP_LDOUBLE_HARD_REG2 = F10_HARD_REG; + +static inline int target_hard_reg_type_ok_p (MIR_reg_t hard_reg, MIR_type_t type) { + assert (hard_reg <= MAX_HARD_REG); + if (type == MIR_T_LD) /* f0,f1,f4,f5,f8,f9,f12,f13 - pair starts */ + return hard_reg >= F0_HARD_REG && (hard_reg - F0_HARD_REG) % 4 <= 1; + return MIR_fp_type_p (type) ? F0_HARD_REG <= hard_reg && hard_reg <= F15_HARD_REG + : hard_reg < F0_HARD_REG; +} + +static inline int target_fixed_hard_reg_p (MIR_reg_t hard_reg) { + assert (hard_reg <= MAX_HARD_REG); + return (hard_reg == FP_HARD_REG + || hard_reg == SP_HARD_REG + /* don't bother to allocate R0 as it has special meaning for base and index reg: */ + || hard_reg == R0_HARD_REG || hard_reg == TEMP_INT_HARD_REG1 + || hard_reg == TEMP_INT_HARD_REG2 || hard_reg == TEMP_FLOAT_HARD_REG1 + || hard_reg == TEMP_FLOAT_HARD_REG2 || hard_reg == TEMP_DOUBLE_HARD_REG1 + || hard_reg == TEMP_DOUBLE_HARD_REG2 || hard_reg == TEMP_LDOUBLE_HARD_REG1 + || hard_reg == TEMP_LDOUBLE_HARD_REG2); +} diff --git a/mir/mir-varr.h b/mir/mir-varr.h index 051a09fd..e6227b94 100644 --- a/mir/mir-varr.h +++ b/mir/mir-varr.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ #ifndef MIR_VARR_H @@ -91,6 +91,11 @@ static inline void MIR_VARR_NO_RETURN mir_varr_error (const char *message) { return varr->els_num; \ } \ \ + static inline size_t VARR_OP_DEF (T, capacity) (const VARR (T) * varr) { \ + VARR_ASSERT (varr, "size", T); \ + return varr->size; \ + } \ + \ static inline T *VARR_OP_DEF (T, addr) (const VARR (T) * varr) { \ VARR_ASSERT (varr, "addr", T); \ return &varr->varr[0]; \ @@ -160,6 +165,7 @@ static inline void MIR_VARR_NO_RETURN mir_varr_error (const char *message) { #define VARR_CREATE(T, V, L) (VARR_OP (T, create) (&(V), L)) #define VARR_DESTROY(T, V) (VARR_OP (T, destroy) (&(V))) #define VARR_LENGTH(T, V) (VARR_OP (T, length) (V)) +#define VARR_CAPACITY(T, V) (VARR_OP (T, capacity) (V)) #define VARR_ADDR(T, V) (VARR_OP (T, addr) (V)) #define VARR_LAST(T, V) (VARR_OP (T, last) (V)) #define VARR_GET(T, V, I) (VARR_OP (T, get) (V, I)) diff --git a/mir/mir-x86_64.c b/mir/mir-x86_64.c index 86664ec0..68cb4de4 100644 --- a/mir/mir-x86_64.c +++ b/mir/mir-x86_64.c @@ -1,7 +1,9 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ +#include "mir-x86_64.h" + /* RBLK args are always passed by address. BLK0 first is copied on the caller stack and passed implicitly. BLK1 is passed in general regs @@ -79,7 +81,7 @@ void va_block_arg_builtin (void *res, void *p, size_t s, uint64_t ncase) { u[1].i = *(uint64_t *) ((char *) va->reg_save_area + va->gp_offset); va->gp_offset += 8; } - memcpy (res, &u, s); + if (res != NULL) memcpy (res, &u, s); return; case 2: u[0].d = *(double *) ((char *) va->reg_save_area + va->fp_offset); @@ -88,7 +90,7 @@ void va_block_arg_builtin (void *res, void *p, size_t s, uint64_t ncase) { u[1].d = *(double *) ((char *) va->reg_save_area + va->fp_offset); va->fp_offset += 16; } - memcpy (res, &u, s); + if (res != NULL) memcpy (res, &u, s); return; case 3: case 4: @@ -102,15 +104,15 @@ void va_block_arg_builtin (void *res, void *p, size_t s, uint64_t ncase) { } va->fp_offset += 8; va->gp_offset += 8; - memcpy (res, &u, s); + if (res != NULL) memcpy (res, &u, s); return; default: break; } - memcpy (res, a, s); + if (res != NULL) memcpy (res, a, s); va->overflow_arg_area += size / 8; } -void va_start_interp_builtin (MIR_context_t ctx, void *p, void *a) { +void va_start_interp_builtin (MIR_context_t ctx MIR_UNUSED, void *p, void *a) { struct x86_64_va_list *va = p; va_list *vap = a; @@ -134,7 +136,7 @@ void *va_arg_builtin (void *p, uint64_t t) { void va_block_arg_builtin (void *res, void *p, size_t s, uint64_t ncase) { struct x86_64_va_list *va = p; void *a = s <= 8 ? va->arg_area : *(void **) va->arg_area; /* pass by pointer */ - memcpy (res, a, s); + if (res != NULL) memcpy (res, a, s); va->arg_area++; } @@ -148,26 +150,50 @@ void va_start_interp_builtin (MIR_context_t ctx, void *p, void *a) { #endif -void va_end_interp_builtin (MIR_context_t ctx, void *p) {} +void va_end_interp_builtin (MIR_context_t ctx MIR_UNUSED, void *p MIR_UNUSED) {} + +static const uint8_t short_jmp_pattern[] = { + 0xe9, 0, 0, 0, 0, /* 0x0: jmp rel32 */ + 0, 0, 0, 0, 0, 0, 0, 0 /* 0x5: abs address holder */ +}; +static const uint8_t long_jmp_pattern[] = { + 0x49, 0xbb, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x0: movabsq 0, r11 */ + 0x41, 0xff, 0xe3, /* 0xa: jmpq *%r11 */ +}; /* r11=
; jump *r11 */ void *_MIR_get_thunk (MIR_context_t ctx) { void *res; - static const uint8_t pattern[] = { - 0x49, 0xbb, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x0: movabsq 0, r11 */ - 0x41, 0xff, 0xe3, /* 0x14: jmpq *%r11 */ - }; - res = _MIR_publish_code (ctx, pattern, sizeof (pattern)); + assert (sizeof (short_jmp_pattern) == sizeof (long_jmp_pattern)); + res = _MIR_publish_code (ctx, short_jmp_pattern, sizeof (short_jmp_pattern)); return res; } +void *_MIR_get_thunk_addr (MIR_context_t ctx MIR_UNUSED, void *thunk) { + void *addr; + int short_p = *(unsigned char *) thunk == 0xe9; + memcpy ((char *) &addr, (char *) thunk + (short_p ? 5 : 2), sizeof (addr)); + return addr; +} + void _MIR_redirect_thunk (MIR_context_t ctx, void *thunk, void *to) { - _MIR_update_code (ctx, thunk, 1, 2, to); + int64_t disp = (char *) to - ((char *) thunk + 5); + int short_p = INT32_MIN <= disp && disp <= INT32_MAX; + uint8_t pattern[sizeof (short_jmp_pattern)]; + if (short_p) { + memcpy (pattern, short_jmp_pattern, sizeof (short_jmp_pattern)); + memcpy (pattern + 1, &disp, 4); /* little endian */ + memcpy (pattern + 5, &to, 8); + } else { + memcpy (pattern, long_jmp_pattern, sizeof (long_jmp_pattern)); + memcpy (pattern + 2, &to, 8); + } + _MIR_change_code (ctx, thunk, pattern, sizeof (short_jmp_pattern)); } static const uint8_t save_pat[] = { #ifndef _WIN32 - 0x48, 0x81, 0xec, 0x80, 0, 0, 0, /*sub $0x80,%rsp */ + 0x48, 0x81, 0xec, 0x80, 0, 0, 0, /*sub $0x88,%rsp */ 0xf3, 0x0f, 0x7f, 0x04, 0x24, /*movdqu %xmm0,(%rsp) */ 0xf3, 0x0f, 0x7f, 0x4c, 0x24, 0x10, /*movdqu %xmm1,0x10(%rsp) */ 0xf3, 0x0f, 0x7f, 0x54, 0x24, 0x20, /*movdqu %xmm2,0x20(%rsp) */ @@ -183,10 +209,10 @@ static const uint8_t save_pat[] = { 0x56, /*push %rsi */ 0x57, /*push %rdi */ #else - 0x48, 0x89, 0x4c, 0x24, 0x08, /*mov %rcx,0x08(%rsp) */ - 0x48, 0x89, 0x54, 0x24, 0x10, /*mov %rdx,0x10(%rsp) */ - 0x4c, 0x89, 0x44, 0x24, 0x18, /*mov %r8, 0x18(%rsp) */ - 0x4c, 0x89, 0x4c, 0x24, 0x20, /*mov %r9, 0x20(%rsp) */ + 0x48, 0x89, 0x4c, 0x24, 0x08, /*mov %rcx,0x08(%rsp) */ + 0x48, 0x89, 0x54, 0x24, 0x10, /*mov %rdx,0x10(%rsp) */ + 0x4c, 0x89, 0x44, 0x24, 0x18, /*mov %r8, 0x18(%rsp) */ + 0x4c, 0x89, 0x4c, 0x24, 0x20, /*mov %r9, 0x20(%rsp) */ #endif }; @@ -208,14 +234,14 @@ static const uint8_t restore_pat[] = { 0xf3, 0x0f, 0x6f, 0x7c, 0x24, 0x70, /*movdqu 0x70(%rsp),%xmm7 */ 0x48, 0x81, 0xc4, 0x80, 0, 0, 0, /*add $0x80,%rsp */ #else - 0x48, 0x8b, 0x4c, 0x24, 0x08, /*mov 0x08(%rsp),%rcx */ - 0x48, 0x8b, 0x54, 0x24, 0x10, /*mov 0x10(%rsp),%rdx */ - 0x4c, 0x8b, 0x44, 0x24, 0x18, /*mov 0x18(%rsp),%r8 */ - 0x4c, 0x8b, 0x4c, 0x24, 0x20, /*mov 0x20(%rsp),%r9 */ - 0xf3, 0x0f, 0x7e, 0x44, 0x24, 0x08, /*movq 0x08(%rsp),%xmm0*/ - 0xf3, 0x0f, 0x7e, 0x4c, 0x24, 0x10, /*movq 0x10(%rsp),%xmm1*/ - 0xf3, 0x0f, 0x7e, 0x54, 0x24, 0x18, /*movq 0x18(%rsp),%xmm2*/ - 0xf3, 0x0f, 0x7e, 0x5c, 0x24, 0x20, /*movq 0x20(%rsp),%xmm3*/ + 0x48, 0x8b, 0x4c, 0x24, 0x08, /*mov 0x08(%rsp),%rcx */ + 0x48, 0x8b, 0x54, 0x24, 0x10, /*mov 0x10(%rsp),%rdx */ + 0x4c, 0x8b, 0x44, 0x24, 0x18, /*mov 0x18(%rsp),%r8 */ + 0x4c, 0x8b, 0x4c, 0x24, 0x20, /*mov 0x20(%rsp),%r9 */ + 0xf3, 0x0f, 0x7e, 0x44, 0x24, 0x08, /*movq 0x08(%rsp),%xmm0*/ + 0xf3, 0x0f, 0x7e, 0x4c, 0x24, 0x10, /*movq 0x10(%rsp),%xmm1*/ + 0xf3, 0x0f, 0x7e, 0x54, 0x24, 0x18, /*movq 0x18(%rsp),%xmm2*/ + 0xf3, 0x0f, 0x7e, 0x5c, 0x24, 0x20, /*movq 0x20(%rsp),%xmm3*/ #endif }; @@ -353,7 +379,7 @@ static void gen_st80 (VARR (uint8_t) * insn_varr, uint32_t src_offset) { r10=mem[rbx,]; res_reg=mem[r10]; ... pop rbx; pop r12; ret. */ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, size_t nargs, - _MIR_arg_desc_t *arg_descs, size_t arg_vars_num) { + _MIR_arg_desc_t *arg_descs, size_t arg_vars_num MIR_UNUSED) { static const uint8_t prolog[] = { #ifndef _WIN32 0x41, 0x54, /* pushq %r12 */ @@ -362,13 +388,13 @@ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, s 0x49, 0x89, 0xfb, /* mov $rdi, $r11 -- fun addr */ 0x48, 0x89, 0xf3, /* mov $rsi, $rbx -- result/arg addresses */ #else - /* 0x0: */ 0x41, 0x54, /* pushq %r12 */ - /* 0x2: */ 0x53, /* pushq %rbx */ - /* 0x3: */ 0x55, /* push %rbp */ - /* 0x4: */ 0x48, 0x89, 0xe5, /* mov %rsp,%rbp */ - /* 0x7: */ 0x48, 0x81, 0xec, 0, 0, 0, 0, /* subq , %rsp */ - /* 0xe: */ 0x49, 0x89, 0xcb, /* mov $rcx, $r11 -- fun addr */ - /* 0x11: */ 0x48, 0x89, 0xd3, /* mov $rdx, $rbx -- result/arg addresses */ + /* 0x0: */ 0x41, 0x54, /* pushq %r12 */ + /* 0x2: */ 0x53, /* pushq %rbx */ + /* 0x3: */ 0x55, /* push %rbp */ + /* 0x4: */ 0x48, 0x89, 0xe5, /* mov %rsp,%rbp */ + /* 0x7: */ 0x48, 0x81, 0xec, 0, 0, 0, 0, /* subq , %rsp */ + /* 0xe: */ 0x49, 0x89, 0xcb, /* mov $rcx, $r11 -- fun addr */ + /* 0x11: */ 0x48, 0x89, 0xd3, /* mov $rdx, $rbx -- result/arg addresses */ #endif }; static const uint8_t call_end[] = { @@ -396,7 +422,7 @@ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, s #else static const uint8_t iregs[] = {1, 2, 8, 9}; /* rcx, rdx, r8, r9 */ static const uint32_t max_iregs = 4, max_xregs = 4; - uint32_t blk_offset = nargs < 4 ? 32 : nargs * 8, sp_offset = 32; /* spill area */ + uint32_t blk_offset = nargs < 4 ? 32 : (uint32_t) nargs * 8, sp_offset = 32; /* spill area */ #endif uint32_t n_iregs = 0, n_xregs = 0, n_fregs, qwords; uint8_t *addr; @@ -410,29 +436,30 @@ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, s if ((MIR_T_I8 <= type && type <= MIR_T_U64) || type == MIR_T_P || type == MIR_T_RBLK) { if (n_iregs < max_iregs) { - gen_mov (code, (i + nres) * sizeof (long double), iregs[n_iregs++], TRUE); + gen_mov (code, (uint32_t) ((i + nres) * sizeof (long double)), iregs[n_iregs++], TRUE); #ifdef _WIN32 n_xregs++; #endif } else { - gen_ldst (code, sp_offset, (i + nres) * sizeof (long double), TRUE); + gen_ldst (code, sp_offset, (uint32_t) ((i + nres) * sizeof (long double)), TRUE); sp_offset += 8; } } else if (type == MIR_T_F || type == MIR_T_D) { if (n_xregs < max_xregs) { - gen_movxmm (code, (i + nres) * sizeof (long double), n_xregs++, type == MIR_T_F, TRUE); + gen_movxmm (code, (uint32_t) ((i + nres) * sizeof (long double)), n_xregs++, + type == MIR_T_F, TRUE); #ifdef _WIN32 - gen_mov (code, (i + nres) * sizeof (long double), iregs[n_iregs++], TRUE); + gen_mov (code, (uint32_t) ((i + nres) * sizeof (long double)), iregs[n_iregs++], TRUE); #endif } else { - gen_ldst (code, sp_offset, (i + nres) * sizeof (long double), type == MIR_T_D); + gen_ldst (code, sp_offset, (uint32_t) ((i + nres) * sizeof (long double)), type == MIR_T_D); sp_offset += 8; } } else if (type == MIR_T_LD) { - gen_ldst80 (code, sp_offset, (i + nres) * sizeof (long double)); + gen_ldst80 (code, sp_offset, (uint32_t) ((i + nres) * sizeof (long double))); sp_offset += 16; } else if (MIR_blk_type_p (type)) { - qwords = (arg_descs[i].size + 7) / 8; + qwords = (uint32_t) ((arg_descs[i].size + 7) / 8); #ifndef _WIN32 if (type == MIR_T_BLK + 1 && n_iregs + qwords <= max_iregs) { assert (qwords <= 2); @@ -472,7 +499,8 @@ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, s sp_offset += qwords * 8; #else if (qwords <= 1) { - gen_mov (code, (i + nres) * sizeof (long double), 12, TRUE); /* r12 = mem[disp + rbx] */ + gen_mov (code, (uint32_t) ((i + nres) * sizeof (long double)), 12, + TRUE); /* r12 = mem[disp + rbx] */ if (n_iregs < max_iregs) { gen_mov2 (code, 0, iregs[n_iregs++], TRUE); /* arg_reg = mem[r12] */ n_xregs++; @@ -483,7 +511,7 @@ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, s } } else { /* r12 = mem[disp + rbx]; mem[rsp+blk_offset + nw] = r10 = mem[r12 + nw]; */ - gen_blk_mov (code, blk_offset, (i + nres) * sizeof (long double), qwords); + gen_blk_mov (code, blk_offset, (uint32_t) ((i + nres) * sizeof (long double)), qwords); if (n_iregs < max_iregs) { gen_add (code, blk_offset, iregs[n_iregs++]); /* arg_reg = sp + blk_offset */ n_xregs++; @@ -524,11 +552,13 @@ void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_types, s for (size_t i = 0; i < nres; i++) { if (((MIR_T_I8 <= res_types[i] && res_types[i] <= MIR_T_U64) || res_types[i] == MIR_T_P) && n_iregs < 2) { - gen_mov (code, i * sizeof (long double), n_iregs++ == 0 ? 0 : 2, FALSE); /* rax or rdx */ + gen_mov (code, (uint32_t) (i * sizeof (long double)), n_iregs++ == 0 ? 0 : 2, + FALSE); /* rax or rdx */ } else if ((res_types[i] == MIR_T_F || res_types[i] == MIR_T_D) && n_xregs < 2) { - gen_movxmm (code, i * sizeof (long double), n_xregs++, res_types[i] == MIR_T_F, FALSE); + gen_movxmm (code, (uint32_t) (i * sizeof (long double)), n_xregs++, res_types[i] == MIR_T_F, + FALSE); } else if (res_types[i] == MIR_T_LD && n_fregs < 2) { - gen_st80 (code, i * sizeof (long double)); + gen_st80 (code, (uint32_t) (i * sizeof (long double))); } else { MIR_get_error_func (ctx) (MIR_ret_error, "x86-64 can not handle this combination of return values"); @@ -669,57 +699,272 @@ void *_MIR_get_interp_shim (MIR_context_t ctx, MIR_item_t func_item, void *handl return res; } -/* save regs; r10 = call hook_address (ctx, called_func); restore regs; jmp *r10 - */ +/* push rsi,rdi;rsi=called_func,rdi=ctx;r10=hook_address;jmp wrapper_end; */ void *_MIR_get_wrapper (MIR_context_t ctx, MIR_item_t called_func, void *hook_address) { - static const uint8_t push_rax[] = {0x50, /*push %rax */}; +#ifndef _WIN32 + static const uint8_t start_pat[] = { + 0x56, /* push %rsi */ + 0x57, /* push %rdi */ + 0x48, 0xbe, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs called_func,%rsi */ + 0x48, 0xbf, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs ctx,%rdi */ + 0x49, 0xba, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs ,%r10 */ + 0xe9, 0, 0, 0, 0, /* 0x0: jmp rel32 */ + }; + size_t call_func_offset = 4, ctx_offset = 14, hook_offset = 24, rel32_offset = 33; +#else + static const uint8_t start_pat[] = { + 0x48, 0x89, 0x4c, 0x24, 0x08, /* mov %rcx,0x08(%rsp) */ + 0x48, 0x89, 0x54, 0x24, 0x10, /* mov %rdx,0x10(%rsp) */ + 0x48, 0xba, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs called_func,%rdx */ + 0x48, 0xb9, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs ctx,%rcx */ + 0x49, 0xba, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs ,%r10*/ + 0xe9, 0, 0, 0, 0, /* 0x0: jmp rel32 */ + }; + size_t call_func_offset = 2, ctx_offset = 12, hook_offset = 22, rel32_offset = 31; +#endif + uint8_t *addr; + VARR (uint8_t) * code; + void *res; + + VARR_CREATE (uint8_t, code, 128); + addr = push_insns (code, start_pat, sizeof (start_pat)); + memcpy (addr + call_func_offset, &called_func, sizeof (void *)); + memcpy (addr + ctx_offset, &ctx, sizeof (void *)); + memcpy (addr + hook_offset, &hook_address, sizeof (void *)); + res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); + VARR_DESTROY (uint8_t, code); + int64_t off = (uint8_t *) wrapper_end_addr - ((uint8_t *) res + rel32_offset + 4); + assert (INT32_MIN <= off && off <= INT32_MAX); + _MIR_change_code (ctx, (uint8_t *) res + rel32_offset, (uint8_t *) &off, 4); /* LE */ + return res; +} + +void *_MIR_get_wrapper_end (MIR_context_t ctx) { +#ifndef _WIN32 + static const uint8_t wrap_end[] = { + 0x50, /*push %rax */ + 0x53, /*push %rbx */ + 0x48, 0x89, 0xe0, /*mov %rsp,%rax */ + 0x48, 0x89, 0xc3, /*mov %rax,%rbx */ + 0x48, 0x83, 0xe0, 0x0f, /*and $0xf,%rax */ + 0x48, 0x05, 0x80, 0, 0, 0, /*add $0x80,%rax */ + 0x48, 0x29, 0xc4, /*sub %rax,%rsp -- aligned now */ + 0xf3, 0x0f, 0x7f, 0x04, 0x24, /*movdqu %xmm0,(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x4c, 0x24, 0x10, /*movdqu %xmm1,0x10(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x54, 0x24, 0x20, /*movdqu %xmm2,0x20(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x5c, 0x24, 0x30, /*movdqu %xmm3,0x30(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x64, 0x24, 0x40, /*movdqu %xmm4,0x40(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x6c, 0x24, 0x50, /*movdqu %xmm5,0x50(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x74, 0x24, 0x60, /*movdqu %xmm6,0x60(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x7c, 0x24, 0x70, /*movdqu %xmm7,0x70(%rsp) */ + 0x41, 0x51, /*push %r9 */ + 0x41, 0x50, /*push %r8 */ + 0x51, /*push %rcx */ + 0x52, /*push %rdx */ + 0x41, 0xff, 0xd2, /*callq *%r10 */ + 0x49, 0x89, 0xc2, /*mov %rax,%r10 */ + 0x5a, /*pop %rdx */ + 0x59, /*pop %rcx */ + 0x41, 0x58, /*pop %r8 */ + 0x41, 0x59, /*pop %r9 */ + 0xf3, 0x0f, 0x6f, 0x04, 0x24, /*movdqu (%rsp),%xmm0 */ + 0xf3, 0x0f, 0x6f, 0x4c, 0x24, 0x10, /*movdqu 0x10(%rsp),%xmm1 */ + 0xf3, 0x0f, 0x6f, 0x54, 0x24, 0x20, /*movdqu 0x20(%rsp),%xmm2 */ + 0xf3, 0x0f, 0x6f, 0x5c, 0x24, 0x30, /*movdqu 0x30(%rsp),%xmm3 */ + 0xf3, 0x0f, 0x6f, 0x64, 0x24, 0x40, /*movdqu 0x40(%rsp),%xmm4 */ + 0xf3, 0x0f, 0x6f, 0x6c, 0x24, 0x50, /*movdqu 0x50(%rsp),%xmm5 */ + 0xf3, 0x0f, 0x6f, 0x74, 0x24, 0x60, /*movdqu 0x60(%rsp),%xmm6 */ + 0xf3, 0x0f, 0x6f, 0x7c, 0x24, 0x70, /*movdqu 0x70(%rsp),%xmm7 */ + 0x48, 0x89, 0xdc, /*mov %rbx,%rsp */ + 0x5b, /*pop %rbx */ + 0x58, /*pop %rax */ + 0x5f, /*pop %rdi */ + 0x5e, /*pop %rsi */ + 0x41, 0xff, 0xe2, /*jmpq *%r10 */ + }; +#else static const uint8_t wrap_end[] = { + 0x4c, 0x89, 0x44, 0x24, 0x18, /*mov %r8, 0x18(%rsp) */ + 0x4c, 0x89, 0x4c, 0x24, 0x20, /*mov %r9, 0x20(%rsp) */ + 0x50, /*push %rax */ + 0x55, /*push %rbp */ + 0x48, 0x89, 0xe5, /*mov %rsp,%rbp */ + 0x48, 0x89, 0xe0, /*mov %rsp,%rax */ + 0x48, 0x83, 0xe0, 0x0f, /*and $0xf,%rax */ + 0x48, 0x05, 0x28, 0, 0, 0, /*add $0x40,%rax */ + 0x48, 0x29, 0xc4, /*sub %rax,%rsp -- aligned now */ + 0x66, 0x0f, 0xd6, 0x04, 0x24, /*movq %xmm0,(%rsp) */ + 0x66, 0x0f, 0xd6, 0x4c, 0x24, 0x08, /*movq %xmm1,0x8(%rsp) */ + 0x66, 0x0f, 0xd6, 0x54, 0x24, 0x10, /*movq %xmm2,0x10(%rsp) */ + 0x66, 0x0f, 0xd6, 0x5c, 0x24, 0x18, /*movq %xmm3,0x18(%rsp) */ + 0x41, 0xff, 0xd2, /*callq *%r10 */ + 0x49, 0x89, 0xc2, /*mov %rax,%r10 */ + 0xf3, 0x0f, 0x7e, 0x04, 0x24, /*movq (%rsp),%xmm0*/ + 0xf3, 0x0f, 0x7e, 0x4c, 0x24, 0x08, /*movq 0x8(%rsp),%xmm1*/ + 0xf3, 0x0f, 0x7e, 0x54, 0x24, 0x10, /*movq 0x10(%rsp),%xmm2*/ + 0xf3, 0x0f, 0x7e, 0x5c, 0x24, 0x18, /*movq 0x18(%rsp),%xmm3*/ + 0x48, 0x89, 0xec, /*mov %rbp,%rsp */ + 0x5d, /*pop %rbp */ + 0x58, /*pop %rax */ + 0x48, 0x8b, 0x4c, 0x24, 0x08, /*mov 0x08(%rsp),%rcx */ + 0x48, 0x8b, 0x54, 0x24, 0x10, /*mov 0x10(%rsp),%rdx */ + 0x4c, 0x8b, 0x44, 0x24, 0x18, /*mov 0x18(%rsp),%r8 */ + 0x4c, 0x8b, 0x4c, 0x24, 0x20, /*mov 0x20(%rsp),%r9 */ + 0x41, 0xff, 0xe2, /*jmpq *%r10 */ + }; +#endif + VARR (uint8_t) * code; + void *res; + + VARR_CREATE (uint8_t, code, 128); + push_insns (code, wrap_end, sizeof (wrap_end)); + res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); + VARR_DESTROY (uint8_t, code); + return res; +} + +/* r10=; jump rex32 ??? mutex free */ +void *_MIR_get_bb_thunk (MIR_context_t ctx, void *bb_version, void *handler) { + void *res; + int32_t disp; + static const uint8_t pattern[] = { + 0x49, 0xba, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x0: movabsq 0, r10 */ + 0xe9, 0, 0, 0, 0, /* 0xa: jmpq */ + }; + res = _MIR_publish_code (ctx, pattern, sizeof (pattern)); + _MIR_update_code (ctx, res, 1, 2, bb_version); + disp = (int32_t) ((char *) handler - ((char *) res + sizeof (pattern))); + _MIR_change_code (ctx, (uint8_t *) res + 11, (uint8_t *) &disp, 4); + return res; +} + +/* change to jmp rex32(to) */ +void _MIR_replace_bb_thunk (MIR_context_t ctx, void *thunk, void *to) { + uint8_t op = 0xe9; /* jmpq */ + int32_t disp; + _MIR_change_code (ctx, (uint8_t *) thunk, &op, 1); /* jmpq */ + disp = (int32_t) ((char *) to - ((char *) thunk + 5)); + _MIR_change_code (ctx, (uint8_t *) thunk + 1, (uint8_t *) &disp, 4); +} + +static const uint8_t save_pat2[] = { #ifndef _WIN32 - 0x58, /*pop %rax */ + 0x48, 0x81, 0xec, 0x80, 0, 0, 0, /*sub $0x80,%rsp */ + 0xf3, 0x0f, 0x7f, 0x04, 0x24, /*movdqu %xmm0,(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x4c, 0x24, 0x10, /*movdqu %xmm1,0x10(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x54, 0x24, 0x20, /*movdqu %xmm2,0x20(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x5c, 0x24, 0x30, /*movdqu %xmm3,0x30(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x64, 0x24, 0x40, /*movdqu %xmm4,0x40(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x6c, 0x24, 0x50, /*movdqu %xmm5,0x50(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x74, 0x24, 0x60, /*movdqu %xmm6,0x60(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x7c, 0x24, 0x70, /*movdqu %xmm7,0x70(%rsp) */ + 0x41, 0x51, /*push %r9 */ + 0x41, 0x50, /*push %r8 */ + 0x51, /*push %rcx */ + 0x52, /*push %rdx */ + 0x56, /*push %rsi */ + 0x57, /*push %rdi */ +#else + 0x48, 0x89, 0x4c, 0x24, 0x08, /*mov %rcx,0x08(%rsp) */ + 0x48, 0x89, 0x54, 0x24, 0x10, /*mov %rdx,0x10(%rsp) */ + 0x4c, 0x89, 0x44, 0x24, 0x18, /*mov %r8, 0x18(%rsp) */ + 0x4c, 0x89, 0x4c, 0x24, 0x20, /*mov %r9, 0x20(%rsp) */ + 0x48, 0x81, 0xec, 0x80, 0, 0, 0, /*sub $0x60,%rsp */ + 0xf3, 0x0f, 0x7f, 0x04, 0x24, /*movdqu %xmm0,(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x4c, 0x24, 0x10, /*movdqu %xmm1,0x10(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x54, 0x24, 0x20, /*movdqu %xmm2,0x20(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x5c, 0x24, 0x30, /*movdqu %xmm3,0x30(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x64, 0x24, 0x40, /*movdqu %xmm4,0x40(%rsp) */ + 0xf3, 0x0f, 0x7f, 0x6c, 0x24, 0x50, /*movdqu %xmm5,0x50(%rsp) */ +#endif + 0x50, /*push %rax */ + 0x41, 0x53, /*push %r11 */ +}; + +static const uint8_t restore_pat2[] = { + 0x41, 0x5b, /*pop %r11 */ + 0x58, /*pop %rax */ +#ifndef _WIN32 + 0x5f, /*pop %rdi */ + 0x5e, /*pop %rsi */ + 0x5a, /*pop %rdx */ + 0x59, /*pop %rcx */ + 0x41, 0x58, /*pop %r8 */ + 0x41, 0x59, /*pop %r9 */ + 0xf3, 0x0f, 0x6f, 0x04, 0x24, /*movdqu (%rsp),%xmm0 */ + 0xf3, 0x0f, 0x6f, 0x4c, 0x24, 0x10, /*movdqu 0x10(%rsp),%xmm1 */ + 0xf3, 0x0f, 0x6f, 0x54, 0x24, 0x20, /*movdqu 0x20(%rsp),%xmm2 */ + 0xf3, 0x0f, 0x6f, 0x5c, 0x24, 0x30, /*movdqu 0x30(%rsp),%xmm3 */ + 0xf3, 0x0f, 0x6f, 0x64, 0x24, 0x40, /*movdqu 0x40(%rsp),%xmm4 */ + 0xf3, 0x0f, 0x6f, 0x6c, 0x24, 0x50, /*movdqu 0x50(%rsp),%xmm5 */ + 0xf3, 0x0f, 0x6f, 0x74, 0x24, 0x60, /*movdqu 0x60(%rsp),%xmm6 */ + 0xf3, 0x0f, 0x6f, 0x7c, 0x24, 0x70, /*movdqu 0x70(%rsp),%xmm7 */ + 0x48, 0x81, 0xc4, 0x80, 0, 0, 0, /*add $0x80,%rsp */ +#else + 0xf3, 0x0f, 0x6f, 0x04, 0x24, /*movdqu (%rsp),%xmm0 */ + 0xf3, 0x0f, 0x6f, 0x4c, 0x24, 0x10, /*movdqu 0x10(%rsp),%xmm1 */ + 0xf3, 0x0f, 0x6f, 0x54, 0x24, 0x20, /*movdqu 0x20(%rsp),%xmm2 */ + 0xf3, 0x0f, 0x6f, 0x5c, 0x24, 0x30, /*movdqu 0x30(%rsp),%xmm3 */ + 0xf3, 0x0f, 0x6f, 0x64, 0x24, 0x40, /*movdqu 0x40(%rsp),%xmm4 */ + 0xf3, 0x0f, 0x6f, 0x6c, 0x24, 0x50, /*movdqu 0x50(%rsp),%xmm5 */ + 0x48, 0x81, 0xc4, 0x80, 0, 0, 0, /*add $0x60,%rsp */ + 0x48, 0x8b, 0x4c, 0x24, 0x08, /*mov 0x08(%rsp),%rcx */ + 0x48, 0x8b, 0x54, 0x24, 0x10, /*mov 0x10(%rsp),%rdx */ + 0x4c, 0x8b, 0x44, 0x24, 0x18, /*mov 0x18(%rsp),%r8 */ + 0x4c, 0x8b, 0x4c, 0x24, 0x20, /*mov 0x20(%rsp),%r9 */ #endif +}; + +/* save all clobbered regs but 10; r10 = call hook_address (data, r10); restore regs; jmp *r10 + r10 is a generator temp reg which is not used across bb borders. */ +void *_MIR_get_bb_wrapper (MIR_context_t ctx, void *data, void *hook_address) { + static const uint8_t wrap_end[] = { 0x41, 0xff, 0xe2, /*jmpq *%r10 */ }; static const uint8_t call_pat[] = #ifndef _WIN32 { - 0x48, 0xbe, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs called_func,%rsi */ - 0x48, 0xbf, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs ctx,%rdi */ - 0x49, 0xba, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs ,%r10 */ - 0x41, 0xff, 0xd2, /* callq *%r10 */ - 0x49, 0x89, 0xc2, /* mov %rax,%r10 */ + 0x4c, 0x89, 0xd6, /* mov %r10,%rsi */ + 0x48, 0xbf, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs data,%rdi */ + 0x49, 0xba, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs ,%r10 */ + 0x48, 0x89, 0xe2, /* mov %rsp,%rdx */ + 0x48, 0x83, 0xe2, 0x0f, /* and $0xf,%rdx */ + 0x74, 0x07, /* je 10 */ + 0x52, /* push %rdx */ + 0x41, 0xff, 0xd2, /* callq *%r10 */ + 0x5a, /* pop %rdx */ + 0xeb, 0x03, /* jmp 13 */ + 0x41, 0xff, 0xd2, /* l: callq *%r10 */ + 0x49, 0x89, 0xc2, /* l2:mov %rax,%r10 */ }; - size_t call_func_offset = 2, ctx_offset = 12, hook_offset = 22; + size_t data_offset = 5, hook_offset = 15; #else { 0x55, /* push %rbp */ 0x48, 0x89, 0xe5, /* mov %rsp,%rbp */ - 0x48, 0xba, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs called_func,%rdx */ - 0x48, 0xb9, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs ctx,%rcx */ + 0x4c, 0x89, 0xd2, /* mov %r10,%rdx */ + 0x48, 0xb9, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs data,%rcx */ 0x49, 0xba, 0, 0, 0, 0, 0, 0, 0, 0, /* movabs ,%r10*/ 0x50, /* push %rax */ 0x48, 0x83, 0xec, 0x28, /* sub 40,%rsp */ - 0x41, 0xff, 0xd2, /* callq *%r10 */ - 0x49, 0x89, 0xc2, /* mov %rax,%r10 */ - 0x48, 0x83, 0xc4, 0x28, /* add 40,%rsp */ - 0x58, /* pop %rax */ - 0x5d, /* pop %rbp */ + 0x41, 0xff, 0xd2, /* callq *%r10 ???align for unaligned sp */ + 0x49, 0x89, 0xc2, /* mov %rax,%r10 */ + 0x48, 0x83, 0xc4, 0x28, /* add 40,%rsp */ + 0x58, /* pop %rax */ + 0x5d, /* pop %rbp */ }; - size_t call_func_offset = 6, ctx_offset = 16, hook_offset = 26; + size_t data_offset = 9, hook_offset = 19; #endif uint8_t *addr; VARR (uint8_t) * code; void *res; VARR_CREATE (uint8_t, code, 128); -#ifndef _WIN32 - push_insns (code, push_rax, sizeof (push_rax)); -#endif - push_insns (code, save_pat, sizeof (save_pat)); + push_insns (code, save_pat2, sizeof (save_pat2)); addr = push_insns (code, call_pat, sizeof (call_pat)); - memcpy (addr + call_func_offset, &called_func, sizeof (void *)); - memcpy (addr + ctx_offset, &ctx, sizeof (void *)); + memcpy (addr + data_offset, &data, sizeof (void *)); memcpy (addr + hook_offset, &hook_address, sizeof (void *)); - push_insns (code, restore_pat, sizeof (restore_pat)); + push_insns (code, restore_pat2, sizeof (restore_pat2)); push_insns (code, wrap_end, sizeof (wrap_end)); res = _MIR_publish_code (ctx, VARR_ADDR (uint8_t, code), VARR_LENGTH (uint8_t, code)); VARR_DESTROY (uint8_t, code); diff --git a/mir/mir-x86_64.h b/mir/mir-x86_64.h new file mode 100644 index 00000000..306eff94 --- /dev/null +++ b/mir/mir-x86_64.h @@ -0,0 +1,59 @@ +/* This file is a part of MIR project. + Copyright (C) 2018-2024 Vladimir Makarov . + A common include file for mir-x86_64.c and mir-gen-x86_64.c +*/ + +#include "mir.h" + +#define HREG_EL(h) h##_HARD_REG +#define REP_SEP , +enum { + REP8 (HREG_EL, AX, CX, DX, BX, SP, BP, SI, DI), + REP8 (HREG_EL, R8, R9, R10, R11, R12, R13, R14, R15), + REP8 (HREG_EL, XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7), + REP8 (HREG_EL, XMM8, XMM9, XMM10, XMM11, XMM12, XMM13, XMM14, XMM15), + REP2 (HREG_EL, ST0, ST1), +}; +#undef REP_SEP + +static const char *const target_hard_reg_names[] = { + "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", "r8", + "r9", "r10", "r11", "r12", "r13", "r14", "r15", "xmm0", "xmm1", + "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", + "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", "st0", "st1", +}; + +#define MAX_HARD_REG ST1_HARD_REG + +/* Hard regs not used in machinized code, preferably call used ones. */ +static const MIR_reg_t TEMP_INT_HARD_REG1 = R10_HARD_REG, TEMP_INT_HARD_REG2 = R11_HARD_REG; +#ifndef _WIN32 +static const MIR_reg_t TEMP_FLOAT_HARD_REG1 = XMM8_HARD_REG, TEMP_FLOAT_HARD_REG2 = XMM9_HARD_REG; +static const MIR_reg_t TEMP_DOUBLE_HARD_REG1 = XMM8_HARD_REG, TEMP_DOUBLE_HARD_REG2 = XMM9_HARD_REG; +#else +static const MIR_reg_t TEMP_FLOAT_HARD_REG1 = XMM4_HARD_REG, TEMP_FLOAT_HARD_REG2 = XMM5_HARD_REG; +static const MIR_reg_t TEMP_DOUBLE_HARD_REG1 = XMM4_HARD_REG, TEMP_DOUBLE_HARD_REG2 = XMM5_HARD_REG; +#endif +static const MIR_reg_t TEMP_LDOUBLE_HARD_REG1 = MIR_NON_VAR; +static const MIR_reg_t TEMP_LDOUBLE_HARD_REG2 = MIR_NON_VAR; + +static inline int target_hard_reg_type_ok_p (MIR_reg_t hard_reg, MIR_type_t type) { + assert (hard_reg <= MAX_HARD_REG); + /* For LD we need x87 stack regs and it is too complicated so no + hard register allocation for LD: */ + if (type == MIR_T_LD) return FALSE; + return MIR_int_type_p (type) ? hard_reg < XMM0_HARD_REG : hard_reg >= XMM0_HARD_REG; +} + +static inline int target_fixed_hard_reg_p (MIR_reg_t hard_reg) { + assert (hard_reg <= MAX_HARD_REG); + return (hard_reg == BP_HARD_REG || hard_reg == SP_HARD_REG || hard_reg == TEMP_INT_HARD_REG1 + || hard_reg == TEMP_INT_HARD_REG2 || hard_reg == TEMP_FLOAT_HARD_REG1 + || hard_reg == TEMP_FLOAT_HARD_REG2 || hard_reg == TEMP_DOUBLE_HARD_REG1 + || hard_reg == TEMP_DOUBLE_HARD_REG2 || hard_reg == ST0_HARD_REG + || hard_reg == ST1_HARD_REG); +} + +static inline int target_locs_num (MIR_reg_t loc, MIR_type_t type) { + return loc > MAX_HARD_REG && type == MIR_T_LD ? 2 : 1; +} diff --git a/mir/mir.c b/mir/mir.c index 033f7dd8..b38bee66 100644 --- a/mir/mir.c +++ b/mir/mir.c @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ #include "mir.h" @@ -19,47 +19,52 @@ struct gen_ctx; struct c2mir_ctx; struct string_ctx; struct reg_ctx; +struct alias_ctx; struct simplify_ctx; struct machine_code_ctx; struct io_ctx; struct scan_ctx; +struct hard_reg_ctx; struct interp_ctx; struct MIR_context { struct gen_ctx *gen_ctx; /* should be the 1st member */ struct c2mir_ctx *c2mir_ctx; /* should be the 2nd member */ -#if MIR_PARALLEL_GEN - mir_mutex_t ctx_mutex; -#endif MIR_error_func_t error_func; + int func_redef_permission_p; /* when true loaded func can be redfined lately */ VARR (size_t) * insn_nops; /* constant after initialization */ VARR (MIR_proto_t) * unspec_protos; /* protos of unspec insns (set only during initialization) */ VARR (char) * temp_string; - VARR (uint8_t) * temp_data; + VARR (uint8_t) * temp_data, *used_label_p; HTAB (MIR_item_t) * module_item_tab; /* Module to keep items potentially used by all modules: */ struct MIR_module environment_module; MIR_module_t curr_module; MIR_func_t curr_func; - int curr_label_num; + size_t curr_label_num; DLIST (MIR_module_t) all_modules; VARR (MIR_module_t) * modules_to_link; + VARR (MIR_op_t) * temp_ops; struct string_ctx *string_ctx; struct reg_ctx *reg_ctx; + struct alias_ctx *alias_ctx; struct simplify_ctx *simplify_ctx; struct machine_code_ctx *machine_code_ctx; struct io_ctx *io_ctx; struct scan_ctx *scan_ctx; + struct hard_reg_ctx *hard_reg_ctx; struct interp_ctx *interp_ctx; - void *setjmp_addr; /* used in interpreter to call setjmp directly not from a shim and FFI */ + void *setjmp_addr; /* used in interpreter to call setjmp directly not from a shim and FFI */ + void *wrapper_end_addr; /* used by generator */ }; -#define ctx_mutex ctx->ctx_mutex #define error_func ctx->error_func +#define func_redef_permission_p ctx->func_redef_permission_p #define unspec_protos ctx->unspec_protos #define insn_nops ctx->insn_nops #define temp_string ctx->temp_string #define temp_data ctx->temp_data +#define used_label_p ctx->used_label_p #define module_item_tab ctx->module_item_tab #define environment_module ctx->environment_module #define curr_module ctx->curr_module @@ -67,7 +72,9 @@ struct MIR_context { #define curr_label_num ctx->curr_label_num #define all_modules ctx->all_modules #define modules_to_link ctx->modules_to_link +#define temp_ops ctx->temp_ops #define setjmp_addr ctx->setjmp_addr +#define wrapper_end_addr ctx->wrapper_end_addr static void util_error (MIR_context_t ctx, const char *message); #define MIR_VARR_ERROR util_error @@ -76,6 +83,7 @@ static void util_error (MIR_context_t ctx, const char *message); #include "mir-hash.h" #include "mir-htab.h" #include "mir-reduce.h" +#include "mir-bitmap.h" #include #include #include @@ -87,7 +95,8 @@ static void interp_init (MIR_context_t ctx); static void finish_func_interpretation (MIR_item_t func_item); static void interp_finish (MIR_context_t ctx); -static void MIR_NO_RETURN default_error (enum MIR_error_type error_type, const char *format, ...) { +static void MIR_NO_RETURN default_error (enum MIR_error_type error_type MIR_UNUSED, + const char *format, ...) { va_list ap; va_start (ap, format); @@ -105,7 +114,7 @@ static void MIR_NO_RETURN MIR_UNUSED util_error (MIR_context_t ctx, const char * #define TEMP_REG_NAME_PREFIX "t" #define TEMP_ITEM_NAME_PREFIX ".lc" -int _MIR_reserved_ref_name_p (MIR_context_t ctx, const char *name) { +int _MIR_reserved_ref_name_p (MIR_context_t ctx MIR_UNUSED, const char *name) { return strncmp (name, TEMP_ITEM_NAME_PREFIX, strlen (TEMP_ITEM_NAME_PREFIX)) == 0; } @@ -166,6 +175,10 @@ static const struct insn_desc insn_descs[] = { {MIR_FNEG, "fneg", {MIR_OP_FLOAT | OUT_FLAG, MIR_OP_FLOAT, MIR_OP_BOUND}}, {MIR_DNEG, "dneg", {MIR_OP_DOUBLE | OUT_FLAG, MIR_OP_DOUBLE, MIR_OP_BOUND}}, {MIR_LDNEG, "ldneg", {MIR_OP_LDOUBLE | OUT_FLAG, MIR_OP_LDOUBLE, MIR_OP_BOUND}}, + {MIR_ADDR, "addr", {MIR_OP_INT | OUT_FLAG, MIR_OP_REG, MIR_OP_BOUND}}, /* MIR_OP_REG! */ + {MIR_ADDR8, "addr8", {MIR_OP_INT | OUT_FLAG, MIR_OP_REG, MIR_OP_BOUND}}, /* MIR_OP_REG! */ + {MIR_ADDR16, "addr16", {MIR_OP_INT | OUT_FLAG, MIR_OP_REG, MIR_OP_BOUND}}, /* MIR_OP_REG! */ + {MIR_ADDR32, "addr32", {MIR_OP_INT | OUT_FLAG, MIR_OP_REG, MIR_OP_BOUND}}, /* MIR_OP_REG! */ {MIR_ADD, "add", {MIR_OP_INT | OUT_FLAG, MIR_OP_INT, MIR_OP_INT, MIR_OP_BOUND}}, {MIR_ADDS, "adds", {MIR_OP_INT | OUT_FLAG, MIR_OP_INT, MIR_OP_INT, MIR_OP_BOUND}}, {MIR_FADD, "fadd", {MIR_OP_FLOAT | OUT_FLAG, MIR_OP_FLOAT, MIR_OP_FLOAT, MIR_OP_BOUND}}, @@ -242,6 +255,14 @@ static const struct insn_desc insn_descs[] = { {MIR_FGE, "fge", {MIR_OP_INT | OUT_FLAG, MIR_OP_FLOAT, MIR_OP_FLOAT, MIR_OP_BOUND}}, {MIR_DGE, "dge", {MIR_OP_INT | OUT_FLAG, MIR_OP_DOUBLE, MIR_OP_DOUBLE, MIR_OP_BOUND}}, {MIR_LDGE, "ldge", {MIR_OP_INT | OUT_FLAG, MIR_OP_LDOUBLE, MIR_OP_LDOUBLE, MIR_OP_BOUND}}, + {MIR_ADDO, "addo", {MIR_OP_INT | OUT_FLAG, MIR_OP_INT, MIR_OP_INT, MIR_OP_BOUND}}, + {MIR_ADDOS, "addos", {MIR_OP_INT | OUT_FLAG, MIR_OP_INT, MIR_OP_INT, MIR_OP_BOUND}}, + {MIR_SUBO, "subo", {MIR_OP_INT | OUT_FLAG, MIR_OP_INT, MIR_OP_INT, MIR_OP_BOUND}}, + {MIR_SUBOS, "subos", {MIR_OP_INT | OUT_FLAG, MIR_OP_INT, MIR_OP_INT, MIR_OP_BOUND}}, + {MIR_MULO, "mulo", {MIR_OP_INT | OUT_FLAG, MIR_OP_INT, MIR_OP_INT, MIR_OP_BOUND}}, + {MIR_MULOS, "mulos", {MIR_OP_INT | OUT_FLAG, MIR_OP_INT, MIR_OP_INT, MIR_OP_BOUND}}, + {MIR_UMULO, "umulo", {MIR_OP_INT | OUT_FLAG, MIR_OP_INT, MIR_OP_INT, MIR_OP_BOUND}}, + {MIR_UMULOS, "umulos", {MIR_OP_INT | OUT_FLAG, MIR_OP_INT, MIR_OP_INT, MIR_OP_BOUND}}, {MIR_JMP, "jmp", {MIR_OP_LABEL, MIR_OP_BOUND}}, {MIR_BT, "bt", {MIR_OP_LABEL, MIR_OP_INT, MIR_OP_BOUND}}, {MIR_BTS, "bts", {MIR_OP_LABEL, MIR_OP_INT, MIR_OP_BOUND}}, @@ -285,10 +306,18 @@ static const struct insn_desc insn_descs[] = { {MIR_FBGE, "fbge", {MIR_OP_LABEL, MIR_OP_FLOAT, MIR_OP_FLOAT, MIR_OP_BOUND}}, {MIR_DBGE, "dbge", {MIR_OP_LABEL, MIR_OP_DOUBLE, MIR_OP_DOUBLE, MIR_OP_BOUND}}, {MIR_LDBGE, "ldbge", {MIR_OP_LABEL, MIR_OP_LDOUBLE, MIR_OP_LDOUBLE, MIR_OP_BOUND}}, + {MIR_BO, "bo", {MIR_OP_LABEL, MIR_OP_BOUND}}, + {MIR_UBO, "ubo", {MIR_OP_LABEL, MIR_OP_BOUND}}, + {MIR_BNO, "bno", {MIR_OP_LABEL, MIR_OP_BOUND}}, + {MIR_UBNO, "ubno", {MIR_OP_LABEL, MIR_OP_BOUND}}, + {MIR_LADDR, "laddr", {MIR_OP_INT, MIR_OP_LABEL, MIR_OP_BOUND}}, + {MIR_JMPI, "jmpi", {MIR_OP_INT, MIR_OP_BOUND}}, {MIR_CALL, "call", {MIR_OP_BOUND}}, {MIR_INLINE, "inline", {MIR_OP_BOUND}}, + {MIR_JCALL, "jcall", {MIR_OP_BOUND}}, {MIR_SWITCH, "switch", {MIR_OP_BOUND}}, {MIR_RET, "ret", {MIR_OP_BOUND}}, + {MIR_JRET, "jret", {MIR_OP_INT, MIR_OP_BOUND}}, {MIR_ALLOCA, "alloca", {MIR_OP_INT | OUT_FLAG, MIR_OP_INT, MIR_OP_BOUND}}, {MIR_BSTART, "bstart", {MIR_OP_INT | OUT_FLAG, MIR_OP_BOUND}}, {MIR_BEND, "bend", {MIR_OP_INT, MIR_OP_BOUND}}, @@ -300,6 +329,10 @@ static const struct insn_desc insn_descs[] = { {MIR_VA_END, "va_end", {MIR_OP_INT, MIR_OP_BOUND}}, {MIR_LABEL, "label", {MIR_OP_BOUND}}, {MIR_UNSPEC, "unspec", {MIR_OP_BOUND}}, + {MIR_PRSET, "prset", {MIR_OP_UNDEF, MIR_OP_INT, MIR_OP_BOUND}}, + {MIR_PRBEQ, "prbeq", {MIR_OP_LABEL, MIR_OP_UNDEF, MIR_OP_INT, MIR_OP_BOUND}}, + {MIR_PRBNE, "prbne", {MIR_OP_LABEL, MIR_OP_UNDEF, MIR_OP_INT, MIR_OP_BOUND}}, + {MIR_USE, "use", {MIR_OP_BOUND}}, {MIR_PHI, "phi", {MIR_OP_BOUND}}, {MIR_INVALID_INSN, "invalid-insn", {MIR_OP_BOUND}}, }; @@ -324,6 +357,12 @@ static MIR_op_mode_t type2mode (MIR_type_t type) { : MIR_OP_INT); } +int64_t _MIR_addr_offset (MIR_context_t ctx MIR_UNUSED, MIR_insn_code_t code) { + int v = 1; + if (code == MIR_ADDR || *(char *) &v != 0) return 0; + return code == MIR_ADDR8 ? 7 : code == MIR_ADDR16 ? 6 : 4; +} + /* New Page */ typedef struct string { @@ -342,10 +381,10 @@ struct string_ctx { #define strings ctx->string_ctx->strings #define string_tab ctx->string_ctx->string_tab -static htab_hash_t str_hash (string_t str, void *arg) { - return mir_hash (str.str.s, str.str.len, 0); +static htab_hash_t str_hash (string_t str, void *arg MIR_UNUSED) { + return (htab_hash_t) mir_hash (str.str.s, str.str.len, 0); } -static int str_eq (string_t str1, string_t str2, void *arg) { +static int str_eq (string_t str1, string_t str2, void *arg MIR_UNUSED) { return str1.str.len == str2.str.len && memcmp (str1.str.s, str2.str.s, str1.str.len) == 0; } @@ -357,8 +396,8 @@ static void string_init (VARR (string_t) * *strs, HTAB (string_t) * *str_tab) { HTAB_CREATE (string_t, *str_tab, 1000, str_hash, str_eq, NULL); } -static int string_find (VARR (string_t) * *strs, HTAB (string_t) * *str_tab, MIR_str_t str, - string_t *s) { +static int string_find (VARR (string_t) * *strs MIR_UNUSED, HTAB (string_t) * *str_tab, + MIR_str_t str, string_t *s) { string_t string; string.str = str; @@ -399,12 +438,37 @@ static void string_finish (VARR (string_t) * *strs, HTAB (string_t) * *str_tab) HTAB_DESTROY (string_t, *str_tab); } +/* Functions to work with aliases. */ + +struct alias_ctx { + VARR (string_t) * aliases; + HTAB (string_t) * alias_tab; +}; + +#define aliases ctx->alias_ctx->aliases +#define alias_tab ctx->alias_ctx->alias_tab + +MIR_alias_t MIR_alias (MIR_context_t ctx, const char *name) { + return (MIR_alias_t) string_store (ctx, &aliases, &alias_tab, + (MIR_str_t){strlen (name) + 1, name}) + .num; +} + +const char *MIR_alias_name (MIR_context_t ctx, MIR_alias_t alias) { + if (alias == 0) return ""; + if (alias >= VARR_LENGTH (string_t, aliases)) + MIR_get_error_func (ctx) (MIR_alloc_error, "Wrong alias number"); + return VARR_ADDR (string_t, aliases)[alias].str.s; +} + /* New Page */ +/* We attribute global vars to func as func can be inlined from different module. */ typedef struct reg_desc { - char *name; /* 1st key for the name2rdn hash tab */ MIR_type_t type; - MIR_reg_t reg; /* 1st key reg2rdn hash tab */ + MIR_reg_t reg; /* key reg2rdn hash tab */ + char *name; /* key for the name2rdn hash tab */ + char *hard_reg_name; /* NULL unless tied global, key for hrn2rdn */ } reg_desc_t; DEF_VARR (reg_desc_t); @@ -414,6 +478,7 @@ DEF_HTAB (size_t); typedef struct func_regs { VARR (reg_desc_t) * reg_descs; HTAB (size_t) * name2rdn_tab; + HTAB (size_t) * hrn2rdn_tab; HTAB (size_t) * reg2rdn_tab; } *func_regs_t; @@ -428,7 +493,21 @@ static htab_hash_t name2rdn_hash (size_t rdn, void *arg) { func_regs_t func_regs = arg; reg_desc_t *addr = VARR_ADDR (reg_desc_t, func_regs->reg_descs); - return mir_hash (addr[rdn].name, strlen (addr[rdn].name), 0); + return (htab_hash_t) mir_hash (addr[rdn].name, strlen (addr[rdn].name), 0); +} + +static int hrn2rdn_eq (size_t rdn1, size_t rdn2, void *arg) { + func_regs_t func_regs = arg; + reg_desc_t *addr = VARR_ADDR (reg_desc_t, func_regs->reg_descs); + + return strcmp (addr[rdn1].hard_reg_name, addr[rdn2].hard_reg_name) == 0; +} + +static htab_hash_t hrn2rdn_hash (size_t rdn, void *arg) { + func_regs_t func_regs = arg; + reg_desc_t *addr = VARR_ADDR (reg_desc_t, func_regs->reg_descs); + + return (htab_hash_t) mir_hash (addr[rdn].hard_reg_name, strlen (addr[rdn].hard_reg_name), 0); } static int reg2rdn_eq (size_t rdn1, size_t rdn2, void *arg) { @@ -442,31 +521,41 @@ static htab_hash_t reg2rdn_hash (size_t rdn, void *arg) { func_regs_t func_regs = arg; reg_desc_t *addr = VARR_ADDR (reg_desc_t, func_regs->reg_descs); - return mir_hash_finish (mir_hash_step (mir_hash_init (0), addr[rdn].reg)); + return (htab_hash_t) mir_hash_finish (mir_hash_step (mir_hash_init (0), addr[rdn].reg)); } static void func_regs_init (MIR_context_t ctx, MIR_func_t func) { func_regs_t func_regs; - reg_desc_t rd = {NULL, MIR_T_I64, 0}; + reg_desc_t rd = {MIR_T_I64, 0, NULL, NULL}; if ((func_regs = func->internal = malloc (sizeof (struct func_regs))) == NULL) MIR_get_error_func (ctx) (MIR_alloc_error, "Not enough memory for func regs info"); VARR_CREATE (reg_desc_t, func_regs->reg_descs, 50); VARR_PUSH (reg_desc_t, func_regs->reg_descs, rd); /* for 0 reg */ HTAB_CREATE (size_t, func_regs->name2rdn_tab, 100, name2rdn_hash, name2rdn_eq, func_regs); + HTAB_CREATE (size_t, func_regs->hrn2rdn_tab, 10, hrn2rdn_hash, hrn2rdn_eq, func_regs); HTAB_CREATE (size_t, func_regs->reg2rdn_tab, 100, reg2rdn_hash, reg2rdn_eq, func_regs); } +static int target_locs_num (MIR_reg_t loc, MIR_type_t type); +static int target_hard_reg_type_ok_p (MIR_reg_t hard_reg, MIR_type_t type); +static int target_fixed_hard_reg_p (MIR_reg_t hard_reg); + static MIR_reg_t create_func_reg (MIR_context_t ctx, MIR_func_t func, const char *name, - MIR_reg_t reg, MIR_type_t type, int any_p, char **name_ptr) { + const char *hard_reg_name, MIR_reg_t reg, MIR_type_t type, + int any_p, char **name_ptr) { func_regs_t func_regs = func->internal; - reg_desc_t rd; + MIR_module_t func_module; + reg_desc_t rd, *rd_ref; size_t rdn, tab_rdn; int htab_res; + MIR_reg_t hr; if (!any_p && _MIR_reserved_name_p (ctx, name)) MIR_get_error_func (ctx) (MIR_reserved_name_error, "redefining a reserved name %s", name); - rd.name = (char *) name; + rd.name = (char *) get_ctx_str (ctx, name); + if (hard_reg_name != NULL) hard_reg_name = get_ctx_str (ctx, hard_reg_name); + rd.hard_reg_name = (char *) hard_reg_name; rd.type = type; rd.reg = reg; /* 0 is reserved */ rdn = VARR_LENGTH (reg_desc_t, func_regs->reg_descs); @@ -475,25 +564,53 @@ static MIR_reg_t create_func_reg (MIR_context_t ctx, MIR_func_t func, const char VARR_POP (reg_desc_t, func_regs->reg_descs); MIR_get_error_func (ctx) (MIR_repeated_decl_error, "Repeated reg declaration %s", name); } - if ((rd.name = malloc (strlen (name) + 1)) == NULL) - MIR_get_error_func (ctx) (MIR_alloc_error, "Not enough memory for reg names"); - VARR_ADDR (reg_desc_t, func_regs->reg_descs)[rdn].name = *name_ptr = rd.name; - strcpy (*name_ptr, name); + if (hard_reg_name != NULL) { + if ((hr = _MIR_get_hard_reg (ctx, hard_reg_name)) == MIR_NON_VAR) { + MIR_get_error_func (ctx) (MIR_hard_reg_error, "unknown hard reg %s", hard_reg_name); + } else if (!target_hard_reg_type_ok_p (hr, type)) { + MIR_get_error_func (ctx) (MIR_hard_reg_error, + "reg %s tied to hard reg %s can not be of type %s", name, + hard_reg_name, MIR_type_str (ctx, type)); + } else if (target_fixed_hard_reg_p (hr)) { + MIR_get_error_func (ctx) (MIR_hard_reg_error, + "reg %s can not be tied to reserved hard reg %s", name, + hard_reg_name); + } else if (target_locs_num (hr, type) > 1) + MIR_get_error_func (ctx) (MIR_hard_reg_error, "reg %s tied to %s requires more one hard reg", + name, hard_reg_name); + if (HTAB_DO (size_t, func_regs->hrn2rdn_tab, rdn, HTAB_FIND, tab_rdn)) { + rd_ref = &VARR_ADDR (reg_desc_t, func_regs->reg_descs)[tab_rdn]; + if (type != rd_ref->type) + MIR_get_error_func (ctx) (MIR_repeated_decl_error, + "regs %s and %s tied to hard reg %s have different types", name, + rd_ref->name, hard_reg_name); + /* Use always one reg for global vars assigned to hard regs: */ + VARR_POP (reg_desc_t, func_regs->reg_descs); + *name_ptr = rd_ref->name; + return rd_ref->reg; + } + func_module = func->func_item->module; + if (func_module->data == NULL) func_module->data = bitmap_create2 (128); + bitmap_set_bit_p (func_module->data, hr); /* hard regs used for globals */ + } + *name_ptr = rd.name; htab_res = HTAB_DO (size_t, func_regs->name2rdn_tab, rdn, HTAB_INSERT, tab_rdn); mir_assert (!htab_res); + if (hard_reg_name != NULL) { + htab_res = HTAB_DO (size_t, func_regs->hrn2rdn_tab, rdn, HTAB_INSERT, tab_rdn); + mir_assert (!htab_res); + } htab_res = HTAB_DO (size_t, func_regs->reg2rdn_tab, rdn, HTAB_INSERT, tab_rdn); mir_assert (!htab_res); return reg; } -static void func_regs_finish (MIR_context_t ctx, MIR_func_t func) { +static void func_regs_finish (MIR_context_t ctx MIR_UNUSED, MIR_func_t func) { func_regs_t func_regs = func->internal; - char *name; - for (size_t i = 0; i < VARR_LENGTH (reg_desc_t, func_regs->reg_descs); i++) - if ((name = VARR_GET (reg_desc_t, func_regs->reg_descs, i).name) != NULL) free (name); VARR_DESTROY (reg_desc_t, func_regs->reg_descs); HTAB_DESTROY (size_t, func_regs->name2rdn_tab); + HTAB_DESTROY (size_t, func_regs->hrn2rdn_tab); HTAB_DESTROY (size_t, func_regs->reg2rdn_tab); free (func->internal); func->internal = NULL; @@ -505,7 +622,7 @@ static void push_data (MIR_context_t ctx, uint8_t *els, size_t size) { for (size_t i = 0; i < size; i++) VARR_PUSH (uint8_t, temp_data, els[i]); } -const char *MIR_item_name (MIR_context_t ctx, MIR_item_t item) { +const char *MIR_item_name (MIR_context_t ctx MIR_UNUSED, MIR_item_t item) { mir_assert (item != NULL); switch (item->item_type) { case MIR_func_item: return item->u.func->name; @@ -516,12 +633,13 @@ const char *MIR_item_name (MIR_context_t ctx, MIR_item_t item) { case MIR_bss_item: return item->u.bss->name; case MIR_data_item: return item->u.data->name; case MIR_ref_data_item: return item->u.ref_data->name; + case MIR_lref_data_item: return item->u.lref_data->name; case MIR_expr_data_item: return item->u.expr_data->name; default: mir_assert (FALSE); return NULL; } } -MIR_func_t MIR_get_item_func (MIR_context_t ctx, MIR_item_t item) { +MIR_func_t MIR_get_item_func (MIR_context_t ctx MIR_UNUSED, MIR_item_t item) { mir_assert (item != NULL); if (item->item_type == MIR_func_item) { return item->u.func; @@ -549,12 +667,18 @@ void MIR_set_error_func (MIR_context_t ctx, MIR_error_func_t func) { // ?? atom error_func = func; } -static htab_hash_t item_hash (MIR_item_t it, void *arg) { - return mir_hash_finish ( +int MIR_get_func_redef_permission_p (MIR_context_t ctx) { return func_redef_permission_p; } + +void MIR_set_func_redef_permission (MIR_context_t ctx, int enable_p) { // ?? atomic access + func_redef_permission_p = enable_p; +} + +static htab_hash_t item_hash (MIR_item_t it, void *arg MIR_UNUSED) { + return (htab_hash_t) mir_hash_finish ( mir_hash_step (mir_hash_step (mir_hash_init (28), (uint64_t) MIR_item_name (NULL, it)), (uint64_t) it->module)); } -static int item_eq (MIR_item_t it1, MIR_item_t it2, void *arg) { +static int item_eq (MIR_item_t it1, MIR_item_t it2, void *arg MIR_UNUSED) { return it1->module == it2->module && MIR_item_name (NULL, it1) == MIR_item_name (NULL, it2); } @@ -592,12 +716,11 @@ static void init_module (MIR_context_t ctx, MIR_module_t m, const char *name) { static void code_init (MIR_context_t ctx); static void code_finish (MIR_context_t ctx); -static void parallel_error (MIR_context_t ctx, const char *err_message) { - MIR_get_error_func (ctx) (MIR_parallel_error, err_message); -} - double _MIR_get_api_version (void) { return MIR_API_VERSION; } +static void hard_reg_name_init (MIR_context_t ctx); +static void hard_reg_name_finish (MIR_context_t ctx); + MIR_context_t _MIR_init (void) { MIR_context_t ctx; @@ -605,29 +728,34 @@ MIR_context_t _MIR_init (void) { if ((ctx = malloc (sizeof (struct MIR_context))) == NULL) default_error (MIR_alloc_error, "Not enough memory for ctx"); ctx->string_ctx = NULL; + ctx->alias_ctx = NULL; ctx->reg_ctx = NULL; ctx->simplify_ctx = NULL; ctx->machine_code_ctx = NULL; ctx->io_ctx = NULL; ctx->scan_ctx = NULL; + ctx->hard_reg_ctx = NULL; ctx->interp_ctx = NULL; - if (mir_mutex_init (&ctx_mutex, NULL)) parallel_error (ctx, "error in mutex init"); #ifndef NDEBUG for (MIR_insn_code_t c = 0; c < MIR_INVALID_INSN; c++) mir_assert (c == insn_descs[c].code); #endif error_func = default_error; + func_redef_permission_p = FALSE; curr_module = NULL; curr_func = NULL; curr_label_num = 0; - if ((ctx->string_ctx = malloc (sizeof (struct string_ctx))) == NULL) + if ((ctx->string_ctx = malloc (sizeof (struct string_ctx))) == NULL + || (ctx->alias_ctx = malloc (sizeof (struct string_ctx))) == NULL) MIR_get_error_func (ctx) (MIR_alloc_error, "Not enough memory for ctx"); string_init (&strings, &string_tab); + string_init (&aliases, &alias_tab); VARR_CREATE (MIR_proto_t, unspec_protos, 0); check_and_prepare_insn_descs (ctx); DLIST_INIT (MIR_module_t, all_modules); simplify_init (ctx); VARR_CREATE (char, temp_string, 64); VARR_CREATE (uint8_t, temp_data, 512); + VARR_CREATE (uint8_t, used_label_p, 512); #if !MIR_NO_IO io_init (ctx); #endif @@ -635,29 +763,37 @@ MIR_context_t _MIR_init (void) { scan_init (ctx); #endif VARR_CREATE (MIR_module_t, modules_to_link, 0); + VARR_CREATE (MIR_op_t, temp_ops, 0); init_module (ctx, &environment_module, ".environment"); HTAB_CREATE (MIR_item_t, module_item_tab, 512, item_hash, item_eq, NULL); setjmp_addr = NULL; code_init (ctx); + wrapper_end_addr = _MIR_get_wrapper_end (ctx); /* should be after code_init */ + hard_reg_name_init (ctx); interp_init (ctx); return ctx; } -void MIR_remove_insn (MIR_context_t ctx, MIR_item_t func_item, MIR_insn_t insn) { +static void remove_insn (MIR_context_t ctx, MIR_item_t func_item, MIR_insn_t insn, + DLIST (MIR_insn_t) * insns) { mir_assert (func_item != NULL); if (func_item->item_type != MIR_func_item) MIR_get_error_func (ctx) (MIR_wrong_param_value_error, "MIR_remove_insn: wrong func item"); - DLIST_REMOVE (MIR_insn_t, func_item->u.func->insns, insn); + DLIST_REMOVE (MIR_insn_t, *insns, insn); free (insn); } +void MIR_remove_insn (MIR_context_t ctx, MIR_item_t func_item, MIR_insn_t insn) { + remove_insn (ctx, func_item, insn, &func_item->u.func->insns); +} + static void remove_func_insns (MIR_context_t ctx, MIR_item_t func_item, DLIST (MIR_insn_t) * insns) { MIR_insn_t insn; mir_assert (func_item->item_type == MIR_func_item); while ((insn = DLIST_HEAD (MIR_insn_t, *insns)) != NULL) { - MIR_remove_insn (ctx, func_item, insn); + remove_insn (ctx, func_item, insn, insns); } } @@ -667,6 +803,7 @@ static void remove_item (MIR_context_t ctx, MIR_item_t item) { remove_func_insns (ctx, item, &item->u.func->insns); remove_func_insns (ctx, item, &item->u.func->original_insns); VARR_DESTROY (MIR_var_t, item->u.func->vars); + if (item->u.func->global_vars != NULL) VARR_DESTROY (MIR_var_t, item->u.func->global_vars); func_regs_finish (ctx, item->u.func); free (item->u.func); break; @@ -685,6 +822,10 @@ static void remove_item (MIR_context_t ctx, MIR_item_t item) { if (item->addr != NULL && item->section_head_p) free (item->addr); free (item->u.ref_data); break; + case MIR_lref_data_item: + if (item->addr != NULL && item->section_head_p) free (item->addr); + free (item->u.lref_data); + break; case MIR_expr_data_item: if (item->addr != NULL && item->section_head_p) free (item->addr); free (item->u.expr_data); @@ -706,7 +847,7 @@ static void remove_module (MIR_context_t ctx, MIR_module_t module, int free_modu DLIST_REMOVE (MIR_item_t, module->items, item); remove_item (ctx, item); } - if (module->data != NULL) free (module->data); + if (module->data != NULL) bitmap_destroy (module->data); if (free_module_p) free (module); } @@ -725,6 +866,7 @@ void MIR_finish (MIR_context_t ctx) { remove_all_modules (ctx); HTAB_DESTROY (MIR_item_t, module_item_tab); VARR_DESTROY (MIR_module_t, modules_to_link); + VARR_DESTROY (MIR_op_t, temp_ops); #if !MIR_NO_SCAN scan_finish (ctx); #endif @@ -732,6 +874,7 @@ void MIR_finish (MIR_context_t ctx) { io_finish (ctx); #endif VARR_DESTROY (uint8_t, temp_data); + VARR_DESTROY (uint8_t, used_label_p); VARR_DESTROY (char, temp_string); while (VARR_LENGTH (MIR_proto_t, unspec_protos) != 0) { MIR_proto_t proto = VARR_POP (MIR_proto_t, unspec_protos); @@ -740,9 +883,11 @@ void MIR_finish (MIR_context_t ctx) { } VARR_DESTROY (MIR_proto_t, unspec_protos); string_finish (&strings, &string_tab); + string_finish (&aliases, &alias_tab); simplify_finish (ctx); VARR_DESTROY (size_t, insn_nops); code_finish (ctx); + hard_reg_name_finish (ctx); if (curr_func != NULL) MIR_get_error_func (ctx) (MIR_finish_error, "finish when function %s is not finished", curr_func->name); @@ -750,7 +895,7 @@ void MIR_finish (MIR_context_t ctx) { MIR_get_error_func (ctx) (MIR_finish_error, "finish when module %s is not finished", curr_module->name); free (ctx->string_ctx); - if (mir_mutex_destroy (&ctx_mutex)) parallel_error (ctx, "error in mutex destroy"); + free (ctx->alias_ctx); free (ctx); ctx = NULL; } @@ -808,7 +953,7 @@ const char *MIR_type_str (MIR_context_t ctx, MIR_type_t tp) { static const char *mode_str (MIR_op_mode_t mode) { switch (mode) { case MIR_OP_REG: return "reg"; - case MIR_OP_HARD_REG: return "hard_reg"; + case MIR_OP_VAR: return "var"; case MIR_OP_INT: return "int"; case MIR_OP_UINT: return "uint"; case MIR_OP_FLOAT: return "float"; @@ -817,7 +962,7 @@ static const char *mode_str (MIR_op_mode_t mode) { case MIR_OP_REF: return "ref"; case MIR_OP_STR: return "str"; case MIR_OP_MEM: return "mem"; - case MIR_OP_HARD_REG_MEM: return "hard_reg_mem"; + case MIR_OP_VAR_MEM: return "var_mem"; case MIR_OP_LABEL: return "label"; case MIR_OP_BOUND: return "bound"; case MIR_OP_UNDEF: return "undef"; @@ -874,6 +1019,7 @@ static MIR_item_t add_item (MIR_context_t ctx, MIR_item_t item) { case MIR_bss_item: case MIR_data_item: case MIR_ref_data_item: + case MIR_lref_data_item: case MIR_expr_data_item: case MIR_func_item: if (item->item_type == MIR_export_item) { @@ -980,7 +1126,7 @@ static MIR_type_t canon_type (MIR_type_t type) { return type; } -size_t _MIR_type_size (MIR_context_t ctx, MIR_type_t type) { +size_t _MIR_type_size (MIR_context_t ctx MIR_UNUSED, MIR_type_t type) { switch (type) { case MIR_T_I8: return sizeof (int8_t); case MIR_T_U8: return sizeof (uint8_t); @@ -1059,6 +1205,38 @@ MIR_item_t MIR_new_ref_data (MIR_context_t ctx, const char *name, MIR_item_t ref return item; } +MIR_item_t MIR_new_lref_data (MIR_context_t ctx, const char *name, MIR_label_t label, + MIR_label_t label2, int64_t disp) { + MIR_item_t tab_item, item = create_item (ctx, MIR_lref_data_item, "lref data"); + MIR_lref_data_t lref_data; + + if (label == NULL) { + free (item); + MIR_get_error_func (ctx) (MIR_alloc_error, "null label for lref data %s", + name == NULL ? "" : name); + } + item->u.lref_data = lref_data = malloc (sizeof (struct MIR_lref_data)); + if (lref_data == NULL) { + free (item); + MIR_get_error_func (ctx) (MIR_alloc_error, "Not enough memory for creation of lref data %s", + name == NULL ? "" : name); + } + if (name != NULL) name = get_ctx_str (ctx, name); + lref_data->name = name; + lref_data->label = label; + lref_data->label2 = label2; + lref_data->disp = disp; + lref_data->orig_label = lref_data->orig_label2 = NULL; + lref_data->next = NULL; + if (name == NULL) { + DLIST_APPEND (MIR_item_t, curr_module->items, item); + } else if ((tab_item = add_item (ctx, item)) != item) { + free (item); + item = tab_item; + } + return item; +} + MIR_item_t MIR_new_expr_data (MIR_context_t ctx, const char *name, MIR_item_t expr_item) { MIR_item_t tab_item, item = create_item (ctx, MIR_expr_data_item, "expr data"); MIR_expr_data_t expr_data; @@ -1099,7 +1277,7 @@ static MIR_proto_t create_proto (MIR_context_t ctx, const char *name, size_t nre proto->name = get_ctx_str (ctx, name); proto->res_types = (MIR_type_t *) ((char *) proto + sizeof (struct MIR_proto)); if (nres != 0) memcpy (proto->res_types, res_types, nres * sizeof (MIR_type_t)); - proto->nres = nres; + proto->nres = (uint32_t) nres; proto->vararg_p = vararg_p != 0; VARR_CREATE (MIR_var_t, proto->args, nargs); for (size_t i = 0; i < nargs; i++) { @@ -1200,7 +1378,7 @@ static MIR_item_t new_func_arr (MIR_context_t ctx, const char *name, size_t nres } func->name = get_ctx_str (ctx, name); func->func_item = func_item; - func->nres = nres; + func->nres = (uint32_t) nres; func->res_types = (MIR_type_t *) ((char *) func + sizeof (struct MIR_func)); for (size_t i = 0; i < nres; i++) func->res_types[i] = canon_type (res_types[i]); tab_item = add_item (ctx, func_item); @@ -1208,20 +1386,23 @@ static MIR_item_t new_func_arr (MIR_context_t ctx, const char *name, size_t nres DLIST_INIT (MIR_insn_t, func->insns); DLIST_INIT (MIR_insn_t, func->original_insns); VARR_CREATE (MIR_var_t, func->vars, nargs + 8); - func->nargs = nargs; + func->global_vars = NULL; + func->nargs = (uint32_t) nargs; func->last_temp_num = 0; func->vararg_p = vararg_p != 0; - func->expr_p = FALSE; + func->expr_p = func->jret_p = FALSE; func->n_inlines = 0; func->machine_code = func->call_addr = NULL; + func->first_lref = NULL; func_regs_init (ctx, func); for (size_t i = 0; i < nargs; i++) { char *stored_name; MIR_type_t type = canon_type (vars[i].type); - - create_func_reg (ctx, func, vars[i].name, i + 1, - type == MIR_T_F || type == MIR_T_D || type == MIR_T_LD ? type : MIR_T_I64, - FALSE, &stored_name); + MIR_reg_t reg + = create_func_reg (ctx, func, vars[i].name, NULL, (MIR_reg_t) (i + 1), + type == MIR_T_F || type == MIR_T_D || type == MIR_T_LD ? type : MIR_T_I64, + FALSE, &stored_name); + mir_assert (i + 1 == reg); vars[i].name = stored_name; VARR_PUSH (MIR_var_t, func->vars, vars[i]); } @@ -1272,24 +1453,46 @@ MIR_item_t MIR_new_vararg_func (MIR_context_t ctx, const char *name, size_t nres return func_item; } -MIR_reg_t MIR_new_func_reg (MIR_context_t ctx, MIR_func_t func, MIR_type_t type, const char *name) { +static MIR_reg_t new_func_reg (MIR_context_t ctx, MIR_func_t func, MIR_type_t type, + const char *name, const char *hard_reg_name) { MIR_var_t var; - MIR_reg_t res; + MIR_reg_t res, reg; char *stored_name; + if (func == NULL) + MIR_get_error_func (ctx) (MIR_reg_type_error, "func can not be NULL for new reg creation"); if (type != MIR_T_I64 && type != MIR_T_F && type != MIR_T_D && type != MIR_T_LD) - MIR_get_error_func (ctx) (MIR_reg_type_error, "wrong type for register %s: got '%s'", name, + MIR_get_error_func (ctx) (MIR_reg_type_error, "wrong type for var %s: got '%s'", name, type_str (ctx, type)); - mir_assert (func != NULL); - res = create_func_reg (ctx, func, name, VARR_LENGTH (MIR_var_t, func->vars) + 1, type, FALSE, - &stored_name); + reg = (MIR_reg_t) VARR_LENGTH (MIR_var_t, func->vars) + 1; + if (func->global_vars != NULL) reg += (MIR_reg_t) VARR_LENGTH (MIR_var_t, func->global_vars); + res = create_func_reg (ctx, func, name, hard_reg_name, reg, type, FALSE, &stored_name); + if (res != reg) return res; /* already exists */ var.type = type; var.name = stored_name; - VARR_PUSH (MIR_var_t, func->vars, var); + if (hard_reg_name == NULL) { + VARR_PUSH (MIR_var_t, func->vars, var); + } else { + if (func->global_vars == NULL) VARR_CREATE (MIR_var_t, func->global_vars, 8); + VARR_PUSH (MIR_var_t, func->global_vars, var); + } return res; } -static reg_desc_t *find_rd_by_name (MIR_context_t ctx, const char *name, MIR_func_t func) { +MIR_reg_t MIR_new_func_reg (MIR_context_t ctx, MIR_func_t func, MIR_type_t type, const char *name) { + return new_func_reg (ctx, func, type, name, NULL); +} + +MIR_reg_t MIR_new_global_func_reg (MIR_context_t ctx, MIR_func_t func, MIR_type_t type, + const char *name, const char *hard_reg_name) { + if (hard_reg_name == NULL) + MIR_get_error_func (ctx) (MIR_hard_reg_error, + "global var %s should have non-null hard reg name", name); + return new_func_reg (ctx, func, type, name, hard_reg_name); +} + +static reg_desc_t *find_rd_by_name (MIR_context_t ctx MIR_UNUSED, const char *name, + MIR_func_t func) { func_regs_t func_regs = func->internal; size_t rdn, temp_rdn; reg_desc_t rd; @@ -1328,9 +1531,10 @@ static reg_desc_t *find_rd_by_reg (MIR_context_t ctx, MIR_reg_t reg, MIR_func_t void MIR_finish_func (MIR_context_t ctx) { int expr_p = TRUE; - MIR_insn_t insn; + MIR_insn_t insn, prev_insn; MIR_insn_code_t code; const char *func_name; + int ret_p = FALSE, jret_p = FALSE; if (curr_func == NULL) MIR_get_error_func (ctx) (MIR_no_func_error, "finish of non-existing function"); @@ -1344,23 +1548,55 @@ void MIR_finish_func (MIR_context_t ctx) { int out_p, can_be_out_p; code = insn->code; - if (code == MIR_PHI) { + if (code == MIR_RET) ret_p = TRUE; + if (code == MIR_JRET) jret_p = TRUE; + if (code == MIR_PHI || code == MIR_USE) { curr_func = NULL; - MIR_get_error_func (ctx) (MIR_vararg_func_error, "phi can be used only internally"); + MIR_get_error_func (ctx) (MIR_vararg_func_error, "use or phi can be used only internally"); } else if (!curr_func->vararg_p && code == MIR_VA_START) { curr_func = NULL; MIR_get_error_func (ctx) (MIR_vararg_func_error, "va_start is not in vararg function"); + } else if (code == MIR_JRET && curr_func->nres != 0) { + curr_func = NULL; + MIR_get_error_func ( + ctx) (MIR_vararg_func_error, + "func %s: in insn '%s': function should not have results in this case", func_name, + insn_descs[code].name); + } else if ((code == MIR_JRET && ret_p) || (code == MIR_RET && jret_p)) { + curr_func = NULL; + MIR_get_error_func (ctx) (MIR_vararg_func_error, "func %s: mix of RET and JRET insns", + func_name); } else if (code == MIR_RET && actual_nops != curr_func->nres) { - int nres = curr_func->nres; - curr_func = NULL; MIR_get_error_func ( ctx) (MIR_vararg_func_error, "func %s: in instruction '%s': number of operands in return does not " "correspond number of function returns. Expected %d, got %d", - func_name, insn_descs[code].name, nres, actual_nops); - } else if (MIR_call_code_p (code)) + func_name, insn_descs[code].name, curr_func->nres, actual_nops); + } else if (MIR_call_code_p (code)) { expr_p = FALSE; + } else if (code == MIR_BO || code == MIR_UBO || code == MIR_BNO || code == MIR_UBNO) { + for (prev_insn = DLIST_PREV (MIR_insn_t, insn); prev_insn != NULL; + prev_insn = DLIST_PREV (MIR_insn_t, prev_insn)) + if (prev_insn->code != MIR_MOV || prev_insn->ops[1].mode != MIR_OP_REG) break; + if (prev_insn == NULL || !MIR_overflow_insn_code_p (prev_insn->code)) + MIR_get_error_func (ctx) (MIR_invalid_insn_error, + "func %s: instruction '%s' has no previous overflow insn " + "separated only by stores and reg moves", + func_name, insn_descs[code].name); + else if ((code == MIR_UBO || code == MIR_UBNO) + && (prev_insn->code == MIR_MULO || prev_insn->code == MIR_MULOS)) + MIR_get_error_func ( + ctx) (MIR_invalid_insn_error, + "func %s: unsigned overflow branch '%s' consumes flag of signed overflow insn '%s'", + func_name, insn_descs[code].name, insn_descs[prev_insn->code].name); + else if ((code == MIR_BO || code == MIR_BNO) + && (prev_insn->code == MIR_UMULO || prev_insn->code == MIR_UMULOS)) + MIR_get_error_func ( + ctx) (MIR_invalid_insn_error, + "func %s: signed overflow branch '%s' consumes flag of unsigned overflow insn '%s'", + func_name, insn_descs[code].name, insn_descs[prev_insn->code].name); + } for (i = 0; i < actual_nops; i++) { if (code == MIR_UNSPEC && i == 0) { mir_assert (insn->ops[i].mode == MIR_OP_INT); @@ -1385,11 +1621,11 @@ void MIR_finish_func (MIR_context_t ctx) { if (code == MIR_SWITCH) { out_p = FALSE; expected_mode = i == 0 ? MIR_OP_INT : MIR_OP_LABEL; - } else if (code != MIR_RET) { - expected_mode = MIR_insn_op_mode (ctx, insn, i, &out_p); - } else { + } else if (code == MIR_RET) { out_p = FALSE; expected_mode = type2mode (curr_func->res_types[i]); + } else { + expected_mode = MIR_insn_op_mode (ctx, insn, i, &out_p); } can_be_out_p = TRUE; switch (insn->ops[i].mode) { @@ -1439,8 +1675,8 @@ void MIR_finish_func (MIR_context_t ctx) { } mode = type2mode (insn->ops[i].u.mem.type); break; - case MIR_OP_HARD_REG: - case MIR_OP_HARD_REG_MEM: + case MIR_OP_VAR: + case MIR_OP_VAR_MEM: expr_p = FALSE; mode = expected_mode; mir_assert (FALSE); /* Should not be here */ @@ -1457,6 +1693,12 @@ void MIR_finish_func (MIR_context_t ctx) { || ((code == MIR_VA_ARG || code == MIR_VA_BLOCK_ARG) && i == 1) || (code == MIR_VA_END && i == 1))) { /* a special case: va_list as undef type mem */ insn->ops[i].value_mode = expected_mode; + } else if (expected_mode == MIR_OP_REG) { + if (insn->ops[i].mode != MIR_OP_REG && insn->ops[i].mode != MIR_OP_VAR) + MIR_get_error_func ( + ctx) (MIR_op_mode_error, + "func %s: in instruction '%s': expected reg for operand #%d. Got '%s'", func_name, + insn_descs[code].name, i + 1, mode_str (insn->ops[i].mode)); } else if (expected_mode != MIR_OP_UNDEF && (mode == MIR_OP_UINT ? MIR_OP_INT : mode) != expected_mode) { curr_func = NULL; @@ -1474,7 +1716,27 @@ void MIR_finish_func (MIR_context_t ctx) { } } } + if (!ret_p && !jret_p + && ((insn = DLIST_TAIL (MIR_insn_t, curr_func->insns)) == NULL || insn->code != MIR_JMP)) { + VARR_TRUNC (MIR_op_t, temp_ops, 0); + for (size_t i = 0; i < curr_func->nres; i++) { /* add absent ret */ + MIR_op_t op; + if (curr_func->res_types[i] == MIR_T_F) + op = MIR_new_float_op (ctx, 0.0f); + else if (curr_func->res_types[i] == MIR_T_D) + op = MIR_new_double_op (ctx, 0.0); + else if (curr_func->res_types[i] == MIR_T_LD) + op = MIR_new_ldouble_op (ctx, 0.0); + else + op = MIR_new_int_op (ctx, 0); + VARR_PUSH (MIR_op_t, temp_ops, op); + } + MIR_append_insn (ctx, curr_func->func_item, + MIR_new_insn_arr (ctx, MIR_RET, curr_func->nres, + VARR_ADDR (MIR_op_t, temp_ops))); + } curr_func->expr_p = expr_p; + curr_func->jret_p = jret_p; curr_func = NULL; } @@ -1484,9 +1746,10 @@ void MIR_finish_module (MIR_context_t ctx) { curr_module = NULL; } -static void setup_global (MIR_context_t ctx, const char *name, void *addr, MIR_item_t def) { +static int setup_global (MIR_context_t ctx, const char *name, void *addr, MIR_item_t def) { MIR_item_t item, tab_item; MIR_module_t saved = curr_module; + int redef_p = FALSE; curr_module = &environment_module; /* Use import for proto representation: */ @@ -1494,6 +1757,7 @@ static void setup_global (MIR_context_t ctx, const char *name, void *addr, MIR_i if ((tab_item = item_tab_find (ctx, MIR_item_name (ctx, item), &environment_module)) != item && tab_item != NULL) { free (item); + redef_p = TRUE; } else { HTAB_DO (MIR_item_t, module_item_tab, item, HTAB_INSERT, tab_item); DLIST_APPEND (MIR_item_t, environment_module.items, item); @@ -1502,6 +1766,7 @@ static void setup_global (MIR_context_t ctx, const char *name, void *addr, MIR_i tab_item->addr = addr; tab_item->ref_def = def; curr_module = saved; + return redef_p; } static void undefined_interface (MIR_context_t ctx) { @@ -1527,6 +1792,9 @@ static MIR_item_t load_bss_data_section (MIR_context_t ctx, MIR_item_t item, int else if (curr_item->item_type == MIR_ref_data_item && (curr_item == item || curr_item->u.ref_data->name == NULL)) section_size += _MIR_type_size (ctx, MIR_T_P); + else if (curr_item->item_type == MIR_lref_data_item + && (curr_item == item || curr_item->u.lref_data->name == NULL)) + section_size += _MIR_type_size (ctx, MIR_T_P); else if (curr_item->item_type == MIR_expr_data_item && (curr_item == item || curr_item->u.expr_data->name == NULL)) { expr_item = curr_item->u.expr_data->expr_item; @@ -1539,6 +1807,8 @@ static MIR_item_t load_bss_data_section (MIR_context_t ctx, MIR_item_t item, int section_size += _MIR_type_size (ctx, expr_item->u.func->res_types[0]); } else break; + if (section_size % 8 != 0) + section_size += 8 - section_size % 8; /* we might use 64-bit copying of data */ if ((item->addr = malloc (section_size)) == NULL) { name = MIR_item_name (ctx, item); MIR_get_error_func (ctx) (MIR_alloc_error, "Not enough memory to allocate data/bss %s", @@ -1566,6 +1836,11 @@ static MIR_item_t load_bss_data_section (MIR_context_t ctx, MIR_item_t item, int curr_item->u.ref_data->load_addr = addr; curr_item->addr = addr; addr += _MIR_type_size (ctx, MIR_T_P); + } else if (curr_item->item_type == MIR_lref_data_item + && (curr_item == item || curr_item->u.lref_data->name == NULL)) { + curr_item->u.lref_data->load_addr = addr; + curr_item->addr = addr; + addr += _MIR_type_size (ctx, MIR_T_P); } else if (curr_item->item_type == MIR_expr_data_item && (curr_item == item || curr_item->u.expr_data->name == NULL)) { expr_item = curr_item->u.expr_data->expr_item; @@ -1579,14 +1854,51 @@ static MIR_item_t load_bss_data_section (MIR_context_t ctx, MIR_item_t item, int return last_item; } +static void link_module_lrefs (MIR_context_t ctx, MIR_module_t m) { + for (MIR_item_t item = DLIST_HEAD (MIR_item_t, m->items); item != NULL; + item = DLIST_NEXT (MIR_item_t, item)) { + if (item->item_type != MIR_func_item) continue; + for (MIR_insn_t insn = DLIST_HEAD (MIR_insn_t, item->u.func->insns); insn != NULL; + insn = DLIST_NEXT (MIR_insn_t, insn)) + if (insn->code == MIR_LABEL) insn->data = item->u.func; + } + for (MIR_item_t item = DLIST_HEAD (MIR_item_t, m->items); item != NULL; + item = DLIST_NEXT (MIR_item_t, item)) { + if (item->item_type == MIR_lref_data_item) { + MIR_lref_data_t lref_data = item->u.lref_data; + MIR_label_t lab = lref_data->label, lab2 = lref_data->label2; + MIR_func_t func = (MIR_func_t) lab->data; + if (lab->data == NULL) + MIR_get_error_func (ctx) (MIR_wrong_lref_error, "A label not from any function in lref %s", + lref_data->name == NULL ? "" : lref_data->name); + else if (lab2 != NULL && lab2->data != func) + MIR_get_error_func (ctx) (MIR_wrong_lref_error, + "Labels from different functions in lref %s", + lref_data->name == NULL ? "" : lref_data->name); + lref_data->next = func->first_lref; + func->first_lref = lref_data; + } + } + for (MIR_item_t item = DLIST_HEAD (MIR_item_t, m->items); item != NULL; + item = DLIST_NEXT (MIR_item_t, item)) { + if (item->item_type != MIR_func_item) continue; + for (MIR_insn_t insn = DLIST_HEAD (MIR_insn_t, item->u.func->insns); insn != NULL; + insn = DLIST_NEXT (MIR_insn_t, insn)) + if (insn->code == MIR_LABEL) insn->data = NULL; + } +} + void MIR_load_module (MIR_context_t ctx, MIR_module_t m) { + int lref_p = FALSE; mir_assert (m != NULL); for (MIR_item_t item = DLIST_HEAD (MIR_item_t, m->items); item != NULL; item = DLIST_NEXT (MIR_item_t, item)) { MIR_item_t first_item = item; if (item->item_type == MIR_bss_item || item->item_type == MIR_data_item - || item->item_type == MIR_ref_data_item || item->item_type == MIR_expr_data_item) { + || item->item_type == MIR_ref_data_item || item->item_type == MIR_lref_data_item + || item->item_type == MIR_expr_data_item) { + if (item->item_type == MIR_lref_data_item) lref_p = TRUE; item = load_bss_data_section (ctx, item, FALSE); } else if (item->item_type == MIR_func_item) { if (item->addr == NULL) { @@ -1601,9 +1913,19 @@ void MIR_load_module (MIR_context_t ctx, MIR_module_t m) { mir_assert (first_item->item_type != MIR_export_item && first_item->item_type != MIR_import_item && first_item->item_type != MIR_forward_item); - setup_global (ctx, MIR_item_name (ctx, first_item), first_item->addr, first_item); + if (setup_global (ctx, MIR_item_name (ctx, first_item), first_item->addr, first_item) + && item->item_type == MIR_func_item + && !func_redef_permission_p +#ifdef __APPLE__ + /* macosx can have multiple equal external inline definitions of the same function: */ + && strncmp (item->u.func->name, "__darwin", 8) != 0 +#endif + ) + MIR_get_error_func (ctx) (MIR_repeated_decl_error, "func %s is prohibited for redefinition", + item->u.func->name); } } + if (lref_p) link_module_lrefs (ctx, m); VARR_PUSH (MIR_module_t, modules_to_link, m); } @@ -1616,6 +1938,7 @@ void MIR_load_external (MIR_context_t ctx, const char *name, void *addr) { setup_global (ctx, name, addr, NULL); } +static void simplify_module_init (MIR_context_t ctx); static int simplify_func (MIR_context_t ctx, MIR_item_t func_item, int mem_float_p); static void process_inlines (MIR_context_t ctx, MIR_item_t func_item); @@ -1639,6 +1962,7 @@ void MIR_link (MIR_context_t ctx, void (*set_interface) (MIR_context_t ctx, MIR_ for (size_t i = 0; i < VARR_LENGTH (MIR_module_t, modules_to_link); i++) { m = VARR_GET (MIR_module_t, modules_to_link, i); + simplify_module_init (ctx); for (item = DLIST_HEAD (MIR_item_t, m->items); item != NULL; item = DLIST_NEXT (MIR_item_t, item)) if (item->item_type == MIR_func_item) { @@ -1686,6 +2010,7 @@ void MIR_link (MIR_context_t ctx, void (*set_interface) (MIR_context_t ctx, MIR_ memcpy (item->u.ref_data->load_addr, &addr, _MIR_type_size (ctx, MIR_T_P)); continue; } + /* lref data are set up in interpreter or generator */ if (item->item_type != MIR_expr_data_item) continue; expr_item = item->u.expr_data->expr_item; MIR_interp (ctx, expr_item, &res, 0); @@ -1741,7 +2066,7 @@ static size_t insn_code_nops (MIR_context_t ctx, MIR_insn_code_t code) { /* 0 fo return VARR_GET (size_t, insn_nops, code); } -size_t MIR_insn_nops (MIR_context_t ctx, MIR_insn_t insn) { +size_t MIR_insn_nops (MIR_context_t ctx MIR_UNUSED, MIR_insn_t insn) { mir_assert (insn != NULL); return insn->nops; } @@ -1766,14 +2091,20 @@ MIR_op_mode_t MIR_insn_op_mode (MIR_context_t ctx, MIR_insn_t insn, size_t nop, *out_p = FALSE; /* to remove unitialized warning */ if (nop >= nops) return MIR_OP_BOUND; mir_assert (out_p != NULL); - if (code == MIR_RET || code == MIR_SWITCH) { - *out_p = FALSE; + switch (code) { + case MIR_RET: + case MIR_SWITCH: /* should be already checked in MIR_finish_func */ - return nop == 0 && code == MIR_SWITCH ? MIR_OP_INT : insn->ops[nop].mode; - } else if (code == MIR_PHI) { - *out_p = nop == 0; - return insn->ops[nop].mode; - } else if (MIR_call_code_p (code) || code == MIR_UNSPEC) { + return nop == 0 && code != MIR_RET ? MIR_OP_INT : insn->ops[nop].mode; + case MIR_ADDR: + case MIR_ADDR8: + case MIR_ADDR16: + case MIR_ADDR32: *out_p = nop == 0; return nop == 0 ? MIR_OP_INT : insn->ops[nop].mode; + case MIR_PHI: *out_p = nop == 0; return insn->ops[nop].mode; + case MIR_USE: return insn->ops[nop].mode; + case MIR_CALL: + case MIR_INLINE: + case MIR_UNSPEC: { MIR_op_t proto_op; MIR_proto_t proto; size_t args_start; @@ -1800,9 +2131,12 @@ MIR_op_mode_t MIR_insn_op_mode (MIR_context_t ctx, MIR_insn_t insn, size_t nop, return type2mode (proto->res_types[nop - args_start]); return type2mode (VARR_GET (MIR_var_t, proto->args, nop - args_start - proto->nres).type); } - mode = insn_descs[code].op_modes[nop]; - *out_p = (mode & OUT_FLAG) != 0; - return *out_p ? mode ^ OUT_FLAG : mode; + default: + mode = insn_descs[code].op_modes[nop]; + if ((mode & OUT_FLAG) == 0) return mode; + *out_p = TRUE; + return mode ^ OUT_FLAG; + } } static MIR_insn_t create_insn (MIR_context_t ctx, size_t nops, MIR_insn_code_t code) { @@ -1855,10 +2189,10 @@ MIR_insn_t MIR_new_insn_arr (MIR_context_t ctx, MIR_insn_code_t code, size_t nop MIR_insn_t insn; MIR_proto_t proto; size_t args_start, narg, i = 0, expected_nops = insn_code_nops (ctx, code); - mir_assert (ops != NULL); + mir_assert (nops == 0 || ops != NULL); - if (!MIR_call_code_p (code) && code != MIR_UNSPEC && code != MIR_PHI && code != MIR_RET - && code != MIR_SWITCH && nops != expected_nops) { + if (!MIR_call_code_p (code) && code != MIR_UNSPEC && code != MIR_USE && code != MIR_PHI + && code != MIR_RET && code != MIR_SWITCH && nops != expected_nops) { MIR_get_error_func (ctx) (MIR_ops_num_error, "wrong number of operands for insn %s", insn_descs[code].name); } else if (code == MIR_SWITCH) { @@ -1899,7 +2233,7 @@ MIR_insn_t MIR_new_insn_arr (MIR_context_t ctx, MIR_insn_code_t code, size_t nop ctx) (MIR_wrong_type_error, "arg of %s is block type memory but param is not of block type", code == MIR_UNSPEC ? "unspec" : "call"); - } else if (VARR_GET (MIR_var_t, proto->args, narg).size != ops[i].u.mem.disp) { + } else if (VARR_GET (MIR_var_t, proto->args, narg).size != (size_t) ops[i].u.mem.disp) { MIR_get_error_func ( ctx) (MIR_wrong_type_error, "different sizes (%lu vs %lu) of arg and param block memory in %s insn", @@ -1924,9 +2258,19 @@ MIR_insn_t MIR_new_insn_arr (MIR_context_t ctx, MIR_insn_code_t code, size_t nop if (ops[2].mode != MIR_OP_MEM) MIR_get_error_func (ctx) (MIR_op_mode_error, "3rd operand of va_arg should be any memory with given type"); + } else if (code == MIR_PRSET) { + if (ops[1].mode != MIR_OP_INT) + MIR_get_error_func (ctx) (MIR_op_mode_error, "property should be a integer operand"); + } else if (code == MIR_PRBEQ || code == MIR_PRBNE) { + if (ops[2].mode != MIR_OP_INT) + MIR_get_error_func (ctx) (MIR_op_mode_error, "property should be a integer operand"); + if (ops[1].mode != MIR_OP_REG && ops[1].mode != MIR_OP_MEM) + MIR_get_error_func ( + ctx) (MIR_op_mode_error, + "2nd operand of property branch should be any memory or reg with given type"); } insn = create_insn (ctx, nops, code); - insn->nops = nops; + insn->nops = (unsigned int) nops; for (i = 0; i < nops; i++) insn->ops[i] = ops[i]; return insn; } @@ -1943,14 +2287,14 @@ MIR_insn_t MIR_new_insn (MIR_context_t ctx, MIR_insn_code_t code, ...) { va_list argp; size_t nops = insn_code_nops (ctx, code); - if (code == MIR_PHI) + if (code == MIR_USE || code == MIR_PHI) MIR_get_error_func (ctx) (MIR_call_op_error, - "Use only MIR_new_insn_arr for creating a phi insn"); + "Use only MIR_new_insn_arr for creating use or phi insn"); else if (MIR_call_code_p (code) || code == MIR_UNSPEC || code == MIR_RET || code == MIR_SWITCH) MIR_get_error_func ( ctx) (MIR_call_op_error, "Use only MIR_new_insn_arr or MIR_new_{call,unspec,ret}_insn for creating a " - "call/unspec/ret/switch insn"); + "call/unspec/ret/jret/switch insn"); va_start (argp, code); return new_insn (ctx, code, nops, argp); } @@ -1962,6 +2306,13 @@ MIR_insn_t MIR_new_call_insn (MIR_context_t ctx, size_t nops, ...) { return new_insn (ctx, MIR_CALL, nops, argp); } +MIR_insn_t MIR_new_jcall_insn (MIR_context_t ctx, size_t nops, ...) { + va_list argp; + + va_start (argp, nops); + return new_insn (ctx, MIR_JCALL, nops, argp); +} + MIR_insn_t MIR_new_ret_insn (MIR_context_t ctx, size_t nops, ...) { va_list argp; @@ -2014,6 +2365,8 @@ static MIR_insn_t create_label (MIR_context_t ctx, int64_t label_num) { MIR_insn_t MIR_new_label (MIR_context_t ctx) { return create_label (ctx, ++curr_label_num); } +void _MIR_free_insn (MIR_context_t ctx MIR_UNUSED, MIR_insn_t insn) { free (insn); } + static MIR_reg_t new_temp_reg (MIR_context_t ctx, MIR_type_t type, MIR_func_t func) { char reg_name[30]; @@ -2033,15 +2386,7 @@ static MIR_reg_t new_temp_reg (MIR_context_t ctx, MIR_type_t type, MIR_func_t fu } MIR_reg_t _MIR_new_temp_reg (MIR_context_t ctx, MIR_type_t type, MIR_func_t func) { - MIR_reg_t temp; - - /* It can be used by generator in parallel from different processed - funcs in the same ctx and uses data, e.g. string tab, common for - ctx. So it should be thread safe. */ - if (mir_mutex_lock (&ctx_mutex)) parallel_error (ctx, "error in mutex lock"); - temp = new_temp_reg (ctx, type, func); - if (mir_mutex_unlock (&ctx_mutex)) parallel_error (ctx, "error in mutex unlock"); - return temp; + return new_temp_reg (ctx, type, func); } static reg_desc_t *get_func_rd_by_name (MIR_context_t ctx, const char *reg_name, MIR_func_t func) { @@ -2072,6 +2417,11 @@ const char *MIR_reg_name (MIR_context_t ctx, MIR_reg_t reg, MIR_func_t func) { return get_func_rd_by_reg (ctx, reg, func)->name; } +const char *MIR_reg_hard_reg_name (MIR_context_t ctx, MIR_reg_t reg, MIR_func_t func) { + const reg_desc_t *rd = get_func_rd_by_reg (ctx, reg, func); + return rd->hard_reg_name; +} + /* Functions to create operands. */ static void init_op (MIR_op_t *op, MIR_op_mode_t mode) { @@ -2079,7 +2429,7 @@ static void init_op (MIR_op_t *op, MIR_op_mode_t mode) { op->data = NULL; } -MIR_op_t MIR_new_reg_op (MIR_context_t ctx, MIR_reg_t reg) { +MIR_op_t MIR_new_reg_op (MIR_context_t ctx MIR_UNUSED, MIR_reg_t reg) { MIR_op_t op; init_op (&op, MIR_OP_REG); @@ -2087,15 +2437,15 @@ MIR_op_t MIR_new_reg_op (MIR_context_t ctx, MIR_reg_t reg) { return op; } -MIR_op_t _MIR_new_hard_reg_op (MIR_context_t ctx, MIR_reg_t hard_reg) { /* used only internally */ +MIR_op_t _MIR_new_var_op (MIR_context_t ctx MIR_UNUSED, MIR_reg_t var) { /* used only internally */ MIR_op_t op; - init_op (&op, MIR_OP_HARD_REG); - op.u.hard_reg = hard_reg; + init_op (&op, MIR_OP_VAR); + op.u.var = var; return op; } -MIR_op_t MIR_new_int_op (MIR_context_t ctx, int64_t i) { +MIR_op_t MIR_new_int_op (MIR_context_t ctx MIR_UNUSED, int64_t i) { MIR_op_t op; init_op (&op, MIR_OP_INT); @@ -2103,7 +2453,7 @@ MIR_op_t MIR_new_int_op (MIR_context_t ctx, int64_t i) { return op; } -MIR_op_t MIR_new_uint_op (MIR_context_t ctx, uint64_t u) { +MIR_op_t MIR_new_uint_op (MIR_context_t ctx MIR_UNUSED, uint64_t u) { MIR_op_t op; init_op (&op, MIR_OP_UINT); @@ -2111,7 +2461,7 @@ MIR_op_t MIR_new_uint_op (MIR_context_t ctx, uint64_t u) { return op; } -MIR_op_t MIR_new_float_op (MIR_context_t ctx, float f) { +MIR_op_t MIR_new_float_op (MIR_context_t ctx MIR_UNUSED, float f) { MIR_op_t op; mir_assert (sizeof (float) == 4); /* IEEE single */ @@ -2120,7 +2470,7 @@ MIR_op_t MIR_new_float_op (MIR_context_t ctx, float f) { return op; } -MIR_op_t MIR_new_double_op (MIR_context_t ctx, double d) { +MIR_op_t MIR_new_double_op (MIR_context_t ctx MIR_UNUSED, double d) { MIR_op_t op; mir_assert (sizeof (double) == 8); /* IEEE double */ @@ -2129,7 +2479,7 @@ MIR_op_t MIR_new_double_op (MIR_context_t ctx, double d) { return op; } -MIR_op_t MIR_new_ldouble_op (MIR_context_t ctx, long double ld) { +MIR_op_t MIR_new_ldouble_op (MIR_context_t ctx MIR_UNUSED, long double ld) { MIR_op_t op; #if defined(_WIN32) || __SIZEOF_LONG_DOUBLE__ == 8 @@ -2141,7 +2491,7 @@ MIR_op_t MIR_new_ldouble_op (MIR_context_t ctx, long double ld) { return op; } -MIR_op_t MIR_new_ref_op (MIR_context_t ctx, MIR_item_t item) { +MIR_op_t MIR_new_ref_op (MIR_context_t ctx MIR_UNUSED, MIR_item_t item) { MIR_op_t op; init_op (&op, MIR_OP_REF); @@ -2157,8 +2507,9 @@ MIR_op_t MIR_new_str_op (MIR_context_t ctx, MIR_str_t str) { return op; } -MIR_op_t MIR_new_mem_op (MIR_context_t ctx, MIR_type_t type, MIR_disp_t disp, MIR_reg_t base, - MIR_reg_t index, MIR_scale_t scale) { +static MIR_op_t new_mem_op (MIR_context_t ctx MIR_UNUSED, MIR_type_t type, MIR_disp_t disp, + MIR_reg_t base, MIR_reg_t index, MIR_scale_t scale, MIR_alias_t alias, + MIR_alias_t nonalias) { MIR_op_t op; init_op (&op, MIR_OP_MEM); @@ -2167,23 +2518,52 @@ MIR_op_t MIR_new_mem_op (MIR_context_t ctx, MIR_type_t type, MIR_disp_t disp, MI op.u.mem.base = base; op.u.mem.index = index; op.u.mem.scale = scale; + op.u.mem.nloc = 0; + op.u.mem.alias = alias; + op.u.mem.nonalias = nonalias; return op; } -MIR_op_t _MIR_new_hard_reg_mem_op (MIR_context_t ctx, MIR_type_t type, MIR_disp_t disp, - MIR_reg_t base, MIR_reg_t index, MIR_scale_t scale) { +MIR_op_t MIR_new_mem_op (MIR_context_t ctx, MIR_type_t type, MIR_disp_t disp, MIR_reg_t base, + MIR_reg_t index, MIR_scale_t scale) { + return new_mem_op (ctx, type, disp, base, index, scale, 0, 0); +} + +MIR_op_t MIR_new_alias_mem_op (MIR_context_t ctx, MIR_type_t type, MIR_disp_t disp, MIR_reg_t base, + MIR_reg_t index, MIR_scale_t scale, MIR_alias_t alias, + MIR_alias_t nonalias) { + return new_mem_op (ctx, type, disp, base, index, scale, alias, nonalias); +} + +static MIR_op_t new_var_mem_op (MIR_context_t ctx MIR_UNUSED, MIR_type_t type, MIR_disp_t disp, + MIR_reg_t base, MIR_reg_t index, MIR_scale_t scale, + MIR_alias_t alias, MIR_alias_t nonalias) { MIR_op_t op; - init_op (&op, MIR_OP_HARD_REG_MEM); - op.u.hard_reg_mem.type = type; - op.u.hard_reg_mem.disp = disp; - op.u.hard_reg_mem.base = base; - op.u.hard_reg_mem.index = index; - op.u.hard_reg_mem.scale = scale; + init_op (&op, MIR_OP_VAR_MEM); + op.u.var_mem.type = type; + op.u.var_mem.disp = disp; + op.u.var_mem.base = base; + op.u.var_mem.index = index; + op.u.var_mem.scale = scale; + op.u.var_mem.nloc = 0; + op.u.var_mem.alias = alias; + op.u.var_mem.nonalias = nonalias; return op; } -MIR_op_t MIR_new_label_op (MIR_context_t ctx, MIR_label_t label) { +MIR_op_t _MIR_new_var_mem_op (MIR_context_t ctx, MIR_type_t type, MIR_disp_t disp, MIR_reg_t base, + MIR_reg_t index, MIR_scale_t scale) { + return new_var_mem_op (ctx, type, disp, base, index, scale, 0, 0); +} + +MIR_op_t _MIR_new_alias_var_mem_op (MIR_context_t ctx, MIR_type_t type, MIR_disp_t disp, + MIR_reg_t base, MIR_reg_t index, MIR_scale_t scale, + MIR_alias_t alias, MIR_alias_t nonalias) { + return new_var_mem_op (ctx, type, disp, base, index, scale, alias, nonalias); +} + +MIR_op_t MIR_new_label_op (MIR_context_t ctx MIR_UNUSED, MIR_label_t label) { MIR_op_t op; init_op (&op, MIR_OP_LABEL); @@ -2195,7 +2575,7 @@ int MIR_op_eq_p (MIR_context_t ctx, MIR_op_t op1, MIR_op_t op2) { if (op1.mode != op2.mode) return FALSE; switch (op1.mode) { case MIR_OP_REG: return op1.u.reg == op2.u.reg; - case MIR_OP_HARD_REG: return op1.u.hard_reg == op2.u.hard_reg; + case MIR_OP_VAR: return op1.u.var == op2.u.var; case MIR_OP_INT: return op1.u.i == op2.u.i; case MIR_OP_UINT: return op1.u.u == op2.u.u; case MIR_OP_FLOAT: return op1.u.f == op2.u.f; @@ -2211,13 +2591,11 @@ int MIR_op_eq_p (MIR_context_t ctx, MIR_op_t op1, MIR_op_t op2) { return (op1.u.mem.type == op2.u.mem.type && op1.u.mem.disp == op2.u.mem.disp && op1.u.mem.base == op2.u.mem.base && op1.u.mem.index == op2.u.mem.index && (op1.u.mem.index == 0 || op1.u.mem.scale == op2.u.mem.scale)); - case MIR_OP_HARD_REG_MEM: - return (op1.u.hard_reg_mem.type == op2.u.hard_reg_mem.type - && op1.u.hard_reg_mem.disp == op2.u.hard_reg_mem.disp - && op1.u.hard_reg_mem.base == op2.u.hard_reg_mem.base - && op1.u.hard_reg_mem.index == op2.u.hard_reg_mem.index - && (op1.u.hard_reg_mem.index == MIR_NON_HARD_REG - || op1.u.hard_reg_mem.scale == op2.u.hard_reg_mem.scale)); + case MIR_OP_VAR_MEM: + return (op1.u.var_mem.type == op2.u.var_mem.type && op1.u.var_mem.disp == op2.u.var_mem.disp + && op1.u.var_mem.base == op2.u.var_mem.base + && op1.u.var_mem.index == op2.u.var_mem.index + && (op1.u.var_mem.index == MIR_NON_VAR || op1.u.var_mem.scale == op2.u.var_mem.scale)); case MIR_OP_LABEL: return op1.u.label == op2.u.label; default: mir_assert (FALSE); /* we should not have other operands here */ } @@ -2225,12 +2603,12 @@ int MIR_op_eq_p (MIR_context_t ctx, MIR_op_t op1, MIR_op_t op2) { } htab_hash_t MIR_op_hash_step (MIR_context_t ctx, htab_hash_t h, MIR_op_t op) { - h = mir_hash_step (h, (uint64_t) op.mode); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) op.mode); switch (op.mode) { - case MIR_OP_REG: return mir_hash_step (h, (uint64_t) op.u.reg); - case MIR_OP_HARD_REG: return mir_hash_step (h, (uint64_t) op.u.hard_reg); - case MIR_OP_INT: return mir_hash_step (h, (uint64_t) op.u.i); - case MIR_OP_UINT: return mir_hash_step (h, (uint64_t) op.u.u); + case MIR_OP_REG: return (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.reg); + case MIR_OP_VAR: return (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.var); + case MIR_OP_INT: return (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.i); + case MIR_OP_UINT: return (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.u); case MIR_OP_FLOAT: { union { double d; @@ -2238,9 +2616,9 @@ htab_hash_t MIR_op_hash_step (MIR_context_t ctx, htab_hash_t h, MIR_op_t op) { } u; u.d = op.u.f; - return mir_hash_step (h, u.u); + return (htab_hash_t) mir_hash_step (h, u.u); } - case MIR_OP_DOUBLE: return mir_hash_step (h, op.u.u); + case MIR_OP_DOUBLE: return (htab_hash_t) mir_hash_step (h, op.u.u); case MIR_OP_LDOUBLE: { union { long double ld; @@ -2248,29 +2626,29 @@ htab_hash_t MIR_op_hash_step (MIR_context_t ctx, htab_hash_t h, MIR_op_t op) { } u; u.ld = op.u.ld; - return mir_hash_step (mir_hash_step (h, u.u[0]), u.u[1]); + return (htab_hash_t) mir_hash_step (mir_hash_step (h, u.u[0]), u.u[1]); } case MIR_OP_REF: if (op.u.ref->item_type == MIR_export_item || op.u.ref->item_type == MIR_import_item) - return mir_hash_step (h, (uint64_t) MIR_item_name (ctx, op.u.ref)); - return mir_hash_step (h, (uint64_t) op.u.ref); - case MIR_OP_STR: return mir_hash_step (h, (uint64_t) op.u.str.s); + return (htab_hash_t) mir_hash_step (h, (uint64_t) MIR_item_name (ctx, op.u.ref)); + return (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.ref); + case MIR_OP_STR: return (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.str.s); case MIR_OP_MEM: - h = mir_hash_step (h, (uint64_t) op.u.mem.type); - h = mir_hash_step (h, (uint64_t) op.u.mem.disp); - h = mir_hash_step (h, (uint64_t) op.u.mem.base); - h = mir_hash_step (h, (uint64_t) op.u.mem.index); - if (op.u.mem.index != 0) h = mir_hash_step (h, (uint64_t) op.u.mem.scale); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.mem.type); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.mem.disp); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.mem.base); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.mem.index); + if (op.u.mem.index != 0) h = (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.mem.scale); break; - case MIR_OP_HARD_REG_MEM: - h = mir_hash_step (h, (uint64_t) op.u.hard_reg_mem.type); - h = mir_hash_step (h, (uint64_t) op.u.hard_reg_mem.disp); - h = mir_hash_step (h, (uint64_t) op.u.hard_reg_mem.base); - h = mir_hash_step (h, (uint64_t) op.u.hard_reg_mem.index); - if (op.u.hard_reg_mem.index != MIR_NON_HARD_REG) - h = mir_hash_step (h, (uint64_t) op.u.hard_reg_mem.scale); + case MIR_OP_VAR_MEM: + h = (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.var_mem.type); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.var_mem.disp); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.var_mem.base); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.var_mem.index); + if (op.u.var_mem.index != MIR_NON_VAR) + h = (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.var_mem.scale); break; - case MIR_OP_LABEL: return mir_hash_step (h, (uint64_t) op.u.label); + case MIR_OP_LABEL: return (htab_hash_t) mir_hash_step (h, (uint64_t) op.u.label); default: mir_assert (FALSE); /* we should not have other operands here */ } return h; @@ -2308,10 +2686,11 @@ void MIR_insert_insn_before (MIR_context_t ctx, MIR_item_t func_item, MIR_insn_t DLIST_INSERT_BEFORE (MIR_insn_t, func_item->u.func->insns, before, insn); } -static void store_labels_for_duplication (MIR_context_t ctx, VARR (MIR_insn_t) * labels, +static void store_labels_for_duplication (MIR_context_t ctx MIR_UNUSED, VARR (MIR_insn_t) * labels, VARR (MIR_insn_t) * branch_insns, MIR_insn_t insn, MIR_insn_t new_insn) { - if (MIR_branch_code_p (insn->code) || insn->code == MIR_SWITCH) { + if (MIR_any_branch_code_p (insn->code) || insn->code == MIR_LADDR || insn->code == MIR_PRBEQ + || insn->code == MIR_PRBNE) { VARR_PUSH (MIR_insn_t, branch_insns, new_insn); } else if (insn->code == MIR_LABEL) { mir_assert (insn->data == NULL); @@ -2320,7 +2699,7 @@ static void store_labels_for_duplication (MIR_context_t ctx, VARR (MIR_insn_t) * } } -static void redirect_duplicated_labels (MIR_context_t ctx, VARR (MIR_insn_t) * labels, +static void redirect_duplicated_labels (MIR_context_t ctx MIR_UNUSED, VARR (MIR_insn_t) * labels, VARR (MIR_insn_t) * branch_insns) { MIR_insn_t insn; @@ -2328,9 +2707,13 @@ static void redirect_duplicated_labels (MIR_context_t ctx, VARR (MIR_insn_t) * l size_t start_label_nop = 0, bound_label_nop = 1, n; insn = VARR_POP (MIR_insn_t, branch_insns); + if (insn->code == MIR_JMPI) continue; if (insn->code == MIR_SWITCH) { start_label_nop = 1; bound_label_nop = start_label_nop + insn->nops - 1; + } else if (insn->code == MIR_LADDR) { + start_label_nop = 1; + bound_label_nop = 2; } for (n = start_label_nop; n < bound_label_nop; n++) insn->ops[n].u.label = insn->ops[n].u.label->data; @@ -2360,6 +2743,12 @@ void _MIR_duplicate_func_insns (MIR_context_t ctx, MIR_item_t func_item) { DLIST_APPEND (MIR_insn_t, func->insns, new_insn); store_labels_for_duplication (ctx, labels, branch_insns, insn, new_insn); } + for (MIR_lref_data_t lref = func->first_lref; lref != NULL; lref = lref->next) { + lref->orig_label = lref->label; + lref->orig_label2 = lref->label2; + lref->label = lref->label->data; + if (lref->label2 != NULL) lref->label2 = lref->label2->data; + } redirect_duplicated_labels (ctx, labels, branch_insns); VARR_DESTROY (MIR_insn_t, labels); VARR_DESTROY (MIR_insn_t, branch_insns); @@ -2389,6 +2778,11 @@ void _MIR_restore_func_insns (MIR_context_t ctx, MIR_item_t func_item) { MIR_remove_insn (ctx, func_item, insn); func->insns = func->original_insns; DLIST_INIT (MIR_insn_t, func->original_insns); + for (MIR_lref_data_t lref = func->first_lref; lref != NULL; lref = lref->next) { + lref->label = lref->orig_label; + lref->label2 = lref->orig_label2; + lref->orig_label = lref->orig_label2 = NULL; + } } static void set_item_name (MIR_item_t item, const char *name) { @@ -2402,6 +2796,7 @@ static void set_item_name (MIR_item_t item, const char *name) { case MIR_bss_item: item->u.bss->name = name; break; case MIR_data_item: item->u.data->name = name; break; case MIR_ref_data_item: item->u.ref_data->name = name; break; + case MIR_lref_data_item: item->u.lref_data->name = name; break; case MIR_expr_data_item: item->u.expr_data->name = name; break; default: mir_assert (FALSE); } @@ -2441,15 +2836,38 @@ void MIR_change_module_ctx (MIR_context_t old_ctx, MIR_module_t m, MIR_context_t if (item->item_type == MIR_proto_item) { change_var_names (new_ctx, item->u.proto->args); } else if (item->item_type == MIR_func_item) { + func_regs_t func_regs = item->u.func->internal; + reg_desc_t *rds = VARR_ADDR (reg_desc_t, func_regs->reg_descs); + + for (size_t i = 1; i < VARR_LENGTH (reg_desc_t, func_regs->reg_descs); i++) { + rds[i].name = (char *) get_ctx_str (new_ctx, rds[i].name); + if (rds[i].hard_reg_name != NULL) + rds[i].hard_reg_name = (char *) get_ctx_str (new_ctx, rds[i].hard_reg_name); + } change_var_names (new_ctx, item->u.func->vars); + if (item->u.func->global_vars != NULL) change_var_names (new_ctx, item->u.func->global_vars); + for (MIR_insn_t insn = DLIST_HEAD (MIR_insn_t, item->u.func->insns); insn != NULL; - insn = DLIST_NEXT (MIR_insn_t, insn)) + insn = DLIST_NEXT (MIR_insn_t, insn)) { for (size_t i = 0; i < insn->nops; i++) { - if ((mode = insn->ops[i].mode) == MIR_OP_STR) + if ((mode = insn->ops[i].mode) == MIR_OP_STR) { insn->ops[i].u.str = get_ctx_string (new_ctx, insn->ops[i].u.str).str; + } else if (mode == MIR_OP_MEM) { + if (insn->ops[i].u.mem.alias != 0) + insn->ops[i].u.mem.alias + = MIR_alias (new_ctx, MIR_alias_name (old_ctx, insn->ops[i].u.mem.alias)); + if (insn->ops[i].u.mem.nonalias != 0) + insn->ops[i].u.mem.nonalias + = MIR_alias (new_ctx, MIR_alias_name (old_ctx, insn->ops[i].u.mem.nonalias)); + } } + } } } +#undef curr_label_num + if (new_ctx->curr_label_num < old_ctx->curr_label_num) + new_ctx->curr_label_num = old_ctx->curr_label_num; +#define curr_label_num ctx->curr_label_num } static void output_type (MIR_context_t ctx, FILE *f, MIR_type_t tp) { @@ -2464,11 +2882,17 @@ static void output_reg (MIR_context_t ctx, FILE *f, MIR_func_t func, MIR_reg_t r fprintf (f, "%s", MIR_reg_name (ctx, reg, func)); } -static void output_hard_reg (FILE *f, MIR_reg_t reg) { fprintf (f, "hr%u", reg); } +static void output_hard_reg (FILE *f, MIR_reg_t hreg) { fprintf (f, "hr%u", hreg); } + +static int var_is_reg_p (MIR_reg_t var); +static MIR_reg_t var2reg (MIR_reg_t var); +static void output_var (MIR_context_t ctx, FILE *f, MIR_func_t func, MIR_reg_t var) { + var_is_reg_p (var) ? output_reg (ctx, f, func, var2reg (var)) : output_hard_reg (f, var); +} static void output_label (MIR_context_t ctx, FILE *f, MIR_func_t func, MIR_label_t label); -static void out_str (FILE *f, MIR_str_t str) { +void MIR_output_str (MIR_context_t ctx MIR_UNUSED, FILE *f, MIR_str_t str) { fprintf (f, "\""); for (size_t i = 0; i < str.len; i++) if (str.s[i] == '\\') @@ -2497,15 +2921,15 @@ static void out_str (FILE *f, MIR_str_t str) { void MIR_output_op (MIR_context_t ctx, FILE *f, MIR_op_t op, MIR_func_t func) { switch (op.mode) { case MIR_OP_REG: output_reg (ctx, f, func, op.u.reg); break; - case MIR_OP_HARD_REG: output_hard_reg (f, op.u.hard_reg); break; + case MIR_OP_VAR: output_var (ctx, f, func, op.u.var); break; case MIR_OP_INT: fprintf (f, "%" PRId64, op.u.i); break; case MIR_OP_UINT: fprintf (f, "%" PRIu64, op.u.u); break; case MIR_OP_FLOAT: fprintf (f, "%.*ef", FLT_MANT_DIG, op.u.f); break; case MIR_OP_DOUBLE: fprintf (f, "%.*e", DBL_MANT_DIG, op.u.d); break; case MIR_OP_LDOUBLE: fprintf (f, "%.*LeL", LDBL_MANT_DIG, op.u.ld); break; case MIR_OP_MEM: - case MIR_OP_HARD_REG_MEM: { - MIR_reg_t no_reg = op.mode == MIR_OP_MEM ? 0 : MIR_NON_HARD_REG; + case MIR_OP_VAR_MEM: { + MIR_reg_t no_reg = op.mode == MIR_OP_MEM ? 0 : MIR_NON_VAR; output_type (ctx, f, op.u.mem.type); fprintf (f, ":"); @@ -2513,18 +2937,11 @@ void MIR_output_op (MIR_context_t ctx, FILE *f, MIR_op_t op, MIR_func_t func) { output_disp (f, op.u.mem.disp); if (op.u.mem.base != no_reg || op.u.mem.index != no_reg) { fprintf (f, "("); - if (op.u.mem.base != no_reg) { - if (op.mode == MIR_OP_MEM) - output_reg (ctx, f, func, op.u.mem.base); - else - output_hard_reg (f, op.u.hard_reg_mem.base); - } + if (op.u.mem.base != no_reg) + (op.mode == MIR_OP_MEM ? output_reg : output_var) (ctx, f, func, op.u.mem.base); if (op.u.mem.index != no_reg) { fprintf (f, ", "); - if (op.mode == MIR_OP_MEM) - output_reg (ctx, f, func, op.u.mem.index); - else - output_hard_reg (f, op.u.hard_reg_mem.index); + (op.mode == MIR_OP_MEM ? output_reg : output_var) (ctx, f, func, op.u.mem.index); if (op.u.mem.scale != 1) { fprintf (f, ", "); output_scale (f, op.u.mem.scale); @@ -2532,13 +2949,18 @@ void MIR_output_op (MIR_context_t ctx, FILE *f, MIR_op_t op, MIR_func_t func) { } fprintf (f, ")"); } + if (op.u.mem.alias != 0 || op.u.mem.nonalias != 0) { + fprintf (f, ":"); + if (op.u.mem.alias != 0) fprintf (f, "%s", MIR_alias_name (ctx, op.u.mem.alias)); + if (op.u.mem.nonalias != 0) fprintf (f, ":%s", MIR_alias_name (ctx, op.u.mem.nonalias)); + } break; } case MIR_OP_REF: if (op.u.ref->module != func->func_item->module) fprintf (f, "%s.", op.u.ref->module->name); fprintf (f, "%s", MIR_item_name (ctx, op.u.ref)); break; - case MIR_OP_STR: out_str (f, op.u.str); break; + case MIR_OP_STR: MIR_output_str (ctx, f, op.u.str); break; case MIR_OP_LABEL: output_label (ctx, f, func, op.u.label); break; default: mir_assert (FALSE); } @@ -2591,15 +3013,63 @@ static void output_func_proto (MIR_context_t ctx, FILE *f, size_t nres, MIR_type fprintf (f, "\n"); } +static void output_vars (MIR_context_t ctx, FILE *f, MIR_func_t func, VARR (MIR_var_t) * vars, + size_t start, size_t vars_num, const char *prefix) { + if (vars == NULL || vars_num == 0) return; + for (size_t i = 0; i < vars_num; i++) { + MIR_var_t var = VARR_GET (MIR_var_t, vars, i + start); + if (i % 8 == 0) { + if (i != 0) fprintf (f, "\n"); + fprintf (f, "\t%s\t", prefix); + } + if (i % 8 != 0) fprintf (f, ", "); + fprintf (f, "%s:%s", MIR_type_str (ctx, var.type), var.name); + MIR_reg_t reg = MIR_reg (ctx, var.name, func); + const char *hard_reg_name = MIR_reg_hard_reg_name (ctx, reg, func); + if (hard_reg_name != NULL) fprintf (f, ":%s", hard_reg_name); + } + fprintf (f, "\n"); +} + +void _MIR_output_data_item_els (MIR_context_t ctx, FILE *f, MIR_item_t item, int c_p) { + mir_assert (item->item_type == MIR_data_item); + MIR_data_t data = item->u.data; + for (size_t i = 0; i < data->nel; i++) { + switch (data->el_type) { + case MIR_T_I8: fprintf (f, "%" PRId8, ((int8_t *) data->u.els)[i]); break; + case MIR_T_U8: fprintf (f, "%" PRIu8, ((uint8_t *) data->u.els)[i]); break; + case MIR_T_I16: fprintf (f, "%" PRId16, ((int16_t *) data->u.els)[i]); break; + case MIR_T_U16: fprintf (f, "%" PRIu16, ((uint16_t *) data->u.els)[i]); break; + case MIR_T_I32: fprintf (f, "%" PRId32, ((int32_t *) data->u.els)[i]); break; + case MIR_T_U32: fprintf (f, "%" PRIu32, ((uint32_t *) data->u.els)[i]); break; + case MIR_T_I64: fprintf (f, "%" PRId64, ((int64_t *) data->u.els)[i]); break; + case MIR_T_U64: fprintf (f, "%" PRIu64, ((uint64_t *) data->u.els)[i]); break; + case MIR_T_F: fprintf (f, "%.*ef", FLT_MANT_DIG, ((float *) data->u.els)[i]); break; + case MIR_T_D: fprintf (f, "%.*e", DBL_MANT_DIG, ((double *) data->u.els)[i]); break; + case MIR_T_LD: + fprintf (f, "%.*LeL", LDBL_MANT_DIG, ((long double *) data->u.els)[i]); + break; + /* only ptr as ref ??? */ + case MIR_T_P: fprintf (f, "0x%" PRIxPTR, ((uintptr_t *) data->u.els)[i]); break; + default: mir_assert (FALSE); + } + if (i + 1 < data->nel) fprintf (f, ", "); + } + if (data->el_type == MIR_T_U8 && data->nel != 0 && data->u.els[data->nel - 1] == '\0') { + fprintf (f, c_p ? "/* " : " # "); /* print possible string as a comment */ + MIR_output_str (ctx, f, (MIR_str_t){data->nel, (char *) data->u.els}); + if (c_p) fprintf (f, " */"); + } +} + void MIR_output_item (MIR_context_t ctx, FILE *f, MIR_item_t item) { MIR_insn_t insn; MIR_func_t func; MIR_proto_t proto; - MIR_var_t var; MIR_data_t data; MIR_ref_data_t ref_data; MIR_expr_data_t expr_data; - size_t i, nlocals; + size_t vars_num, nglobal; mir_assert (f != NULL && item != NULL); if (item->item_type == MIR_export_item) { @@ -2626,6 +3096,17 @@ void MIR_output_item (MIR_context_t ctx, FILE *f, MIR_item_t item) { (int64_t) ref_data->disp); return; } + if (item->item_type == MIR_lref_data_item) { + MIR_lref_data_t lref_data = item->u.lref_data; + if (lref_data->name != NULL) fprintf (f, "%s:", lref_data->name); + mir_assert (lref_data->label->ops[0].mode == MIR_OP_INT); + fprintf (f, "\tlref\tL%" PRId64, lref_data->label->ops[0].u.i); + mir_assert (lref_data->label2 == NULL || lref_data->label2->ops[0].mode == MIR_OP_INT); + if (lref_data->label2 != NULL) fprintf (f, ", L%" PRId64, lref_data->label2->ops[0].u.i); + if (lref_data->disp != 0) fprintf (f, ", %" PRId64, (int64_t) lref_data->disp); + fprintf (f, "\n"); + return; + } if (item->item_type == MIR_expr_data_item) { expr_data = item->u.expr_data; if (expr_data->name != NULL) fprintf (f, "%s:", expr_data->name); @@ -2635,31 +3116,7 @@ void MIR_output_item (MIR_context_t ctx, FILE *f, MIR_item_t item) { data = item->u.data; if (data->name != NULL) fprintf (f, "%s:", data->name); fprintf (f, "\t%s\t", MIR_type_str (ctx, data->el_type)); - for (size_t i = 0; i < data->nel; i++) { - switch (data->el_type) { - case MIR_T_I8: fprintf (f, "%" PRId8, ((int8_t *) data->u.els)[i]); break; - case MIR_T_U8: fprintf (f, "%" PRIu8, ((uint8_t *) data->u.els)[i]); break; - case MIR_T_I16: fprintf (f, "%" PRId16, ((int16_t *) data->u.els)[i]); break; - case MIR_T_U16: fprintf (f, "%" PRIu16, ((uint16_t *) data->u.els)[i]); break; - case MIR_T_I32: fprintf (f, "%" PRId32, ((int32_t *) data->u.els)[i]); break; - case MIR_T_U32: fprintf (f, "%" PRIu32, ((uint32_t *) data->u.els)[i]); break; - case MIR_T_I64: fprintf (f, "%" PRId64, ((int64_t *) data->u.els)[i]); break; - case MIR_T_U64: fprintf (f, "%" PRIu64, ((uint64_t *) data->u.els)[i]); break; - case MIR_T_F: fprintf (f, "%.*ef", FLT_MANT_DIG, ((float *) data->u.els)[i]); break; - case MIR_T_D: fprintf (f, "%.*e", DBL_MANT_DIG, ((double *) data->u.els)[i]); break; - case MIR_T_LD: - fprintf (f, "%.*LeL", LDBL_MANT_DIG, ((long double *) data->u.els)[i]); - break; - /* only ptr as ref ??? */ - case MIR_T_P: fprintf (f, "0x%" PRIxPTR, ((uintptr_t *) data->u.els)[i]); break; - default: mir_assert (FALSE); - } - if (i + 1 < data->nel) fprintf (f, ", "); - } - if (data->el_type == MIR_T_U8 && data->nel != 0 && data->u.els[data->nel - 1] == '\0') { - fprintf (f, " # "); /* print possible string as a comment */ - out_str (f, (MIR_str_t){data->nel, (char *) data->u.els}); - } + _MIR_output_data_item_els (ctx, f, item, FALSE); fprintf (f, "\n"); return; } @@ -2673,17 +3130,12 @@ void MIR_output_item (MIR_context_t ctx, FILE *f, MIR_item_t item) { func = item->u.func; fprintf (f, "%s:\tfunc\t", func->name); output_func_proto (ctx, f, func->nres, func->res_types, func->nargs, func->vars, func->vararg_p); - nlocals = VARR_LENGTH (MIR_var_t, func->vars) - func->nargs; - for (i = 0; i < nlocals; i++) { - var = VARR_GET (MIR_var_t, func->vars, i + func->nargs); - if (i % 8 == 0) { - if (i != 0) fprintf (f, "\n"); - fprintf (f, "\tlocal\t"); - } - fprintf (f, i % 8 == 0 ? "%s:%s" : ", %s:%s", MIR_type_str (ctx, var.type), var.name); - } - fprintf (f, "\n# %u arg%s, %u local%s\n", func->nargs, func->nargs == 1 ? "" : "s", - (unsigned) nlocals, nlocals == 1 ? "" : "s"); + vars_num = VARR_LENGTH (MIR_var_t, func->vars) - func->nargs; + nglobal = func->global_vars == NULL ? 0 : VARR_LENGTH (MIR_var_t, func->global_vars); + output_vars (ctx, f, func, func->vars, func->nargs, vars_num, "local"); + output_vars (ctx, f, func, func->global_vars, 0, nglobal, "global"); + fprintf (f, "\n# %u arg%s, %ld local%s, %ld global%s\n", func->nargs, func->nargs == 1 ? "" : "s", + (long) vars_num, vars_num == 1 ? "" : "s", (long) nglobal, nglobal == 1 ? "" : "s"); for (insn = DLIST_HEAD (MIR_insn_t, func->insns); insn != NULL; insn = DLIST_NEXT (MIR_insn_t, insn)) MIR_output_insn (ctx, f, insn, func, TRUE); @@ -2730,15 +3182,22 @@ DEF_HTAB (val_t); struct simplify_ctx { HTAB (val_t) * val_tab; - VARR (MIR_insn_t) * temp_insns, *labels; /* temp_insns is for branch or ret insns */ + /* temp_insns is for branch or ret insns */ + VARR (MIR_insn_t) * temp_insns, *cold_insns, *labels; VARR (MIR_reg_t) * inline_reg_map; - size_t inlined_calls, inline_insns_before, inline_insns_after; + VARR (MIR_insn_t) * anchors; + VARR (size_t) * alloca_sizes; + size_t new_label_num, inlined_calls, inline_insns_before, inline_insns_after; }; #define val_tab ctx->simplify_ctx->val_tab #define temp_insns ctx->simplify_ctx->temp_insns +#define cold_insns ctx->simplify_ctx->cold_insns #define labels ctx->simplify_ctx->labels #define inline_reg_map ctx->simplify_ctx->inline_reg_map +#define anchors ctx->simplify_ctx->anchors +#define alloca_sizes ctx->simplify_ctx->alloca_sizes +#define new_label_num ctx->simplify_ctx->new_label_num #define inlined_calls ctx->simplify_ctx->inlined_calls #define inline_insns_before ctx->simplify_ctx->inline_insns_before #define inline_insns_after ctx->simplify_ctx->inline_insns_after @@ -2747,11 +3206,11 @@ static htab_hash_t val_hash (val_t v, void *arg) { MIR_context_t ctx = arg; htab_hash_t h; - h = mir_hash_step (mir_hash_init (0), (uint64_t) v.code); - h = mir_hash_step (h, (uint64_t) v.type); + h = (htab_hash_t) mir_hash_step (mir_hash_init (0), (uint64_t) v.code); + h = (htab_hash_t) mir_hash_step (h, (uint64_t) v.type); h = MIR_op_hash_step (ctx, h, v.op1); if (v.code != MIR_INSN_BOUND) h = MIR_op_hash_step (ctx, h, v.op2); - return mir_hash_finish (h); + return (htab_hash_t) mir_hash_finish (h); } static int val_eq (val_t v1, val_t v2, void *arg) { @@ -2766,12 +3225,17 @@ static void simplify_init (MIR_context_t ctx) { MIR_get_error_func (ctx) (MIR_alloc_error, "Not enough memory for ctx"); HTAB_CREATE (val_t, val_tab, 512, val_hash, val_eq, ctx); VARR_CREATE (MIR_insn_t, temp_insns, 0); + VARR_CREATE (MIR_insn_t, cold_insns, 0); VARR_CREATE (MIR_insn_t, labels, 0); VARR_CREATE (MIR_reg_t, inline_reg_map, 256); + VARR_CREATE (MIR_insn_t, anchors, 32); + VARR_CREATE (size_t, alloca_sizes, 32); inlined_calls = inline_insns_before = inline_insns_after = 0; } static void simplify_finish (MIR_context_t ctx) { + VARR_DESTROY (size_t, alloca_sizes); + VARR_DESTROY (MIR_insn_t, anchors); VARR_DESTROY (MIR_reg_t, inline_reg_map); #if 0 if (inlined_calls != 0) @@ -2781,11 +3245,17 @@ static void simplify_finish (MIR_context_t ctx) { #endif VARR_DESTROY (MIR_insn_t, labels); VARR_DESTROY (MIR_insn_t, temp_insns); + VARR_DESTROY (MIR_insn_t, cold_insns); HTAB_DESTROY (val_t, val_tab); free (ctx->simplify_ctx); ctx->simplify_ctx = NULL; } +static void simplify_module_init (MIR_context_t ctx) { + new_label_num = 0; + VARR_TRUNC (uint8_t, used_label_p, 0); +} + static void vn_empty (MIR_context_t ctx) { HTAB_CLEAR (val_t, val_tab); } static MIR_reg_t vn_add_val (MIR_context_t ctx, MIR_func_t func, MIR_type_t type, @@ -2802,12 +3272,20 @@ static MIR_reg_t vn_add_val (MIR_context_t ctx, MIR_func_t func, MIR_type_t type return val.reg; } -void _MIR_get_temp_item_name (MIR_context_t ctx, MIR_module_t module, char *buff, size_t buff_len) { +void _MIR_get_temp_item_name (MIR_context_t ctx MIR_UNUSED, MIR_module_t module, char *buff, + size_t buff_len) { mir_assert (module != NULL); module->last_temp_item_num++; snprintf (buff, buff_len, "%s%u", TEMP_ITEM_NAME_PREFIX, (unsigned) module->last_temp_item_num); } +static MIR_insn_code_t get_type_move_code (MIR_type_t type) { + return (type == MIR_T_F ? MIR_FMOV + : type == MIR_T_D ? MIR_DMOV + : type == MIR_T_LD ? MIR_LDMOV + : MIR_MOV); +} + static void simplify_op (MIR_context_t ctx, MIR_item_t func_item, MIR_insn_t insn, int nop, int out_p, MIR_insn_code_t code, int keep_ref_p, int mem_float_p) { mir_assert (insn != NULL && func_item != NULL); @@ -2818,8 +3296,8 @@ static void simplify_op (MIR_context_t ctx, MIR_item_t func_item, MIR_insn_t ins MIR_op_mode_t value_mode = op->value_mode; int move_p = code == MIR_MOV || code == MIR_FMOV || code == MIR_DMOV || code == MIR_LDMOV; - if (code == MIR_PHI) return; /* do nothing: it is a phi insn */ - if (code == MIR_UNSPEC && nop == 0) return; /* do nothing: it is an unspec code */ + if (code == MIR_PHI || code == MIR_USE) return; /* do nothing: it is use or phi insn */ + if (code == MIR_UNSPEC && nop == 0) return; /* do nothing: it is an unspec code */ if (MIR_call_code_p (code)) { if (nop == 0) return; /* do nothing: it is a prototype */ if (nop == 1 && op->mode == MIR_OP_REF @@ -2827,9 +3305,12 @@ static void simplify_op (MIR_context_t ctx, MIR_item_t func_item, MIR_insn_t ins return; /* do nothing: it is an immediate operand */ } if (code == MIR_VA_ARG && nop == 2) return; /* do nothing: this operand is used as a type */ + if ((code == MIR_PRBEQ || code == MIR_PRBNE) && nop == 2) return; /* it is a property */ + if (code == MIR_PRSET && nop == 1) return; /* it is a property */ switch (op->mode) { case MIR_OP_REF: if (keep_ref_p) break; + /* falls through */ case MIR_OP_INT: case MIR_OP_UINT: case MIR_OP_FLOAT: @@ -2883,20 +3364,46 @@ static void simplify_op (MIR_context_t ctx, MIR_item_t func_item, MIR_insn_t ins : MIR_T_I64); new_op = MIR_new_reg_op (ctx, vn_add_val (ctx, func, type, MIR_INSN_BOUND, *op, *op)); MIR_insert_insn_before (ctx, func_item, insn, - MIR_new_insn (ctx, - type == MIR_T_F ? MIR_FMOV - : type == MIR_T_D ? MIR_DMOV - : type == MIR_T_LD ? MIR_LDMOV - : MIR_MOV, - new_op, *op)); + MIR_new_insn (ctx, get_type_move_code (type), new_op, *op)); *op = new_op; break; case MIR_OP_REG: - case MIR_OP_HARD_REG: + if (MIR_reg_hard_reg_name (ctx, op->u.reg, func) == NULL) break; + int another_nop = nop == 0 ? 1 : 0; + if (move_p && insn->ops[another_nop].mode == MIR_OP_REG + && MIR_reg_hard_reg_name (ctx, insn->ops[another_nop].u.reg, func) == NULL) + break; + type = MIR_reg_type (ctx, op->u.reg, func); + new_op = MIR_new_reg_op (ctx, vn_add_val (ctx, func, type, MIR_INSN_BOUND, *op, *op)); + if (out_p) { + MIR_insert_insn_after (ctx, func_item, insn, + MIR_new_insn (ctx, get_type_move_code (type), *op, new_op)); + } else { + MIR_insert_insn_before (ctx, func_item, insn, + MIR_new_insn (ctx, get_type_move_code (type), new_op, *op)); + } + *op = new_op; + break; + case MIR_OP_VAR: case MIR_OP_LABEL: break; /* Do nothing */ case MIR_OP_MEM: { + MIR_op_t reg_op; MIR_reg_t addr_reg = 0; + if (op->u.mem.base != 0 && MIR_reg_hard_reg_name (ctx, op->u.mem.base, func) != NULL) { + reg_op = MIR_new_reg_op (ctx, op->u.mem.base); + new_op + = MIR_new_reg_op (ctx, vn_add_val (ctx, func, MIR_T_I64, MIR_INSN_BOUND, reg_op, reg_op)); + MIR_insert_insn_before (ctx, func_item, insn, MIR_new_insn (ctx, MIR_MOV, new_op, reg_op)); + op->u.mem.base = new_op.u.reg; + } + if (op->u.mem.index != 0 && MIR_reg_hard_reg_name (ctx, op->u.mem.index, func) != NULL) { + reg_op = MIR_new_reg_op (ctx, op->u.mem.index); + new_op + = MIR_new_reg_op (ctx, vn_add_val (ctx, func, MIR_T_I64, MIR_INSN_BOUND, reg_op, reg_op)); + MIR_insert_insn_before (ctx, func_item, insn, MIR_new_insn (ctx, MIR_MOV, new_op, reg_op)); + op->u.mem.index = new_op.u.reg; + } mem_op = *op; type = mem_op.u.mem.type; if (op->u.mem.base != 0 && op->u.mem.disp == 0 @@ -2974,10 +3481,7 @@ static void simplify_op (MIR_context_t ctx, MIR_item_t func_item, MIR_insn_t ins || mem_op.u.mem.type == MIR_T_LD ? mem_op.u.mem.type : MIR_T_I64); - code = (type == MIR_T_F ? MIR_FMOV - : type == MIR_T_D ? MIR_DMOV - : type == MIR_T_LD ? MIR_LDMOV - : MIR_MOV); + code = get_type_move_code (type); new_op = MIR_new_reg_op (ctx, vn_add_val (ctx, func, type, MIR_INSN_BOUND, mem_op, mem_op)); if (out_p) new_insn = MIR_new_insn (ctx, code, mem_op, new_op); @@ -3004,9 +3508,8 @@ static void simplify_insn (MIR_context_t ctx, MIR_item_t func_item, MIR_insn_t i for (i = 0; i < nops; i++) { MIR_insn_op_mode (ctx, insn, i, &out_p); - simplify_op (ctx, func_item, insn, i, out_p, code, - (insn->code == MIR_INLINE || insn->code == MIR_CALL) && i == 1 && keep_ref_p, - mem_float_p); + simplify_op (ctx, func_item, insn, (int) i, out_p, code, + MIR_call_code_p (insn->code) && i == 1 && keep_ref_p, mem_float_p); } } @@ -3017,33 +3520,20 @@ static void make_one_ret (MIR_context_t ctx, MIR_item_t func_item) { MIR_op_t reg_op, ret_reg_op; MIR_func_t func = func_item->u.func; MIR_type_t *res_types = func->res_types; - MIR_insn_t ret_label, insn, first_ret_insn = NULL; - VARR (MIR_op_t) * ret_ops; - int one_last_ret_p; - - one_last_ret_p - = (VARR_LENGTH (MIR_insn_t, temp_insns) == 1 - && VARR_GET (MIR_insn_t, temp_insns, 0) == DLIST_TAIL (MIR_insn_t, func->insns)); - ret_label = NULL; - if (one_last_ret_p) { - first_ret_insn = VARR_GET (MIR_insn_t, temp_insns, 0); - } else { + MIR_insn_t ret_label = NULL, insn, last_ret_insn; + VARR (MIR_insn_t) *ret_insns = temp_insns; + VARR (MIR_op_t) *ret_ops = temp_ops; + + if (VARR_LENGTH (MIR_insn_t, ret_insns) == 0) return; /* jcall/jret func */ + last_ret_insn = VARR_LAST (MIR_insn_t, ret_insns); + VARR_TRUNC (MIR_op_t, ret_ops, 0); + if (VARR_LENGTH (MIR_insn_t, ret_insns) > 1) { ret_label = MIR_new_label (ctx); - MIR_append_insn (ctx, func_item, ret_label); + MIR_insert_insn_before (ctx, func_item, last_ret_insn, ret_label); } - VARR_CREATE (MIR_op_t, ret_ops, 16); - for (i = 0; i < func->nres; i++) { - if (one_last_ret_p) { - ret_reg_op = first_ret_insn->ops[i]; - } else { - mov_code = (res_types[i] == MIR_T_F ? MIR_FMOV - : res_types[i] == MIR_T_D ? MIR_DMOV - : res_types[i] == MIR_T_LD ? MIR_LDMOV - : MIR_MOV); - ret_reg = new_temp_reg (ctx, mov_code == MIR_MOV ? MIR_T_I64 : res_types[i], func); - ret_reg_op = MIR_new_reg_op (ctx, ret_reg); - VARR_PUSH (MIR_op_t, ret_ops, ret_reg_op); - } + for (i = 0; i < func->nres; i++) { /* generate ext insn before last ret */ + ret_reg_op = last_ret_insn->ops[i]; + VARR_PUSH (MIR_op_t, ret_ops, ret_reg_op); switch (res_types[i]) { case MIR_T_I8: ext_code = MIR_EXT8; break; case MIR_T_U8: ext_code = MIR_UEXT8; break; @@ -3054,44 +3544,46 @@ static void make_one_ret (MIR_context_t ctx, MIR_item_t func_item) { default: ext_code = MIR_INVALID_INSN; break; } if (ext_code == MIR_INVALID_INSN) continue; - if (one_last_ret_p) - MIR_insert_insn_before (ctx, func_item, first_ret_insn, - MIR_new_insn (ctx, ext_code, ret_reg_op, ret_reg_op)); - else - MIR_append_insn (ctx, func_item, MIR_new_insn (ctx, ext_code, ret_reg_op, ret_reg_op)); - } - if (!one_last_ret_p) { - MIR_append_insn (ctx, func_item, - MIR_new_insn_arr (ctx, MIR_RET, func->nres, VARR_ADDR (MIR_op_t, ret_ops))); - for (i = 0; i < VARR_LENGTH (MIR_insn_t, temp_insns); i++) { - insn = VARR_GET (MIR_insn_t, temp_insns, i); - mir_assert (func->nres == MIR_insn_nops (ctx, insn)); - for (j = 0; j < func->nres; j++) { - mov_code = (res_types[j] == MIR_T_F ? MIR_FMOV - : res_types[j] == MIR_T_D ? MIR_DMOV - : res_types[j] == MIR_T_LD ? MIR_LDMOV - : MIR_MOV); - reg_op = insn->ops[j]; - mir_assert (reg_op.mode == MIR_OP_REG); - ret_reg_op = VARR_GET (MIR_op_t, ret_ops, j); - MIR_insert_insn_before (ctx, func_item, insn, - MIR_new_insn (ctx, mov_code, ret_reg_op, reg_op)); - } + mov_code = get_type_move_code (res_types[i]); + ret_reg = _MIR_new_temp_reg (ctx, mov_code == MIR_MOV ? MIR_T_I64 : res_types[i], func); + ret_reg_op = MIR_new_reg_op (ctx, ret_reg); + MIR_insert_insn_before (ctx, func_item, last_ret_insn, + MIR_new_insn (ctx, ext_code, ret_reg_op, last_ret_insn->ops[i])); + last_ret_insn->ops[i] = ret_reg_op; + } + /* change ret insns (except last one) to moves & jumps */ + for (i = 0; i < VARR_LENGTH (MIR_insn_t, ret_insns); i++) { + insn = VARR_GET (MIR_insn_t, ret_insns, i); + if (insn == last_ret_insn) continue; + mir_assert (insn->code == MIR_RET || func->nres == MIR_insn_nops (ctx, insn)); + for (j = 0; j < func->nres; j++) { + mov_code = get_type_move_code (res_types[j]); + reg_op = insn->ops[j]; + mir_assert (reg_op.mode == MIR_OP_REG); + ret_reg_op = VARR_GET (MIR_op_t, ret_ops, j); MIR_insert_insn_before (ctx, func_item, insn, - MIR_new_insn (ctx, MIR_JMP, MIR_new_label_op (ctx, ret_label))); - MIR_remove_insn (ctx, func_item, insn); + MIR_new_insn (ctx, mov_code, ret_reg_op, reg_op)); } + MIR_insert_insn_before (ctx, func_item, insn, + MIR_new_insn (ctx, MIR_JMP, MIR_new_label_op (ctx, ret_label))); + MIR_remove_insn (ctx, func_item, insn); } - VARR_DESTROY (MIR_op_t, ret_ops); +} + +static void mark_used_label (MIR_context_t ctx, MIR_label_t label) { + int64_t label_num = label->ops[0].u.i; + while (label_num >= (int64_t) VARR_LENGTH (uint8_t, used_label_p)) + VARR_PUSH (uint8_t, used_label_p, FALSE); + VARR_SET (uint8_t, used_label_p, label_num, TRUE); } static void remove_unused_and_enumerate_labels (MIR_context_t ctx, MIR_item_t func_item) { - int64_t new_label_num = 0; for (size_t i = 0; i < VARR_LENGTH (MIR_insn_t, labels); i++) { MIR_insn_t label = VARR_GET (MIR_insn_t, labels, i); int64_t label_num = label->ops[0].u.i; - if (label_num < VARR_LENGTH (uint8_t, temp_data) && VARR_GET (uint8_t, temp_data, label_num)) { + if (label_num < (int64_t) VARR_LENGTH (uint8_t, used_label_p) + && VARR_GET (uint8_t, used_label_p, label_num)) { label->ops[0] = MIR_new_int_op (ctx, new_label_num++); continue; } @@ -3126,6 +3618,12 @@ MIR_insn_code_t MIR_reverse_branch_code (MIR_insn_code_t code) { case MIR_BGES: return MIR_BLTS; case MIR_UBGE: return MIR_UBLT; case MIR_UBGES: return MIR_UBLTS; + case MIR_BO: return MIR_BNO; + case MIR_UBO: return MIR_UBNO; + case MIR_BNO: return MIR_BO; + case MIR_UBNO: return MIR_UBO; + case MIR_PRBEQ: return MIR_PRBNE; + case MIR_PRBNE: return MIR_PRBEQ; default: return MIR_INSN_BOUND; } } @@ -3181,15 +3679,12 @@ static int simplify_func (MIR_context_t ctx, MIR_item_t func_item, int mem_float } if (ext_code != MIR_INVALID_INSN) { MIR_reg_t reg = MIR_reg (ctx, var.name, func); - MIR_insn_t new_insn - = MIR_new_insn (ctx, ext_code, MIR_new_reg_op (ctx, reg), MIR_new_reg_op (ctx, reg)); - + new_insn = MIR_new_insn (ctx, ext_code, MIR_new_reg_op (ctx, reg), MIR_new_reg_op (ctx, reg)); MIR_prepend_insn (ctx, func_item, new_insn); } } VARR_TRUNC (MIR_insn_t, temp_insns, 0); VARR_TRUNC (MIR_insn_t, labels, 0); - VARR_TRUNC (uint8_t, temp_data, 0); for (insn = DLIST_HEAD (MIR_insn_t, func->insns); insn != NULL; insn = next_insn) { MIR_insn_code_t code = insn->code; MIR_op_t temp_op; @@ -3232,12 +3727,13 @@ static int simplify_func (MIR_context_t ctx, MIR_item_t func_item, int mem_float insn->ops[1].u.i = overall_size; next_insn = DLIST_NEXT (MIR_insn_t, insn); /* to process the current and new insns */ } - if (code == MIR_INLINE || code == MIR_CALL) inline_p = TRUE; + if (MIR_call_code_p (code)) inline_p = TRUE; if ((MIR_int_branch_code_p (code) || code == MIR_JMP) && insn->ops[0].mode == MIR_OP_LABEL && skip_labels (next_insn, insn->ops[0].u.label) == insn->ops[0].u.label) { /* BR L|JMP L; L: => L: Also Remember signaling NAN*/ MIR_remove_insn (ctx, func_item, insn); - } else if (((code == MIR_MUL || code == MIR_MULS || code == MIR_DIV || code == MIR_DIVS) + } else if (((code == MIR_MUL || code == MIR_MULS || code == MIR_MULO || code == MIR_MULOS + || code == MIR_DIV || code == MIR_DIVS) && insn->ops[2].mode == MIR_OP_INT && insn->ops[2].u.i == 1) || ((code == MIR_ADD || code == MIR_ADDS || code == MIR_SUB || code == MIR_SUBS || code == MIR_OR || code == MIR_ORS || code == MIR_XOR || code == MIR_XORS @@ -3286,21 +3782,21 @@ static int simplify_func (MIR_context_t ctx, MIR_item_t func_item, int mem_float next_insn = insn; continue; } else { - if (MIR_branch_code_p (code) || code == MIR_SWITCH) { - int64_t label_num; + if ((MIR_any_branch_code_p (code) && code != MIR_JMPI) || code == MIR_LADDR + || code == MIR_PRBEQ || code == MIR_PRBNE) { size_t start_label_nop = 0, bound_label_nop = 1, n; - if (code == MIR_SWITCH) { + if (code == MIR_LADDR) { + start_label_nop = 1; + bound_label_nop = 2; + } else if (code == MIR_SWITCH) { start_label_nop = 1; bound_label_nop = start_label_nop + insn->nops - 1; } for (n = start_label_nop; n < bound_label_nop; n++) { label = last_label (insn->ops[n].u.label); if (label != insn->ops[n].u.label) insn->ops[n].u.label = label; - label_num = label->ops[0].u.i; - while (label_num >= VARR_LENGTH (uint8_t, temp_data)) - VARR_PUSH (uint8_t, temp_data, FALSE); - VARR_SET (uint8_t, temp_data, label_num, TRUE); + mark_used_label (ctx, label); } } simplify_insn (ctx, func_item, insn, TRUE, mem_float_p); @@ -3308,6 +3804,10 @@ static int simplify_func (MIR_context_t ctx, MIR_item_t func_item, int mem_float jmps_num = 0; } make_one_ret (ctx, func_item); + for (MIR_lref_data_t lref = func->first_lref; lref != NULL; lref = lref->next) { + mark_used_label (ctx, lref->label); + if (lref->label2 != NULL) mark_used_label (ctx, lref->label2); + } remove_unused_and_enumerate_labels (ctx, func_item); #if 0 fprintf (stderr, "+++++ Function after simplification:\n"); @@ -3341,17 +3841,31 @@ static void set_inline_reg_map (MIR_context_t ctx, MIR_reg_t old_reg, MIR_reg_t /* Simple alloca analysis. Return top alloca insn with const size. If there are other allocas return true through non_top_alloca_p. Should we consider bstart/bend too? */ -static MIR_insn_t func_alloca_features (MIR_context_t ctx, MIR_func_t func, int *non_top_alloca_p, - int64_t *alloca_size) { +static MIR_insn_t func_alloca_features (MIR_context_t ctx, MIR_func_t func, int *top_alloca_used_p, + int *non_top_alloca_p, int64_t *alloca_size) { int set_top_alloca_p = TRUE; + MIR_reg_t alloca_reg; MIR_op_t *op_ref; MIR_insn_t top_alloca = NULL, insn, prev_insn; + *top_alloca_used_p = FALSE; if (non_top_alloca_p != NULL) *non_top_alloca_p = FALSE; for (insn = DLIST_HEAD (MIR_insn_t, func->insns); insn != NULL; insn = DLIST_NEXT (MIR_insn_t, insn)) { if (insn->code == MIR_LABEL && set_top_alloca_p) set_top_alloca_p = FALSE; - if (insn->code != MIR_ALLOCA) continue; + if (insn->code != MIR_ALLOCA) { + if (top_alloca == NULL || *top_alloca_used_p) continue; + alloca_reg = top_alloca->ops[0].u.reg; + for (size_t i = 0; i < insn->nops; i++) + if ((insn->ops[i].mode == MIR_OP_REG && insn->ops[i].u.reg == alloca_reg) + || (insn->ops[i].mode == MIR_OP_MEM + && (insn->ops[i].u.mem.base == alloca_reg + || insn->ops[i].u.mem.index == alloca_reg))) { + *top_alloca_used_p = TRUE; + break; + } + continue; + } op_ref = &insn->ops[1]; if (insn->ops[1].mode == MIR_OP_REG && (prev_insn = DLIST_PREV (MIR_insn_t, insn)) != NULL && prev_insn->code == MIR_MOV && MIR_op_eq_p (ctx, prev_insn->ops[0], insn->ops[1])) @@ -3359,11 +3873,13 @@ static MIR_insn_t func_alloca_features (MIR_context_t ctx, MIR_func_t func, int if (op_ref->mode != MIR_OP_INT && op_ref->mode != MIR_OP_UINT) op_ref = NULL; if (!set_top_alloca_p || op_ref == NULL) { if (non_top_alloca_p != NULL) *non_top_alloca_p = TRUE; - return top_alloca; + if (top_alloca == NULL) return NULL; + } else { + top_alloca = insn; + if (insn->ops[0].mode != MIR_OP_REG) *top_alloca_used_p = TRUE; + set_top_alloca_p = FALSE; + if (alloca_size != NULL) *alloca_size = op_ref->u.i; } - top_alloca = insn; - set_top_alloca_p = FALSE; - if (alloca_size != NULL) *alloca_size = op_ref->u.i; } return top_alloca; } @@ -3412,42 +3928,97 @@ static long add_blk_move (MIR_context_t ctx, MIR_item_t func_item, MIR_insn_t be return label_num; } +static void rename_regs (MIR_context_t ctx, MIR_func_t func, MIR_func_t called_func, + VARR (MIR_var_t) * vars, size_t nvars) { + char buff[50]; + const char *hard_reg_name; + MIR_var_t var; + MIR_type_t type; + MIR_reg_t old_reg, new_reg; + + if (vars == NULL) return; + for (size_t i = 0; i < nvars; i++) { + VARR_TRUNC (char, temp_string, 0); + sprintf (buff, ".c%d_", func->n_inlines); + VARR_PUSH_ARR (char, temp_string, buff, strlen (buff)); + var = VARR_GET (MIR_var_t, vars, i); + type + = (var.type == MIR_T_F || var.type == MIR_T_D || var.type == MIR_T_LD ? var.type : MIR_T_I64); + old_reg = MIR_reg (ctx, var.name, called_func); + VARR_PUSH_ARR (char, temp_string, var.name, strlen (var.name) + 1); + if ((hard_reg_name = MIR_reg_hard_reg_name (ctx, old_reg, called_func)) != NULL) { + new_reg + = MIR_new_global_func_reg (ctx, func, type, VARR_ADDR (char, temp_string), hard_reg_name); + } else { + new_reg = MIR_new_func_reg (ctx, func, type, VARR_ADDR (char, temp_string)); + } + set_inline_reg_map (ctx, old_reg, new_reg); + } +} + +static void change_inline_insn_regs (MIR_context_t ctx, MIR_insn_t new_insn) { + size_t i, actual_nops; + actual_nops = MIR_insn_nops (ctx, new_insn); + for (i = 0; i < actual_nops; i++) { + switch (new_insn->ops[i].mode) { + case MIR_OP_REG: + new_insn->ops[i].u.reg = VARR_GET (MIR_reg_t, inline_reg_map, new_insn->ops[i].u.reg); + break; + case MIR_OP_MEM: + if (new_insn->ops[i].u.mem.base != 0) + new_insn->ops[i].u.mem.base + = VARR_GET (MIR_reg_t, inline_reg_map, new_insn->ops[i].u.mem.base); + if (new_insn->ops[i].u.mem.index != 0) + new_insn->ops[i].u.mem.index + = VARR_GET (MIR_reg_t, inline_reg_map, new_insn->ops[i].u.mem.index); + break; + default: /* do nothing */ break; + } + } +} + /* Only simplified code should be inlined because we need already extensions and one return. */ static void process_inlines (MIR_context_t ctx, MIR_item_t func_item) { - int non_top_alloca_p; - int64_t alloca_size = 0, alloca_align, max_func_top_alloca_align; + int non_top_alloca_p, func_top_alloca_used_p, called_func_top_alloca_used_p; + int64_t alloca_size, alloca_align, max_func_top_alloca_align; int64_t init_func_top_alloca_size, curr_func_top_alloca_size, max_func_top_alloca_size; - size_t i, actual_nops, nargs, nvars; + size_t i, nargs, arg_num; MIR_type_t type, *res_types; MIR_var_t var; - MIR_reg_t ret_reg, old_reg, new_reg, temp_reg; + MIR_reg_t ret_reg, temp_reg; MIR_insn_t func_top_alloca, called_func_top_alloca, new_called_func_top_alloca = NULL; MIR_insn_t func_insn, head_func_insn, next_func_insn; - MIR_insn_t call, insn, prev_insn, new_insn, ret_insn, anchor; + MIR_insn_t call, insn, prev_insn, new_insn, ret_insn, anchor, stop_insn; MIR_item_t called_func_item; MIR_func_t func, called_func; size_t original_func_insns_num, func_insns_num, called_func_insns_num; - char buff[50]; - long new_label_num = 0; mir_assert (func_item->item_type == MIR_func_item); vn_empty (ctx); func = func_item->u.func; original_func_insns_num = func_insns_num = DLIST_LENGTH (MIR_insn_t, func->insns); - func_top_alloca = func_alloca_features (ctx, func, NULL, &alloca_size); + func_top_alloca = func_alloca_features (ctx, func, &func_top_alloca_used_p, NULL, &alloca_size); + mir_assert (func_top_alloca != NULL || !func_top_alloca_used_p); init_func_top_alloca_size = curr_func_top_alloca_size = max_func_top_alloca_size = 0; max_func_top_alloca_align = 0; - if (func_top_alloca != NULL) + if (func_top_alloca != NULL && func_top_alloca_used_p) init_func_top_alloca_size = max_func_top_alloca_size = curr_func_top_alloca_size = get_alloca_size_align (alloca_size, &max_func_top_alloca_align); + VARR_TRUNC (MIR_insn_t, anchors, 0); + VARR_TRUNC (size_t, alloca_sizes, 0); + VARR_TRUNC (MIR_insn_t, cold_insns, 0); for (head_func_insn = func_insn = DLIST_HEAD (MIR_insn_t, func->insns); func_insn != NULL; func_insn = next_func_insn) { inline_insns_before++; inline_insns_after++; + while (VARR_LENGTH (MIR_insn_t, anchors) != 0 && VARR_LAST (MIR_insn_t, anchors) == func_insn) { + VARR_POP (MIR_insn_t, anchors); + curr_func_top_alloca_size = VARR_POP (size_t, alloca_sizes); + } next_func_insn = DLIST_NEXT (MIR_insn_t, func_insn); if (func_insn->code == MIR_LABEL) func_insn->ops[0].u.i = new_label_num++; - if (func_insn->code != MIR_INLINE && func_insn->code != MIR_CALL) continue; + if (!MIR_call_code_p (func_insn->code)) continue; call = func_insn; if (call->ops[1].mode != MIR_OP_REF) { simplify_op (ctx, func_item, func_insn, 1, FALSE, func_insn->code, FALSE, TRUE); @@ -3466,9 +4037,9 @@ static void process_inlines (MIR_context_t ctx, MIR_item_t func_item) { } called_func = called_func_item->u.func; called_func_insns_num = DLIST_LENGTH (MIR_insn_t, called_func->insns); - if (called_func->vararg_p - || called_func_insns_num > (func_insn->code == MIR_CALL ? MIR_MAX_INSNS_FOR_CALL_INLINE - : MIR_MAX_INSNS_FOR_INLINE) + if (called_func->first_lref != NULL || called_func->vararg_p || called_func->jret_p + || called_func_insns_num > (func_insn->code != MIR_CALL ? MIR_MAX_INSNS_FOR_INLINE + : MIR_MAX_INSNS_FOR_CALL_INLINE) || (func_insns_num > MIR_MAX_FUNC_INLINE_GROWTH * original_func_insns_num / 100 && func_insns_num > MIR_MAX_CALLER_SIZE_FOR_ANY_GROWTH_INLINE)) { simplify_op (ctx, func_item, func_insn, 1, FALSE, func_insn->code, FALSE, TRUE); @@ -3483,45 +4054,42 @@ static void process_inlines (MIR_context_t ctx, MIR_item_t func_item) { MIR_insert_insn_after (ctx, func_item, call, anchor); } func->n_inlines++; + rename_regs (ctx, func, called_func, called_func->vars, + VARR_LENGTH (MIR_var_t, called_func->vars)); + rename_regs (ctx, func, called_func, called_func->global_vars, + called_func->global_vars == NULL + ? 0 + : VARR_LENGTH (MIR_var_t, called_func->global_vars)); nargs = called_func->nargs; - nvars = VARR_LENGTH (MIR_var_t, called_func->vars); - for (i = 0; i < nvars; i++) { - VARR_TRUNC (char, temp_string, 0); - sprintf (buff, ".c%d_", func->n_inlines); - VARR_PUSH_ARR (char, temp_string, buff, strlen (buff)); - var = VARR_GET (MIR_var_t, called_func->vars, i); + for (i = 2 + called_func->nres, arg_num = 0; arg_num < nargs && i < call->nops; + i++, arg_num++) { /* Parameter passing */ + MIR_op_t op = call->ops[i]; + var = VARR_GET (MIR_var_t, called_func->vars, arg_num); type = (var.type == MIR_T_F || var.type == MIR_T_D || var.type == MIR_T_LD ? var.type : MIR_T_I64); - old_reg = MIR_reg (ctx, var.name, called_func); - VARR_PUSH_ARR (char, temp_string, var.name, strlen (var.name) + 1); - new_reg = MIR_new_func_reg (ctx, func, type, VARR_ADDR (char, temp_string)); - set_inline_reg_map (ctx, old_reg, new_reg); - if (i < nargs && call->nops > i + 2 + called_func->nres) { /* Parameter passing */ - MIR_op_t op = call->ops[i + 2 + called_func->nres]; - - mir_assert (!MIR_all_blk_type_p (type) || (op.mode == MIR_OP_MEM && type == MIR_T_I64)); - if (MIR_blk_type_p (var.type)) { /* alloca and block move: */ - new_label_num - = add_blk_move (ctx, func_item, anchor, MIR_new_reg_op (ctx, new_reg), - MIR_new_reg_op (ctx, op.u.mem.base), var.size, new_label_num); - } else { - if (var.type == MIR_T_RBLK) op = MIR_new_reg_op (ctx, op.u.mem.base); - new_insn = MIR_new_insn (ctx, - type == MIR_T_F ? MIR_FMOV - : type == MIR_T_D ? MIR_DMOV - : type == MIR_T_LD ? MIR_LDMOV - : MIR_MOV, - MIR_new_reg_op (ctx, new_reg), op); - MIR_insert_insn_before (ctx, func_item, anchor, new_insn); - } + const char *old_var_name = var.name; + MIR_reg_t old_reg = MIR_reg (ctx, old_var_name, called_func); + MIR_reg_t new_reg = VARR_GET (MIR_reg_t, inline_reg_map, old_reg); + + mir_assert (!MIR_all_blk_type_p (type) || (op.mode == MIR_OP_MEM && type == MIR_T_I64)); + if (MIR_blk_type_p (var.type)) { /* alloca and block move: */ + new_label_num + = add_blk_move (ctx, func_item, anchor, MIR_new_reg_op (ctx, new_reg), + MIR_new_reg_op (ctx, op.u.mem.base), var.size, (long) new_label_num); + } else { + if (var.type == MIR_T_RBLK) op = MIR_new_reg_op (ctx, op.u.mem.base); + new_insn = MIR_new_insn (ctx, get_type_move_code (type), MIR_new_reg_op (ctx, new_reg), op); + MIR_insert_insn_before (ctx, func_item, anchor, new_insn); } } /* ??? No frame only alloca */ + VARR_PUSH (MIR_insn_t, anchors, anchor); + VARR_PUSH (size_t, alloca_sizes, curr_func_top_alloca_size); /* Add new insns: */ ret_reg = 0; - called_func_top_alloca - = func_alloca_features (ctx, called_func, &non_top_alloca_p, &alloca_size); - if (called_func_top_alloca != NULL) { + called_func_top_alloca = func_alloca_features (ctx, called_func, &called_func_top_alloca_used_p, + &non_top_alloca_p, &alloca_size); + if (called_func_top_alloca != NULL && called_func_top_alloca_used_p) { alloca_size = get_alloca_size_align (alloca_size, &alloca_align); if (max_func_top_alloca_align < alloca_align) { max_func_top_alloca_align = alloca_align; @@ -3535,47 +4103,43 @@ static void process_inlines (MIR_context_t ctx, MIR_item_t func_item) { VARR_TRUNC (MIR_insn_t, temp_insns, 0); VARR_TRUNC (MIR_insn_t, labels, 0); VARR_TRUNC (uint8_t, temp_data, 0); - for (insn = DLIST_HEAD (MIR_insn_t, called_func->insns); insn != NULL; + stop_insn = NULL; + if (!non_top_alloca_p) { /* store cold code when we have no BSTART/BEND */ + for (insn = DLIST_TAIL (MIR_insn_t, called_func->insns); insn != NULL; + insn = DLIST_PREV (MIR_insn_t, insn)) { + if (insn->code == MIR_RET || insn->code == MIR_JRET) break; + inline_insns_after++; + new_insn = MIR_copy_insn (ctx, insn); + change_inline_insn_regs (ctx, new_insn); + store_labels_for_duplication (ctx, labels, temp_insns, insn, new_insn); + VARR_PUSH (MIR_insn_t, cold_insns, new_insn); + } + mir_assert (insn != NULL); + stop_insn = DLIST_NEXT (MIR_insn_t, insn); + } + for (insn = DLIST_HEAD (MIR_insn_t, called_func->insns); insn != stop_insn; insn = DLIST_NEXT (MIR_insn_t, insn)) { + mir_assert (insn->code != MIR_JRET); inline_insns_after++; - actual_nops = MIR_insn_nops (ctx, insn); new_insn = MIR_copy_insn (ctx, insn); /* va insns are possible here as va_list can be passed as arg */ - if (insn == called_func_top_alloca) { - new_called_func_top_alloca = new_insn; - } - for (i = 0; i < actual_nops; i++) switch (new_insn->ops[i].mode) { - case MIR_OP_REG: - new_insn->ops[i].u.reg = VARR_GET (MIR_reg_t, inline_reg_map, new_insn->ops[i].u.reg); - break; - case MIR_OP_MEM: - if (insn->ops[i].u.mem.base != 0) - new_insn->ops[i].u.mem.base - = VARR_GET (MIR_reg_t, inline_reg_map, new_insn->ops[i].u.mem.base); - if (insn->ops[i].u.mem.index != 0) - new_insn->ops[i].u.mem.index - = VARR_GET (MIR_reg_t, inline_reg_map, new_insn->ops[i].u.mem.index); - break; - default: /* do nothing */ break; - } + if (insn == called_func_top_alloca) new_called_func_top_alloca = new_insn; + change_inline_insn_regs (ctx, new_insn); if (new_insn->code != MIR_RET) { MIR_insert_insn_before (ctx, func_item, anchor, new_insn); store_labels_for_duplication (ctx, labels, temp_insns, insn, new_insn); } else { - /* RET should be the last insn after simplification */ - mir_assert (DLIST_NEXT (MIR_insn_t, insn) == NULL && call->ops[0].mode == MIR_OP_REF + size_t actual_nops = MIR_insn_nops (ctx, insn); + /* [J]RET should be the last insn extracting cold code */ + mir_assert (DLIST_NEXT (MIR_insn_t, insn) == stop_insn && call->ops[0].mode == MIR_OP_REF && call->ops[0].u.ref->item_type == MIR_proto_item); mir_assert (called_func->nres == actual_nops); ret_insn = new_insn; for (i = 0; i < actual_nops; i++) { mir_assert (ret_insn->ops[i].mode == MIR_OP_REG); ret_reg = ret_insn->ops[i].u.reg; - new_insn = MIR_new_insn (ctx, - res_types[i] == MIR_T_F ? MIR_FMOV - : res_types[i] == MIR_T_D ? MIR_DMOV - : res_types[i] == MIR_T_LD ? MIR_LDMOV - : MIR_MOV, - call->ops[i + 2], MIR_new_reg_op (ctx, ret_reg)); + new_insn = MIR_new_insn (ctx, get_type_move_code (res_types[i]), call->ops[i + 2], + MIR_new_reg_op (ctx, ret_reg)); MIR_insert_insn_before (ctx, func_item, anchor, new_insn); } free (ret_insn); @@ -3590,41 +4154,63 @@ static void process_inlines (MIR_context_t ctx, MIR_item_t func_item) { MIR_insert_insn_before (ctx, func_item, anchor, new_insn); } if (called_func_top_alloca != NULL) { - // ???? at the ret: curr_func_top_alloca_size -= alloca_size; - if (func_top_alloca == NULL) { - temp_reg = new_temp_reg (ctx, MIR_T_I64, func); - func_top_alloca = MIR_new_insn (ctx, MIR_ALLOCA, new_called_func_top_alloca->ops[0], - MIR_new_reg_op (ctx, temp_reg)); - if (head_func_insn->code != MIR_LABEL) - MIR_insert_insn_before (ctx, func_item, head_func_insn, func_top_alloca); - else - MIR_insert_insn_after (ctx, func_item, head_func_insn, func_top_alloca); - init_func_top_alloca_size = 0; - new_insn - = MIR_new_insn (ctx, MIR_MOV, MIR_new_reg_op (ctx, temp_reg), MIR_new_int_op (ctx, 0)); - MIR_insert_insn_before (ctx, func_item, func_top_alloca, new_insn); + if (called_func_top_alloca_used_p) { + func_top_alloca_used_p = TRUE; + if (func_top_alloca == NULL) { + temp_reg = new_temp_reg (ctx, MIR_T_I64, func); + func_top_alloca = MIR_new_insn (ctx, MIR_ALLOCA, new_called_func_top_alloca->ops[0], + MIR_new_reg_op (ctx, temp_reg)); + if (head_func_insn->code != MIR_LABEL) + MIR_insert_insn_before (ctx, func_item, head_func_insn, func_top_alloca); + else + MIR_insert_insn_after (ctx, func_item, head_func_insn, func_top_alloca); + init_func_top_alloca_size = 0; + new_insn + = MIR_new_insn (ctx, MIR_MOV, MIR_new_reg_op (ctx, temp_reg), MIR_new_int_op (ctx, 0)); + MIR_insert_insn_before (ctx, func_item, func_top_alloca, new_insn); + } + if (curr_func_top_alloca_size - alloca_size == 0) { + new_insn = MIR_new_insn (ctx, MIR_MOV, new_called_func_top_alloca->ops[0], + func_top_alloca->ops[0]); + } else { + temp_reg = new_temp_reg (ctx, MIR_T_I64, func); + new_insn + = MIR_new_insn (ctx, MIR_PTR32 ? MIR_ADDS : MIR_ADD, new_called_func_top_alloca->ops[0], + func_top_alloca->ops[0], MIR_new_reg_op (ctx, temp_reg)); + MIR_insert_insn_after (ctx, func_item, call, new_insn); + new_insn = MIR_new_insn (ctx, MIR_MOV, MIR_new_reg_op (ctx, temp_reg), + MIR_new_int_op (ctx, curr_func_top_alloca_size - alloca_size)); + } + MIR_insert_insn_after (ctx, func_item, call, new_insn); } - temp_reg = new_temp_reg (ctx, MIR_T_I64, func); - new_insn - = MIR_new_insn (ctx, MIR_PTR32 ? MIR_ADDS : MIR_ADD, new_called_func_top_alloca->ops[0], - func_top_alloca->ops[0], MIR_new_reg_op (ctx, temp_reg)); + if (head_func_insn == new_called_func_top_alloca) + head_func_insn = DLIST_NEXT (MIR_insn_t, head_func_insn); MIR_remove_insn (ctx, func_item, new_called_func_top_alloca); - MIR_insert_insn_after (ctx, func_item, call, new_insn); - new_insn = MIR_new_insn (ctx, MIR_MOV, MIR_new_reg_op (ctx, temp_reg), - MIR_new_int_op (ctx, curr_func_top_alloca_size - alloca_size)); - MIR_insert_insn_after (ctx, func_item, call, new_insn); } + if (head_func_insn == call) head_func_insn = DLIST_NEXT (MIR_insn_t, head_func_insn); MIR_remove_insn (ctx, func_item, call); + if (head_func_insn == call) head_func_insn = DLIST_HEAD (MIR_insn_t, func->insns); next_func_insn = (prev_insn == NULL ? DLIST_HEAD (MIR_insn_t, func->insns) : DLIST_NEXT (MIR_insn_t, prev_insn)); } - if (func_top_alloca != NULL && max_func_top_alloca_size != init_func_top_alloca_size) { - temp_reg = new_temp_reg (ctx, MIR_T_I64, func); - new_insn = MIR_new_insn (ctx, MIR_MOV, MIR_new_reg_op (ctx, temp_reg), - MIR_new_int_op (ctx, max_func_top_alloca_size)); - func_top_alloca->ops[1] = MIR_new_reg_op (ctx, temp_reg); - MIR_insert_insn_before (ctx, func_item, func_top_alloca, new_insn); + mir_assert (VARR_LENGTH (MIR_insn_t, anchors) == 0 && VARR_LENGTH (size_t, alloca_sizes) == 0); + if (func_top_alloca != NULL) { + if (!func_top_alloca_used_p) { + MIR_remove_insn (ctx, func_item, func_top_alloca); + } else if (max_func_top_alloca_size != init_func_top_alloca_size) { + temp_reg = new_temp_reg (ctx, MIR_T_I64, func); + new_insn = MIR_new_insn (ctx, MIR_MOV, MIR_new_reg_op (ctx, temp_reg), + MIR_new_int_op (ctx, max_func_top_alloca_size)); + func_top_alloca->ops[1] = MIR_new_reg_op (ctx, temp_reg); + MIR_insert_insn_before (ctx, func_item, func_top_alloca, new_insn); + } } + while (VARR_LENGTH (MIR_insn_t, cold_insns) != 0) { + insn = VARR_POP (MIR_insn_t, cold_insns); + if (insn->code == MIR_LABEL) insn->ops[0].u.i = new_label_num++; + MIR_append_insn (ctx, func_item, insn); + } + if (curr_label_num < new_label_num) curr_label_num = new_label_num; } /* New Page */ @@ -3644,7 +4230,6 @@ MIR_item_t _MIR_builtin_proto (MIR_context_t ctx, MIR_module_t module, const cha MIR_module_t saved_module; va_start (argp, nargs); - if (mir_mutex_lock (&ctx_mutex)) parallel_error (ctx, "error in mutex lock"); saved_module = curr_module; for (i = 0; i < nargs; i++) { args[i].type = va_arg (argp, MIR_type_t); @@ -3662,7 +4247,6 @@ MIR_item_t _MIR_builtin_proto (MIR_context_t ctx, MIR_module_t module, const cha for (i = 0; i < nargs; i++) if (args[i].type != VARR_GET (MIR_var_t, proto_item->u.proto->args, i).type) break; if (i >= nargs) { - if (mir_mutex_unlock (&ctx_mutex)) parallel_error (ctx, "error in mutex unlock"); return proto_item; } } @@ -3677,7 +4261,6 @@ MIR_item_t _MIR_builtin_proto (MIR_context_t ctx, MIR_module_t module, const cha DLIST_REMOVE (MIR_item_t, curr_module->items, proto_item); DLIST_PREPEND (MIR_item_t, curr_module->items, proto_item); /* make it first in the list */ curr_module = saved_module; - if (mir_mutex_unlock (&ctx_mutex)) parallel_error (ctx, "error in mutex unlock"); return proto_item; } @@ -3686,7 +4269,6 @@ MIR_item_t _MIR_builtin_func (MIR_context_t ctx, MIR_module_t module, const char MIR_item_t item, ref_item; MIR_module_t saved_module = curr_module; - if (mir_mutex_lock (&ctx_mutex)) parallel_error (ctx, "error in mutex lock"); name = _MIR_uniq_string (ctx, name); if ((ref_item = item_tab_find (ctx, name, &environment_module)) != NULL) { if (ref_item->item_type != MIR_import_item || ref_item->addr != addr) @@ -3718,7 +4300,6 @@ MIR_item_t _MIR_builtin_func (MIR_context_t ctx, MIR_module_t module, const char item->ref_def = ref_item; curr_module = saved_module; } - if (mir_mutex_unlock (&ctx_mutex)) parallel_error (ctx, "error in mutex unlock"); return item; } @@ -3745,13 +4326,10 @@ static int mem_protect (void *addr, size_t len, int prot) { #if !defined(__APPLE__) || !defined(__aarch64__) return mprotect (addr, len, prot); #else - if (!pthread_jit_write_protect_supported_np ()) { - fprintf (stderr, "unsupported pthread_jit_write_protect_np -- good bye!\n"); - exit (1); - } - if (prot & PROT_WRITE) pthread_jit_write_protect_np (FALSE); + if ((prot & PROT_WRITE) && pthread_jit_write_protect_supported_np ()) + pthread_jit_write_protect_np (FALSE); if (prot & PROT_READ) { - pthread_jit_write_protect_np (TRUE); + if (pthread_jit_write_protect_supported_np ()) pthread_jit_write_protect_np (TRUE); sys_icache_invalidate (addr, len); } else if (0) { if (mprotect (addr, len, prot) != 0) { @@ -3811,14 +4389,10 @@ typedef struct code_holder code_holder_t; DEF_VARR (code_holder_t); struct machine_code_ctx { -#if MIR_PARALLEL_GEN - mir_mutex_t code_mutex; -#endif VARR (code_holder_t) * code_holders; size_t page_size; }; -#define code_mutex ctx->machine_code_ctx->code_mutex #define code_holders ctx->machine_code_ctx->code_holders #define page_size ctx->machine_code_ctx->page_size @@ -3864,7 +4438,7 @@ void _MIR_set_code (size_t prot_start, size_t prot_len, uint8_t *base, size_t nl } #endif -static uint8_t *add_code (MIR_context_t ctx, code_holder_t *ch_ptr, const uint8_t *code, +static uint8_t *add_code (MIR_context_t ctx MIR_UNUSED, code_holder_t *ch_ptr, const uint8_t *code, size_t code_len) { uint8_t *mem = ch_ptr->free; @@ -3883,10 +4457,8 @@ uint8_t *_MIR_publish_code (MIR_context_t ctx, const uint8_t *code, code_holder_t *ch_ptr; uint8_t *res = NULL; - if (mir_mutex_lock (&code_mutex)) parallel_error (ctx, "error in mutex lock"); if ((ch_ptr = get_last_code_holder (ctx, code_len)) != NULL) res = add_code (ctx, ch_ptr, code, code_len); - if (mir_mutex_unlock (&code_mutex)) parallel_error (ctx, "error in mutex unlock"); return res; } @@ -3907,12 +4479,10 @@ void _MIR_change_code (MIR_context_t ctx, uint8_t *addr, const uint8_t *code, start = (size_t) addr / page_size * page_size; len = (size_t) addr + code_len - start; - if (mir_mutex_lock (&code_mutex)) parallel_error (ctx, "error in mutex lock"); reloc.offset = 0; reloc.value = code; _MIR_set_code (start, len, addr, 1, &reloc, code_len); _MIR_flush_code_cache (addr, addr + code_len); - if (mir_mutex_unlock (&code_mutex)) parallel_error (ctx, "error in mutex unlock"); } void _MIR_update_code_arr (MIR_context_t ctx, uint8_t *base, size_t nloc, @@ -3924,10 +4494,8 @@ void _MIR_update_code_arr (MIR_context_t ctx, uint8_t *base, size_t nloc, if (max_offset < relocs[i].offset) max_offset = relocs[i].offset; start = (size_t) base / page_size * page_size; len = (size_t) base + max_offset + sizeof (void *) - start; - if (mir_mutex_lock (&code_mutex)) parallel_error (ctx, "error in mutex lock"); _MIR_set_code (start, len, base, nloc, relocs, 0); _MIR_flush_code_cache (base, base + max_offset + sizeof (void *)); - if (mir_mutex_unlock (&code_mutex)) parallel_error (ctx, "error in mutex unlock"); } void _MIR_update_code (MIR_context_t ctx, uint8_t *base, size_t nloc, ...) { /* thread safe */ @@ -3955,11 +4523,9 @@ static void code_init (MIR_context_t ctx) { MIR_get_error_func (ctx) (MIR_alloc_error, "Not enough memory for ctx"); page_size = mem_page_size (); VARR_CREATE (code_holder_t, code_holders, 128); - if (mir_mutex_init (&code_mutex, NULL)) parallel_error (ctx, "error in mutex init"); } static void code_finish (MIR_context_t ctx) { - if (mir_mutex_destroy (&code_mutex)) parallel_error (ctx, "error in mutex destroy"); while (VARR_LENGTH (code_holder_t, code_holders) != 0) { code_holder_t ch = VARR_POP (code_holder_t, code_holders); mem_unmap (ch.start, ch.bound - ch.start); @@ -4012,6 +4578,9 @@ typedef enum { TAG_EL (TRBLOCK) = TAG_EL (TBLOCK) + MIR_BLK_NUM, TAG_EL (EOI), TAG_EL (EOFILE), /* end of insn with variable number operands (e.g. a call) or end of file */ + REP4 (TAG_EL, ALIAS_MEM_DISP, ALIAS_MEM_BASE, ALIAS_MEM_INDEX, ALIAS_MEM_DISP_BASE), + REP3 (TAG_EL, ALIAS_MEM_DISP_INDEX, ALIAS_MEM_BASE_INDEX, ALIAS_MEM_DISP_BASE_INDEX), + TAG_EL (LAST) = TAG_EL (ALIAS_MEM_DISP_BASE_INDEX), /* unsigned integer 0..127 is kept in one byte. The most significant bit of the byte is 1: */ U0_MASK = 0x7f, U0_FLAG = 0x80, @@ -4023,7 +4592,8 @@ typedef enum { VERSION NSTR (string)* - ( ((label)* (insn code) (operand)* | STRN=(func|local|import|export|forward|) ...) EOI? + ( ((label)* (insn code) (operand)* | STRN=(func|global|local|import|export|forward|) ...) + EOI? )* EOF where @@ -4160,8 +4730,8 @@ static size_t write_int (MIR_context_t ctx, writer_func_t writer, int64_t i) { if (writer == NULL) return 0; nb = int_length (i); assert (nb > 0); - put_byte (ctx, writer, TAG_I1 + nb - 1); - len = put_int (ctx, writer, i, nb) + 1; + put_byte (ctx, writer, TAG_I1 + (int) nb - 1); + len = put_int (ctx, writer, i, (int) nb) + 1; output_int_len += len; return len; } @@ -4171,11 +4741,11 @@ static size_t write_uint (MIR_context_t ctx, writer_func_t writer, uint64_t u) { if (writer == NULL) return 0; if ((nb = uint_length (u)) == 0) { - put_byte (ctx, writer, 0x80 | u); + put_byte (ctx, writer, (int) (0x80 | u)); return 1; } - put_byte (ctx, writer, TAG_U1 + nb - 1); - len = put_uint (ctx, writer, u, nb) + 1; + put_byte (ctx, writer, TAG_U1 + (int) nb - 1); + len = put_uint (ctx, writer, u, (int) nb) + 1; output_int_len += len; return len; } @@ -4225,8 +4795,8 @@ static size_t write_str_tag (MIR_context_t ctx, writer_func_t writer, MIR_str_t nb = uint_length (string.num - 1); mir_assert (nb <= 4); if (nb == 0) nb = 1; - put_byte (ctx, writer, start_tag + nb - 1); - return put_uint (ctx, writer, string.num - 1, nb) + 1; + put_byte (ctx, writer, start_tag + (int) nb - 1); + return put_uint (ctx, writer, string.num - 1, (int) nb) + 1; } static size_t write_str (MIR_context_t ctx, writer_func_t writer, MIR_str_t str) { @@ -4256,8 +4826,8 @@ static size_t write_lab (MIR_context_t ctx, writer_func_t writer, MIR_label_t la nb = uint_length (lab_num); mir_assert (nb <= 4); if (nb == 0) nb = 1; - put_byte (ctx, writer, TAG_LAB1 + nb - 1); - len = put_uint (ctx, writer, lab_num, nb) + 1; + put_byte (ctx, writer, TAG_LAB1 + (int) nb - 1); + len = put_uint (ctx, writer, lab_num, (int) nb) + 1; output_labs_len += len; return len; } @@ -4273,18 +4843,23 @@ static size_t write_op (MIR_context_t ctx, writer_func_t writer, MIR_func_t func case MIR_OP_MEM: { bin_tag_t tag; size_t len; + int alias_p = op.u.mem.alias != 0 || op.u.mem.nonalias != 0; if (op.u.mem.disp != 0) { if (op.u.mem.base != 0) - tag = op.u.mem.index != 0 ? TAG_MEM_DISP_BASE_INDEX : TAG_MEM_DISP_BASE; + tag = op.u.mem.index != 0 + ? (alias_p ? TAG_ALIAS_MEM_DISP_BASE_INDEX : TAG_MEM_DISP_BASE_INDEX) + : (alias_p ? TAG_ALIAS_MEM_DISP_BASE : TAG_MEM_DISP_BASE); else - tag = op.u.mem.index != 0 ? TAG_MEM_DISP_INDEX : TAG_MEM_DISP; + tag = op.u.mem.index != 0 ? (alias_p ? TAG_ALIAS_MEM_DISP_INDEX : TAG_MEM_DISP_INDEX) + : (alias_p ? TAG_ALIAS_MEM_DISP : TAG_MEM_DISP); } else if (op.u.mem.base != 0) { - tag = op.u.mem.index != 0 ? TAG_MEM_BASE_INDEX : TAG_MEM_BASE; + tag = op.u.mem.index != 0 ? (alias_p ? TAG_ALIAS_MEM_BASE_INDEX : TAG_MEM_BASE_INDEX) + : (alias_p ? TAG_ALIAS_MEM_BASE : TAG_MEM_BASE); } else if (op.u.mem.index != 0) { - tag = TAG_MEM_INDEX; + tag = alias_p ? TAG_ALIAS_MEM_INDEX : TAG_MEM_INDEX; } else { - tag = TAG_MEM_DISP; + tag = alias_p ? TAG_ALIAS_MEM_DISP : TAG_MEM_DISP; } put_byte (ctx, writer, tag); len = write_type (ctx, writer, op.u.mem.type) + 1; @@ -4295,6 +4870,10 @@ static size_t write_op (MIR_context_t ctx, writer_func_t writer, MIR_func_t func len += write_reg (ctx, writer, MIR_reg_name (ctx, op.u.mem.index, func)); len += write_uint (ctx, writer, op.u.mem.scale); } + if (alias_p) { + len += write_name (ctx, writer, MIR_alias_name (ctx, op.u.mem.alias)); + len += write_name (ctx, writer, MIR_alias_name (ctx, op.u.mem.nonalias)); + } output_mem_len += len; return len; } @@ -4311,9 +4890,9 @@ static size_t write_insn (MIR_context_t ctx, writer_func_t writer, MIR_func_t fu MIR_insn_code_t code = insn->code; size_t len; - if (code == MIR_UNSPEC || code == MIR_PHI) + if (code == MIR_UNSPEC || code == MIR_USE || code == MIR_PHI) MIR_get_error_func (ctx) (MIR_binary_io_error, - "UNSPEC or PHI is not portable and can not be output"); + "UNSPEC, USE, or PHI is not portable and can not be output"); if (code == MIR_LABEL) return write_lab (ctx, writer, insn); nops = MIR_insn_nops (ctx, insn); len = write_uint (ctx, writer, code); @@ -4328,12 +4907,32 @@ static size_t write_insn (MIR_context_t ctx, writer_func_t writer, MIR_func_t fu return len; } +static size_t write_vars (MIR_context_t ctx, writer_func_t writer, MIR_func_t func, + VARR (MIR_var_t) * vars, size_t start, size_t vars_num, + const char *prefix) { + if (vars_num == 0 || vars == NULL) return 0; + size_t len = 0; + int first_p = TRUE; + for (size_t i = 0; i < vars_num; i++) { + MIR_var_t var = VARR_GET (MIR_var_t, vars, i + start); + if (first_p) len += write_name (ctx, writer, prefix); + first_p = FALSE; + len += write_type (ctx, writer, var.type); + len += write_name (ctx, writer, var.name); + MIR_reg_t reg = MIR_reg (ctx, var.name, func); + const char *hard_reg_name = MIR_reg_hard_reg_name (ctx, reg, func); + if (hard_reg_name != NULL) len += write_name (ctx, writer, hard_reg_name); + } + len += put_byte (ctx, writer, TAG_EOI); + return len; +} + static size_t write_item (MIR_context_t ctx, writer_func_t writer, MIR_item_t item) { MIR_insn_t insn; MIR_func_t func; MIR_proto_t proto; MIR_var_t var; - size_t i, nlocals, len = 0; + size_t i, vars_num, len = 0; if (item->item_type == MIR_import_item) { len += write_name (ctx, writer, "import"); @@ -4371,6 +4970,28 @@ static size_t write_item (MIR_context_t ctx, writer_func_t writer, MIR_item_t it len += write_int (ctx, writer, item->u.ref_data->disp); return len; } + if (item->item_type == MIR_lref_data_item) { + if (item->u.lref_data->name == NULL) { + len += write_name (ctx, writer, "lref"); + } else { + len += write_name (ctx, writer, "nlref"); + len += write_name (ctx, writer, item->u.lref_data->name); + } + mir_assert (item->u.lref_data->label->ops[0].mode == MIR_OP_INT); + mir_assert (item->u.lref_data->label2 == NULL + || (item->u.lref_data->label2->ops[0].mode == MIR_OP_INT + && item->u.lref_data->label2->ops[0].u.i >= 0)); + len += write_int (ctx, writer, item->u.lref_data->label->ops[0].u.i); + if (item->u.lref_data->label2 == NULL) { + len += write_int (ctx, writer, -1); + } else { + mir_assert (item->u.lref_data->label2->ops[0].mode == MIR_OP_INT + && item->u.lref_data->label2->ops[0].u.i >= 0); + len += write_int (ctx, writer, item->u.lref_data->label2->ops[0].u.i); + } + len += write_int (ctx, writer, item->u.lref_data->disp); + return len; + } if (item->item_type == MIR_expr_data_item) { if (item->u.expr_data->name == NULL) { len += write_name (ctx, writer, "expr"); @@ -4441,16 +5062,11 @@ static size_t write_item (MIR_context_t ctx, writer_func_t writer, MIR_item_t it if (MIR_all_blk_type_p (var.type)) len += write_uint (ctx, writer, var.size); } len += put_byte (ctx, writer, TAG_EOI); - nlocals = VARR_LENGTH (MIR_var_t, func->vars) - func->nargs; - if (nlocals != 0) { - len += write_name (ctx, writer, "local"); - for (i = 0; i < nlocals; i++) { - var = VARR_GET (MIR_var_t, func->vars, i + func->nargs); - len += write_type (ctx, writer, var.type); - len += write_name (ctx, writer, var.name); - } - len += put_byte (ctx, writer, TAG_EOI); - } + vars_num = VARR_LENGTH (MIR_var_t, func->vars) - func->nargs; + len += write_vars (ctx, writer, func, func->vars, func->nargs, vars_num, "local"); + len += write_vars (ctx, writer, func, func->global_vars, 0, + func->global_vars == NULL ? 0 : VARR_LENGTH (MIR_var_t, func->global_vars), + "global"); for (insn = DLIST_HEAD (MIR_insn_t, func->insns); insn != NULL; insn = DLIST_NEXT (MIR_insn_t, insn)) len += write_insn (ctx, writer, func, insn); @@ -4580,7 +5196,7 @@ static float get_float (MIR_context_t ctx) { float f; } u; - u.u = get_uint (ctx, sizeof (uint32_t)); + u.u = (uint32_t) get_uint (ctx, (int) sizeof (uint32_t)); return u.f; } @@ -4722,6 +5338,8 @@ static bin_tag_t read_token (MIR_context_t ctx, token_attr_t *attr) { break; REP6 (TAG_CASE, MEM_DISP, MEM_BASE, MEM_INDEX, MEM_DISP_BASE, MEM_DISP_INDEX, MEM_BASE_INDEX) REP3 (TAG_CASE, MEM_DISP_BASE_INDEX, EOI, EOFILE) + REP4 (TAG_CASE, ALIAS_MEM_DISP, ALIAS_MEM_BASE, ALIAS_MEM_INDEX, ALIAS_MEM_DISP_BASE) + REP3 (TAG_CASE, ALIAS_MEM_DISP_INDEX, ALIAS_MEM_BASE_INDEX, ALIAS_MEM_DISP_BASE_INDEX) break; REP8 (TAG_CASE, TI8, TU8, TI16, TU16, TI32, TU32, TI64, TU64) REP5 (TAG_CASE, TF, TD, TP, TV, TRBLOCK) @@ -4764,6 +5382,8 @@ static int read_operand (MIR_context_t ctx, MIR_op_t *op, MIR_item_t func) { MIR_disp_t disp; MIR_reg_t base, index; MIR_scale_t scale; + int alias_p = FALSE; + const char *name; tag = read_token (ctx, &attr); switch (tag) { @@ -4786,7 +5406,7 @@ static int read_operand (MIR_context_t ctx, MIR_op_t *op, MIR_item_t func) { *op = MIR_new_reg_op (ctx, to_reg (ctx, attr.u, func)); break; REP4 (TAG_CASE, NAME1, NAME2, NAME3, NAME4) { - const char *name = to_str (ctx, attr.u).s; + name = to_str (ctx, attr.u).s; MIR_item_t item = item_tab_find (ctx, name, func->module); if (item == NULL) MIR_get_error_func (ctx) (MIR_binary_io_error, "not found item %s", name); @@ -4799,25 +5419,42 @@ static int read_operand (MIR_context_t ctx, MIR_op_t *op, MIR_item_t func) { REP4 (TAG_CASE, LAB1, LAB2, LAB3, LAB4) *op = MIR_new_label_op (ctx, to_lab (ctx, attr.u)); break; - REP7 (TAG_CASE, MEM_DISP, MEM_BASE, MEM_INDEX, MEM_DISP_BASE, MEM_DISP_INDEX, MEM_BASE_INDEX, + REP7 (TAG_CASE, ALIAS_MEM_DISP, ALIAS_MEM_BASE, ALIAS_MEM_INDEX, ALIAS_MEM_DISP_BASE, + ALIAS_MEM_DISP_INDEX, ALIAS_MEM_BASE_INDEX, ALIAS_MEM_DISP_BASE_INDEX) + alias_p = TRUE; + /* falls through */ + case TAG_MEM_DISP: + REP6 (TAG_CASE, MEM_BASE, MEM_INDEX, MEM_DISP_BASE, MEM_DISP_INDEX, MEM_BASE_INDEX, MEM_DISP_BASE_INDEX) t = read_type (ctx, "wrong memory type"); disp = (tag == TAG_MEM_DISP || tag == TAG_MEM_DISP_BASE || tag == TAG_MEM_DISP_INDEX - || tag == TAG_MEM_DISP_BASE_INDEX + || tag == TAG_MEM_DISP_BASE_INDEX || tag == TAG_ALIAS_MEM_DISP + || tag == TAG_ALIAS_MEM_DISP_BASE || tag == TAG_ALIAS_MEM_DISP_INDEX + || tag == TAG_ALIAS_MEM_DISP_BASE_INDEX ? read_disp (ctx) : 0); base = (tag == TAG_MEM_BASE || tag == TAG_MEM_DISP_BASE || tag == TAG_MEM_BASE_INDEX - || tag == TAG_MEM_DISP_BASE_INDEX + || tag == TAG_MEM_DISP_BASE_INDEX || tag == TAG_ALIAS_MEM_BASE + || tag == TAG_ALIAS_MEM_DISP_BASE || tag == TAG_ALIAS_MEM_BASE_INDEX + || tag == TAG_ALIAS_MEM_DISP_BASE_INDEX ? read_reg (ctx, func) : 0); index = 0; scale = 0; if (tag == TAG_MEM_INDEX || tag == TAG_MEM_DISP_INDEX || tag == TAG_MEM_BASE_INDEX - || tag == TAG_MEM_DISP_BASE_INDEX) { + || tag == TAG_MEM_DISP_BASE_INDEX || tag == TAG_ALIAS_MEM_INDEX + || tag == TAG_ALIAS_MEM_DISP_INDEX || tag == TAG_ALIAS_MEM_BASE_INDEX + || tag == TAG_ALIAS_MEM_DISP_BASE_INDEX) { index = read_reg (ctx, func); - scale = read_uint (ctx, "wrong memory index scale"); + scale = (MIR_scale_t) read_uint (ctx, "wrong memory index scale"); } *op = MIR_new_mem_op (ctx, t, disp, base, index, scale); + if (alias_p) { + name = read_name (ctx, func->module, "wrong alias name"); + if (strcmp (name, "") != 0) op->u.mem.alias = MIR_alias (ctx, name); + name = read_name (ctx, func->module, "wrong nonalias name"); + if (strcmp (name, "") != 0) op->u.mem.nonalias = MIR_alias (ctx, name); + } break; case TAG_EOI: return FALSE; default: mir_assert (FALSE); @@ -4867,10 +5504,10 @@ static size_t reduce_reader (void *start, size_t len, void *data) { #endif void MIR_read_with_func (MIR_context_t ctx, int (*const reader) (MIR_context_t)) { - int version; - bin_tag_t tag; + int version, global_p, nlref_p; + bin_tag_t tag, type_tag; token_attr_t attr; - MIR_label_t lab; + MIR_label_t lab, lab2; uint64_t nstr, nres, u; int64_t i; MIR_op_t op; @@ -4885,7 +5522,7 @@ void MIR_read_with_func (MIR_context_t ctx, int (*const reader) (MIR_context_t)) MIR_get_error_func (ctx) (MIR_binary_io_error, "can not alloc data for MIR binary decompression"); #endif - version = read_uint (ctx, "wrong header"); + version = (int) read_uint (ctx, "wrong header"); if (version > CURR_BIN_VERSION) MIR_get_error_func (ctx) (MIR_binary_io_error, "can not read version %d MIR binary: expected %d or less", version, @@ -4987,6 +5624,18 @@ void MIR_read_with_func (MIR_context_t ctx, int (*const reader) (MIR_context_t)) item_name); i = read_int (ctx, "wrong ref disp"); MIR_new_ref_data (ctx, name, item, i); + } else if ((nlref_p = strcmp (name, "nlref") == 0) || strcmp (name, "lref") == 0) { + name = NULL; + if (nlref_p) name = read_name (ctx, module, "wrong lref data name"); + if (VARR_LENGTH (uint64_t, insn_label_string_nums) != 0) + MIR_get_error_func (ctx) (MIR_binary_io_error, "lref data %s should have no labels", + name == NULL ? "" : name); + i = read_int (ctx, "wrong lref label num"); + lab = create_label (ctx, i); + i = read_int (ctx, "wrong 2nd lref label num"); + lab2 = i < 0 ? NULL : create_label (ctx, i); + i = read_int (ctx, "wrong lref disp"); + MIR_new_lref_data (ctx, name, lab, lab2, i); } else if (strcmp (name, "nexpr") == 0 || strcmp (name, "expr") == 0) { name = strcmp (name, "nexpr") == 0 ? read_name (ctx, module, "wrong expr name") : NULL; if (VARR_LENGTH (uint64_t, insn_label_string_nums) != 0) @@ -5035,15 +5684,15 @@ void MIR_read_with_func (MIR_context_t ctx, int (*const reader) (MIR_context_t)) case TAG_U8: switch (type) { case MIR_T_U8: - v.u8 = attr.u; + v.u8 = (uint8_t) attr.u; push_data (ctx, &v.u8, sizeof (uint8_t)); break; case MIR_T_U16: - v.u16 = attr.u; + v.u16 = (uint16_t) attr.u; push_data (ctx, (uint8_t *) &v.u16, sizeof (uint16_t)); break; case MIR_T_U32: - v.u32 = attr.u; + v.u32 = (uint32_t) attr.u; push_data (ctx, (uint8_t *) &v.u32, sizeof (uint32_t)); break; case MIR_T_U64: @@ -5066,15 +5715,15 @@ void MIR_read_with_func (MIR_context_t ctx, int (*const reader) (MIR_context_t)) case TAG_I8: switch (type) { case MIR_T_I8: - v.i8 = attr.i; + v.i8 = (int8_t) attr.i; push_data (ctx, (uint8_t *) &v.i8, sizeof (int8_t)); break; case MIR_T_I16: - v.i16 = attr.i; + v.i16 = (int16_t) attr.i; push_data (ctx, (uint8_t *) &v.i16, sizeof (int16_t)); break; case MIR_T_I32: - v.i32 = attr.i; + v.i32 = (int32_t) attr.i; push_data (ctx, (uint8_t *) &v.i32, sizeof (int32_t)); break; case MIR_T_I64: @@ -5115,17 +5764,29 @@ void MIR_read_with_func (MIR_context_t ctx, int (*const reader) (MIR_context_t)) MIR_new_data (ctx, name, type, VARR_LENGTH (uint8_t, temp_data) / _MIR_type_size (ctx, type), VARR_ADDR (uint8_t, temp_data)); - } else if (strcmp (name, "local") == 0) { - if (func == NULL) MIR_get_error_func (ctx) (MIR_binary_io_error, "local outside func"); + } else if ((global_p = strcmp (name, "global") == 0) || strcmp (name, "local") == 0) { + if (func == NULL) + MIR_get_error_func (ctx) (MIR_binary_io_error, "local/global outside func"); if (VARR_LENGTH (uint64_t, insn_label_string_nums) != 0) - MIR_get_error_func (ctx) (MIR_binary_io_error, "local should have no labels"); + MIR_get_error_func (ctx) (MIR_binary_io_error, "local/global should have no labels"); + tag = read_token (ctx, &attr); for (;;) { - tag = read_token (ctx, &attr); if (tag == TAG_EOI) break; if (TAG_TI8 > tag || tag > TAG_TRBLOCK) - MIR_get_error_func (ctx) (MIR_binary_io_error, "wrong local var type tag %d", tag); - MIR_new_func_reg (ctx, func->u.func, tag_type (tag), - read_name (ctx, module, "wrong local var name")); + MIR_get_error_func (ctx) (MIR_binary_io_error, "wrong local/global var type tag %d", + tag); + name = read_name (ctx, module, "wrong local/global var name"); + type_tag = tag; + tag = read_token (ctx, &attr); + if (!global_p) { + MIR_new_func_reg (ctx, func->u.func, tag_type (type_tag), name); + } else if (TAG_NAME1 <= tag && tag <= TAG_NAME4) { + const char *reg_name = to_str (ctx, get_uint (ctx, tag - TAG_NAME1 + 1)).s; + MIR_new_global_func_reg (ctx, func->u.func, tag_type (type_tag), name, reg_name); + tag = read_token (ctx, &attr); + } else { + MIR_get_error_func (ctx) (MIR_binary_io_error, "global without hard reg name"); + } } } else { MIR_get_error_func (ctx) (MIR_binary_io_error, "unknown insn name %s", name); @@ -5135,11 +5796,11 @@ void MIR_read_with_func (MIR_context_t ctx, int (*const reader) (MIR_context_t)) if (insn_code >= MIR_LABEL) MIR_get_error_func (ctx) (MIR_binary_io_error, "wrong insn code %d", insn_code); - if (insn_code == MIR_UNSPEC || insn_code == MIR_PHI) + if (insn_code == MIR_UNSPEC || insn_code == MIR_USE || insn_code == MIR_PHI) MIR_get_error_func (ctx) (MIR_binary_io_error, - "UNSPEC or PHI is not portable and can not be read"); - for (uint64_t i = 0; i < VARR_LENGTH (uint64_t, insn_label_string_nums); i++) { - lab = to_lab (ctx, VARR_GET (uint64_t, insn_label_string_nums, i)); + "UNSPEC, USE, or PHI is not portable and can not be read"); + for (size_t j = 0; j < VARR_LENGTH (uint64_t, insn_label_string_nums); j++) { + lab = to_lab (ctx, VARR_GET (uint64_t, insn_label_string_nums, j)); MIR_append_insn (ctx, func, lab); } nop = insn_code_nops (ctx, insn_code); @@ -5177,7 +5838,7 @@ void MIR_read (MIR_context_t ctx, FILE *f) { } static void io_init (MIR_context_t ctx) { - mir_assert (TAG_EOFILE < 127); /* see bin_tag_t */ + mir_assert (TAG_LAST < 127); /* see bin_tag_t */ if ((ctx->io_ctx = malloc (sizeof (struct io_ctx))) == NULL) MIR_get_error_func (ctx) (MIR_alloc_error, "Not enough memory for ctx"); VARR_CREATE (MIR_var_t, proto_vars, 0); @@ -5202,6 +5863,11 @@ static void io_finish (MIR_context_t ctx) { #endif /* if !MIR_NO_IO */ /* New Page */ +int _MIR_name_char_p (MIR_context_t ctx MIR_UNUSED, int ch, int first_p) { + if (isalpha (ch) || ch == '_' || ch == '$' || ch == '%' || ch == '.') return TRUE; + return !first_p && isdigit (ch); +} + /* Reading MIR text file */ @@ -5218,11 +5884,11 @@ typedef struct insn_name { MIR_insn_code_t code; } insn_name_t; -static int insn_name_eq (insn_name_t in1, insn_name_t in2, void *arg) { +static int insn_name_eq (insn_name_t in1, insn_name_t in2, void *arg MIR_UNUSED) { return strcmp (in1.name, in2.name) == 0; } -static htab_hash_t insn_name_hash (insn_name_t in, void *arg) { - return mir_hash (in.name, strlen (in.name), 0); +static htab_hash_t insn_name_hash (insn_name_t in, void *arg MIR_UNUSED) { + return (htab_hash_t) mir_hash (in.name, strlen (in.name), 0); } #define TC_EL(t) TC_##t @@ -5250,6 +5916,7 @@ typedef const char *label_name_t; DEF_VARR (label_name_t); typedef struct label_desc { + int def_p; const char *name; MIR_label_t label; } label_desc_t; @@ -5484,7 +6151,7 @@ static void scan_token (MIR_context_t ctx, token_t *token, int (*get_char) (MIR_ case '#': while ((ch = get_char (ctx)) != '\n' && ch != EOF) ; - /* Fall through: */ + /* falls through */ case '\n': token->code = TC_NL; return; case '(': token->code = TC_LEFT_PAR; return; case ')': token->code = TC_RIGHT_PAR; return; @@ -5494,11 +6161,11 @@ static void scan_token (MIR_context_t ctx, token_t *token, int (*get_char) (MIR_ case '"': scan_string (ctx, token, ch, get_char, unget_char); return; default: VARR_TRUNC (char, temp_string, 0); - if (isalpha (ch) || ch == '_' || ch == '$' || ch == '%' || ch == '.') { + if (_MIR_name_char_p (ctx, ch, TRUE)) { do { VARR_PUSH (char, temp_string, ch); ch = get_char (ctx); - } while (isalpha (ch) || isdigit (ch) || ch == '_' || ch == '$' || ch == '%' || ch == '.'); + } while (_MIR_name_char_p (ctx, ch, FALSE)); VARR_PUSH (char, temp_string, '\0'); unget_char (ctx, ch); token->u.name = _MIR_uniq_string (ctx, VARR_ADDR (char, temp_string)); @@ -5532,8 +6199,8 @@ static void scan_token (MIR_context_t ctx, token_t *token, int (*get_char) (MIR_ : strtoull (repr, &end, base)); } mir_assert (*end == '\0'); - if (errno != 0) - ; + if (errno != 0) { + } return; } else { VARR_PUSH (char, temp_string, '\0'); @@ -5543,28 +6210,34 @@ static void scan_token (MIR_context_t ctx, token_t *token, int (*get_char) (MIR_ } } -static int label_eq (label_desc_t l1, label_desc_t l2, void *arg) { +static int label_eq (label_desc_t l1, label_desc_t l2, void *arg MIR_UNUSED) { return strcmp (l1.name, l2.name) == 0; } -static htab_hash_t label_hash (label_desc_t l, void *arg) { - return mir_hash (l.name, strlen (l.name), 0); +static htab_hash_t label_hash (label_desc_t l, void *arg MIR_UNUSED) { + return (htab_hash_t) mir_hash (l.name, strlen (l.name), 0); } -static MIR_label_t create_label_desc (MIR_context_t ctx, const char *name) { +static MIR_label_t create_label_desc (MIR_context_t ctx, const char *name, int def_p) { MIR_label_t label; label_desc_t label_desc; label_desc.name = name; if (HTAB_DO (label_desc_t, label_desc_tab, label_desc, HTAB_FIND, label_desc)) { + if (def_p) { + if (label_desc.def_p) scan_error (ctx, "redefinition of label %s in a module", name); + label_desc.def_p = TRUE; + HTAB_DO (label_desc_t, label_desc_tab, label_desc, HTAB_REPLACE, label_desc); + } label = label_desc.label; } else { label_desc.label = label = MIR_new_label (ctx); + label_desc.def_p = def_p; HTAB_DO (label_desc_t, label_desc_tab, label_desc, HTAB_INSERT, label_desc); } return label; } -static int func_reg_p (MIR_context_t ctx, MIR_func_t func, const char *name) { +static int func_reg_p (MIR_context_t ctx MIR_UNUSED, MIR_func_t func, const char *name) { func_regs_t func_regs = func->internal; size_t rdn, tab_rdn; reg_desc_t rd; @@ -5630,13 +6303,13 @@ static MIR_type_t str2type (const char *type_name) { label : name code : name op : name | int | float | double | long double | mem | str - mem : type ':' addr + mem : type ':' addr aliases addr : disp | [ disp ] '(' sib ')' sib : name | [ name ] ',' name [ ',' scale] disp : int | name scale : int - + aliases : [':' [name] [':' name] ] */ void MIR_scan_string (MIR_context_t ctx, const char *str) { @@ -5649,10 +6322,9 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { MIR_type_t type, data_type = MIR_T_BOUND; MIR_op_t op, *op_addr; MIR_label_t label; - size_t n; - int64_t i; + int64_t i, n; int module_p, end_module_p, proto_p, func_p, end_func_p, dots_p, export_p, import_p, forward_p; - int bss_p, ref_p, expr_p, string_p, local_p, push_op_p, read_p, disp_p; + int bss_p, ref_p, lref_p, expr_p, string_p, global_p, local_p, push_op_p, read_p, disp_p; insn_name_t in, el; VARR_TRUNC (char, error_msg_buf, 0); @@ -5683,7 +6355,8 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { scan_token (ctx, &t, get_string_char, unget_string_char); /* label_names without insn */ } module_p = end_module_p = proto_p = func_p = end_func_p = FALSE; - export_p = import_p = forward_p = bss_p = ref_p = expr_p = string_p = local_p = FALSE; + export_p = import_p = forward_p = bss_p = ref_p = lref_p = expr_p = string_p = FALSE; + global_p = local_p = FALSE; if (strcmp (name, "module") == 0) { module_p = TRUE; if (VARR_LENGTH (label_name_t, label_names) != 1) @@ -5724,6 +6397,10 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { ref_p = TRUE; if (VARR_LENGTH (label_name_t, label_names) > 1) scan_error (ctx, "at most one label should be used for ref"); + } else if (strcmp (name, "lref") == 0) { + lref_p = TRUE; + if (VARR_LENGTH (label_name_t, label_names) > 1) + scan_error (ctx, "at most one label should be used for lref"); } else if (strcmp (name, "expr") == 0) { expr_p = TRUE; if (VARR_LENGTH (label_name_t, label_names) > 1) @@ -5732,11 +6409,11 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { string_p = TRUE; if (VARR_LENGTH (label_name_t, label_names) > 1) scan_error (ctx, "at most one label should be used for string"); - } else if (strcmp (name, "local") == 0) { - local_p = TRUE; - if (func == NULL) scan_error (ctx, "local outside func"); + } else if ((local_p = strcmp (name, "local") == 0) + || (global_p = strcmp (name, "global") == 0)) { + if (func == NULL) scan_error (ctx, "local/global outside func"); if (VARR_LENGTH (label_name_t, label_names) != 0) - scan_error (ctx, "local should have no labels"); + scan_error (ctx, "local/global should have no labels"); } else if ((data_type = str2type (name)) != MIR_T_BOUND) { if (VARR_LENGTH (label_name_t, label_names) > 1) scan_error (ctx, "at most one label should be used for data"); @@ -5745,10 +6422,10 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { if (!HTAB_DO (insn_name_t, insn_name_tab, in, HTAB_FIND, el)) scan_error (ctx, "Unknown insn %s", name); insn_code = el.code; - if (insn_code == MIR_UNSPEC || insn_code == MIR_PHI) - scan_error (ctx, "UNSPEC or PHI is not portable and can not be scanned", name); - for (n = 0; n < VARR_LENGTH (label_name_t, label_names); n++) { - label = create_label_desc (ctx, VARR_GET (label_name_t, label_names, n)); + if (insn_code == MIR_UNSPEC || insn_code == MIR_USE || insn_code == MIR_PHI) + scan_error (ctx, "UNSPEC, USE, or PHI is not portable and can not be scanned", name); + for (n = 0; n < (int64_t) VARR_LENGTH (label_name_t, label_names); n++) { + label = create_label_desc (ctx, VARR_GET (label_name_t, label_names, n), TRUE); if (func != NULL) MIR_append_insn (ctx, func, label); } } @@ -5769,7 +6446,7 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { break; } read_p = FALSE; - if (t.code != TC_COL && !proto_p && !func_p && !local_p) { + if (t.code != TC_COL && !proto_p && !func_p && !local_p && !global_p) { if (export_p) { MIR_new_export (ctx, name); push_op_p = FALSE; @@ -5779,12 +6456,16 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { } else if (forward_p) { MIR_new_forward (ctx, name); push_op_p = FALSE; - } else if (!module_p && !end_module_p && !proto_p && !func_p && !end_func_p && !local_p - && ((MIR_branch_code_p (insn_code) + } else if (lref_p) { + op = MIR_new_label_op (ctx, create_label_desc (ctx, name, FALSE)); + } else if (!module_p && !end_module_p && !end_func_p + && (((MIR_branch_code_p (insn_code) || insn_code == MIR_PRBEQ + || insn_code == MIR_PRBNE) && VARR_LENGTH (MIR_op_t, scan_insn_ops) == 0) + || (insn_code == MIR_LADDR && VARR_LENGTH (MIR_op_t, scan_insn_ops) == 1) || (insn_code == MIR_SWITCH && VARR_LENGTH (MIR_op_t, scan_insn_ops) > 0))) { - op = MIR_new_label_op (ctx, create_label_desc (ctx, name)); + op = MIR_new_label_op (ctx, create_label_desc (ctx, name, FALSE)); } else if (!expr_p && !ref_p && func != NULL && func_reg_p (ctx, func->u.func, name)) { op.mode = MIR_OP_REG; op.u.reg = MIR_reg (ctx, name, func->u.func); @@ -5794,24 +6475,37 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { scan_error (ctx, "undeclared name %s", name); } break; - } - /* Memory, type only, arg, or var */ + } /* Memory, type only, arg, or var */ type = str2type (name); if (type == MIR_T_BOUND) scan_error (ctx, "Unknown type %s", name); - else if (local_p && type != MIR_T_I64 && type != MIR_T_F && type != MIR_T_D + else if ((global_p || local_p) && type != MIR_T_I64 && type != MIR_T_F && type != MIR_T_D && type != MIR_T_LD) - scan_error (ctx, "wrong type %s for local var", name); + scan_error (ctx, "wrong type %s for local/global var", name); op = MIR_new_mem_op (ctx, type, 0, 0, 0, 1); - if (proto_p || func_p || local_p) { + if (proto_p || func_p || global_p || local_p) { if (t.code == TC_COL) { scan_token (ctx, &t, get_string_char, unget_string_char); if (t.code == TC_NAME) { op.u.mem.disp = (MIR_disp_t) t.u.name; - } else if (local_p || t.code != TC_INT || !MIR_all_blk_type_p (type)) { + scan_token (ctx, &t, get_string_char, unget_string_char); + if (!global_p) { + } else if (t.code != TC_COL) { + scan_error (ctx, "global %s without hard register", (const char *) op.u.mem.disp); + } else { + scan_token (ctx, &t, get_string_char, unget_string_char); + if (t.code != TC_NAME) { + scan_error (ctx, "hard register for %s is not a name", (char *) op.data); + } else { + op.data = (void *) t.u.name; + scan_token (ctx, &t, get_string_char, unget_string_char); + } + } + } else if (global_p || local_p || t.code != TC_INT || !MIR_all_blk_type_p (type)) { scan_error (ctx, local_p ? "wrong var" : "wrong arg"); + scan_token (ctx, &t, get_string_char, unget_string_char); } else { - op.u.mem.base = t.u.i; + op.u.mem.base = (MIR_reg_t) t.u.i; if (t.u.i < 0 || t.u.i >= (1ll << sizeof (MIR_reg_t) * 8)) scan_error (ctx, "invalid block arg size"); scan_token (ctx, &t, get_string_char, unget_string_char); @@ -5821,8 +6515,8 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { op.u.mem.disp = (MIR_disp_t) t.u.name; scan_token (ctx, &t, get_string_char, unget_string_char); if (t.code != TC_RIGHT_PAR) scan_error (ctx, "wrong block arg"); + scan_token (ctx, &t, get_string_char, unget_string_char); } - scan_token (ctx, &t, get_string_char, unget_string_char); } } else { scan_token (ctx, &t, get_string_char, unget_string_char); @@ -5850,14 +6544,42 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { if (t.code == TC_COMMA) { scan_token (ctx, &t, get_string_char, unget_string_char); if (t.code != TC_INT) scan_error (ctx, "wrong scale"); - op.u.mem.scale = t.u.i; + op.u.mem.scale = (MIR_scale_t) t.u.i; scan_token (ctx, &t, get_string_char, unget_string_char); } } if (t.code != TC_RIGHT_PAR) scan_error (ctx, "wrong memory op"); scan_token (ctx, &t, get_string_char, unget_string_char); - } else if (!disp_p) + } else if (!disp_p) { scan_error (ctx, "wrong memory"); + } + if (t.code == TC_COL) { + scan_token (ctx, &t, get_string_char, unget_string_char); + if (t.code == TC_COL) { + op.u.mem.alias = 0; + scan_token (ctx, &t, get_string_char, unget_string_char); + if (t.code != TC_NAME) { + scan_error (ctx, "empty nonalias name"); + } else { + op.u.mem.nonalias = MIR_alias (ctx, t.u.name); + scan_token (ctx, &t, get_string_char, unget_string_char); + } + } else if (t.code != TC_NAME) { + scan_error (ctx, "wrong alias name"); + } else { + op.u.mem.alias = MIR_alias (ctx, t.u.name); + scan_token (ctx, &t, get_string_char, unget_string_char); + if (t.code == TC_COL) { + scan_token (ctx, &t, get_string_char, unget_string_char); + if (t.code != TC_NAME) { + scan_error (ctx, "empty nonalias name"); + } else { + op.u.mem.nonalias = MIR_alias (ctx, t.u.name); + scan_token (ctx, &t, get_string_char, unget_string_char); + } + } + } + } } break; } @@ -5884,7 +6606,10 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { default: break; } if (dots_p) break; - if (push_op_p) VARR_PUSH (MIR_op_t, scan_insn_ops, op); + if (push_op_p) { + VARR_PUSH (MIR_op_t, scan_insn_ops, op); + op.data = NULL; /* reset value set up for global */ + } if (read_p) scan_token (ctx, &t, get_string_char, unget_string_char); if (t.code != TC_COMMA) break; scan_token (ctx, &t, get_string_char, unget_string_char); @@ -5896,6 +6621,7 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { if (VARR_LENGTH (MIR_op_t, scan_insn_ops) != 0) scan_error (ctx, "module should have no params"); module = MIR_new_module (ctx, VARR_GET (label_name_t, label_names, 0)); + HTAB_CLEAR (label_desc_t, label_desc_tab); } else if (end_module_p) { if (module == NULL) scan_error (ctx, "standalone endmodule"); if (VARR_LENGTH (MIR_op_t, scan_insn_ops) != 0) @@ -5922,6 +6648,31 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { = (VARR_LENGTH (label_name_t, label_names) == 0 ? NULL : VARR_GET (label_name_t, label_names, 0)); MIR_new_ref_data (ctx, name, op_addr[0].u.ref, op_addr[1].u.i); + } else if (lref_p) { + size_t len = VARR_LENGTH (MIR_op_t, scan_insn_ops); + MIR_label_t lab = NULL, lab2 = NULL; + int64_t disp = 0; + if (len == 0 || len > 3) + scan_error (ctx, "lref should have at least one but at most three operands"); + op_addr = VARR_ADDR (MIR_op_t, scan_insn_ops); + if (op_addr[0].mode != MIR_OP_LABEL) scan_error (ctx, "1st lref operand is not a label"); + lab = op_addr[0].u.label; + if (len == 2) { + if (op_addr[1].mode != MIR_OP_LABEL && op_addr[1].mode != MIR_OP_INT) + scan_error (ctx, "2nd lref operand is not a label or displacement"); + if (op_addr[1].mode == MIR_OP_LABEL) lab2 = op_addr[1].u.label; + if (op_addr[1].mode == MIR_OP_INT) disp = op_addr[1].u.i; + } else if (len == 3) { + if (op_addr[1].mode != MIR_OP_LABEL) scan_error (ctx, "2nd lref operand is not a label"); + if (op_addr[2].mode != MIR_OP_INT) + scan_error (ctx, "3rd lref operand is not a displacement"); + lab2 = op_addr[1].u.label; + disp = op_addr[2].u.i; + } + name + = (VARR_LENGTH (label_name_t, label_names) == 0 ? NULL + : VARR_GET (label_name_t, label_names, 0)); + MIR_new_lref_data (ctx, name, lab, lab2, disp); } else if (expr_p) { if (VARR_LENGTH (MIR_op_t, scan_insn_ops) != 1) scan_error (ctx, "expr should have one operand"); @@ -5972,7 +6723,6 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { VARR_LENGTH (MIR_type_t, scan_types), VARR_ADDR (MIR_type_t, scan_types), VARR_LENGTH (MIR_var_t, scan_vars), VARR_ADDR (MIR_var_t, scan_vars)); - HTAB_CLEAR (label_desc_t, label_desc_tab); } else if (end_func_p) { if (func == NULL) scan_error (ctx, "standalone endfunc"); if (VARR_LENGTH (MIR_op_t, scan_insn_ops) != 0) @@ -5981,14 +6731,20 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { MIR_finish_func (ctx); } else if (export_p || import_p || forward_p) { /* we already created items, now do nothing: */ mir_assert (VARR_LENGTH (MIR_op_t, scan_insn_ops) == 0); - } else if (local_p) { + } else if (global_p || local_p) { op_addr = VARR_ADDR (MIR_op_t, scan_insn_ops); - n = VARR_LENGTH (MIR_op_t, scan_insn_ops); + n = (int64_t) VARR_LENGTH (MIR_op_t, scan_insn_ops); for (i = 0; i < n; i++) { if (op_addr[i].mode != MIR_OP_MEM || (const char *) op_addr[i].u.mem.disp == NULL) - scan_error (ctx, "wrong local var"); - MIR_new_func_reg (ctx, func->u.func, op_addr[i].u.mem.type, - (const char *) op_addr[i].u.mem.disp); + scan_error (ctx, "wrong local/global var"); + if (op_addr[i].data == NULL) { + MIR_new_func_reg (ctx, func->u.func, op_addr[i].u.mem.type, + (const char *) op_addr[i].u.mem.disp); + } else { + MIR_new_global_func_reg (ctx, func->u.func, op_addr[i].u.mem.type, + (const char *) op_addr[i].u.mem.disp, + (const char *) op_addr[i].data); + } } } else if (data_type != MIR_T_BOUND) { union { @@ -6002,7 +6758,7 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { int64_t i64; } v; - n = VARR_LENGTH (MIR_op_t, scan_insn_ops); + n = (int64_t) VARR_LENGTH (MIR_op_t, scan_insn_ops); op_addr = VARR_ADDR (MIR_op_t, scan_insn_ops); VARR_TRUNC (uint8_t, temp_data, 0); for (i = 0; i < n; i++) { @@ -6010,27 +6766,27 @@ void MIR_scan_string (MIR_context_t ctx, const char *str) { scan_error (ctx, "data operand is not of data type"); switch (data_type) { case MIR_T_I8: - v.i8 = op_addr[i].u.i; + v.i8 = (int8_t) op_addr[i].u.i; push_data (ctx, (uint8_t *) &v.i8, sizeof (int8_t)); break; case MIR_T_U8: - v.u8 = op_addr[i].u.u; + v.u8 = (uint8_t) op_addr[i].u.u; push_data (ctx, (uint8_t *) &v.u8, sizeof (uint8_t)); break; case MIR_T_I16: - v.i16 = op_addr[i].u.i; + v.i16 = (int16_t) op_addr[i].u.i; push_data (ctx, (uint8_t *) &v.i16, sizeof (int16_t)); break; case MIR_T_U16: - v.u16 = op_addr[i].u.u; + v.u16 = (uint16_t) op_addr[i].u.u; push_data (ctx, (uint8_t *) &v.u16, sizeof (uint16_t)); break; case MIR_T_I32: - v.i32 = op_addr[i].u.i; + v.i32 = (int32_t) op_addr[i].u.i; push_data (ctx, (uint8_t *) &v.i32, sizeof (int32_t)); break; case MIR_T_U32: - v.u32 = op_addr[i].u.u; + v.u32 = (uint32_t) op_addr[i].u.u; push_data (ctx, (uint8_t *) &v.u32, sizeof (uint32_t)); break; case MIR_T_I64: @@ -6119,7 +6875,7 @@ static void scan_finish (MIR_context_t ctx) { #define pclose _pclose #endif -void _MIR_dump_code (const char *name, int index, uint8_t *code, size_t code_len) { +void _MIR_dump_code (const char *name, uint8_t *code, size_t code_len) { size_t i; int ch; char cfname[50]; @@ -6131,7 +6887,7 @@ void _MIR_dump_code (const char *name, int index, uint8_t *code, size_t code_len #endif if (name != NULL) fprintf (stderr, "%s:", name); - sprintf (cfname, "_mir_%d_%lu.c", index, (unsigned long) getpid ()); + sprintf (cfname, "_mir_%lu.c", (unsigned long) getpid ()); if ((f = fopen (cfname, "w")) == NULL) return; #if defined(__APPLE__) fprintf (f, "unsigned char code[] = {"); @@ -6149,7 +6905,7 @@ void _MIR_dump_code (const char *name, int index, uint8_t *code, size_t code_len cfname, cfname, cfname, cfname, cfname); #endif #else - sprintf (bfname, "_mir_%d_%lu.bin", index, (unsigned long) getpid ()); + sprintf (bfname, "_mir_%lu.bin", (unsigned long) getpid ()); if ((bf = fopen (bfname, "w")) == NULL) return; fprintf (f, "void code (void) {}\n"); for (i = 0; i < code_len; i++) fputc (code[i], bf); @@ -6191,6 +6947,71 @@ void _MIR_dump_code (const char *name, int index, uint8_t *code, size_t code_len #error "undefined or unsupported generation target" #endif +static int var_is_reg_p (MIR_reg_t var) { return var > MAX_HARD_REG; } +static MIR_reg_t var2reg (MIR_reg_t var) { + mir_assert (var_is_reg_p (var)); + return var == MIR_NON_VAR ? 0 : var - MAX_HARD_REG; +} + +struct hard_reg_desc { + const char *name; + int num; +}; +typedef struct hard_reg_desc hard_reg_desc_t; + +DEF_HTAB (hard_reg_desc_t); + +struct hard_reg_ctx { + HTAB (hard_reg_desc_t) * hard_reg_desc_tab; +}; + +#define hard_reg_desc_tab ctx->hard_reg_ctx->hard_reg_desc_tab + +static htab_hash_t hard_reg_desc_hash (hard_reg_desc_t desc, void *arg MIR_UNUSED) { + return (htab_hash_t) mir_hash (desc.name, strlen (desc.name), 0); +} +static int hard_reg_desc_eq (hard_reg_desc_t desc1, hard_reg_desc_t desc2, void *arg MIR_UNUSED) { + return strcmp (desc1.name, desc2.name) == 0; +} + +static void hard_reg_name_init (MIR_context_t ctx) { + hard_reg_desc_t desc, tab_desc; + int res; + + if ((ctx->hard_reg_ctx = malloc (sizeof (struct hard_reg_ctx))) == NULL) + MIR_get_error_func (ctx) (MIR_alloc_error, "Not enough memory for ctx"); + HTAB_CREATE (hard_reg_desc_t, hard_reg_desc_tab, 200, hard_reg_desc_hash, hard_reg_desc_eq, NULL); + for (size_t i = 0; i * sizeof (char *) < sizeof (target_hard_reg_names); i++) { + desc.num = (int) i; + desc.name = target_hard_reg_names[i]; + res = HTAB_DO (hard_reg_desc_t, hard_reg_desc_tab, desc, HTAB_INSERT, tab_desc); + mir_assert (!res); + } +} + +static void hard_reg_name_finish (MIR_context_t ctx) { + HTAB_DESTROY (hard_reg_desc_t, hard_reg_desc_tab); + free (ctx->hard_reg_ctx); + ctx->hard_reg_ctx = NULL; +} + +int _MIR_get_hard_reg (MIR_context_t ctx, const char *hard_reg_name) { + hard_reg_desc_t desc, tab_desc; + + desc.name = hard_reg_name; + if (!HTAB_DO (hard_reg_desc_t, hard_reg_desc_tab, desc, HTAB_FIND, tab_desc)) return -1; + return tab_desc.num; +} + +static MIR_UNUSED const char *get_hard_reg_name (MIR_context_t ctx MIR_UNUSED, int hard_reg) { + if (hard_reg > MAX_HARD_REG || target_fixed_hard_reg_p (hard_reg)) return NULL; + return target_hard_reg_names[hard_reg]; +} + +void *_MIR_get_module_global_var_hard_regs (MIR_context_t ctx MIR_UNUSED, MIR_module_t module) { + return module->data; +} + /* New Page */ #include "mir-interp.c" diff --git a/mir/mir.h b/mir/mir.h index 820397b0..95c365c0 100644 --- a/mir/mir.h +++ b/mir/mir.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov . + Copyright (C) 2018-2024 Vladimir Makarov . */ #ifndef MIR_H @@ -21,7 +21,7 @@ extern "C" { #include "mir-varr.h" #include "mir-htab.h" -#define MIR_API_VERSION 0.1 +#define MIR_API_VERSION 0.2 #ifdef NDEBUG static inline int mir_assert (int cond) { return 0 && cond; } @@ -41,15 +41,6 @@ static inline int mir_assert (int cond) { return 0 && cond; } #define MIR_NO_SCAN 0 #endif -#ifndef MIR_PARALLEL_GEN -#define MIR_PARALLEL_GEN 0 -#endif - -#if MIR_PARALLEL_GEN && defined(_WIN32) /* TODO: Win thread primitives ??? */ -#undef MIR_PARALLEL_GEN -#define MIR_PARALLEL_GEN 0 -#endif - #ifdef __GNUC__ #define MIR_UNUSED __attribute__ ((unused)) #else @@ -69,10 +60,10 @@ static inline int mir_assert (int cond) { return 0 && cond; } #define ERR_EL(e) MIR_##e##_error typedef enum MIR_error_type { REP8 (ERR_EL, no, syntax, binary_io, alloc, finish, no_module, nested_module, no_func), - REP4 (ERR_EL, func, vararg_func, nested_func, wrong_param_value), + REP5 (ERR_EL, func, vararg_func, nested_func, wrong_param_value, hard_reg), REP5 (ERR_EL, reserved_name, import_export, undeclared_func_reg, repeated_decl, reg_type), REP6 (ERR_EL, wrong_type, unique_reg, undeclared_op_ref, ops_num, call_op, unspec_op), - REP6 (ERR_EL, ret, op_mode, out_op, invalid_insn, ctx_change, parallel) + REP6 (ERR_EL, wrong_lref, ret, op_mode, out_op, invalid_insn, ctx_change) } MIR_error_type_t; #ifdef __GNUC__ @@ -84,31 +75,6 @@ typedef enum MIR_error_type { typedef void MIR_NO_RETURN (*MIR_error_func_t) (MIR_error_type_t error_type, const char *format, ...); -#if MIR_PARALLEL_GEN -#include -typedef pthread_mutex_t mir_mutex_t; -typedef pthread_cond_t mir_cond_t; -typedef pthread_attr_t mir_thread_attr_t; -#define mir_thread_create(m, attr, f, arg) pthread_create (m, attr, f, arg) -#define mir_thread_join(t, r) pthread_join (t, r) -#define mir_mutex_init(m, a) pthread_mutex_init (m, a) -#define mir_mutex_destroy(m) pthread_mutex_destroy (m) -#define mir_mutex_lock(m) pthread_mutex_lock (m) -#define mir_mutex_unlock(m) pthread_mutex_unlock (m) -#define mir_cond_init(m, a) pthread_cond_init (m, a) -#define mir_cond_destroy(m) pthread_cond_destroy (m) -#define mir_cond_wait(c, m) pthread_cond_wait (c, m) -#define mir_cond_signal(c) pthread_cond_signal (c) -#define mir_cond_broadcast(c) pthread_cond_broadcast (c) -#define mir_thread_attr_init(a) pthread_attr_init (a) -#define mir_thread_attr_setstacksize(a, s) pthread_attr_setstacksize (a, s) -#else -#define mir_mutex_init(m, a) 0 -#define mir_mutex_destroy(m) 0 -#define mir_mutex_lock(m) 0 -#define mir_mutex_unlock(m) 0 -#endif - #define INSN_EL(i) MIR_##i /* The most MIR insns have destination operand and one or two source @@ -131,6 +97,7 @@ typedef enum { REP3 (INSN_EL, F2I, D2I, LD2I), /* Float or (long) double to integer conversion */ REP6 (INSN_EL, F2D, F2LD, D2F, D2LD, LD2F, LD2D), /* Float, (long) double conversions */ REP5 (INSN_EL, NEG, NEGS, FNEG, DNEG, LDNEG), /* Changing sign */ + REP4 (INSN_EL, ADDR, ADDR8, ADDR16, ADDR32), /* reg addr in natural mode or given integer mode */ /* 3 operand insn: */ REP5 (INSN_EL, ADD, ADDS, FADD, DADD, LDADD), /* Addition */ REP5 (INSN_EL, SUB, SUBS, FSUB, DSUB, LDSUB), /* Subtraction */ @@ -145,6 +112,7 @@ typedef enum { REP7 (INSN_EL, LE, LES, ULE, ULES, FLE, DLE, LDLE), /* Less or equal */ REP7 (INSN_EL, GT, GTS, UGT, UGTS, FGT, DGT, LDGT), /* Greater then */ REP7 (INSN_EL, GE, GES, UGE, UGES, FGE, DGE, LDGE), /* Greater or equal */ + REP8 (INSN_EL, ADDO, ADDOS, SUBO, SUBOS, MULO, MULOS, UMULO, UMULOS), /* setting overflow flag */ /* Unconditional (1 operand) and conditional (2 operands) branch insns. The first operand is a label. */ REP5 (INSN_EL, JMP, BT, BTS, BF, BFS), @@ -156,26 +124,34 @@ typedef enum { REP7 (INSN_EL, BLE, BLES, UBLE, UBLES, FBLE, DBLE, LDBLE), REP7 (INSN_EL, BGT, BGTS, UBGT, UBGTS, FBGT, DBGT, LDBGT), REP7 (INSN_EL, BGE, BGES, UBGE, UBGES, FBGE, DBGE, LDBGE), + REP2 (INSN_EL, BO, UBO), /* branch on overflow: prev insn should be overflow add/sub */ + REP2 (INSN_EL, BNO, UBNO), /* branch on not overflow: prev insn should be overflow add/sub */ + INSN_EL (LADDR), /* put label address (2nd op) into the 1st op */ + INSN_EL (JMPI), /* indirect jump to the label whose address stored in the 1st op */ /* 1st operand is a prototype, 2nd one is ref or op containing func address, 3rd and subsequent ops are optional result (if result in the prototype is not of void type), call arguments. */ - REP2 (INSN_EL, CALL, INLINE), + REP3 (INSN_EL, CALL, INLINE, JCALL), /* 1st operand is an index, subsequent ops are labels to which goto according the index (1st label has index zero). The insn behavior is undefined if there is no label for the index. */ INSN_EL (SWITCH), - /* 1 operand insn: */ INSN_EL (RET), + INSN_EL (JRET), /* return by jumping to address of the operand */ + /* 1 operand insn: */ INSN_EL (ALLOCA), /* 2 operands: result address and size */ REP2 (INSN_EL, BSTART, BEND), /* block start: result addr; block end: addr from block start */ /* Special insns: */ INSN_EL (VA_ARG), /* result is arg address, operands: va_list addr and memory */ - INSN_EL (VA_BLOCK_ARG), /* result is arg address, operands: va_list addr and integer (size) */ + INSN_EL (VA_BLOCK_ARG), /* result is arg address, operands: va_list addr, integer (size), and + integer (block type) */ INSN_EL (VA_START), - INSN_EL (VA_END), /* operand is va_list */ - INSN_EL (LABEL), /* One immediate operand is unique label number */ - INSN_EL (UNSPEC), /* First operand unspec code and the rest are args */ - INSN_EL (PHI), /* Used only internally in the generator, the first operand is output */ + INSN_EL (VA_END), /* operand is va_list */ + INSN_EL (LABEL), /* One immediate operand is unique label number */ + INSN_EL (UNSPEC), /* First operand unspec code and the rest are args */ + REP3 (INSN_EL, PRSET, PRBEQ, PRBNE), /* work with properties */ + INSN_EL (USE), /* Used only internally in the generator, all operands are input */ + INSN_EL (PHI), /* Used only internally in the generator, the first operand is output */ INSN_EL (INVALID_INSN), INSN_EL (INSN_BOUND), /* Should be the last */ } MIR_insn_code_t; @@ -223,7 +199,7 @@ typedef int64_t MIR_disp_t; /* Address displacement in memory */ typedef uint32_t MIR_reg_t; #define MIR_MAX_REG_NUM UINT32_MAX -#define MIR_NON_HARD_REG MIR_MAX_REG_NUM +#define MIR_NON_VAR MIR_MAX_REG_NUM /* Immediate in immediate moves. */ typedef union { @@ -234,15 +210,19 @@ typedef union { long double ld; } MIR_imm_t; -/* Memory: mem:type[base + index * scale + disp]. It also can be - memory with hard regs but such memory used only internally. An - integer type memory value expands to int64_t value when the insn is - executed. */ +typedef uint32_t MIR_alias_t; /* unique number of alias name */ + +/* Memory: mem:type[base + index * scale + disp]. It also can be memory with vars + (regs and hard regs) but such memory used only internally. An integer type memory + value expands to int64_t value when the insn is executed. */ typedef struct { MIR_type_t type : 8; MIR_scale_t scale; - /* 0 means no reg for memory. MIR_NON_HARD_REG means no reg for - hard reg memory. */ + MIR_alias_t alias; /* 0 may alias any memory, memory with the same alias is aliased */ + MIR_alias_t nonalias; /* 0 for ignoring, memory with the same nonalias is not aliased */ + /* Used internally: mem operand with the same nonzero nloc always refers to the same memory */ + uint32_t nloc; + /* 0 and MIR_NON_VAR means no reg for correspondingly for memory and var memory. */ MIR_reg_t base, index; MIR_disp_t disp; } MIR_mem_t; @@ -255,8 +235,8 @@ typedef const char *MIR_name_t; /* Operand mode */ typedef enum { - REP8 (OP_EL, UNDEF, REG, HARD_REG, INT, UINT, FLOAT, DOUBLE, LDOUBLE), - REP6 (OP_EL, REF, STR, MEM, HARD_REG_MEM, LABEL, BOUND), + REP8 (OP_EL, UNDEF, REG, VAR, INT, UINT, FLOAT, DOUBLE, LDOUBLE), + REP6 (OP_EL, REF, STR, MEM, VAR_MEM, LABEL, BOUND), } MIR_op_mode_t; typedef struct MIR_item *MIR_item_t; @@ -271,13 +251,13 @@ typedef struct MIR_str MIR_str_t; /* An insn operand */ typedef struct { void *data; /* Aux data */ - MIR_op_mode_t mode; + MIR_op_mode_t mode : 8; /* Defined after MIR_func_finish. Only MIR_OP_INT, MIR_OP_UINT, MIR_OP_FLOAT, MIR_OP_DOUBLE, MIR_OP_LDOUBLE: */ - MIR_op_mode_t value_mode; + MIR_op_mode_t value_mode : 8; union { MIR_reg_t reg; - MIR_reg_t hard_reg; /* Used only internally */ + MIR_reg_t var; /* Used only internally */ int64_t i; uint64_t u; float f; @@ -286,7 +266,7 @@ typedef struct { MIR_item_t ref; /* non-export/non-forward after simplification */ MIR_str_t str; MIR_mem_t mem; - MIR_mem_t hard_reg_mem; /* Used only internally */ + MIR_mem_t var_mem; /* Used only internally */ MIR_label_t label; } u; } MIR_op_t; @@ -323,13 +303,16 @@ typedef struct MIR_func { DLIST (MIR_insn_t) insns, original_insns; uint32_t nres, nargs, last_temp_num, n_inlines; MIR_type_t *res_types; - char vararg_p; /* flag of variable number of arguments */ - char expr_p; /* flag of that the func can be used as a linker expression */ - VARR (MIR_var_t) * vars; /* args and locals but temps */ - void *machine_code; /* address of generated machine code or NULL */ - void *call_addr; /* address to call the function, it can be the same as machine_code */ - void *internal; /* internal data structure */ -} * MIR_func_t; + char vararg_p; /* flag of variable number of arguments */ + char expr_p; /* flag of that the func can be used as a linker expression */ + char jret_p; /* flag of jcall/jret func, set up after MIR_func_finish */ + VARR (MIR_var_t) * vars; /* args and locals but temps */ + VARR (MIR_var_t) * global_vars; /* can be NULL */ + void *machine_code; /* address of generated machine code or NULL */ + void *call_addr; /* address to call the function, it can be the same as machine_code */ + void *internal; /* internal data structure */ + struct MIR_lref_data *first_lref; /* label addr data of the func: defined by module load */ +} *MIR_func_t; typedef struct MIR_proto { const char *name; @@ -337,7 +320,7 @@ typedef struct MIR_proto { MIR_type_t *res_types; /* != MIR_T_UNDEF */ char vararg_p; /* flag of variable number of arguments */ VARR (MIR_var_t) * args; /* args name can be NULL */ -} * MIR_proto_t; +} *MIR_proto_t; typedef struct MIR_data { const char *name; /* can be NULL */ @@ -347,25 +330,35 @@ typedef struct MIR_data { long double d; /* for alignment of temporary literals */ uint8_t els[1]; } u; -} * MIR_data_t; +} *MIR_data_t; typedef struct MIR_ref_data { const char *name; /* can be NULL */ MIR_item_t ref_item; /* base */ int64_t disp; /* disp relative to base */ void *load_addr; -} * MIR_ref_data_t; +} *MIR_ref_data_t; + +typedef struct MIR_lref_data { /* describing [name:]lref lab[,label2][,disp] = lab-lab2+disp */ + const char *name; /* can be NULL */ + MIR_label_t label; /* base */ + MIR_label_t label2; /* can be NULL */ + MIR_label_t orig_label, orig_label2; /* used to restore original func lrefs */ + int64_t disp; /* disp relative to base */ + void *load_addr; /* where is the value placed */ + struct MIR_lref_data *next; /* next label addr related to the same func */ +} *MIR_lref_data_t; typedef struct MIR_expr_data { const char *name; /* can be NULL */ MIR_item_t expr_item; /* a special function can be called during linking */ void *load_addr; -} * MIR_expr_data_t; +} *MIR_expr_data_t; typedef struct MIR_bss { const char *name; /* can be NULL */ uint64_t len; -} * MIR_bss_t; +} *MIR_bss_t; typedef struct MIR_module *MIR_module_t; @@ -375,8 +368,8 @@ DEF_DLIST_LINK (MIR_item_t); #define ITEM_EL(i) MIR_##i##_item typedef enum { - REP8 (ITEM_EL, func, proto, import, export, forward, data, ref_data, expr_data), - ITEM_EL (bss), + REP8 (ITEM_EL, func, proto, import, export, forward, data, ref_data, lref_data), + REP2 (ITEM_EL, expr_data, bss), } MIR_item_type_t; #undef ERR_EL @@ -409,6 +402,7 @@ struct MIR_item { MIR_name_t forward_id; MIR_data_t data; MIR_ref_data_t ref_data; + MIR_lref_data_t lref_data; MIR_expr_data_t expr_data; MIR_bss_t bss; } u; @@ -444,7 +438,7 @@ static inline int MIR_FP_branch_code_p (MIR_insn_code_t code) { } static inline int MIR_call_code_p (MIR_insn_code_t code) { - return code == MIR_CALL || code == MIR_INLINE; + return code == MIR_CALL || code == MIR_INLINE || code == MIR_JCALL; } static inline int MIR_int_branch_code_p (MIR_insn_code_t code) { @@ -453,13 +447,27 @@ static inline int MIR_int_branch_code_p (MIR_insn_code_t code) { || code == MIR_BLTS || code == MIR_UBLT || code == MIR_UBLTS || code == MIR_BLE || code == MIR_BLES || code == MIR_UBLE || code == MIR_UBLES || code == MIR_BGT || code == MIR_BGTS || code == MIR_UBGT || code == MIR_UBGTS || code == MIR_BGE - || code == MIR_BGES || code == MIR_UBGE || code == MIR_UBGES); + || code == MIR_BGES || code == MIR_UBGE || code == MIR_UBGES || code == MIR_BO + || code == MIR_UBO || code == MIR_BNO || code == MIR_UBNO); } static inline int MIR_branch_code_p (MIR_insn_code_t code) { return (code == MIR_JMP || MIR_int_branch_code_p (code) || MIR_FP_branch_code_p (code)); } +static inline int MIR_any_branch_code_p (MIR_insn_code_t code) { + return (MIR_branch_code_p (code) || code == MIR_JMPI || code == MIR_SWITCH); +} + +static inline int MIR_addr_code_p (MIR_insn_code_t code) { + return (code == MIR_ADDR || code == MIR_ADDR8 || code == MIR_ADDR16 || code == MIR_ADDR32); +} + +static inline int MIR_overflow_insn_code_p (MIR_insn_code_t code) { + return (code == MIR_ADDO || code == MIR_ADDOS || code == MIR_SUBO || code == MIR_SUBOS + || code == MIR_MULO || code == MIR_MULOS || code == MIR_UMULO || code == MIR_UMULOS); +} + extern double _MIR_get_api_version (void); extern MIR_context_t _MIR_init (void); @@ -489,6 +497,9 @@ extern MIR_item_t MIR_new_string_data (MIR_context_t ctx, const char *name, MIR_str_t str); /* name can be NULL */ extern MIR_item_t MIR_new_ref_data (MIR_context_t ctx, const char *name, MIR_item_t item, int64_t disp); /* name can be NULL */ +extern MIR_item_t MIR_new_lref_data (MIR_context_t ctx, const char *name, MIR_label_t label, + MIR_label_t label2, + int64_t disp); /* name and label2 can be NULL */ extern MIR_item_t MIR_new_expr_data (MIR_context_t ctx, const char *name, MIR_item_t expr_item); /* name can be NULL */ extern MIR_item_t MIR_new_proto_arr (MIR_context_t ctx, const char *name, size_t nres, @@ -511,16 +522,22 @@ extern const char *MIR_item_name (MIR_context_t ctx, MIR_item_t item); extern MIR_func_t MIR_get_item_func (MIR_context_t ctx, MIR_item_t item); extern MIR_reg_t MIR_new_func_reg (MIR_context_t ctx, MIR_func_t func, MIR_type_t type, const char *name); +extern MIR_reg_t MIR_new_global_func_reg (MIR_context_t ctx, MIR_func_t func, MIR_type_t type, + const char *name, const char *hard_reg_name); extern void MIR_finish_func (MIR_context_t ctx); extern void MIR_finish_module (MIR_context_t ctx); extern MIR_error_func_t MIR_get_error_func (MIR_context_t ctx); extern void MIR_set_error_func (MIR_context_t ctx, MIR_error_func_t func); +extern int MIR_get_func_redef_permission_p (MIR_context_t ctx); +extern void MIR_set_func_redef_permission (MIR_context_t ctx, int flag_p); + extern MIR_insn_t MIR_new_insn_arr (MIR_context_t ctx, MIR_insn_code_t code, size_t nops, MIR_op_t *ops); extern MIR_insn_t MIR_new_insn (MIR_context_t ctx, MIR_insn_code_t code, ...); extern MIR_insn_t MIR_new_call_insn (MIR_context_t ctx, size_t nops, ...); +extern MIR_insn_t MIR_new_jcall_insn (MIR_context_t ctx, size_t nops, ...); extern MIR_insn_t MIR_new_ret_insn (MIR_context_t ctx, size_t nops, ...); extern MIR_insn_t MIR_copy_insn (MIR_context_t ctx, MIR_insn_t insn); @@ -533,6 +550,10 @@ extern MIR_insn_t MIR_new_label (MIR_context_t ctx); extern MIR_reg_t MIR_reg (MIR_context_t ctx, const char *reg_name, MIR_func_t func); extern MIR_type_t MIR_reg_type (MIR_context_t ctx, MIR_reg_t reg, MIR_func_t func); extern const char *MIR_reg_name (MIR_context_t ctx, MIR_reg_t reg, MIR_func_t func); +extern const char *MIR_reg_hard_reg_name (MIR_context_t ctx, MIR_reg_t reg, MIR_func_t func); + +extern const char *MIR_alias_name (MIR_context_t ctx, MIR_alias_t alias); +extern MIR_alias_t MIR_alias (MIR_context_t ctx, const char *name); extern MIR_op_t MIR_new_reg_op (MIR_context_t ctx, MIR_reg_t reg); extern MIR_op_t MIR_new_int_op (MIR_context_t ctx, int64_t v); @@ -544,6 +565,9 @@ extern MIR_op_t MIR_new_ref_op (MIR_context_t ctx, MIR_item_t item); extern MIR_op_t MIR_new_str_op (MIR_context_t ctx, MIR_str_t str); extern MIR_op_t MIR_new_mem_op (MIR_context_t ctx, MIR_type_t type, MIR_disp_t disp, MIR_reg_t base, MIR_reg_t index, MIR_scale_t scale); +extern MIR_op_t MIR_new_alias_mem_op (MIR_context_t ctx, MIR_type_t type, MIR_disp_t disp, + MIR_reg_t base, MIR_reg_t index, MIR_scale_t scale, + MIR_alias_t alias, MIR_alias_t noalias); extern MIR_op_t MIR_new_label_op (MIR_context_t ctx, MIR_label_t label); extern int MIR_op_eq_p (MIR_context_t ctx, MIR_op_t op1, MIR_op_t op2); extern htab_hash_t MIR_op_hash_step (MIR_context_t ctx, htab_hash_t h, MIR_op_t op); @@ -561,6 +585,7 @@ extern void MIR_change_module_ctx (MIR_context_t old_ctx, MIR_module_t m, MIR_co extern MIR_insn_code_t MIR_reverse_branch_code (MIR_insn_code_t code); extern const char *MIR_type_str (MIR_context_t ctx, MIR_type_t tp); +extern void MIR_output_str (MIR_context_t ctx, FILE *f, MIR_str_t str); extern void MIR_output_op (MIR_context_t ctx, FILE *f, MIR_op_t op, MIR_func_t func); extern void MIR_output_insn (MIR_context_t ctx, FILE *f, MIR_insn_t insn, MIR_func_t func, int newline_p); @@ -615,6 +640,8 @@ extern MIR_context_t _MIR_init (void); extern const char *_MIR_uniq_string (MIR_context_t ctx, const char *str); extern int _MIR_reserved_ref_name_p (MIR_context_t ctx, const char *name); extern int _MIR_reserved_name_p (MIR_context_t ctx, const char *name); +extern int64_t _MIR_addr_offset (MIR_context_t ctx, MIR_insn_code_t code); +extern void _MIR_free_insn (MIR_context_t ctx, MIR_insn_t insn); extern MIR_reg_t _MIR_new_temp_reg (MIR_context_t ctx, MIR_type_t type, MIR_func_t func); /* for internal use only */ extern size_t _MIR_type_size (MIR_context_t ctx, MIR_type_t type); @@ -627,19 +654,22 @@ extern void _MIR_register_unspec_insn (MIR_context_t ctx, uint64_t code, const c extern void _MIR_duplicate_func_insns (MIR_context_t ctx, MIR_item_t func_item); extern void _MIR_restore_func_insns (MIR_context_t ctx, MIR_item_t func_item); +extern void _MIR_output_data_item_els (MIR_context_t ctx, FILE *f, MIR_item_t item, int c_p); extern void _MIR_get_temp_item_name (MIR_context_t ctx, MIR_module_t module, char *buff, size_t buff_len); -extern MIR_op_t _MIR_new_hard_reg_op (MIR_context_t ctx, MIR_reg_t hard_reg); +extern MIR_op_t _MIR_new_var_op (MIR_context_t ctx, MIR_reg_t var); -extern MIR_op_t _MIR_new_hard_reg_mem_op (MIR_context_t ctx, MIR_type_t type, MIR_disp_t disp, - MIR_reg_t base, MIR_reg_t index, MIR_scale_t scale); +extern MIR_op_t _MIR_new_var_mem_op (MIR_context_t ctx, MIR_type_t type, MIR_disp_t disp, + MIR_reg_t base, MIR_reg_t index, MIR_scale_t scale); +extern MIR_op_t _MIR_new_alias_var_mem_op (MIR_context_t ctx, MIR_type_t type, MIR_disp_t disp, + MIR_reg_t base, MIR_reg_t index, MIR_scale_t scale, + MIR_alias_t alias, MIR_alias_t no_alias); extern MIR_item_t _MIR_builtin_proto (MIR_context_t ctx, MIR_module_t module, const char *name, size_t nres, MIR_type_t *res_types, size_t nargs, ...); extern MIR_item_t _MIR_builtin_func (MIR_context_t ctx, MIR_module_t module, const char *name, void *addr); - extern void _MIR_flush_code_cache (void *start, void *bound); extern uint8_t *_MIR_publish_code (MIR_context_t ctx, const uint8_t *code, size_t code_len); extern uint8_t *_MIR_get_new_code_addr (MIR_context_t ctx, size_t size); @@ -677,10 +707,20 @@ extern void *_MIR_get_ff_call (MIR_context_t ctx, size_t nres, MIR_type_t *res_t _MIR_arg_desc_t *arg_descs, size_t arg_vars_num); extern void *_MIR_get_interp_shim (MIR_context_t ctx, MIR_item_t func_item, void *handler); extern void *_MIR_get_thunk (MIR_context_t ctx); +extern void *_MIR_get_thunk_addr (MIR_context_t ctx, void *thunk); extern void _MIR_redirect_thunk (MIR_context_t ctx, void *thunk, void *to); +extern void *_MIR_get_jmpi_thunk (MIR_context_t ctx, void **res_loc, void *res, void *cont); extern void *_MIR_get_wrapper (MIR_context_t ctx, MIR_item_t called_func, void *hook_address); +extern void *_MIR_get_wrapper_end (MIR_context_t ctx); +extern void *_MIR_get_bb_thunk (MIR_context_t ctx, void *bb_version, void *handler); +extern void _MIR_replace_bb_thunk (MIR_context_t ctx, void *thunk, void *to); +extern void *_MIR_get_bb_wrapper (MIR_context_t ctx, void *data, void *hook_address); + +extern int _MIR_name_char_p (MIR_context_t ctx, int ch, int first_p); +extern void _MIR_dump_code (const char *name, uint8_t *code, size_t code_len); -extern void _MIR_dump_code (const char *name, int index, uint8_t *code, size_t code_len); +extern int _MIR_get_hard_reg (MIR_context_t ctx, const char *hard_reg_name); +extern void *_MIR_get_module_global_var_hard_regs (MIR_context_t ctx, MIR_module_t module); #ifdef __cplusplus } diff --git a/mir/real-time.h b/mir/real-time.h index 620a196f..1ff07cb6 100644 --- a/mir/real-time.h +++ b/mir/real-time.h @@ -1,5 +1,5 @@ /* This file is a part of MIR project. - Copyright (C) 2018-2023 Vladimir Makarov and logzero + Copyright (C) 2018-2024 Vladimir Makarov and logzero */ #ifndef _WIN32 diff --git a/src/ravi_mirjit.c b/src/ravi_mirjit.c index 961195c9..6f46ef8f 100644 --- a/src/ravi_mirjit.c +++ b/src/ravi_mirjit.c @@ -379,12 +379,12 @@ void *mir_get_func(MIR_context_t ctx, MIR_module_t module, const char *func_name fprintf(stderr, "Error: Compiled function %s not found\n", func_name); exit(1); } - return MIR_gen(ctx, 0, main_func); + return MIR_gen(ctx, main_func); } void mir_prepare(MIR_context_t ctx, int optlevel) { c2mir_init(ctx); - MIR_gen_init(ctx, optlevel); + MIR_gen_init(ctx); } void mir_cleanup(MIR_context_t ctx) { From 4c18d953fb7560019beb84624e249a5dee75abd7 Mon Sep 17 00:00:00 2001 From: dibyendumajumdar Date: Tue, 4 Jun 2024 22:30:02 +0100 Subject: [PATCH 2/5] issue #169 apply bug fix from upstream --- mir/mir-gen.c | 5 ++++- mir/mir.c | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mir/mir-gen.c b/mir/mir-gen.c index 3dd75934..b17a5d73 100644 --- a/mir/mir-gen.c +++ b/mir/mir-gen.c @@ -2100,7 +2100,10 @@ static int clone_bbs (gen_ctx_t gen_ctx) { } }); } - if (res) enumerate_bbs (gen_ctx); + if (res) { + remove_unreachable_bbs (gen_ctx); + enumerate_bbs (gen_ctx); + } return res; } diff --git a/mir/mir.c b/mir/mir.c index b38bee66..db7b4dbf 100644 --- a/mir/mir.c +++ b/mir/mir.c @@ -5863,14 +5863,14 @@ static void io_finish (MIR_context_t ctx) { #endif /* if !MIR_NO_IO */ /* New Page */ + +/* Reading MIR text file */ + int _MIR_name_char_p (MIR_context_t ctx MIR_UNUSED, int ch, int first_p) { if (isalpha (ch) || ch == '_' || ch == '$' || ch == '%' || ch == '.') return TRUE; return !first_p && isdigit (ch); } - -/* Reading MIR text file */ - #if !MIR_NO_SCAN #include @@ -6906,7 +6906,7 @@ void _MIR_dump_code (const char *name, uint8_t *code, size_t code_len) { #endif #else sprintf (bfname, "_mir_%lu.bin", (unsigned long) getpid ()); - if ((bf = fopen (bfname, "w")) == NULL) return; + if ((bf = fopen (bfname, "wb")) == NULL) return; fprintf (f, "void code (void) {}\n"); for (i = 0; i < code_len; i++) fputc (code[i], bf); fclose (f); From 39bb69dd0c47e4bfdcc6b4aa1d02e9c634e965d9 Mon Sep 17 00:00:00 2001 From: Dibyendu Majumdar Date: Sun, 9 Jun 2024 11:45:00 +0100 Subject: [PATCH 3/5] #258 Set opt level --- src/ravi_mirjit.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ravi_mirjit.c b/src/ravi_mirjit.c index 6f46ef8f..16aa0c66 100644 --- a/src/ravi_mirjit.c +++ b/src/ravi_mirjit.c @@ -385,6 +385,7 @@ void *mir_get_func(MIR_context_t ctx, MIR_module_t module, const char *func_name void mir_prepare(MIR_context_t ctx, int optlevel) { c2mir_init(ctx); MIR_gen_init(ctx); + MIR_gen_set_optimize_level(ctx, optlevel); } void mir_cleanup(MIR_context_t ctx) { From aacdd171f5a191fb1bc970efae097e17d2abc68f Mon Sep 17 00:00:00 2001 From: Dibyendu Majumdar Date: Sun, 9 Jun 2024 11:32:26 +0100 Subject: [PATCH 4/5] #257 Check if proto-code is available before accessing it (cherry picked from commit 25953d5b5318fa6f9307010583323f8cf434298b) --- src/ldebug.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ldebug.c b/src/ldebug.c index 569c6ae8..e78051fa 100644 --- a/src/ldebug.c +++ b/src/ldebug.c @@ -44,6 +44,9 @@ static const char *funcnamefromcode (lua_State *L, CallInfo *ci, static int currentpc (CallInfo *ci) { lua_assert(isLua(ci)); + /* If this function was compiled using the source to JIT then no code available */ + if (ci_func(ci)->p->code == NULL) + return 0; return pcRel(ci->u.l.savedpc, ci_func(ci)->p); } From 2c6f95ddeee52516e7b22ab8c9c6fbe8e2f968fd Mon Sep 17 00:00:00 2001 From: dibyendumajumdar Date: Sat, 15 Jun 2024 23:24:20 +0100 Subject: [PATCH 5/5] #258 apply upstream fix --- mir/mir-gen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mir/mir-gen.c b/mir/mir-gen.c index b17a5d73..85de5e9c 100644 --- a/mir/mir-gen.c +++ b/mir/mir-gen.c @@ -6131,7 +6131,7 @@ static int consider_move_vars_only (gen_ctx_t gen_ctx) { bitmap_clear (temp_bitmap); scan_vars_num = 0; scan_collected_moves (gen_ctx); - return scan_vars_num > 0; + return scan_vars_num > 0 && scan_vars_num < MIR_MAX_COALESCE_VARS; } static void add_bb_insn_dead_vars (gen_ctx_t gen_ctx) {