From 230df85b529e0dd0f97ebb14a83a6dedaa77fe31 Mon Sep 17 00:00:00 2001 From: Fireboyd78 Date: Tue, 30 Jan 2024 18:32:28 -0800 Subject: [PATCH 1/7] [Lucid Reloaded]: QoL updates, bugfixes, improvements, performance enhancements + more! Originally this started out as a personal side project of mine that I didn't expect to do much work on. But just like that, before you know it.. I ended up rewriting a majority of Lucid and majorly improved performance! Lucid is one of my most-used plugins in IDA, so this is my way of showing apprecation to Markus for his amazing work :) Considering I never planned on doing this much work, there may be some changes that are not listed below. [Changelist]: - Bumped version number up to 0.2 - Added my username to the list of authors when Lucid is loaded - Cleaned up LOTS of code, but there's still plenty of work to be done - Modernized some of the Python to 3.x (TODO: type hinting!!!) - Fixed quite a few bugs within the code that would cause exceptions, especially under different platforms (like MIPS) - Added 'Developer mode' checkbox for showing additional information of instructions. - Added 'Refresh view' button to force a refresh of the microcode viewer. - Added ability to navigate the lines within the explorer using the keyboard :) - Added some more block flags to display + changed formatting up a bit - Fixed the microcode view not being updated when the pseudocode is refreshed - Sub-instruction graph is now fully synchronized with the microcode view, and will update in realtime - Optimized generation of TokenCell objects and their derivatives - Created specialized helper classes for iterating through tokens - Created specialized helper class for registering/notifying callbacks - Miscellaneous QoL improvements that may not be listed here - Probably introduced a random bug or two (or five) somewhere.. only one way to find out! ;) --- plugins/lucid/core.py | 8 +- plugins/lucid/microtext.py | 482 +++++++++++++++++++++------------- plugins/lucid/text.py | 358 ++++++++++++++++++------- plugins/lucid/ui/explorer.py | 277 ++++++++++--------- plugins/lucid/ui/subtree.py | 172 +++++++++++- plugins/lucid/ui/sync.py | 12 +- plugins/lucid/util/options.py | 234 +++++++++++++++++ plugins/lucid/util/python.py | 43 ++- 8 files changed, 1155 insertions(+), 431 deletions(-) create mode 100644 plugins/lucid/util/options.py diff --git a/plugins/lucid/core.py b/plugins/lucid/core.py index d2c2f7e..d854adf 100644 --- a/plugins/lucid/core.py +++ b/plugins/lucid/core.py @@ -20,9 +20,9 @@ class LucidCore(object): PLUGIN_NAME = "Lucid" - PLUGIN_VERSION = "0.1.1" - PLUGIN_AUTHORS = "Markus Gaasedelen" - PLUGIN_DATE = "2020" + PLUGIN_VERSION = "0.2.0" + PLUGIN_AUTHORS = "Markus Gaasedelen, Fireboyd78" + PLUGIN_DATE = "2024" def __init__(self, defer_load=False): self.loaded = False @@ -102,7 +102,7 @@ def interactive_view_microcode(self, ctx=None): """ current_address = ida_kernwin.get_screen_ea() if current_address == ida_idaapi.BADADDR: - print("Could not open Microcode Explorer (bad cursor address)") + ida_kernwin.warning("Could not open Microcode Explorer (bad cursor address)") return # diff --git a/plugins/lucid/microtext.py b/plugins/lucid/microtext.py index 89ab91b..8bbdfe3 100644 --- a/plugins/lucid/microtext.py +++ b/plugins/lucid/microtext.py @@ -1,10 +1,15 @@ +from typing import Any +import ida_funcs import ida_lines import ida_idaapi import ida_hexrays +import ida_hexrays as hr # M.K. 1/23/2024 +import ida_kernwin from lucid.text import TextCell, TextToken, TextLine, TextBlock from lucid.util.ida import tag_text -from lucid.util.hexrays import get_mmat_name +from lucid.util.hexrays import get_mmat_name, get_microcode +from lucid.util.options import OptionListener, OptionProvider #----------------------------------------------------------------------------- # Microtext @@ -42,6 +47,14 @@ MAGIC_BLK_DNU = 0x1235 MAGIC_BLK_VAL = 0x1236 MAGIC_BLK_TERM = 0x1237 + + +MicrocodeOptions = OptionProvider({ + # options defined here can be accessed like normal class members ;) + 'developer_mode': False, + 'verbose': False, +}) + class BlockHeaderLine(TextLine): """ @@ -160,7 +173,9 @@ class MicroInstructionToken(TextToken): FLAGS = ida_hexrays.SHINS_VALNUM | ida_hexrays.SHINS_SHORT def __init__(self, insn, index, parent_token): - super(MicroInstructionToken, self).__init__(insn._print(self.FLAGS), parent=parent_token) + insn_text = insn._print(self.FLAGS) + + super(MicroInstructionToken, self).__init__(insn_text, parent=parent_token) self.index = index self.insn = insn self._generate_from_insn() @@ -194,6 +209,7 @@ def _create_subop(self, mop): return subop class InstructionCommentToken(TextToken): + """ A container token for micro-instruction comment text. """ @@ -211,7 +227,28 @@ def _generate_from_ins(self, blk, insn, usedef): # append the instruction address items.append(AddressToken(insn.ea)) - + + if MicrocodeOptions.developer_mode: + insn_flags = { + hr.IPROP_ASSERT: "ASSERT", + hr.IPROP_PERSIST: "PERSIST", + hr.IPROP_MBARRIER: "MBARRIER", + hr.IPROP_OPTIONAL: "OPT", + hr.IPROP_COMBINED: "COMB", + hr.IPROP_DONT_PROP: "NO_PROP", + hr.IPROP_DONT_COMB: "NO_COMB", + hr.IPROP_INV_JX: "INV_JX", + hr.IPROP_FPINSN: "FPINSN", + hr.IPROP_EXTSTX: "EXTSTX", + hr.IPROP_FARCALL: "FARCALL", + hr.IPROP_TAILCALL: "TAILCALL", + hr.IPROP_MULTI_MOV: "MULTI_MOV", + hr.IPROP_WAS_NORET: "WAS_NORET", + } + + if insn_tokens := [TextCell(name) for flag, name in insn_flags.items() if insn.iprops & flag]: + items.extend([TextCell(" ")] + [x for flag in insn_tokens for x in (TextCell(" +"), flag)]) + # append the use/def list if usedef: use_def_tokens = self._generate_use_def(blk, insn) @@ -234,15 +271,14 @@ def _generate_use_def(self, blk, insn): # use list must_use = blk.build_use_list(insn, ida_hexrays.MUST_ACCESS) may_use = blk.build_use_list(insn, ida_hexrays.MAY_ACCESS) - - use_str = generate_mlist_str(must_use, may_use) - items.append(TextCell(" u=%-13s" % use_str)) + if use_str := generate_mlist_str(must_use, may_use): + items.append(TextCell(" u=%-13s" % use_str)) # def list must_def = blk.build_def_list(insn, ida_hexrays.MUST_ACCESS) may_def = blk.build_def_list(insn, ida_hexrays.MAY_ACCESS) - def_str = generate_mlist_str(must_def, may_def) - items.append(TextCell(" d=%-13s" % def_str)) + if def_str := generate_mlist_str(must_def, may_def): + items.append(TextCell(" d=%-13s" % def_str)) return items @@ -267,19 +303,16 @@ class MicroBlockText(TextBlock): High level text wrapper of a micro-block (mblock_t). """ - def __init__(self, blk, verbose=False): + def __init__(self, blk): super(MicroBlockText, self).__init__() self.instructions = [] - self.verbose = verbose self.blk = blk self.refresh() - def refresh(self, verbose=None): + def refresh(self): """ Regenerate the micro-block text. """ - if verbose is not None: - self.verbose = verbose self._generate_from_blk() self._generate_lines() self._generate_token_address_map() @@ -325,24 +358,51 @@ def _generate_header_lines(self): } blk_type = type_names[blk.type] + blk_props = { + hr.MBL_PRIV: "PRIVATE", + hr.MBL_FAKE: "FAKE", + hr.MBL_NORET: "NORET", + hr.MBL_DSLOT: "DSLOT", + hr.MBL_GOTO: "GOTO", + hr.MBL_TCAL: "TAILCALL", + } + blk_flags ={ + hr.MBL_KEEP: "KEEP", + hr.MBL_PROP: "PROP", + hr.MBL_COMB: "COMB", + hr.MBL_PUSH: "PUSH", + hr.MBL_CALL: "CALL", + hr.MBL_DMT64: "DMT_64BIT", + hr.MBL_INCONST: "INCONST", + hr.MBL_BACKPROP: "BACKPROP", + hr.MBL_VALRANGES: "VALRANGES", + } # block properties prop_tokens = [] - - if blk.flags & ida_hexrays.MBL_DSLOT: - prop_tokens.append(TextCell("DSLOT")) - if blk.flags & ida_hexrays.MBL_NORET: - prop_tokens.append(TextCell("NORET")) - if blk.needs_propagation(): - prop_tokens.append(TextCell("PROP")) - if blk.flags & ida_hexrays.MBL_COMB: - prop_tokens.append(TextCell("COMB")) - if blk.flags & ida_hexrays.MBL_PUSH: - prop_tokens.append(TextCell("PUSH")) - if blk.flags & ida_hexrays.MBL_TCAL: - prop_tokens.append(TextCell("TAILCALL")) - if blk.flags & ida_hexrays.MBL_FAKE: - prop_tokens.append(TextCell("FAKE")) + flag_tokens = [] + + for flag, name in blk_props.items(): + if blk.flags & flag: + prop_tokens.append(TextCell(name)) + for flag, name in blk_flags.items(): + if blk.flags & flag: + flag_tokens.append(TextCell(name)) + + #if blk.flags & ida_hexrays.MBL_DSLOT: + # prop_tokens.append(TextCell("DSLOT")) + #if blk.flags & ida_hexrays.MBL_NORET: + # prop_tokens.append(TextCell("NORET")) + #if blk.needs_propagation(): + # prop_tokens.append(TextCell("PROP")) + #if blk.flags & ida_hexrays.MBL_COMB: + # prop_tokens.append(TextCell("COMB")) + #if blk.flags & ida_hexrays.MBL_PUSH: + # prop_tokens.append(TextCell("PUSH")) + #if blk.flags & ida_hexrays.MBL_TCAL: + # prop_tokens.append(TextCell("TAILCALL")) + #if blk.flags & ida_hexrays.MBL_FAKE: + # prop_tokens.append(TextCell("FAKE")) # misc block info prop_tokens = [x for prop in prop_tokens for x in (prop, TextCell(" "))] @@ -365,12 +425,26 @@ def _generate_header_lines(self): edge_tokens = [TextCell("- ")] + inbound_tokens + outbound_tokens lines.append(BlockHeaderLine(edge_tokens, MAGIC_BLK_EDGE, parent=self)) + if flag_tokens: + last_token = len(flag_tokens) - 1 + if last_token: + def _get_splitter(i): + return TextCell(" | ") if i != last_token else TextCell("]") + + flag_tokens_split = [x for i,flag in enumerate(flag_tokens) for x in (flag, _get_splitter(i))] + lines.append(BlockHeaderLine([TextCell("FLAGS: [")] + flag_tokens_split, MAGIC_BLK_UDNR, parent=self)) + else: + lines.append(BlockHeaderLine([TextCell("FLAGS: [")] + flag_tokens + [TextCell("]")], MAGIC_BLK_UDNR, parent=self)) + # only generate use/def comments if in verbose mode - if self.verbose: + if MicrocodeOptions.verbose: if not blk.lists_ready(): lines.append(BlockHeaderLine([TextCell("- USE-DEF LISTS ARE NOT READY")], MAGIC_BLK_UDNR, parent=self)) + elif use_defs := self._generate_use_def(blk): + lines.extend(use_defs) else: - lines.extend(self._generate_use_def(blk)) + lines.append(BlockHeaderLine([TextCell("- USE-DEF LISTS ARE EMPTY")], MAGIC_BLK_UDNR, parent=self)) + return lines @@ -379,21 +453,18 @@ def _generate_use_def(self, blk): Generate use/def comments for this block. """ lines = [] - - # use list - use_str = generate_mlist_str(blk.mustbuse, blk.maybuse) - if use_str: - lines.append(BlockHeaderLine([TextCell("- USE: %s" % use_str)], MAGIC_BLK_USE, parent=self)) - - # def list - def_str = generate_mlist_str(blk.mustbdef, blk.maybdef) - if def_str: - lines.append(BlockHeaderLine([TextCell("- DEF: %s" % def_str)], MAGIC_BLK_DEF, parent=self)) - - # dnu list - dnu_str = generate_mlist_str(blk.dnu) - if dnu_str: - lines.append(BlockHeaderLine([TextCell("- DNU: %s" % dnu_str)], MAGIC_BLK_DNU, parent=self)) + + usedef_lists = { + 'USE': (MAGIC_BLK_USE, (blk.mustbuse, blk.maybuse)), # use list + 'DEF': (MAGIC_BLK_DEF, (blk.mustbdef, blk.maybdef)), # def list + 'DNU': (MAGIC_BLK_DNU, (blk.dnu, None)), # dnu list + } + + for mblkname, mblkdata in usedef_lists.items(): + blkmagic, (blkmust, blkmay) = mblkdata + if list_str := generate_mlist_str(blkmust, blkmay): + line = BlockHeaderLine([TextCell(f"- {mblkname}: {list_str}")], blkmagic, parent=self) + lines.append(line) return lines @@ -402,7 +473,7 @@ def _generate_token_line(self, idx, ins_token): Generate a block/index prefixed line for a given instruction token. """ prefix_token = LinePrefixToken(self.blk.serial, idx) - cmt_token = InstructionCommentToken(self.blk, ins_token.insn, self.verbose) + cmt_token = InstructionCommentToken(self.blk, ins_token.insn) cmt_padding = max(50 - (len(prefix_token.text) + len(ins_token.text)), 1) padding_token = TextCell(" " * cmt_padding) @@ -456,18 +527,42 @@ class MicrocodeText(TextBlock): High level text wrapper of a micro-block-array (mba_t). """ - def __init__(self, mba, verbose=False): + def __init__(self, maturity): super(MicrocodeText, self).__init__() - self.verbose = verbose - self.mba = mba + self.maturity = maturity + + @classmethod + def create(cls, func, maturity): + """ + Create a new instance of the class and generate the microcode text. + """ + mtext = MicrocodeText(maturity) + mtext.func = func + mtext.reinit() + return mtext + + def copy(self): + """ + Create a copy of the microcode and generate its text. + """ + mtext = MicrocodeText(self.maturity) + mtext.func = self.func + mtext.mba = self.mba + mtext.refresh() + return mtext + + def reinit(self): + """ + Reinitialize the underlying microcode and regenerate text. + """ + self.func = ida_funcs.get_func(self.func.start_ea) + self.mba = get_microcode(self.func, self.maturity) self.refresh() - def refresh(self, verbose=None): + def refresh(self, maturity=None): """ Regenerate the microcode text. """ - if verbose is not None: - self.verbose = verbose self._generate_from_mba() self._generate_lines() self._generate_token_address_map() @@ -480,7 +575,7 @@ def _generate_from_mba(self): for blk_idx in range(self.mba.qty): blk = self.mba.get_mblock(blk_idx) - blk_token = MicroBlockText(blk, self.verbose) + blk_token = MicroBlockText(blk) blks.append(blk_token) self.blks = blks @@ -489,12 +584,23 @@ def _generate_lines(self): """ Populate the line array for this mba_t. """ - lines = [] - - for blk in self.blks: - lines.extend(blk.lines) - - self.lines = lines + self.lines = [line for blk in self.blks for line in blk.lines] + + def iter_block_token_preds(self, blk_token): + """ + Iterate through the blocks specified by the tokens predecessors. + """ + blk = blk_token.blk + for serial in range(blk.npred()): + yield self.blks[blk.pred(serial)] + + def iter_block_token_succs(self, blk_token): + """ + Iterate through the blocks specified by the tokens successors. + """ + blk = blk_token.blk + for serial in range(blk.nsucc()): + yield self.blks[blk.succ(serial)] def get_block_for_line(self, line): """ @@ -540,31 +646,36 @@ def generate_mlist_str(must, maybe=None): """ Generate the use/def string given must-use and maybe-use lists. """ - must_regs = must.reg.dstr().split(",") - must_mems = must.mem.dstr().split(",") - - maybe_regs = maybe.reg.dstr().split(",") if maybe else [] - maybe_mems = maybe.mem.dstr().split(",") if maybe else [] - - for splits in [must_regs, must_mems, maybe_regs, maybe_mems]: - splits[:] = list(filter(None, splits))[:] # lol - - maybe_regs = list(filter(lambda x: x not in must_regs, maybe_regs)) - maybe_mems = list(filter(lambda x: x not in must_mems, maybe_mems)) - - must_str = ', '.join(must_regs + must_mems) - maybe_str = ', '.join(maybe_regs + maybe_mems) - - if must_str and maybe_str: - full_str = "%s (%s)" % (must_str, maybe_str) - elif must_str: - full_str = must_str - elif maybe_str: - full_str = "(%s)" % maybe_str - else: - full_str = "" - return full_str + def get_usage_lists(must_use, may_use): + must_uses = list(filter(None, must_use.dstr().split(","))) + + def may_use_valid(may_use): + return may_use and may_use not in must_uses + + if not must_uses: + may_use_valid = None # no point in checking if in an empty list + + may_uses = list(filter(may_use_valid, may_use.dstr().split(","))) if may_use else [] + + return must_uses, may_uses + + must_regs, maybe_regs = get_usage_lists(must.reg, maybe.reg if maybe else None) + must_mems, maybe_mems = get_usage_lists(must.mem, maybe.mem if maybe else None) + + must_uses = must_regs + must_mems + maybe_uses = maybe_regs + maybe_mems + + if not must_uses and not maybe_uses: + return None + + must_str = ', '.join(must_uses) if must_uses else None + maybe_str = ', '.join(maybe_uses) if maybe_uses else None + + if maybe_str: + maybe_str = '({})'.format(maybe_str) + + return ' '.join(filter(None, (must_str, maybe_str))) def find_similar_block(blk_token_src, mtext_dst): """ @@ -576,20 +687,23 @@ def find_similar_block(blk_token_src, mtext_dst): # search through all the blocks in the target mba/mtext for a similar block for blk_token_dst in mtext_dst.blks: blk_dst = blk_token_dst.blk - - # 1 for 1 block match (start addr, end addr) - if (blk_dst.start == blk_src.start and blk_dst.end == blk_src.end): - return blk_token_dst - - # matching block starts - # TODO/COMMENT: explain the serial != 0 case - elif (blk_dst.start == blk_src.start and blk_dst.serial != 0): - fallbacks.append(blk_token_dst) - - # block got merged into another block + + fallback = None + + if blk_dst.start == blk_src.start: + if blk_dst.end == blk_src.end: + # 1 for 1 block match (start addr, end addr) + return blk_token_dst + if blk_dst.serial != 0: + # matching block starts + # TODO/COMMENT: explain the serial != 0 case + fallback = blk_token_dst elif (blk_dst.start < blk_src.start < blk_dst.end): - fallbacks.append(blk_token_dst) + # block got merged into another block + fallback = blk_token_dst + if fallback: + fallbacks.append(fallback) # # there doesn't appear to be any blocks in this mtext that seem similar to # the given block. this should seldom happen.. if ever ? @@ -655,57 +769,53 @@ def translate_block_header_position(position, mtext_src, mtext_dst): # get the line the given position falls within on the source mtext line = mtext_src.lines[line_num] + # get the block the given position falls within on the source mtext blk_token_src = mtext_src.get_block_for_line(line) # find a block in the dest mtext that seems to match the source block - blk_token_dst = find_similar_block(blk_token_src, mtext_dst) - - if blk_token_dst: + if blk_token_dst := find_similar_block(blk_token_src, mtext_dst): ins_src = set([x.address for x in blk_token_src.instructions]) - ins_dst = set([x.address for x in blk_token_dst.instructions]) - translate_header = (ins_src == ins_dst or blk_token_dst.blk.start == blk_token_src.blk.start) - else: - translate_header = False - - # - # if we think we have found a suitable matching block, translate the given - # position from the src block header to the destination one - # - - if translate_header: - - # get the equivalent header line from the destination block - line_dst = blk_token_dst.get_special_line(line.type) - + ins_dst = set([x.address for x in blk_token_dst.instructions]) + # - # if a matching header line doesn't exist in the dest, attempt - # to match the line 'depth' into the header instead.. this will - # help with the illusion of the 'stationary' user cursor + # if we think we have found a suitable matching block, translate the given + # position from the src block header to the destination one # - if not line_dst: - line_idx = blk_token_src.lines.index(line) + if (ins_src == ins_dst) or (blk_token_dst.blk.start == blk_token_src.blk.start): - try: + # get the equivalent header line from the destination block + line_dst = blk_token_dst.get_special_line(line.type) - line_dst = blk_token_dst.lines[line_idx] - if not line_dst.type: - raise # - # either the destination block didn't have enough lines - # to fufill the 'stationary' illusion, or the target - # line wasn't a block header line. in these cases, just - # fallback to mapping the cursor to the top of the block + # if a matching header line doesn't exist in the dest, attempt + # to match the line 'depth' into the header instead.. this will + # help with the illusion of the 'stationary' user cursor # - except: - line_dst = blk_token_dst.lines[0] + if not line_dst: + line_idx = blk_token_src.lines.index(line) + + try: + + line_dst = blk_token_dst.lines[line_idx] + if not line_dst.type: + raise + # + # either the destination block didn't have enough lines + # to fufill the 'stationary' illusion, or the target + # line wasn't a block header line. in these cases, just + # fallback to mapping the cursor to the top of the block + # + + except: + line_dst = blk_token_dst.lines[0] - # return the target/ - line_num_dst = mtext_dst.lines.index(line_dst) - return (line_num_dst, 0, y) + # return the target/ + line_num_dst = mtext_dst.lines.index(line_dst) + return (line_num_dst, 0, y) # # if the block header the cursor was on in the source mtext has @@ -720,9 +830,8 @@ def translate_block_header_position(position, mtext_src, mtext_dst): # for ins_token in blk_token_src.instructions: - tokens_dst = mtext_dst.get_tokens_for_address(ins_token.address) - if tokens_dst: - line_num, x = mtext_dst.get_pos_of_token(tokens_dst[0]) + if token_dst := mtext_dst.get_first_token_for_address(ins_token.address): + line_num, x = mtext_dst.get_pos_of_token(token_dst) return (line_num, x, y) # @@ -737,7 +846,6 @@ def translate_instruction_position(position, mtext_src, mtext_dst): Translate an instruction position from one mtext to another. """ line_num, x, y = position - token_src = mtext_src.get_token_at_position(line_num, x) address_src = mtext_src.get_address_at_position(line_num, x) # @@ -745,8 +853,7 @@ def translate_instruction_position(position, mtext_src, mtext_dst): # current address # - line_nums_dst = mtext_dst.get_line_nums_for_address(address_src) - if not line_nums_dst: + if not mtext_dst.line_nums_contain_address(address_src): return None # get the line the given position falls within on the source mtext @@ -754,7 +861,6 @@ def translate_instruction_position(position, mtext_src, mtext_dst): # get the block the given position falls within on the source mtext blk_token_src = mtext_src.get_block_for_line(line) - blk_src = blk_token_src.blk # find a block in the dest mtext that seems to match the source block blk_token_dst = find_similar_block(blk_token_src, mtext_dst) @@ -766,11 +872,7 @@ def translate_instruction_position(position, mtext_src, mtext_dst): # our position in the source mtext. # - if blk_token_dst: - blk_dst = blk_token_dst.blk - tokens = blk_token_dst.get_tokens_for_address(address_src) - else: - tokens = [] + tokens = blk_token_dst.get_tokens_for_address(address_src) if blk_token_dst else [] # # no tokens matching the target address in the 'similar' dest block (or @@ -782,39 +884,46 @@ def translate_instruction_position(position, mtext_src, mtext_dst): tokens = mtext_dst.get_tokens_for_address(address_src) assert tokens, "This should never happen because line_nums_dst... ?" + token_src = mtext_src.get_token_at_position(line_num, x) + # compute the relative cursor address into the token text _, x_base_src = mtext_src.get_pos_of_token(token_src) x_rel = (x - x_base_src) - + # 1 for 1 token match for token in tokens: if token.text == token_src.text: line_num_dst, x_dst = mtext_dst.get_pos_of_token(token) x_dst += x_rel return (line_num_dst, x_dst, y) - - # common 'ancestor', eg the target token actually got its address from an ancestor - token_src_ancestor = token_src.ancestor_with_address() - for token in tokens: - line_num, x_dst = mtext_dst.get_pos_of_token(token) - if token.text == token_src_ancestor.text: - line_num, x_dst = mtext_dst.get_pos_of_token(token) - x_dst_base = token.text.index(token_src.text) - x_dst += x_dst_base + x_rel # oof - return (line_num, x_dst, y) - - # last ditch effort, try to land on a text that matches the target token - for token in tokens: + + def get_best_ancestor_token(): + # common 'ancestor', eg the target token actually got its address from an ancestor + token_src_ancestor = token_src.ancestor_with_address() + for token in tokens: + if token.text == token_src_ancestor.text: + return token + # last ditch effort, try to land on a text that matches the target token + for token in tokens: + if token_src.text in token.text: + return token + return None + + if token := get_best_ancestor_token(): line_num, x_dst = mtext_dst.get_pos_of_token(token) - if token_src.text in token.text: - line_num, x_dst = mtext_dst.get_pos_of_token(token) - x_dst_base = token.text.index(token_src.text) - x_dst += x_dst_base + x_rel # oof - return (line_num, x_dst, y) + x_dst_base = token.text.index(token_src.text) + x_dst += x_dst_base + x_rel # oof + return (line_num, x_dst, y) # yolo, just land on whatever token available line_num, x = mtext_dst.get_pos_of_token(tokens[0]) return (line_num, x, y) + + + + + + #----------------------------------------------------------------------------- # Position Remapping @@ -855,12 +964,10 @@ def remap_mtext_position(position, mtext_src, mtext_dst): # line_max = len(mtext_dst.lines) - if position[0] < line_max: - line_num = position[0] - else: + if line_num >= line_max: line_num = max(line_max - 1, 0) - - return (line_num, position[1], position[2]) + + return (line_num, x, y) def remap_block_header_position(position, mtext_src, mtext_dst): """ @@ -880,22 +987,19 @@ def remap_block_header_position(position, mtext_src, mtext_dst): if blk_token in blks_visited: continue - blk_token_dst = find_similar_block(blk_token, mtext_dst) - if blk_token_dst: + if blk_token_dst := find_similar_block(blk_token, mtext_dst): line_num, x = mtext_dst.get_pos_of_token(blk_token_dst.lines[0]) return (line_num, x, y) - remap_tokens = [blk_token.instructions[0]] if blk_token.instructions else [] - - for token in remap_tokens: - insn_line_num, insn_x = mtext_src.get_pos_of_token(token) - projection = remap_instruction_position((insn_line_num, insn_x, y), mtext_src, mtext_dst) - if projection: + if blk_token.instructions: + remap_token = blk_token.instructions[0] + insn_line_num, insn_x = mtext_src.get_pos_of_token(remap_token) + if projection := remap_instruction_position((insn_line_num, insn_x, y), mtext_src, mtext_dst): return (projection[0], projection[1], y) - for blk_serial in range(blk_token.blk.nsucc()): - blk_token_succ = mtext_src.blks[blk_token.blk.succ(blk_serial)] - if blk_token_succ in blks_visited or blk_token_succ in blks_to_visit: + for blk_token_succ in mtext_src.iter_block_token_succs(blk_token): + if blk_token_succ in blks_visited \ + or blk_token_succ in blks_to_visit: continue blks_to_visit.append(blk_token_succ) @@ -910,20 +1014,20 @@ def remap_instruction_position(position, mtext_src, mtext_dst): # the block in the source mtext where the given position resides blk_token_src = mtext_src.get_block_for_line(line) - blk_src = blk_token_src.blk - ins_token_src = mtext_src.get_ins_for_line(line) + pred_addresses = [x.address for x in blk_token_src.instructions[:ins_token_src.index]] succ_addresses = [x.address for x in blk_token_src.instructions[ins_token_src.index+1:]] - remap_targets = succ_addresses + pred_addresses[::-1] - - for blk_serial in range(blk_src.nsucc()): - remap_targets += [x.address for x in mtext_src.blks[blk_src.succ(blk_serial)].instructions] - - for address in remap_targets: - new_tokens = mtext_dst.get_tokens_for_address(address) - if new_tokens: - line_num, x = mtext_dst.get_pos_of_token(new_tokens[0]) + + def iter_possible_remap_targets(): + yield from succ_addresses + yield from pred_addresses[::-1] + for blk_token_succ in mtext_src.iter_block_token_succs(blk_token_src): + yield from [x.address for x in blk_token_succ.instructions] + + for address in iter_possible_remap_targets(): + if token := mtext_dst.get_first_token_for_address(address): + line_num, x = mtext_dst.get_pos_of_token(token) return (line_num, x, y) # diff --git a/plugins/lucid/text.py b/plugins/lucid/text.py index 3f5009d..78cfcdb 100644 --- a/plugins/lucid/text.py +++ b/plugins/lucid/text.py @@ -1,4 +1,5 @@ import collections +from collections.abc import Iterable import ida_lines import ida_idaapi @@ -30,12 +31,17 @@ def __init__(self, text="", parent=None): # public attributes self.parent = parent self.address = ida_idaapi.BADADDR + + def get_children(self): + """ + Returns any children of this text cell. Default behavior is to return nothing. + """ + return None def ancestor_with_address(self): """ Return the first parent of this cell that has a valid address. """ - address = ida_idaapi.BADADDR token = self.parent # iterate upwards through the target token parents until an address is found @@ -61,6 +67,174 @@ def tagged_text(self): """ return self._tagged_text + +class TokenAddressIterator: + """ + Custom helper class for iterating over tokens that exist at a specific address. + """ + + @staticmethod + def is_valid(token): + return token and issubclass(type(token), TextCell) + + def __init__(self, items, address, max_count = 0): + addrmap = False + + if isinstance(items, dict): + if addresses := items.get(address, []): + items = addresses + addrmap = True + else: + items = items.items() + + if issubclass(type(items), Iterable): + items = list(items) + else: + raise TypeError(items) + + self._addrmap = addrmap + self._items = items + self._address = address + + assert max_count >= 0, "max_count cannot be a negative number" + + self._max_count = max_count + self._setup_iter() + + def _setup_iter(self): + self._myiter = filter(self.is_valid, self._items) + self._subiter = None + self._exhausted = False + self._remain = self._max_count if self._max_count else 0 + + def __iter__(self): + self._setup_iter() + return self + + def _get_next_token(self): + if self._subiter: + # try to return the next sub-token + try: + return next(self._subiter) + except StopIteration as e: + # no more sub-tokens are left :( + self._subiter = None + + if not self._myiter: + # well, this is strange...I'm sure it'll happen lol + raise Exception("iterator undefined?!") + + # grab the next token + return next(self._myiter) + + def __next__(self): + addrmap = self._addrmap # items already of the target address? + while not self._exhausted: + try: + token = self._get_next_token() + if not token: + # end of subtokens + continue + + if not addrmap and token.address > self._address: + # too far ahead, stop iterating through our tokens + raise StopIteration + + if issubclass(type(token), TextToken): + # prepare to check its subtokens next + if items := token.get_token_items(): + self._subiter = TokenAddressIterator(items, self._address) + + if addrmap or token.address == self._address: + # only get up to max_count tokens, if specified + if self._remain: + self._remain -= 1 + if not self._remain: + self._exhausted = True + # we found a match! + return token + except StopIteration as e: + # we're all out of tokens now :( + self._exhausted = True + if self._exhausted: + raise StopIteration + + +class TokenRange: + """ + Custom helper class for managing token ranges. + """ + + def __init__(self, token, start, end): + self._token = token + self._issubclass = issubclass(type(token), TextToken) + self._start = start + self._end = end + + def __contains__(self, index): + return index >= self._start and index < self._end + + def empty(self): + return not self._token + + def get_index(self, token): + """ + Returns the index of the specified token, if it exists; otherwise, None + """ + + if self._token == token: + return self._start + if self._issubclass: + if index := self._token.get_index_of_token(token) is not None: + return self._start + index + return None + + def get_token(self, x_index): + """ + Returns the token at the specified index, if it exists; otherwise, None + """ + + if self.empty() or x_index not in self: + return None + + token = self._token + + # + # if the matching child token does not derive from a TextToken, it is + # probably a TextCell which cannot nest other tokens. so we can simply + # return the found token as it is a leaf + # + + if not self._issubclass: + return token + + # + # the matching token must derive from a TextToken or something + # capable of nesting tokens, so recurse downwards through the text + # structure to see if there is a deeper, more precise token that + # can be returned + # + + return token.get_token_at_index(x_index - self._start) + + + @property + def token(self): + return self._token + + @property + def start(self): + return self._start + + @property + def end(self): + return self._end + + @property + def length(self): + return self._end - self._start + + class TextToken(TextCell): """ A text element that can nest similar text-based elements. @@ -88,31 +262,47 @@ def _generate_token_ranges(self): token_ranges = [] parsing_offset = 0 + debug_lines = [f"**** {len(self.items)} tokens"] for token in self.items: - token_index = self.text[parsing_offset:].index(token.text) - token_start = parsing_offset + token_index + debug_lines.append(f"** find token '{token.text}' in '{self.text}'\n\tat {parsing_offset}: '{self.text[parsing_offset:]}'") + + token_start = self.text.index(token.text, parsing_offset) token_end = token_start + len(token.text) - token_ranges.append((range(token_start, token_end), token)) + + debug_lines.append(f"** - start={token_start}, end={token_end} => '{self.text[token_start:token_end]}'") + + token_range = TokenRange(token, token_start, token_end) + token_ranges.append(token_range) + + if token_start > 2 and self.text.find(',', parsing_offset, token_start) > parsing_offset: + debug_lines.append("**** skipped setting the parsing offset!") + continue + parsing_offset = token_end + + #print('\n'.join(debug_lines)) self._token_ranges = token_ranges #------------------------------------------------------------------------- # Textual APIs #------------------------------------------------------------------------- - - def get_tokens_for_address(self, address): + + def get_token_items(self): + return self.items + + def get_tokens_for_address(self, address, max_count = 0): """ Return all (child) tokens matching the given address. """ - found = [self] if self.address == address else [] - for token in self.items: - if not issubclass(type(token), TextToken): - if token.address == address: - found.append(token) - continue - found.extend(token.get_tokens_for_address(address)) - return found + it = TokenAddressIterator(self.get_token_items(), address, max_count=max_count) + return list(it) + + def get_first_token_for_address(self, address): + """ + Return first (child) token matching the given address. + """ + return next(iter(self.get_tokens_for_address(address, 1) + [None])) def get_index_of_token(self, target_token): """ @@ -121,14 +311,9 @@ def get_index_of_token(self, target_token): if target_token == self: return 0 - for token_range, token in self._token_ranges: - if token == target_token: - return token_range[0] - if not issubclass(type(token), TextToken): - continue - found = token.get_index_of_token(target_token) - if found is not None: - return found + token_range[0] + for token_range in self._token_ranges: + if (index := token_range.get_index(target_token)) is not None: + return index return None @@ -143,40 +328,13 @@ def get_token_at_index(self, x_index): # if the given index falls within any of them # - for token_range, token in self._token_ranges: - - # skip 'blank' children - if not token.text: - continue - - if x_index in token_range: - break - - # - # if the given index does not fall within a child token range, the - # given index must fall on text that makes up this token itself - # - - else: - return self - - # - # if the matching child token does not derive from a TextToken, it is - # probably a TextCell which cannot nest other tokens. so we can simply - # return the found token as it is a leaf - # - - if not issubclass(type(token), TextToken): - return token - - # - # the matching token must derive from a TextToken or something - # capable of nesting tokens, so recurse downwards through the text - # structure to see if there is a deeper, more precise token that - # can be returned - # + for token_range in self._token_ranges: + if (token := token_range.get_token(x_index)) is not None: + #print(f"**** token '{token.text}' found at {x_index} in '{self.text}'") + return token - return token.get_token_at_index(x_index - token_range[0]) + #print(f"**** token not found at {x_index} in '{self.text}'") + return self def get_address_at_index(self, x_index): """ @@ -267,27 +425,22 @@ def _generate_token_address_map(self): """ Generate a map of token --> address. """ - to_visit = [] - for line_idx, line in enumerate(self.lines): - to_visit.append((line_idx, line)) - - line_map = collections.defaultdict(list) + addr_map = collections.defaultdict(list) - - while to_visit: - line_idx, token = to_visit.pop(0) - line_map[line_idx].append(token) + + def _iter_token(token): + yield token for subtoken in token.items: - line_map[line_idx].append(subtoken) - if not issubclass(type(subtoken), TextToken): - continue - to_visit.append((line_idx, subtoken)) - if subtoken.address == ida_idaapi.BADADDR: - continue - addr_map[subtoken.address].append(subtoken) - + yield subtoken + if issubclass(type(subtoken), TextToken): + if subtoken.address != ida_idaapi.BADADDR: + addr_map[subtoken.address].append(subtoken) + yield from _iter_token(subtoken) + self._ea2token = addr_map - self._line2token = line_map + self._line2token = { + line_idx:list(_iter_token(token)) for line_idx, token in enumerate(self.lines) + } #------------------------------------------------------------------------- # Properties @@ -305,21 +458,30 @@ def tagged_text(self): # Textual APIs #------------------------------------------------------------------------- + def get_line_token(self, line_num): + """ + Return the token at the given line number. + """ + if not (0 <= line_num < len(self.lines)): + return None + return self.lines[line_num] + def get_token_at_position(self, line_num, x_index): """ Return the token at the given text position. """ - if not(0 <= line_num < len(self.lines)): - return None - return self.lines[line_num].get_token_at_index(x_index) + if line := self.get_line_token(line_num): + return line.get_token_at_index(x_index) + return None def get_address_at_position(self, line_num, x_index): """ Return the mapped address of the given text position. """ - if not(0 <= line_num < len(self.lines)): - return ida_idaapi.BADADDR - return self.lines[line_num].get_address_at_index(x_index) + if line := self.get_line_token(line_num): + return line.get_address_at_index(x_index) + # TODO: explain why ? + return ida_idaapi.BADADDR def get_pos_of_token(self, target_token): """ @@ -328,30 +490,44 @@ def get_pos_of_token(self, target_token): for line_num, tokens in self._line2token.items(): if target_token in tokens: return (line_num, self.lines[line_num].get_index_of_token(target_token)) + raise Exception(f"**** target_token '{target_token.text}' NOT found!!!") return None - def get_tokens_for_address(self, address): + def get_tokens_for_address(self, address, max_count = 0): + """ + Return all (child) tokens matching the given address. + """ + it = TokenAddressIterator(self._ea2token, address, max_count=max_count) + return list(it) + + def get_first_token_for_address(self, address): + """ + Return first (child) token matching the given address. + """ + return next(iter(self.get_tokens_for_address(address, 1) + [None])) + + def line_nums_contain_address(self, address): """ - Return the list of tokens matching the given address. + Returns whether the address exists within any of the line numbers. """ - return self._ea2token.get(address, []) + for _,tokens in self._line2token.items(): + for token in tokens: + if token.address == address: + return True + return False def get_line_nums_for_address(self, address): """ Return a list of line numbers which contain tokens matching the given address. """ line_nums = set() - for line_idx, tokens in self._line2token.items(): - for token in tokens: - if token.address == address: - line_nums.add(line_idx) - return list(line_nums) + if items := self._line2token.items(): + line_nums = {line for line,tokens in items \ + for _ in filter(lambda t: t.address == address, tokens)} + return line_nums def get_addresses_for_line_num(self, line_num): """ Return a list of addresses contained by tokens on the given line number. """ - addresses = set() - for token in self._line2token.get(line_num, []): - addresses.add(token.address) - return list(addresses) \ No newline at end of file + return set(token.address for token in self._line2token.get(line_num, [])) \ No newline at end of file diff --git a/plugins/lucid/ui/explorer.py b/plugins/lucid/ui/explorer.py index 3429a3e..c173a42 100644 --- a/plugins/lucid/ui/explorer.py +++ b/plugins/lucid/ui/explorer.py @@ -11,9 +11,10 @@ from lucid.ui.sync import MicroCursorHighlight from lucid.ui.subtree import MicroSubtreeView -from lucid.util.python import register_callback, notify_callback +from lucid.util.python import register_callback, notify_callback, CallbackHandler from lucid.util.hexrays import get_microcode, get_mmat, get_mmat_name, get_mmat_levels -from lucid.microtext import MicrocodeText, MicroInstructionToken, MicroOperandToken, AddressToken, BlockNumberToken, translate_mtext_position, remap_mtext_position +from lucid.util.options import OptionListener, OptionProvider +from lucid.microtext import MicrocodeOptions, MicrocodeText, MicroInstructionToken, MicroOperandToken, AddressToken, BlockNumberToken, translate_mtext_position, remap_mtext_position #------------------------------------------------------------------------------ # Microcode Explorer @@ -35,6 +36,7 @@ class MicrocodeExplorer(object): """ def __init__(self): + self.graph = None self.model = MicrocodeExplorerModel() self.view = MicrocodeExplorerView(self, self.model) self.view._code_sync.enable_sync(True) # XXX/HACK @@ -52,13 +54,27 @@ def show_subtree(self, insn_token): """ Show the sub-instruction graph for the given instruction token. """ - graph = MicroSubtreeView(insn_token.insn) + if self.graph: + self.graph.show() + return + + graph = MicroSubtreeView(insn_token.insn, self) graph.show() - - # TODO/HACK: this is dumb, but moving it breaks my centering code so - # i'll figure it out later... - gv = ida_graph.get_graph_viewer(graph.GetWidget()) - ida_graph.viewer_set_titlebar_height(gv, 15) + + self.graph = graph + + def update_subtree(self): + """ + Updates the sub-instruction graph if it is currently opened. + """ + if self.graph: + self.graph.update_insn() + + def free_subtree(self): + """ + Frees references to the sub-instruction graph. + """ + self.graph = None #------------------------------------------------------------------------- # View Toggles @@ -74,12 +90,13 @@ def set_highlight_mutual(self, status): self.view._code_sync.unhook() ida_kernwin.refresh_idaview_anyway() - def set_verbose(self, status): + def set_option(self, name, value): """ - Toggle the verbosity of the printed microcode text. + Sets the named microcode option with the given value. """ - self.model.verbose = status - ida_kernwin.refresh_idaview_anyway() + if not isinstance(name, str): + raise TypeError(name) + MicrocodeOptions[name] = value #------------------------------------------------------------------------- # View Controls @@ -94,8 +111,7 @@ def select_function(self, address): return False for maturity in get_mmat_levels(): - mba = get_microcode(func, maturity) - mtext = MicrocodeText(mba, self.model.verbose) + mtext = MicrocodeText.create(func, maturity) self.model.update_mtext(mtext, maturity) self.view.refresh() @@ -113,18 +129,19 @@ def select_address(self, address): """ Select a token in the microcode view matching the given address. """ - tokens = self.model.mtext.get_tokens_for_address(address) - if not tokens: + token = self.model.mtext.get_first_token_for_address(address) + if not token: return None - token_line_num, token_x = self.model.mtext.get_pos_of_token(tokens[0]) + token_line_num, token_x = self.model.mtext.get_pos_of_token(token) rel_y = self.model.current_position[2] - if self.model.current_position[2] == 0: + if rel_y == 0: rel_y = 30 - self.model.current_position = (token_line_num, token_x, rel_y) - return tokens[0] + self.select_position(token_line_num, token_x, rel_y) + + return token def select_position(self, line_num, x, y): """ @@ -140,18 +157,26 @@ def activate_position(self, line_num, x, y): Activate (eg. double click) the given text position in the microcode view. """ token = self.model.mtext.get_token_at_position(line_num, x) + if not token: + return if isinstance(token, AddressToken): ida_kernwin.jumpto(token.target_address, -1, 0) - return - - if isinstance(token, BlockNumberToken) or (isinstance(token, MicroOperandToken) and token.mop.t == ida_hexrays.mop_b): - blk_idx = token.blk_idx if isinstance(token, BlockNumberToken) else token.mop.b + else: + blk_idx = None + + if isinstance(token, BlockNumberToken): + blk_idx = token.blk_idx + elif isinstance(token, MicroOperandToken) and token.mop.t == ida_hexrays.mop_b: + blk_idx = token.mop.b + + if blk_idx is None: + return + blk_token = self.model.mtext.blks[blk_idx] blk_line_num, _ = self.model.mtext.get_pos_of_token(blk_token.lines[0]) - self.model.current_position = (blk_line_num, 0, y) - self.view._code_view.Jump(*self.model.current_position) - return + + self.select_position(blk_line_num, 0, y) class MicrocodeExplorerModel(object): """ @@ -163,7 +188,7 @@ class MicrocodeExplorerModel(object): """ def __init__(self): - + # # 'mtext' is short for MicrocodeText objects (see microtext.py) # @@ -194,28 +219,36 @@ def __init__(self): self._active_maturity = ida_hexrays.MMAT_GENERATED - # this flag tracks the verbosity toggle state - self._verbose = False - #---------------------------------------------------------------------- # Callbacks #---------------------------------------------------------------------- - - self._mtext_refreshed_callbacks = [] - self._position_changed_callbacks = [] - self._maturity_changed_callbacks = [] + + self.mtext_changed = CallbackHandler(self, name="mtext changed") + self.mtext_refreshed = CallbackHandler(self, name="mtext refreshed") + self.position_changed = CallbackHandler(self, name="position changed") + self.maturity_changed = CallbackHandler(self, name="maturity changed") + #------------------------------------------------------------------------- # Read-Only Properties #------------------------------------------------------------------------- @property - def mtext(self): + def mtext(self) -> MicrocodeText: """ Return the microcode text mapping for the current maturity level. """ return self._mtext[self._active_maturity] + @mtext.setter + def mtext(self, mtext): + """ + Set the microcode text mapping for the current maturity level. + """ + old_mtext = self.mtext + self._mtext[self._active_maturity] = mtext + self.mtext_changed(old_mtext, mtext) + @property def current_line(self): """ @@ -273,28 +306,7 @@ def current_position(self, value): Set the cursor position of the viewport. """ self._gen_cursors(value, self.active_maturity) - self._notify_position_changed() - - @property - def verbose(self): - """ - Return the microcode verbosity status of the viewport. - """ - return self._verbose - - @verbose.setter - def verbose(self, value): - """ - Set the verbosity of the microcode displayed by the viewport. - """ - if self._verbose == value: - return - - # update the active verbosity setting - self._verbose = value - - # verbosity must have changed, so force a mtext refresh - self.refresh_mtext() + self.position_changed() @property def active_maturity(self): @@ -309,34 +321,44 @@ def active_maturity(self, new_maturity): Set the active microcode maturity level. """ self._active_maturity = new_maturity - self._notify_maturity_changed() + self.maturity_changed() #---------------------------------------------------------------------- # Misc #---------------------------------------------------------------------- - + def update_mtext(self, mtext, maturity): """ Set the mtext for a given microcode maturity level. """ self._mtext[maturity] = mtext self._view_cursors[maturity] = ViewCursor(0, 0, 0) - - def refresh_mtext(self): + + def refresh_mtext(self, reinit = False, force_all = False): """ Regenerate the rendered text for all microcode maturity levels. - - TODO: This is a bit sloppy, and is basically only used for the - verbosity toggle. """ + old_mtext = None + new_mtext = None + + # XXX: this is also a bit hacky, unfortunately for maturity, mtext in self._mtext.items(): - if maturity == self.active_maturity: - new_mtext = MicrocodeText(mtext.mba, self.verbose) - self._mtext[maturity] = new_mtext - self.current_position = translate_mtext_position(self.current_position, mtext, new_mtext) - continue - mtext.refresh(self.verbose) - self._notify_mtext_refreshed() + if maturity == self.active_maturity or force_all: + if not reinit: + old_mtext = mtext + else: + # fully rebuild it + mtext.reinit() + else: + # refresh it since it's out of view + mtext.refresh() + + if old_mtext: + # make a new copy of it + self.mtext = new_mtext = old_mtext.copy() + self.current_position = translate_mtext_position(self.current_position, old_mtext, new_mtext) + + self.mtext_refreshed() def _gen_cursors(self, position, mmat_src): """ @@ -354,17 +376,15 @@ def _gen_cursors(self, position, mmat_src): # map the cursor backwards from the source maturity mmat_lower = range(mmat_first, mmat_src)[::-1] - current_maturity = mmat_src - for next_maturity in mmat_lower: - self._transfer_cursor(current_maturity, next_maturity) - current_maturity = next_maturity - + # map the cursor forward from the source maturity mmat_higher = range(mmat_src+1, mmat_final + 1) - current_maturity = mmat_src - for next_maturity in mmat_higher: - self._transfer_cursor(current_maturity, next_maturity) - current_maturity = next_maturity + + for mmat_range in (mmat_lower, mmat_higher): + current_maturity = mmat_src + for next_maturity in mmat_range: + self._transfer_cursor(current_maturity, next_maturity) + current_maturity = next_maturity def _transfer_cursor(self, mmat_src, mmat_dst): """ @@ -389,64 +409,37 @@ def _transfer_cursor(self, mmat_src, mmat_dst): # Callbacks #---------------------------------------------------------------------- - def mtext_refreshed(self, callback): - """ - Subscribe a callback for mtext refresh events. - """ - register_callback(self._mtext_refreshed_callbacks, callback) - - def _notify_mtext_refreshed(self): - """ - Notify listeners of a mtext refresh event. - """ - notify_callback(self._mtext_refreshed_callbacks) - - def position_changed(self, callback): - """ - Subscribe a callback for cursor position changed events. - """ - register_callback(self._position_changed_callbacks, callback) - - def _notify_position_changed(self): - """ - Notify listeners of a cursor position changed event. - """ - notify_callback(self._position_changed_callbacks) - - def maturity_changed(self, callback): - """ - Subscribe a callback for maturity changed events. - """ - register_callback(self._maturity_changed_callbacks, callback) - - def _notify_maturity_changed(self): - """ - Notify listeners of a maturity changed event. - """ - notify_callback(self._maturity_changed_callbacks) #----------------------------------------------------------------------------- # UI Components #----------------------------------------------------------------------------- -class MicrocodeExplorerView(QtWidgets.QWidget): +class MicrocodeExplorerView(OptionListener, QtWidgets.QWidget, providers = [ MicrocodeOptions ]): """ The view component of the Microcode Explorer. """ WINDOW_TITLE = "Microcode Explorer" - def __init__(self, controller, model): + def __init__(self, controller: MicrocodeExplorer, model: MicrocodeExplorerModel): super(MicrocodeExplorerView, self).__init__() self.visible = False # the backing model, and controller for this view (eg, mvc pattern) self.model = model self.controller = controller - + # initialize the plugin UI self._ui_init() self._ui_init_signals() + + def notify_change(self, option_name, option_value, **kwargs): + """ + Implementation of OptionListener.notify_change for when a microcode option has been updated. + """ + #print(f"**** notify_change {option_name} = {option_value} (IN OPTIONS={bool(option_name in MicrocodeOptions)})") + self.model.refresh_mtext() + ida_kernwin.refresh_idaview_anyway() #-------------------------------------------------------------------------- # Pseudo Widget Functions @@ -508,6 +501,17 @@ def widget_invisible(_, twidget): def widget_visible(_, twidget): if twidget == self._twidget: self.visible = True + def postprocess_action(_, *args): + # XXX: seemingly the only way to allow the explorer to navigate via keyboard events... + # (maybe this should be hooked elsewhere?) + if not self._code_view or not self._code_view.IsFocused(): + return + + old_line = self.model.current_line + new_line = self._code_view.GetLineNo() + + if new_line != old_line: + self.controller.select_position(*self._code_view.GetPos()) # install the widget lifetime hooks self._ui_hooks = ExplorerUIHooks() @@ -525,7 +529,7 @@ def _ui_init_code(self): """ self._code_view = MicrocodeView(self.model) self._code_sync = MicroCursorHighlight(self.controller, self.model) - self._code_sync.track_view(self._code_view.widget) + self._code_sync.track_view(self._code_view) def _ui_init_settings(self): """ @@ -536,12 +540,18 @@ def _ui_init_settings(self): self._checkbox_verbose = QtWidgets.QCheckBox("Show use/def") self._checkbox_sync = QtWidgets.QCheckBox("Sync hexrays") self._checkbox_sync.setCheckState(QtCore.Qt.Checked) + self._checkbox_devmode = QtWidgets.QCheckBox("Developer mode") + + self._refresh_button = QtWidgets.QPushButton("Refresh view") + self._refresh_button.setFixedSize(120, 60) self._groupbox_settings = QtWidgets.QGroupBox("Settings") layout = QtWidgets.QVBoxLayout() layout.addWidget(self._checkbox_cursor) layout.addWidget(self._checkbox_verbose) layout.addWidget(self._checkbox_sync) + layout.addWidget(self._checkbox_devmode) + layout.addWidget(self._refresh_button) self._groupbox_settings.setLayout(layout) def _ui_layout(self): @@ -565,20 +575,34 @@ def _ui_init_signals(self): self._maturity_list.currentItemChanged.connect(lambda x, y: self.controller.select_maturity(x.text())) self._code_view.connect_signals(self.controller) self._code_view.OnClose = self.hide # HACK + + self._refresh_button.clicked.connect(lambda: self.reinit()) # checkboxes self._checkbox_cursor.stateChanged.connect(lambda x: self.controller.set_highlight_mutual(bool(x))) - self._checkbox_verbose.stateChanged.connect(lambda x: self.controller.set_verbose(bool(x))) + self._checkbox_verbose.stateChanged.connect(lambda x: self.controller.set_option('verbose', bool(x))) self._checkbox_sync.stateChanged.connect(lambda x: self._code_sync.enable_sync(bool(x))) + self._checkbox_devmode.stateChanged.connect(lambda x: self.controller.set_option('developer_mode', bool(x))) # model signals - self.model.mtext_refreshed(self.refresh) - self.model.maturity_changed(self.refresh) + self.model.mtext_refreshed += self.refresh + self.model.maturity_changed += self.refresh + + # XXX: bit of a hack placing it here... a lot of stuff needs to be rewritten :/ + self.model.maturity_changed += self.controller.update_subtree #-------------------------------------------------------------------------- # Misc #-------------------------------------------------------------------------- + def reinit(self, force_all = False): + """ + Fully reinitializes the microcode explorer UI based on the model state. + May optionally force all of the maturity levels to update. + """ + self.model.refresh_mtext(reinit = True, force_all=force_all) + self.refresh() + def refresh(self): """ Refresh the microcode explorer UI based on the model state. @@ -638,13 +662,16 @@ class MicrocodeView(ida_kernwin.simplecustviewer_t): def __init__(self, model): super(MicrocodeView, self).__init__() self.model = model + self.new_line = None self.Create() def connect_signals(self, controller): self.controller = controller self.OnCursorPosChanged = lambda: controller.select_position(*self.GetPos()) self.OnDblClick = lambda _: controller.activate_position(*self.GetPos()) - self.model.position_changed(self.refresh_cursor) + + self.model.position_changed += self.refresh_cursor + self.model.position_changed += self.controller.update_subtree def refresh(self): self.ClearLines() @@ -672,7 +699,7 @@ def OnCursorPosChanged(self): def OnDblClick(self, shift): pass - + def OnPopup(self, form, popup_handle): controller = self.controller diff --git a/plugins/lucid/ui/subtree.py b/plugins/lucid/ui/subtree.py index f679709..2fe6572 100644 --- a/plugins/lucid/ui/subtree.py +++ b/plugins/lucid/ui/subtree.py @@ -1,11 +1,95 @@ import ida_graph import ida_moves import ida_hexrays +import ida_hexrays as hr import ida_kernwin from PyQt5 import QtWidgets, QtCore from lucid.util.hexrays import get_mcode_name, get_mopt_name +from lucid.microtext import MicrocodeOptions + +class MOPHelper: + + MOP_RESOLVERS = { + hr.mnumber_t: lambda mnum: mnum.value, + hr.minsn_t: lambda minsn: minsn.dstr(), + hr.stkvar_ref_t: lambda stkvar: stkvar.off, + hr.mcallinfo_t: lambda mcall: mcall.dstr(), + hr.lvar_ref_t: lambda lvar: f"{lvar.off}:{lvar.idx}", + hr.mop_addr_t: lambda maddr: f"({maddr.insize},{maddr.outsize})", + hr.fnumber_t: lambda fnum: f"{fnum.fnum}({fnum.nbytes})", + hr.scif_t: lambda scif: scif.name, + } + + MOP_TYPES = { + hr.mop_z: None, + hr.mop_r: int, + hr.mop_n: hr.mnumber_t, + hr.mop_d: hr.minsn_t, + hr.mop_S: hr.stkvar_ref_t, + hr.mop_v: int, + hr.mop_b: int, + hr.mop_f: hr.mcallinfo_t, + hr.mop_l: hr.lvar_ref_t, + hr.mop_a: hr.mop_addr_t, + hr.mop_h: str, + hr.mop_str: str, + hr.mop_c: hr.mcases_t, + hr.mop_fn: hr.fnumber_t, + hr.mop_p: hr.mop_pair_t, + hr.mop_sc: hr.scif_t, + } + + MOP_FIELDS = { + hr.mop_z: None, + hr.mop_r: 'r', # register number [mreg_t] + hr.mop_n: 'nnn', # immediate value [mnumber_t] + hr.mop_d: 'd', # result (destination) of another instruction [minsn_t] + hr.mop_S: 's', # stack variable [stkvar_ref_t] + hr.mop_v: 'g', # global variable (its linear address) [ea_t] + hr.mop_b: 'b', # block number (used in jmp,call instructions) [int] + hr.mop_f: 'f', # function call information [mcallinfo_t] + hr.mop_l: 'l', # local variable [lvar_ref_t] + hr.mop_a: 'a', # variable whose address is taken [mop_addr_t] + hr.mop_h: 'helper', # helper function name [string] + hr.mop_str: 'cstr', # utf8 string constant, user representation [string] + hr.mop_c: 'c', # cases [mcases_t] + hr.mop_fn: 'fpc', # floating point constant [fnumber_t] + hr.mop_p: 'pair', # operand pair [mop_pair_t] + hr.mop_sc: 'scif', # scattered operand info [scif_t] + } + + @classmethod + def valid_for(cls, mop, value): + type = cls.MOP_TYPES[mop.t] + return isinstance(value, type) + + @classmethod + def get_value(cls, mop): + attr = cls.MOP_FIELDS[mop.t] + if attr: + return getattr(mop, attr) + return None + + @classmethod + def to_string(cls, mop): + if mop.t == hr.mop_p: + return f"mop_pair<{cls.to_string(mop.pair.lop)},{cls.to_string(mop.pair.hop)}>" + elif mop.t == hr.mop_r: + return f"reg<{mop.r}({hr.get_mreg_name(mop.r, mop.size)}.{mop.size})>" + + value = cls.get_value(mop) + + if value == None: + return "" + + val_type = type(value) + if val_type in cls.MOP_RESOLVERS: + resolver = cls.MOP_RESOLVERS[val_type] + value = resolver(value) + + return str(value) #------------------------------------------------------------------------------ # Microinstruction Sub-trees @@ -36,19 +120,45 @@ class MicroSubtreeView(ida_graph.GraphViewer): """ WINDOW_TITLE = "Sub-instruction Graph" - def __init__(self, insn): - super(MicroSubtreeView, self).__init__(self.WINDOW_TITLE, True) + def __init__(self, insn, controller): + super(MicroSubtreeView, self,).__init__(self.WINDOW_TITLE, True) self.insn = insn + self.next_insn = None + self.controller = controller self._populated = False + + # XXX: these will cause crashes if placed here! + #self.controller.model.position_changed += self.update_insn + #self.controller.model.maturity_changed += self.update_insn + + def update_insn(self): + insn_token = self.controller.model.mtext.get_ins_for_line(self.controller.model.current_line) + if insn_token is None or insn_token.insn == self.insn: + return + + self.next_insn = insn_token.insn + + gv = ida_graph.get_graph_viewer(self.GetWidget()) + + self.Refresh() + self._center_graph() + + ida_graph.viewer_fit_window(gv) def show(self): self.Show() - ida_kernwin.set_dock_pos(self.WINDOW_TITLE, "Microcode Explorer", ida_kernwin.DP_INSIDE) + ida_kernwin.set_dock_pos(self.WINDOW_TITLE, self.controller.view.WINDOW_TITLE, ida_kernwin.DP_BOTTOM) - # XXX: bit of a hack for now... lool - QtCore.QTimer.singleShot(50, self._center_graph) + gv = ida_graph.get_graph_viewer(self.GetWidget()) + ida_graph.viewer_set_titlebar_height(gv, 15) - def _center_graph(self): + self.Refresh() + self._center_graph() + + # XXX: absolute hackery fuckery, but it works! + ida_graph.viewer_fit_window(gv) + + def _center_graph(self, fit_window = False): """ Center the sub-tree graph, and set an appropriate zoom level. """ @@ -56,27 +166,47 @@ def _center_graph(self): gv = ida_graph.get_graph_viewer(widget) g = ida_graph.get_viewer_graph(gv) - ida_graph.viewer_fit_window(gv) - ida_graph.refresh_viewer(gv) + #ida_graph.viewer_fit_window(gv) + #ida_graph.refresh_viewer(gv) gli = ida_moves.graph_location_info_t() ida_graph.viewer_get_gli(gli, gv, ida_graph.GLICTL_CENTER) if gli.zoom > 1.5: gli.zoom = 1.5 else: - gli.zoom = gli.zoom * 0.9 + gli.zoom = round(gli.zoom * 0.9, 1) ida_graph.viewer_set_gli(gv, gli, ida_graph.GLICTL_CENTER) + + if fit_window: + ida_graph.viewer_fit_window(gv) #ida_graph.refresh_viewer(gv) + def _get_mop_oprops(self, mop): + text = "" + for oprop, name in [(getattr(ida_hexrays, x), x) for x in filter(lambda y: y.startswith('OPROP_'), dir(ida_hexrays))]: + if mop.oprops & oprop: + text += f" +{name[6:]}" + return text def _insert_mop(self, mop, parent): if mop.t == 0: return -1 text = " " + get_mopt_name(mop.t) + + #if mop.is_insn(): + # text += " (%s)" % get_mcode_name(mop.d.opcode) + + if MicrocodeOptions.developer_mode: + text += f" := {MOPHelper.to_string(mop)}" + oprops = self._get_mop_oprops(mop) + if oprops: + text += ' \n' + self._get_mop_oprops(mop) + if mop.is_insn(): - text += " (%s)" % get_mcode_name(mop.d.opcode) - text += ' \n ' + mop._print() + " " + text += ' \n ' + mop.d._print() + " " + else: + text += ' \n ' + mop._print() + " " node_id = self.AddNode(text) self.AddEdge(parent, node_id) @@ -113,9 +243,25 @@ def _insert_insn(self, insn): self._insert_mop(insn.d, node_id) return node_id + def OnClose(self): + self.controller.free_subtree() + + def OnViewKeydown(self, key, state): + c = chr(key & 0xFF) + + if c == 'C': + self._center_graph(fit_window=True) + + return True + def OnRefresh(self): + if self.next_insn: + self.insn = self.next_insn + self.next_insn = None + self._populated = False + if self._populated: - return + return False self.Clear() twidget = self.GetWidget() @@ -130,4 +276,4 @@ def OnRefresh(self): return True def OnGetText(self, node_id): - return (self._nodes[node_id], self._node_color) \ No newline at end of file + return (self[node_id], self._node_color) \ No newline at end of file diff --git a/plugins/lucid/ui/sync.py b/plugins/lucid/ui/sync.py index 6b16bc1..e2ec487 100644 --- a/plugins/lucid/ui/sync.py +++ b/plugins/lucid/ui/sync.py @@ -55,7 +55,7 @@ def __init__(self, controller, model): self._hxe_hooks.refresh_pseudocode = self.hxe_refresh_pseudocode self._hxe_hooks.close_pseudocode = self.hxe_close_pseudocode self._ui_hooks.get_lines_rendering_info = self.render_lines - self.model.position_changed(self.refresh_hexrays_cursor) + self.model.position_changed += self.refresh_hexrays_cursor def hook(self): self._ui_hooks.hook() @@ -64,8 +64,9 @@ def unhook(self): self._ui_hooks.unhook() self.enable_sync(False) - def track_view(self, widget): - self._code_widget = widget # TODO / temp + def track_view(self, view): + self._code_view = view + self._code_widget = view.widget def enable_sync(self, status): @@ -149,6 +150,9 @@ def hxe_refresh_pseudocode(self, vdui): """ if self.model.current_function != vdui.cfunc.entry_ea: self._sync_microtext(vdui) + else: + # TODO: remove need for force_all (have them queue up for loading as needed) + self.controller.view.reinit(force_all=True) return 0 def hxe_curpos(self, vdui): @@ -289,7 +293,7 @@ def _highlight_microcode(self, lines_out, widget, lines_in): # special case, only highlight the currently selected microcode line (a special line / block header) if self.model.current_line.type: - to_paint.add(self.model.current_position[0]) + to_paint.add(self.model.current_line) target_addresses = [] # 'default' case, target all lines containing the address under the cursor diff --git a/plugins/lucid/util/options.py b/plugins/lucid/util/options.py new file mode 100644 index 0000000..ae348db --- /dev/null +++ b/plugins/lucid/util/options.py @@ -0,0 +1,234 @@ +# +# Custom helper classes created for and specifically-tailored to Lucid. +# by Fireboyd78 - revision date is 2024/01/28 [initial public release] +# +# +# TODO: +# - Documentation/explanation of everything +# - Thorough bug-testing would be nice ;) +# +# +class OptionSet: + + def __init__(self, default_options={}): + options = {} + attrs = {} + if default_options: + if not isinstance(default_options, dict): + raise TypeError(default_options) + for name, value in default_options.items(): + attrs[name] = self._make_attr(name, value, options, default_options) + self._options = options + self._default_options = default_options + self._attrs = attrs + + def _make_attr(self, name, value, options={}, default_options={}): + options[name] = value + def _get_attr(self, name): + return options[name] if name in options else default_options[name] + return _get_attr + + def __getattr__(self, name: str): + try: + return object.__getattribute__(self, name) + except Exception as e: + attrs = self.__dict__.get('_attrs', None) + if not attrs or name not in attrs: + # option attribute not found, re-raise the exception normally + raise e + return attrs[name](self, name) + + def __setattr__(self, name: str, value): + attrs = self.__dict__.get('_attrs', None) + if attrs is None or name not in attrs: + object.__setattr__(self, name, value) + return + self.set(name, value) + + def __contains__(self, name: str): + return self.contains(name) + + def __delitem__(self, name: str): + self.remove(name) + + def __getitem__(self, name: str): + options = self._get_options_list(name) + return options[name] + + def __setitem__(self, name: str, value): + self.set(name, value) + + + def _get_options_list(self, name: str, allow_defaults=True): + options = self._options + if name not in options: + options = self._default_options if allow_defaults else None + if options is None: + raise KeyError(name) + return options + + def _get_option_internal(self, name: str, allow_defaults=True): + options = self._get_options_list(name, allow_defaults=allow_defaults) + if name not in options: + return (False, None) + return (True, options[name]) + + + def clear(self): + self._options = {} + + def changes(self): + changes = [] + for name, value in self._default_options.items(): + present, old_value = self._get_option_internal(name, allow_defaults=False) + if not present or old_value == value: + continue + changes.append((name, old_value, value)) + return changes + + def restore_defaults(self): + changes = self.changes() + for name, _, new_value in changes: + # directly set the value like the constructor does + self._options[name] = new_value + return changes + + def contains(self, name: str, check_defaults=True): + options = self._get_options_list(name, allow_defaults=check_defaults) + if name not in options: + return False + return True + + def remove(self, name: str): + options = self._options + if name not in options: + raise KeyError(name) + del options[name] + + def pop(self, name: str): + _, value = self._get_option_internal(name, allow_defaults=False) + self.remove(name) + return value + + def restore(self, name: str): + default_options = self._default_options + if default_options is None or name not in default_options: + raise KeyError(name, default_options) + self.set(name, default_options[name]) + + def get(self, name: str, default_value=None): + need_default = True if default_value is None else False + present, value = self._get_option_internal(name, allow_defaults=need_default) + if not present: + if default_value is None: + raise KeyError(name) + value = default_value + return value + + def set(self, name: str, value): + self._options[name] = value + + def has_default(self, name: str): + return name in self._default_options + + def get_default(self, name: str): + default_value = self._default_options.get(name, self) + if default_value == self: + raise KeyError(name) + return default_value + + +class OptionListener: + __providers = [] + + def __init_subclass__(cls, /, providers=[], **kwargs): + cls.__providers = providers + + def __new__(cls, *args, **kwargs): + obj = super().__new__(cls, *args, **kwargs) + obj.notify_creation() + return obj + + def __del__(self): + self.notify_deletion() + + def notify_creation(self): + for provider in self.__providers: + provider.add_listener(self) + + def notify_deletion(self): + for provider in self.__providers: + provider.remove_listener(self) + + def notify_change(self, option_name, option_value, **kwargs): + print(f"**** notify_change[{self.__class__.__name__}]: {option_name} = {option_value}") + + +class OptionProvider(OptionSet): + + def __init__(self, default_options={}): + super().__init__(default_options) + self._listeners = [] + + + def _notify_change(self, option_name, option_value, **kwargs): + for listener in self._listeners: + listener.notify_change(option_name, option_value, **kwargs) + + def add_listener(self, listener): + if listener not in self._listeners: + self._listeners.append(listener) + return True + return False + + def remove_listener(self, listener): + if listener not in self._listeners: + return False + self._listeners.remove(listener) + return True + + def clear(self): + if self._listeners: + for name, value in self._options: + if not self.contains(name, check_defaults=False): + continue + super().remove(name) # don't notify listeners + self._notify_change(name, None, deleted=True, old_value=value) + super().clear() + + def refresh(self): + if not self._listeners: + return + for name, value in self._options: + self._notify_change(name, value) + + def reset(self): + self._listeners.clear() + super().restore_defaults() # no need to notify listeners + + def restore_defaults(self): + for name, old_value, new_value in self.changes(): + self._notify_change(name, new_value, reset=True, old_value=old_value) + + def set(self, name: str, value): + present, old_value = self._get_option_internal(name, allow_defaults=False) + if not present: + # notify of new value + super().set(name, value) + self._notify_change(name, value) + return + elif value != old_value: + super().set(name, value) + # notify of updated value + self._notify_change(name, value, old_value=old_value) + + def pop(self, name: str): + old_value = super().pop(name) + self._notify_change(name, None, deleted=True, old_value=old_value) + return old_value + + def remove(self, name: str): + if self._listeners: + self.pop(name) + return + super().remove(name) \ No newline at end of file diff --git a/plugins/lucid/util/python.py b/plugins/lucid/util/python.py index 8513e84..32fab40 100644 --- a/plugins/lucid/util/python.py +++ b/plugins/lucid/util/python.py @@ -93,6 +93,36 @@ def notify_callback(callback_list, *args): for callback_ref in cleanup: callback_list.remove(callback_ref) + +# +# NB: This code is based off of the original callback handling code in 'explorer.py'. +# +class CallbackHandler: + """ + A custom callback handler that supports event subscribers. + """ + def __init__(self, owner, name = "-unspecified-"): + self.__doc__ = f"Callback data for {name} events." + self.owner = owner + self.callbacks_list = [] + self._ready = True + + def __call__(self, *args): + """ + Notify listeners of an event. + """ + if not self._ready: + return object.__call__(self, *args) + notify_callback(self.callbacks_list, *args) + + def __iadd__(self, callback): + """ + Subscribe a callback for the events. + """ + register_callback(self.callbacks_list, callback) + return self + + #------------------------------------------------------------------------------ # Module Reloading #------------------------------------------------------------------------------ @@ -109,15 +139,15 @@ def _recurseive_reload(module, target_name, visited): ignore = ["__builtins__", "__cached__", "__doc__", "__file__", "__loader__", "__name__", "__package__", "__spec__", "__path__"] visited[module.__name__] = module - + for attribute_name in dir(module): - + # skip the stuff we don't care about if attribute_name in ignore: continue attribute_value = getattr(module, attribute_name) - + if type(attribute_value) == ModuleType: attribute_module_name = attribute_value.__name__ attribute_module = attribute_value @@ -130,8 +160,11 @@ def _recurseive_reload(module, target_name, visited): #print("TODO: should probably try harder to reload this...", attribute_name, type(attribute_value)) continue else: - #print("UNKNOWN TYPE TO RELOAD", attribute_name, type(attribute_value)) - raise ValueError("OH NOO RELOADING IS HARD") + print("**** OH NOO RELOADING IS HARD") + attribute_module_name = attribute_value.__class__.__name__ + attribute_module = attribute_value.__class__.__module__ + print("UNKNOWN TYPE TO RELOAD", attribute_name, type(attribute_value)) + return if not target_name in attribute_module_name: #print(" - Not a module of interest...") From eb65578d15881cd54c9e91d3c41029f4d9168e00 Mon Sep 17 00:00:00 2001 From: Fireboyd78 Date: Wed, 31 Jan 2024 04:04:11 -0800 Subject: [PATCH 2/7] [Lucid Reloaded]: More performance enhancements when generating initial microcode. - No longer hangs when loading into very large functions - Different maturity levels will now be loaded when the user requests them - Added a queue system for handling pending maturity level changes - **TODO:** Fix cursor synchronization between maturity levels - it was fully dependent on expecting every maturity level to be available at once... --- plugins/lucid/microtext.py | 21 +++-- plugins/lucid/ui/explorer.py | 142 ++++++++++++++++++++++++---------- plugins/lucid/ui/sync.py | 46 +++++------ plugins/lucid/util/hexrays.py | 2 +- 4 files changed, 140 insertions(+), 71 deletions(-) diff --git a/plugins/lucid/microtext.py b/plugins/lucid/microtext.py index 8bbdfe3..215cc31 100644 --- a/plugins/lucid/microtext.py +++ b/plugins/lucid/microtext.py @@ -530,33 +530,39 @@ class MicrocodeText(TextBlock): def __init__(self, maturity): super(MicrocodeText, self).__init__() self.maturity = maturity + self.premade = False @classmethod def create(cls, func, maturity): """ - Create a new instance of the class and generate the microcode text. + Create a new instance of the class. Does not generate any of its contents. """ mtext = MicrocodeText(maturity) mtext.func = func - mtext.reinit() + mtext.mba = get_microcode(func, maturity) + mtext.premade = True return mtext def copy(self): """ - Create a copy of the microcode and generate its text. + Create a copy of the microcode. Does not generate any of its contents. """ mtext = MicrocodeText(self.maturity) mtext.func = self.func mtext.mba = self.mba - mtext.refresh() + mtext.premade = True return mtext def reinit(self): """ Reinitialize the underlying microcode and regenerate text. """ - self.func = ida_funcs.get_func(self.func.start_ea) - self.mba = get_microcode(self.func, self.maturity) + if not self.premade: + # get the most up-to-date microcode + self.func = ida_funcs.get_func(self.func.start_ea) + self.mba = get_microcode(self.func, self.maturity) + else: # do a one-time skip if we were just created/copied + self.premade = False self.refresh() def refresh(self, maturity=None): @@ -573,6 +579,9 @@ def _generate_from_mba(self): """ blks = [] + if not self.mba: + raise Exception("The requested microcode block is invalid.") + for blk_idx in range(self.mba.qty): blk = self.mba.get_mblock(blk_idx) blk_token = MicroBlockText(blk) diff --git a/plugins/lucid/ui/explorer.py b/plugins/lucid/ui/explorer.py index c173a42..19a7df5 100644 --- a/plugins/lucid/ui/explorer.py +++ b/plugins/lucid/ui/explorer.py @@ -111,19 +111,17 @@ def select_function(self, address): return False for maturity in get_mmat_levels(): - mtext = MicrocodeText.create(func, maturity) - self.model.update_mtext(mtext, maturity) + self.model.init_function(func, maturity) self.view.refresh() ida_kernwin.refresh_idaview_anyway() return True - def select_maturity(self, maturity_name): + def select_maturity(self, maturity): """ Switch the microcode view to the specified maturity level. """ - self.model.active_maturity = get_mmat(maturity_name) - #self.view.refresh() + self.model.active_maturity = maturity def select_address(self, address): """ @@ -177,6 +175,14 @@ def activate_position(self, line_num, x, y): blk_line_num, _ = self.model.mtext.get_pos_of_token(blk_token.lines[0]) self.select_position(blk_line_num, 0, y) + + def regenerate_microtext(self): + self.model.queue_rebuild(active_only=False) + self.view.refresh() + + def synchronize_microtext(self, vdui: ida_hexrays.vdui_t): + address = vdui.cfunc.entry_ea + self.select_function(address) class MicrocodeExplorerModel(object): """ @@ -218,13 +224,15 @@ def __init__(self): # self._active_maturity = ida_hexrays.MMAT_GENERATED + + self._rebuild_queue = {x: False for x in get_mmat_levels()} + self._refresh_queue = {x: False for x in get_mmat_levels()} #---------------------------------------------------------------------- # Callbacks #---------------------------------------------------------------------- self.mtext_changed = CallbackHandler(self, name="mtext changed") - self.mtext_refreshed = CallbackHandler(self, name="mtext refreshed") self.position_changed = CallbackHandler(self, name="position changed") self.maturity_changed = CallbackHandler(self, name="maturity changed") @@ -245,9 +253,8 @@ def mtext(self, mtext): """ Set the microcode text mapping for the current maturity level. """ - old_mtext = self.mtext self._mtext[self._active_maturity] = mtext - self.mtext_changed(old_mtext, mtext) + self.mtext_changed() @property def current_line(self): @@ -320,6 +327,10 @@ def active_maturity(self, new_maturity): """ Set the active microcode maturity level. """ + old_maturity = self._active_maturity + if new_maturity == old_maturity: + return + self._active_maturity = new_maturity self.maturity_changed() @@ -327,38 +338,84 @@ def active_maturity(self, new_maturity): # Misc #---------------------------------------------------------------------- + def queue_rebuild(self, active_only = False): + if not active_only: + # queue all maturities for a full rebuild + # they will only be rebuilt once active + for maturity in get_mmat_levels(): + self._rebuild_queue[maturity] = True + else: + # force the active maturity to be rebuilt + self._rebuild_queue[self.active_maturity] = True + + def queue_refresh(self, active_only = False): + if not active_only: + # queue all maturities for a full rebuild + # they will only be rebuilt once active + for maturity in get_mmat_levels(): + self._refresh_queue[maturity] = True + else: + # force the active maturity to be rebuilt + self._refresh_queue[self.active_maturity] = True + + def init_function(self, func, maturity): + mtext = MicrocodeText.create(func, maturity) + self.update_mtext(mtext, maturity) + def update_mtext(self, mtext, maturity): """ Set the mtext for a given microcode maturity level. """ self._mtext[maturity] = mtext + self._rebuild_queue[maturity] = True # needs to be generated self._view_cursors[maturity] = ViewCursor(0, 0, 0) - def refresh_mtext(self, reinit = False, force_all = False): + def redraw_mtext(self, maturity): """ - Regenerate the rendered text for all microcode maturity levels. + Redraws the rendered text for the microcode maturity level. """ - old_mtext = None - new_mtext = None + if maturity != self.active_maturity: + return False # XXX: Performance optimization. + + self._mtext[maturity].refresh() + self._refresh_queue[maturity] = False - # XXX: this is also a bit hacky, unfortunately - for maturity, mtext in self._mtext.items(): - if maturity == self.active_maturity or force_all: - if not reinit: - old_mtext = mtext - else: - # fully rebuild it - mtext.reinit() - else: - # refresh it since it's out of view - mtext.refresh() + return True + + def rebuild_mtext(self, maturity): + """ + Regenerate the rendered text for the microcode maturity level. + """ + if maturity != self.active_maturity: + return False # XXX: Performance optimization. - if old_mtext: - # make a new copy of it - self.mtext = new_mtext = old_mtext.copy() + if maturity in self._rebuild_queue: + # fully rebuild our active maturity + self.mtext.reinit() + self._rebuild_queue[maturity] = False + self._refresh_queue[maturity] = False # no need to redraw since it was rebuilt + else: + # make a new copy of it and translate the active cursor + # this will ensure a proper refresh of the microcode + old_mtext = self.mtext + new_mtext = old_mtext.copy() + self.update_mtext(new_mtext, maturity) self.current_position = translate_mtext_position(self.current_position, old_mtext, new_mtext) - self.mtext_refreshed() + return True + + def refresh_mtext(self): + """ + Updates the rendered text for the microcode as needed. + """ + for maturity, needs_rebuild in self._rebuild_queue.items(): + if not needs_rebuild: + continue + self.rebuild_mtext(maturity) + for maturity, needs_redraw in self._refresh_queue.items(): + if not needs_redraw: + continue + self.redraw_mtext(maturity) def _gen_cursors(self, position, mmat_src): """ @@ -390,6 +447,10 @@ def _transfer_cursor(self, mmat_src, mmat_dst): """ Translate the cursor position from one maturity to the next. """ + + if mmat_src in self._rebuild_queue or mmat_dst in self._rebuild_queue: + return + position = self._view_cursors[mmat_src].viewport_position mapped = self._view_cursors[mmat_src].mapped @@ -438,8 +499,8 @@ def notify_change(self, option_name, option_value, **kwargs): Implementation of OptionListener.notify_change for when a microcode option has been updated. """ #print(f"**** notify_change {option_name} = {option_value} (IN OPTIONS={bool(option_name in MicrocodeOptions)})") - self.model.refresh_mtext() - ida_kernwin.refresh_idaview_anyway() + self.model.queue_refresh() + self.refresh() #-------------------------------------------------------------------------- # Pseudo Widget Functions @@ -572,7 +633,13 @@ def _ui_init_signals(self): """ Connect UI signals. """ - self._maturity_list.currentItemChanged.connect(lambda x, y: self.controller.select_maturity(x.text())) + + def _maturity_changed(item): + maturity = self._maturity_list.row(item) + 1 + self.controller.select_maturity(maturity) + + + self._maturity_list.currentItemChanged.connect(_maturity_changed) self._code_view.connect_signals(self.controller) self._code_view.OnClose = self.hide # HACK @@ -585,30 +652,27 @@ def _ui_init_signals(self): self._checkbox_devmode.stateChanged.connect(lambda x: self.controller.set_option('developer_mode', bool(x))) # model signals - self.model.mtext_refreshed += self.refresh + self.model.mtext_changed += self.reinit self.model.maturity_changed += self.refresh - # XXX: bit of a hack placing it here... a lot of stuff needs to be rewritten :/ - self.model.maturity_changed += self.controller.update_subtree - #-------------------------------------------------------------------------- # Misc #-------------------------------------------------------------------------- - def reinit(self, force_all = False): + def reinit(self): """ Fully reinitializes the microcode explorer UI based on the model state. - May optionally force all of the maturity levels to update. """ - self.model.refresh_mtext(reinit = True, force_all=force_all) + self.model.queue_rebuild(active_only=True) self.refresh() def refresh(self): """ Refresh the microcode explorer UI based on the model state. """ - self._maturity_list.setCurrentRow(self.model.active_maturity - 1) + self.model.refresh_mtext() self._code_view.refresh() + self.controller.update_subtree() class LayerListWidget(QtWidgets.QListWidget): """ @@ -680,7 +744,7 @@ def refresh(self): self.refresh_cursor() def refresh_cursor(self): - if not self.model.current_position: + if not self.model.current_cursor or not self.model.current_position: return self.Jump(*self.model.current_position) diff --git a/plugins/lucid/ui/sync.py b/plugins/lucid/ui/sync.py index e2ec487..31f0648 100644 --- a/plugins/lucid/ui/sync.py +++ b/plugins/lucid/ui/sync.py @@ -68,6 +68,12 @@ def track_view(self, view): self._code_view = view self._code_widget = view.widget + def sync_if_needed(self, vdui, may_regenerate=False): + if self.model.current_function != vdui.cfunc.entry_ea: + self.controller.synchronize_microtext(vdui) + elif may_regenerate: + self.controller.regenerate_microtext() + def enable_sync(self, status): # nothing to do @@ -81,8 +87,8 @@ def enable_sync(self, status): if status: self._hxe_hooks.hook() self._cache_active_vdui() - if self._last_vdui and (self.model.current_function != self._last_vdui.cfunc.entry_ea): - self._sync_microtext(self._last_vdui) + if self._last_vdui: + self.sync_if_needed(self._last_vdui, may_regenerate=False) # syncing disabled else: @@ -148,11 +154,7 @@ def hxe_refresh_pseudocode(self, vdui): """ (Event) A Hex-Rays pseudocode window was refreshed/changed. """ - if self.model.current_function != vdui.cfunc.entry_ea: - self._sync_microtext(vdui) - else: - # TODO: remove need for force_all (have them queue up for loading as needed) - self.controller.view.reinit(force_all=True) + self.sync_if_needed(vdui, may_regenerate=True) return 0 def hxe_curpos(self, vdui): @@ -162,17 +164,15 @@ def hxe_curpos(self, vdui): self._hexrays_origin = False self._hexrays_addresses = self._get_active_vdui_addresses(vdui) - if self.model.current_function != vdui.cfunc.entry_ea: - self._sync_microtext(vdui) + self.sync_if_needed(vdui, may_regenerate=False) - if self._ignore_move: - # TODO put a refresh here ? - return 0 - self._hexrays_origin = True - if not self._hexrays_addresses: + self._hexrays_origin = True if not self._ignore_move and self._hexrays_addresses else False + + if self._hexrays_origin: + self.controller.select_address(self._hexrays_addresses[0]) + elif not self._ignore_move: ida_kernwin.refresh_idaview_anyway() - return 0 - self.controller.select_address(self._hexrays_addresses[0]) + return 0 def render_lines(self, lines_out, widget, lines_in): @@ -266,6 +266,9 @@ def _highlight_hexrays(self, lines_out, widget, lines_in): """ Highlight lines in the given Hex-Rays window according to the synchronized addresses. """ + if not self.model.current_cursor: + return + vdui = ida_hexrays.get_widget_vdui(widget) if self._hexrays_addresses or self._hexrays_origin: self._highlight_lines(lines_out, set([vdui.cpos.lnnum]), lines_in) @@ -274,7 +277,7 @@ def _highlight_microcode(self, lines_out, widget, lines_in): """ Highlight lines in the given microcode window according to the synchronized addresses. """ - if not self.model.mtext.lines: + if not self.model.mtext.lines or not self.model.current_cursor: return to_paint = set() @@ -314,11 +317,4 @@ def _highlight_microcode(self, lines_out, widget, lines_in): to_paint.add(line_num) - self._highlight_lines(lines_out, to_paint, lines_in) - - def _sync_microtext(self, vdui): - """ - TODO: this probably should just be a func in the controller - """ - self.controller.select_function(vdui.cfunc.entry_ea) - self.controller.view.refresh() \ No newline at end of file + self._highlight_lines(lines_out, to_paint, lines_in) \ No newline at end of file diff --git a/plugins/lucid/util/hexrays.py b/plugins/lucid/util/hexrays.py index 528438e..8209d8a 100644 --- a/plugins/lucid/util/hexrays.py +++ b/plugins/lucid/util/hexrays.py @@ -15,7 +15,7 @@ def get_microcode(func, maturity): hf = ida_hexrays.hexrays_failure_t() ml = ida_hexrays.mlist_t() ida_hexrays.mark_cfunc_dirty(func.start_ea) - mba = ida_hexrays.gen_microcode(mbr, hf, ml, ida_hexrays.DECOMP_NO_WAIT, maturity) + mba = ida_hexrays.gen_microcode(mbr, hf, ml, ida_hexrays.DECOMP_NO_WAIT | ida_hexrays.DECOMP_ALL_BLKS, maturity) if not mba: print("0x%08X: %s" % (hf.errea, hf.desc())) return None From 7d624bae02605c9c8b8cf1938dfc2da7844ff2a4 Mon Sep 17 00:00:00 2001 From: Fireboyd78 Date: Thu, 1 Feb 2024 01:57:46 -0800 Subject: [PATCH 3/7] [Lucid Reloaded]: Fixed cursor synchronization when switching between maturity levels. - No longer dependent on having all maturity levels loaded at once - Transfers the cursor to the correct position upon the maturity level being changed --- plugins/lucid/microtext.py | 5 ++++ plugins/lucid/ui/explorer.py | 56 +++++++++++++++++++++++++----------- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/plugins/lucid/microtext.py b/plugins/lucid/microtext.py index 215cc31..53f6c98 100644 --- a/plugins/lucid/microtext.py +++ b/plugins/lucid/microtext.py @@ -531,6 +531,7 @@ def __init__(self, maturity): super(MicrocodeText, self).__init__() self.maturity = maturity self.premade = False + self.generation = 0 @classmethod def create(cls, func, maturity): @@ -553,6 +554,9 @@ def copy(self): mtext.premade = True return mtext + def is_pending(self): + return self.generation == 0 + def reinit(self): """ Reinitialize the underlying microcode and regenerate text. @@ -572,6 +576,7 @@ def refresh(self, maturity=None): self._generate_from_mba() self._generate_lines() self._generate_token_address_map() + self.generation += 1 def _generate_from_mba(self): """ diff --git a/plugins/lucid/ui/explorer.py b/plugins/lucid/ui/explorer.py index 19a7df5..7e11978 100644 --- a/plugins/lucid/ui/explorer.py +++ b/plugins/lucid/ui/explorer.py @@ -305,6 +305,9 @@ def current_position(self): """ Return the current viewport cursor position (line_num, view_x, view_y). """ + if not self.current_cursor: + return (0, 0, 0) # lol + return self.current_cursor.viewport_position @current_position.setter @@ -332,7 +335,7 @@ def active_maturity(self, new_maturity): return self._active_maturity = new_maturity - self.maturity_changed() + self.maturity_changed(old_maturity) #---------------------------------------------------------------------- # Misc @@ -404,7 +407,7 @@ def rebuild_mtext(self, maturity): return True - def refresh_mtext(self): + def refresh_mtext(self, old_maturity = None): """ Updates the rendered text for the microcode as needed. """ @@ -416,26 +419,46 @@ def refresh_mtext(self): if not needs_redraw: continue self.redraw_mtext(maturity) - + + if old_maturity is None: + # we don't need to move the cursor since no other maturities are loaded + return + + # transition from this maturity to the next one + self._transfer_cursor(old_maturity, self.active_maturity) + def _gen_cursors(self, position, mmat_src): """ Generate the cursors for all levels from a source position and maturity. """ - mmat_levels = get_mmat_levels() - mmat_first, mmat_final = mmat_levels[0], mmat_levels[-1] - - # clear out all the existing cursor mappings - self._view_cursors = {x: None for x in mmat_levels} - + mmat_levels = [] + + for maturity, mtext in self._mtext.items(): + if not mtext or mtext.is_pending(): + continue + mmat_levels.append(maturity) + + if not mmat_levels: + return + + mmat_first = mmat_levels[0] + mmat_final = mmat_levels[-1] + + #print(f"**** MATURITY: first={mmat_first}, final={mmat_final}") + #print(f" - levels: [{','.join([str(n) for n in mmat_levels])}]") + # save the starting cursor - line_num, x, y = position + line_num, x, y = position + + # clear out all the existing cursor mappings + self._view_cursors = {x: None for x in get_mmat_levels()} self._view_cursors[mmat_src] = ViewCursor(line_num, x, y, True) - + # map the cursor backwards from the source maturity - mmat_lower = range(mmat_first, mmat_src)[::-1] + mmat_lower = [mmat for mmat in range(mmat_first, mmat_src) if mmat in mmat_levels][::-1] # map the cursor forward from the source maturity - mmat_higher = range(mmat_src+1, mmat_final + 1) + mmat_higher = [mmat for mmat in range(mmat_src+1, mmat_final+1) if mmat in mmat_levels] for mmat_range in (mmat_lower, mmat_higher): current_maturity = mmat_src @@ -447,8 +470,7 @@ def _transfer_cursor(self, mmat_src, mmat_dst): """ Translate the cursor position from one maturity to the next. """ - - if mmat_src in self._rebuild_queue or mmat_dst in self._rebuild_queue: + if self._mtext[mmat_src].is_pending() or self._mtext[mmat_dst].is_pending(): return position = self._view_cursors[mmat_src].viewport_position @@ -666,11 +688,11 @@ def reinit(self): self.model.queue_rebuild(active_only=True) self.refresh() - def refresh(self): + def refresh(self, old_maturity = None): """ Refresh the microcode explorer UI based on the model state. """ - self.model.refresh_mtext() + self.model.refresh_mtext(old_maturity=old_maturity) self._code_view.refresh() self.controller.update_subtree() From 8c4c3d1aedfec9dc7536e5f1d6f7aa8ed4078066 Mon Sep 17 00:00:00 2001 From: Fireboyd78 Date: Thu, 1 Feb 2024 02:06:08 -0800 Subject: [PATCH 4/7] [Lucid Reloaded]: Fixed support for reloading the plugin while IDA is open. - Bumped version number up to 0.2.3 - Added 'unload' methods to classes where needed in order for them to unregister any dependencies. - Added a 'hotload state' that can be retrieved and later set to restore the environment prior to reloading. (TODO: Delegate loading/saving to the classes + add more stuff to save/restore) - Added some additional info to the 'recursive reload' function when the "OH NOO" condition is met, lol (this honestly made me laugh really hard the first time I saw it) --- plugins/lucid/core.py | 33 ++++++++++++++++++++++++++++++--- plugins/lucid/ui/explorer.py | 19 ++++++++++++++++++- plugins/lucid/ui/sync.py | 4 ++++ plugins/lucid/util/python.py | 4 +--- plugins/lucid_plugin.py | 4 ++-- 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/plugins/lucid/core.py b/plugins/lucid/core.py index d854adf..1e20f5b 100644 --- a/plugins/lucid/core.py +++ b/plugins/lucid/core.py @@ -20,7 +20,7 @@ class LucidCore(object): PLUGIN_NAME = "Lucid" - PLUGIN_VERSION = "0.2.0" + PLUGIN_VERSION = "0.2.3" PLUGIN_AUTHORS = "Markus Gaasedelen, Fireboyd78" PLUGIN_DATE = "2024" @@ -40,7 +40,7 @@ def ready_to_run(self): self._startup_hooks = UIHooks() self._startup_hooks.ready_to_run = self.load - + if defer_load: self._startup_hooks.hook() return @@ -71,7 +71,30 @@ def load(self): # all done, mark the core as loaded self.loaded = True - + + def get_hotload_state(self): + """ + Gets persistent parameters that can be used to restore after a hotload. + """ + state = {} + # TODO: Let the classes handle their state data. + if self.explorer: + explorer_params = { + "active": self.explorer.view.visible, + } + state["explorer"] = explorer_params + return state + + def set_hotload_state(self, state): + """ + Restores saved parameters that were retrieved prior to a hotload. + """ + explorer_params = state.get("explorer", {}) + # TODO: Let the classes handle their state data. + if explorer_params: + if explorer_params.get("active", False): + self.interactive_view_microcode() + def unload(self, from_ida=False): """ Unload the plugin core. @@ -85,6 +108,10 @@ def unload(self, from_ida=False): return print("Unloading %s..." % self.PLUGIN_NAME) + + if self.explorer: + self.explorer.unload() + del self.explorer # mark the core as 'unloaded' and teardown its components self.loaded = False diff --git a/plugins/lucid/ui/explorer.py b/plugins/lucid/ui/explorer.py index 7e11978..d1d5d98 100644 --- a/plugins/lucid/ui/explorer.py +++ b/plugins/lucid/ui/explorer.py @@ -40,7 +40,15 @@ def __init__(self): self.model = MicrocodeExplorerModel() self.view = MicrocodeExplorerView(self, self.model) self.view._code_sync.enable_sync(True) # XXX/HACK - + + def unload(self): + if self.graph: + self.graph.Close() + del self.graph + + self.view.unload() + self.model.unload() + def show(self, address=None): """ Show the microcode explorer. @@ -236,6 +244,10 @@ def __init__(self): self.position_changed = CallbackHandler(self, name="position changed") self.maturity_changed = CallbackHandler(self, name="maturity changed") + def unload(self): + del self.maturity_changed + del self.position_changed + del self.mtext_changed #------------------------------------------------------------------------- # Read-Only Properties @@ -516,6 +528,11 @@ def __init__(self, controller: MicrocodeExplorer, model: MicrocodeExplorerModel) self._ui_init() self._ui_init_signals() + def unload(self): + ida_kernwin.close_widget(self._twidget, ida_kernwin.PluginForm.WCLS_DELETE_LATER) + self._code_sync.unload() + self._ui_hooks.unhook() + def notify_change(self, option_name, option_value, **kwargs): """ Implementation of OptionListener.notify_change for when a microcode option has been updated. diff --git a/plugins/lucid/ui/sync.py b/plugins/lucid/ui/sync.py index 31f0648..d80ce6d 100644 --- a/plugins/lucid/ui/sync.py +++ b/plugins/lucid/ui/sync.py @@ -57,6 +57,10 @@ def __init__(self, controller, model): self._ui_hooks.get_lines_rendering_info = self.render_lines self.model.position_changed += self.refresh_hexrays_cursor + def unload(self): + self._ui_hooks.unhook() + self._hxe_hooks.unhook() + def hook(self): self._ui_hooks.hook() diff --git a/plugins/lucid/util/python.py b/plugins/lucid/util/python.py index 32fab40..c194d0d 100644 --- a/plugins/lucid/util/python.py +++ b/plugins/lucid/util/python.py @@ -160,11 +160,9 @@ def _recurseive_reload(module, target_name, visited): #print("TODO: should probably try harder to reload this...", attribute_name, type(attribute_value)) continue else: - print("**** OH NOO RELOADING IS HARD") + print("**** OH NOO RELOADING IS HARD - UNKNOWN TYPE", attribute_name, type(attribute_value)) attribute_module_name = attribute_value.__class__.__name__ attribute_module = attribute_value.__class__.__module__ - print("UNKNOWN TYPE TO RELOAD", attribute_name, type(attribute_value)) - return if not target_name in attribute_module_name: #print(" - Not a module of interest...") diff --git a/plugins/lucid_plugin.py b/plugins/lucid_plugin.py index 13472a3..7b03f84 100644 --- a/plugins/lucid_plugin.py +++ b/plugins/lucid_plugin.py @@ -64,11 +64,11 @@ def reload(self): """ Hot-reload the plugin core. """ - print("Reloading...") + state = self.core.get_hotload_state() self.core.unload() reload_package(lucid) self.core = lucid.LucidCore() - self.core.interactive_view_microcode() + self.core.set_hotload_state(state) def test(self): """ From c98f5051e94350e8b93fdafe4045c66a339e3d3a Mon Sep 17 00:00:00 2001 From: Fireboyd78 Date: Thu, 1 Feb 2024 02:32:55 -0800 Subject: [PATCH 5/7] [Lucid Reloaded]: Removed assertion preventing moves from far-away maturity levels. - As originally stated, it wasn't strictly necessary, and the code works just fine without it :) --- plugins/lucid/microtext.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/plugins/lucid/microtext.py b/plugins/lucid/microtext.py index 53f6c98..3bf37fb 100644 --- a/plugins/lucid/microtext.py +++ b/plugins/lucid/microtext.py @@ -755,17 +755,6 @@ def translate_mtext_position(position, mtext_src, mtext_dst): """ line_num, x, y = position - # - # while this isn't strictly required, let's enforce it. this basically - # means that we won't allow you to translate a position from maturity - # levels that are more than one degree apart. - # - # eg, no hopping from maturity 0 --> 7 instead, you must translate - # through each layer 0 -> 1 -> 2 -> ... -> 7 - # - - assert abs(mtext_src.mba.maturity - mtext_dst.mba.maturity) <= 1 - # get the line the cursor falls on line = mtext_src.lines[line_num] From 053534da41805ec9d75cb468e357e32eb48e0357 Mon Sep 17 00:00:00 2001 From: Fireboyd78 Date: Thu, 1 Feb 2024 03:22:16 -0800 Subject: [PATCH 6/7] [Lucid Reloaded]: Fixed some issues where callbacks were not being unregistered. --- plugins/lucid/core.py | 5 ++++- plugins/lucid/ui/explorer.py | 18 ++++++++++++++---- plugins/lucid/util/options.py | 3 +++ plugins/lucid/util/python.py | 4 ++++ 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/plugins/lucid/core.py b/plugins/lucid/core.py index 1e20f5b..e36482c 100644 --- a/plugins/lucid/core.py +++ b/plugins/lucid/core.py @@ -1,6 +1,7 @@ import ida_idaapi import ida_kernwin +from lucid.microtext import MicrocodeOptions from lucid.util.ida import UIHooks, IDACtxEntry, hexrays_available from lucid.ui.explorer import MicrocodeExplorer @@ -37,7 +38,9 @@ def __init__(self, defer_load=False): class UIHooks(ida_kernwin.UI_Hooks): def ready_to_run(self): pass - + + MicrocodeOptions.clear_listeners() + self._startup_hooks = UIHooks() self._startup_hooks.ready_to_run = self.load diff --git a/plugins/lucid/ui/explorer.py b/plugins/lucid/ui/explorer.py index d1d5d98..8ae8519 100644 --- a/plugins/lucid/ui/explorer.py +++ b/plugins/lucid/ui/explorer.py @@ -530,8 +530,7 @@ def __init__(self, controller: MicrocodeExplorer, model: MicrocodeExplorerModel) def unload(self): ida_kernwin.close_widget(self._twidget, ida_kernwin.PluginForm.WCLS_DELETE_LATER) - self._code_sync.unload() - self._ui_hooks.unhook() + self._cleanup_hooks() def notify_change(self, option_name, option_value, **kwargs): """ @@ -555,12 +554,15 @@ def show(self): self._code_sync.hook() + def _cleanup_hooks(self): + self._code_sync.unhook() + self._ui_hooks.unhook() + def _cleanup(self): self.visible = False self._twidget = None self.widget = None - self._code_sync.unhook() - self._ui_hooks.unhook() + self._cleanup_hooks() # TODO cleanup controller / model #-------------------------------------------------------------------------- @@ -598,9 +600,11 @@ def widget_invisible(_, twidget): if twidget == self._twidget: self.visible = False self._cleanup() + self.notify_deletion() #XXX/HACK: sigh... def widget_visible(_, twidget): if twidget == self._twidget: self.visible = True + self.notify_creation() #XXX/HACK: sigh... def postprocess_action(_, *args): # XXX: seemingly the only way to allow the explorer to navigate via keyboard events... # (maybe this should be hooked elsewhere?) @@ -642,6 +646,7 @@ def _ui_init_settings(self): self._checkbox_sync.setCheckState(QtCore.Qt.Checked) self._checkbox_devmode = QtWidgets.QCheckBox("Developer mode") + self._refresh_button = QtWidgets.QPushButton("Refresh view") self._refresh_button.setFixedSize(120, 60) @@ -690,6 +695,11 @@ def _maturity_changed(item): self._checkbox_sync.stateChanged.connect(lambda x: self._code_sync.enable_sync(bool(x))) self._checkbox_devmode.stateChanged.connect(lambda x: self.controller.set_option('developer_mode', bool(x))) + if MicrocodeOptions.verbose: + self._checkbox_verbose.setCheckState(QtCore.Qt.Checked) + if MicrocodeOptions.developer_mode: + self._checkbox_devmode.setCheckState(QtCore.Qt.Checked) + # model signals self.model.mtext_changed += self.reinit self.model.maturity_changed += self.refresh diff --git a/plugins/lucid/util/options.py b/plugins/lucid/util/options.py index ae348db..8904cf4 100644 --- a/plugins/lucid/util/options.py +++ b/plugins/lucid/util/options.py @@ -187,6 +187,9 @@ def remove_listener(self, listener): self._listeners.remove(listener) return True + def clear_listeners(self): + self._listeners.clear() + def clear(self): if self._listeners: for name, value in self._options: diff --git a/plugins/lucid/util/python.py b/plugins/lucid/util/python.py index c194d0d..ec59091 100644 --- a/plugins/lucid/util/python.py +++ b/plugins/lucid/util/python.py @@ -107,6 +107,10 @@ def __init__(self, owner, name = "-unspecified-"): self.callbacks_list = [] self._ready = True + def __del__(self): + self.callbacks_list = [] + self._ready = False + def __call__(self, *args): """ Notify listeners of an event. From ce9aa680a1a3e8b7ca4c25b811f36b7492fa2f0a Mon Sep 17 00:00:00 2001 From: gaasedelen Date: Wed, 7 Feb 2024 20:35:16 -0500 Subject: [PATCH 7/7] some basic formatting / cleanup --- plugins/lucid/core.py | 10 +-- plugins/lucid/microtext.py | 112 ++++++++-------------------------- plugins/lucid/ui/subtree.py | 3 +- plugins/lucid/util/hexrays.py | 48 +++++++++++++++ 4 files changed, 78 insertions(+), 95 deletions(-) diff --git a/plugins/lucid/core.py b/plugins/lucid/core.py index e36482c..7b9a469 100644 --- a/plugins/lucid/core.py +++ b/plugins/lucid/core.py @@ -29,18 +29,14 @@ def __init__(self, defer_load=False): self.loaded = False self.explorer = None + MicrocodeOptions.clear_listeners() + # # we can 'defer' the load of the plugin core a little bit. this # ensures that all the other plugins (eg, decompilers) can get loaded # and initialized when opening an idb/bin # - class UIHooks(ida_kernwin.UI_Hooks): - def ready_to_run(self): - pass - - MicrocodeOptions.clear_listeners() - self._startup_hooks = UIHooks() self._startup_hooks.ready_to_run = self.load @@ -61,7 +57,7 @@ def load(self): """ self._startup_hooks.unhook() - # the plugin will only load for decompiler-capabale IDB's / installs + # the plugin will only load for decompiler-capable IDB's / installs if not hexrays_available(): return diff --git a/plugins/lucid/microtext.py b/plugins/lucid/microtext.py index 3bf37fb..cd4d66f 100644 --- a/plugins/lucid/microtext.py +++ b/plugins/lucid/microtext.py @@ -3,12 +3,10 @@ import ida_lines import ida_idaapi import ida_hexrays -import ida_hexrays as hr # M.K. 1/23/2024 -import ida_kernwin from lucid.text import TextCell, TextToken, TextLine, TextBlock from lucid.util.ida import tag_text -from lucid.util.hexrays import get_mmat_name, get_microcode +from lucid.util.hexrays import get_microcode, IPROP_FLAG_NAMES, MBL_TYPE_NAMES, MBL_PROP_NAMES, MBL_FLAG_NAMES from lucid.util.options import OptionListener, OptionProvider #----------------------------------------------------------------------------- @@ -29,6 +27,16 @@ # the microcode. For more information about the Text* classes, see text.py # +#----------------------------------------------------------------------------- +# Options +#----------------------------------------------------------------------------- + +MicrocodeOptions = OptionProvider({ + # options defined here can be accessed like normal class members ;) + 'developer_mode': False, + 'verbose': False, +}) + #----------------------------------------------------------------------------- # Annotation Tokens #----------------------------------------------------------------------------- @@ -47,14 +55,6 @@ MAGIC_BLK_DNU = 0x1235 MAGIC_BLK_VAL = 0x1236 MAGIC_BLK_TERM = 0x1237 - - -MicrocodeOptions = OptionProvider({ - # options defined here can be accessed like normal class members ;) - 'developer_mode': False, - 'verbose': False, -}) - class BlockHeaderLine(TextLine): """ @@ -209,7 +209,6 @@ def _create_subop(self, mop): return subop class InstructionCommentToken(TextToken): - """ A container token for micro-instruction comment text. """ @@ -229,24 +228,8 @@ def _generate_from_ins(self, blk, insn, usedef): items.append(AddressToken(insn.ea)) if MicrocodeOptions.developer_mode: - insn_flags = { - hr.IPROP_ASSERT: "ASSERT", - hr.IPROP_PERSIST: "PERSIST", - hr.IPROP_MBARRIER: "MBARRIER", - hr.IPROP_OPTIONAL: "OPT", - hr.IPROP_COMBINED: "COMB", - hr.IPROP_DONT_PROP: "NO_PROP", - hr.IPROP_DONT_COMB: "NO_COMB", - hr.IPROP_INV_JX: "INV_JX", - hr.IPROP_FPINSN: "FPINSN", - hr.IPROP_EXTSTX: "EXTSTX", - hr.IPROP_FARCALL: "FARCALL", - hr.IPROP_TAILCALL: "TAILCALL", - hr.IPROP_MULTI_MOV: "MULTI_MOV", - hr.IPROP_WAS_NORET: "WAS_NORET", - } - if insn_tokens := [TextCell(name) for flag, name in insn_flags.items() if insn.iprops & flag]: + if insn_tokens := [TextCell(name) for flag, name in IPROP_FLAG_NAMES.items() if insn.iprops & flag]: items.extend([TextCell(" ")] + [x for flag in insn_tokens for x in (TextCell(" +"), flag)]) # append the use/def list @@ -345,64 +328,19 @@ def _generate_header_lines(self): blk, mba = self.blk, self.blk.mba lines = [] - # block type names - type_names = \ - { - ida_hexrays.BLT_NONE: "????", - ida_hexrays.BLT_STOP: "STOP", - ida_hexrays.BLT_0WAY: "0WAY", - ida_hexrays.BLT_1WAY: "1WAY", - ida_hexrays.BLT_2WAY: "2WAY", - ida_hexrays.BLT_NWAY: "NWAY", - ida_hexrays.BLT_XTRN: "XTRN", - } - - blk_type = type_names[blk.type] - blk_props = { - hr.MBL_PRIV: "PRIVATE", - hr.MBL_FAKE: "FAKE", - hr.MBL_NORET: "NORET", - hr.MBL_DSLOT: "DSLOT", - hr.MBL_GOTO: "GOTO", - hr.MBL_TCAL: "TAILCALL", - } - blk_flags ={ - hr.MBL_KEEP: "KEEP", - hr.MBL_PROP: "PROP", - hr.MBL_COMB: "COMB", - hr.MBL_PUSH: "PUSH", - hr.MBL_CALL: "CALL", - hr.MBL_DMT64: "DMT_64BIT", - hr.MBL_INCONST: "INCONST", - hr.MBL_BACKPROP: "BACKPROP", - hr.MBL_VALRANGES: "VALRANGES", - } + blk_type = MBL_TYPE_NAMES[blk.type] # block properties prop_tokens = [] flag_tokens = [] - for flag, name in blk_props.items(): + for flag, name in MBL_PROP_NAMES.items(): if blk.flags & flag: prop_tokens.append(TextCell(name)) - for flag, name in blk_flags.items(): + + for flag, name in MBL_FLAG_NAMES.items(): if blk.flags & flag: flag_tokens.append(TextCell(name)) - - #if blk.flags & ida_hexrays.MBL_DSLOT: - # prop_tokens.append(TextCell("DSLOT")) - #if blk.flags & ida_hexrays.MBL_NORET: - # prop_tokens.append(TextCell("NORET")) - #if blk.needs_propagation(): - # prop_tokens.append(TextCell("PROP")) - #if blk.flags & ida_hexrays.MBL_COMB: - # prop_tokens.append(TextCell("COMB")) - #if blk.flags & ida_hexrays.MBL_PUSH: - # prop_tokens.append(TextCell("PUSH")) - #if blk.flags & ida_hexrays.MBL_TCAL: - # prop_tokens.append(TextCell("TAILCALL")) - #if blk.flags & ida_hexrays.MBL_FAKE: - # prop_tokens.append(TextCell("FAKE")) # misc block info prop_tokens = [x for prop in prop_tokens for x in (prop, TextCell(" "))] @@ -444,7 +382,6 @@ def _get_splitter(i): lines.extend(use_defs) else: lines.append(BlockHeaderLine([TextCell("- USE-DEF LISTS ARE EMPTY")], MAGIC_BLK_UDNR, parent=self)) - return lines @@ -561,12 +498,16 @@ def reinit(self): """ Reinitialize the underlying microcode and regenerate text. """ + + # get the most up-to-date microcode if not self.premade: - # get the most up-to-date microcode self.func = ida_funcs.get_func(self.func.start_ea) self.mba = get_microcode(self.func, self.maturity) - else: # do a one-time skip if we were just created/copied + + # do a one-time skip if we were just created/copied + else: self.premade = False + self.refresh() def refresh(self, maturity=None): @@ -901,15 +842,18 @@ def translate_instruction_position(position, mtext_src, mtext_dst): return (line_num_dst, x_dst, y) def get_best_ancestor_token(): + # common 'ancestor', eg the target token actually got its address from an ancestor token_src_ancestor = token_src.ancestor_with_address() for token in tokens: if token.text == token_src_ancestor.text: return token + # last ditch effort, try to land on a text that matches the target token for token in tokens: if token_src.text in token.text: return token + return None if token := get_best_ancestor_token(): @@ -921,12 +865,6 @@ def get_best_ancestor_token(): # yolo, just land on whatever token available line_num, x = mtext_dst.get_pos_of_token(tokens[0]) return (line_num, x, y) - - - - - - #----------------------------------------------------------------------------- # Position Remapping diff --git a/plugins/lucid/ui/subtree.py b/plugins/lucid/ui/subtree.py index 2fe6572..acd2851 100644 --- a/plugins/lucid/ui/subtree.py +++ b/plugins/lucid/ui/subtree.py @@ -187,7 +187,8 @@ def _get_mop_oprops(self, mop): for oprop, name in [(getattr(ida_hexrays, x), x) for x in filter(lambda y: y.startswith('OPROP_'), dir(ida_hexrays))]: if mop.oprops & oprop: text += f" +{name[6:]}" - return text + return text + def _insert_mop(self, mop, parent): if mop.t == 0: return -1 diff --git a/plugins/lucid/util/hexrays.py b/plugins/lucid/util/hexrays.py index 8209d8a..1717e3d 100644 --- a/plugins/lucid/util/hexrays.py +++ b/plugins/lucid/util/hexrays.py @@ -52,6 +52,54 @@ def get_all_vdui(): MOPT = [(getattr(ida_hexrays, x), x) for x in filter(lambda y: y.startswith('mop_'), dir(ida_hexrays))] MCODE = sorted([(getattr(ida_hexrays, x), x) for x in filter(lambda y: y.startswith('m_'), dir(ida_hexrays))]) +MBL_TYPE_NAMES = { + ida_hexrays.BLT_NONE: "????", + ida_hexrays.BLT_STOP: "STOP", + ida_hexrays.BLT_0WAY: "0WAY", + ida_hexrays.BLT_1WAY: "1WAY", + ida_hexrays.BLT_2WAY: "2WAY", + ida_hexrays.BLT_NWAY: "NWAY", + ida_hexrays.BLT_XTRN: "XTRN", +} + +MBL_PROP_NAMES = { + ida_hexrays.MBL_PRIV: "PRIVATE", + ida_hexrays.MBL_FAKE: "FAKE", + ida_hexrays.MBL_NORET: "NORET", + ida_hexrays.MBL_DSLOT: "DSLOT", + ida_hexrays.MBL_GOTO: "GOTO", + ida_hexrays.MBL_TCAL: "TAILCALL", +} + +MBL_FLAG_NAMES = { + ida_hexrays.MBL_KEEP: "KEEP", + ida_hexrays.MBL_PROP: "PROP", + ida_hexrays.MBL_COMB: "COMB", + ida_hexrays.MBL_PUSH: "PUSH", + ida_hexrays.MBL_CALL: "CALL", + ida_hexrays.MBL_DMT64: "DMT_64BIT", + ida_hexrays.MBL_INCONST: "INCONST", + ida_hexrays.MBL_BACKPROP: "BACKPROP", + ida_hexrays.MBL_VALRANGES: "VALRANGES", +} + +IPROP_FLAG_NAMES = { + ida_hexrays.IPROP_ASSERT: "ASSERT", + ida_hexrays.IPROP_PERSIST: "PERSIST", + ida_hexrays.IPROP_MBARRIER: "MBARRIER", + ida_hexrays.IPROP_OPTIONAL: "OPT", + ida_hexrays.IPROP_COMBINED: "COMB", + ida_hexrays.IPROP_DONT_PROP: "NO_PROP", + ida_hexrays.IPROP_DONT_COMB: "NO_COMB", + ida_hexrays.IPROP_INV_JX: "INV_JX", + ida_hexrays.IPROP_FPINSN: "FPINSN", + ida_hexrays.IPROP_EXTSTX: "EXTSTX", + ida_hexrays.IPROP_FARCALL: "FARCALL", + ida_hexrays.IPROP_TAILCALL: "TAILCALL", + ida_hexrays.IPROP_MULTI_MOV: "MULTI_MOV", + ida_hexrays.IPROP_WAS_NORET: "WAS_NORET", +} + class MatDelta: INCREASING = 1 NEUTRAL = 0