move history deduplication from load time to search time

removes an O(N^2) operation at game load, significantly speeding up load
time

gui/launcher.lua

@@ -128,31 +128,29 @@ local function get_filter_pen()
 end
 
 -- trims the history down to its maximum size, if needed
-local function trim_history(hist, hist_set)
-    if #hist <= HISTORY_SIZE then return end
-    -- we can only ever go over by one, so no need to loop
+local function trim_history(hist)
+    local hist_size = #hist
+    local overage = hist_size - HISTORY_SIZE
+    if overage <= 0 then return end
     -- This is O(N) in the HISTORY_SIZE. if we need to make this more efficient,
     -- we can use a ring buffer.
-    local line = table.remove(hist, 1)
-    -- since all lines are guaranteed to be unique, we can just remove the hash
-    -- from the set instead of, say, decrementing a counter
-    hist_set[line] = nil
+    for i=overage+1,hist_size do
+        hist[i-overage] = hist[i]
+        if i > HISTORY_SIZE then
+            hist[i] = nil
+        end
+    end
 end
 
--- removes duplicate existing history lines and adds the given line to the front
-local function add_history(hist, hist_set, line)
+-- adds the given line to the front of the history as long as it is different from the previous command
+local function add_history(hist, line, defer_trim)
     line = line:trim()
-    if hist_set[line] then
-        for i,v in ipairs(hist) do
-            if v == line then
-                table.remove(hist, i)
-                break
-            end
-        end
-    end
+    local hist_size = #hist
+    if line == hist[hist_size] then return end
     table.insert(hist, line)
-    hist_set[line] = true
-    trim_history(hist, hist_set)
+    if not defer_trim then
+        trim_history(hist)
+    end
 end
 
 local function file_exists(fname)
@@ -161,42 +159,44 @@ end
 
 -- history files are written with the most recent entry on *top*, which the
 -- opposite of what we want. add the file contents to our history in reverse.
-local function add_history_lines(lines, hist, hist_set)
+-- you must manually call trim_history() after this function
+local function add_history_lines(lines, hist)
     for i=#lines,1,-1 do
-        add_history(hist, hist_set, lines[i])
+        add_history(hist, lines[i], true)
     end
 end
 
-local function add_history_file(fname, hist, hist_set)
+-- you must manually call trim_history() after this function
+local function add_history_file(fname, hist)
     if not file_exists(fname) then
         return
     end
     local lines = {}
     for line in io.lines(fname) do
         table.insert(lines, line)
     end
-    add_history_lines(lines, hist, hist_set)
+    add_history_lines(lines, hist)
 end
 
 local function init_history()
-    local hist, hist_set = {}, {}
+    local hist = {}
     -- snarf the console history into our active history. it would be better if
     -- both the launcher and the console were using the same history object so
     -- the sharing would be "live", but we can address that later.
-    add_history_file(CONSOLE_HISTORY_FILE_OLD, hist, hist_set)
-    add_history_file(CONSOLE_HISTORY_FILE, hist, hist_set)
+    add_history_file(CONSOLE_HISTORY_FILE_OLD, hist)
+    add_history_file(CONSOLE_HISTORY_FILE, hist)
 
     -- read in our own command history
-    add_history_lines(dfhack.getCommandHistory(HISTORY_ID, HISTORY_FILE),
-                      hist, hist_set)
+    add_history_lines(dfhack.getCommandHistory(HISTORY_ID, HISTORY_FILE), hist)
 
-    return hist, hist_set
-end
+    trim_history(hist)
 
-if not history then
-    history, history_set = init_history()
+    return hist
 end
 
+-- history is a list of previously run commands, most recent at history[#history]
+history = history or init_history()
+
 local function get_first_word(text)
     local word = text:trim():split(' +')[1]
     if word:startswith(':') then word = word:sub(2) end
@@ -208,7 +208,7 @@ local function get_command_count(command)
 end
 
 local function record_command(line)
-    add_history(history, history_set, line)
+    add_history(history, line)
     local firstword = get_first_word(line)
     user_freq.data[firstword] = (user_freq.data[firstword] or 0) + 1
     user_freq:write()
@@ -470,6 +470,7 @@ EditPanel.ATTRS{
 
 function EditPanel:init()
     self.stack = {}
+    self.seen_search = {}
     self:reset_history_idx()
 
     self:addviews{
@@ -579,16 +580,22 @@ function EditPanel:move_history(delta)
 end
 
 function EditPanel:on_search_text(search_str, next_match)
+    if not next_match then self.seen_search = {} end
     if not search_str or #search_str == 0 then return end
     local start_idx = math.min(self.history_idx - (next_match and 1 or 0),
                                #history)
     for history_idx = start_idx, 1, -1 do
-        if history[history_idx]:find(search_str, 1, true) then
+        local line = history[history_idx]
+        if line:find(search_str, 1, true) then
             self:move_history(history_idx - self.history_idx)
-            return
+            if not self.seen_search[line] then
+                self.seen_search[line] = true
+                return
+            end
         end
     end
     -- no matches. restart at the saved input buffer for the next search.
+    self.seen_search = {}
     self:move_history(#history + 1 - self.history_idx)
 end