Support fulltext search by trigger name and patterns moira-alert#34

moira/api/resources/trigger.py

@@ -126,14 +126,18 @@ def __init__(self, db):
     def render_GET(self, request):
         filter_ok = request.getCookie('moira_filter_ok')
         filter_tags = request.getCookie('moira_filter_tags')
+        filter_words = request.getCookie('moira_filter_words')
         page = request.args.get("p")
         size = request.args.get("size")
+
         page = 0 if page is None else int(page[0])
         size = 10 if size is None else int(size[0])
         filter_ok = False if filter_ok is None else filter_ok == 'true'
         filter_tags = [] if not filter_tags else unquote(filter_tags).split(',')
-        if not filter_ok and len(filter_tags) == 0:
-            triggers, total = yield self.db.getTriggersChecksPage(page * size, size - 1)
+        filter_words = [] if not filter_words else unquote(filter_words).split(',')  
+
+        if any([filter_ok, filter_tags, filter_words]):
+            triggers, total = yield self.db.getFilteredTriggersChecksPage(page, size, filter_ok, filter_tags, filter_words)
         else:
-            triggers, total = yield self.db.getFilteredTriggersChecksPage(page, size, filter_ok, filter_tags)
+            triggers, total = yield self.db.getTriggersChecksPage(page * size, size - 1)
         self.write_json(request, {"list": triggers, "page": page, "size": size, "total": total})

moira/db.py

@@ -11,6 +11,7 @@
 from moira import config
 from moira.cache import cache
 from moira import logs
+from moira.tools.search import get_tokens_for_text, get_tokens_for_pattern
 from moira.trigger import trigger_reformat
 
 _doc_string = """
@@ -41,6 +42,8 @@
     - SORTED SET {23}
     - SET {24}
     - KEY {25}
+    - SET {26}
+    - SET {27}
 """
 
 __docformat__ = 'reStructuredText'
@@ -75,6 +78,8 @@
 TRIGGER_CHECK_LOCK_PREFIX = "moira-metric-check-lock:{0}"
 TRIGGER_IN_BAD_STATE = "moira-bad-state-triggers"
 CHECKS_COUNTER = "moira-selfstate:checks-counter"
+SEARCH_WORD_PREFIX = "moira-search-word:{}"
+TRIGGER_SWORD_PREFIX = "moira-trigger-swords:{0}"
 
 TRIGGER_EVENTS_TTL = 3600 * 24 * 30
 
@@ -102,10 +107,12 @@
     TRIGGER_EVENTS.format("<trigger_id>"),
     TRIGGER_THROTTLING_BEGINNING_PREFIX.format("<trigger_id>"),
     TAG_PREFIX.format("<tag>"),
-    TRIGGER_CHECK_LOCK_PREFIX.format("trigger_id"),
+    TRIGGER_CHECK_LOCK_PREFIX.format("<trigger_id>"),
     TRIGGERS_CHECKS,
     TRIGGER_IN_BAD_STATE,
-    CHECKS_COUNTER
+    CHECKS_COUNTER,
+    SEARCH_WORD_PREFIX.format("<word>"),
+    TRIGGER_SWORD_PREFIX.format("<trigger_id>")
 )
 
 
@@ -421,36 +428,93 @@ def saveTrigger(self, trigger_id, trigger, existing=None):
         if ttl is not None:
             trigger["ttl"] = str(ttl)
         tags = trigger.get("tags", [])
-        t = yield self.rc.multi()
         patterns = trigger.get("patterns", [])
+
+        t = yield self.rc.multi()
+
         cleanup_patterns = []
         if existing is not None:
-            for pattern in [
-                item for item in existing.get(
-                    "patterns",
-                    []) if item not in patterns]:
-                yield t.srem(PATTERN_TRIGGERS_PREFIX.format(pattern), trigger_id)
-                cleanup_patterns.append(pattern)
-            for tag in [
-                item for item in existing.get(
-                    "tags",
-                    []) if item not in tags]:
-                yield self.removeTriggerTag(trigger_id, tag, t)
+            # clean old patterns
+            old_patterns = existing.get("patterns", [])
+            for old_pattern in set(old_patterns) - set(patterns):
+                yield t.srem(PATTERN_TRIGGERS_PREFIX.format(old_pattern), trigger_id)
+                cleanup_patterns.append(old_pattern)
+
+            # clean old tags
+            old_tags = existing.get("tags", [])
+            for old_tag in set(old_tags) - set(tags):
+                yield self.removeTriggerTag(trigger_id, old_tag, t)
+
         yield t.set(TRIGGER_PREFIX.format(trigger_id), anyjson.serialize(trigger))
         yield t.sadd(TRIGGERS, trigger_id)
+
         for pattern in patterns:
             yield t.sadd(PATTERNS, pattern)
             yield t.sadd(PATTERN_TRIGGERS_PREFIX.format(pattern), trigger_id)
+
         for tag in tags:
             yield self.addTriggerTag(trigger_id, tag, t)
+
+        yield self.addTriggerToSearchIndex(trigger_id, trigger, existing)
+
         yield t.commit()
+
         for pattern in cleanup_patterns:
             triggers = yield self.getPatternTriggers(pattern)
             if not triggers:
                 yield self.removePatternTriggers(pattern)
                 yield self.removePattern(pattern)
                 yield self.delPatternMetrics(pattern)
 
+    @defer.inlineCallbacks
+    @docstring_parameters(
+	TRIGGER_SWORD_PREFIX.format("<trigger_id>"),
+        SEARCH_WORD_PREFIX.format("<word>"))
+    def addTriggerToSearchIndex(self, trigger_id, trigger, existing=None):
+        """
+        addTriggerToSearchIndex(self, trigger_id, trigger)
+
+        Creates tokens for trigger:
+            - Update search words set {0}
+	    - Add *trigger_id* to set {1}
+
+        :param trigger: trigger json object
+        :type trigger: dict
+        :param trigger_id: trigger identity
+        :type trigger_id: string
+        """
+
+        tags = trigger.get("tags", [])
+        patterns = trigger.get("patterns", [])
+        title = trigger.get("name", u"")
+
+        tokens = get_tokens_for_text(title)
+        for p in patterns:
+            tokens.extend(get_tokens_for_pattern(p))
+        words = set(tokens)
+
+        pipeline = yield self.rc.pipeline()
+
+        cleanup_words = []
+        if existing is not None:
+            # clean old keys
+            old_words = yield self.rc.smembers(TRIGGER_SWORD_PREFIX.format(trigger_id))
+            cleanup_words = set(old_words) - words
+            for word in cleanup_words:
+                yield self.rc.srem(TRIGGER_SWORD_PREFIX.format(trigger_id), word)
+
+        for word in words:
+            pipeline.sadd(TRIGGER_SWORD_PREFIX.format(trigger_id), word)
+            pipeline.sadd(SEARCH_WORD_PREFIX.format(word), trigger_id)
+
+        yield pipeline.execute_pipeline()
+
+        for word in cleanup_words:
+            word_with_prefix = SEARCH_WORD_PREFIX.format(word)
+            have_triggers = (yield self.rc.scard(word_with_prefix)) == 0
+            if not have_triggers:
+                yield self.rc.delete(word_with_prefix)
+
     @defer.inlineCallbacks
     @docstring_parameters(PATTERNS)
     def getPatterns(self):
@@ -520,13 +584,15 @@ def getTrigger(self, trigger_id):
     @defer.inlineCallbacks
     def _getTriggersChecks(self, triggers_ids):
         triggers = []
+
         pipeline = yield self.rc.pipeline()
         for trigger_id in triggers_ids:
             pipeline.get(TRIGGER_PREFIX.format(trigger_id))
             pipeline.smembers(TRIGGER_TAGS_PREFIX.format(trigger_id))
             pipeline.get(LAST_CHECK_PREFIX.format(trigger_id))
             pipeline.get(TRIGGER_NEXT_PREFIX.format(trigger_id))
         results = yield pipeline.execute_pipeline()
+
         slices = [[triggers_ids[i / 4]] + results[i:i + 4] for i in range(0, len(results), 4)]
         for trigger_id, trigger_json, trigger_tags, last_check, throttling in slices:
             if trigger_json is None:
@@ -536,6 +602,7 @@ def _getTriggersChecks(self, triggers_ids):
             trigger["last_check"] = None if last_check is None else anyjson.deserialize(last_check)
             trigger["throttling"] = long(throttling) if throttling and time.time() < long(throttling) else 0
             triggers.append(trigger)
+
         defer.returnValue(triggers)
 
     @defer.inlineCallbacks
@@ -574,7 +641,7 @@ def getTriggersChecksPage(self, start, size):
 
     @defer.inlineCallbacks
     @docstring_parameters(TRIGGERS_CHECKS)
-    def getFilteredTriggersChecksPage(self, page, size, filter_ok, filter_tags):
+    def getFilteredTriggersChecksPage(self, page, size, filter_ok, filter_tags, filter_words=None):
         """
         getFilteredTriggersChecksPage(self, page, size, filter_ok, filter_tags)
 
@@ -588,26 +655,23 @@ def getFilteredTriggersChecksPage(self, page, size, filter_ok, filter_tags):
         :type filter_ok: boolean
         :param filter_tags: use tag triggers set
         :type filter_tags: list of strings
+        :param filter_words: use words triggers set
+        :type filter_words: list of strings
         :rtype: json
         """
-        filter_sets = map(lambda tag: TAG_TRIGGERS_PREFIX.format(tag), filter_tags)
+        filter_sets = map(TAG_TRIGGERS_PREFIX.format, filter_tags)
+        filter_sets.extend(
+            map(SEARCH_WORD_PREFIX.format, filter_words or [])
+        )
         if filter_ok:
             filter_sets.append(TRIGGER_IN_BAD_STATE)
+
         pipeline = yield self.rc.pipeline()
         pipeline.zrevrange(TRIGGERS_CHECKS, start=0, end=-1)
-        for s in filter_sets:
-            pipeline.smembers(s)
-        triggers_lists = yield pipeline.execute_pipeline()
-        total = []
-        for id in triggers_lists[0]:
-            valid = True
-            for s in triggers_lists[1:]:
-                if id not in s:
-                    valid = False
-                    break
-            if valid:
-                total.append(id)
+        pipeline.sinter(*filter_sets)
+        triggers_lists, filtered_trigger_set = yield pipeline.execute_pipeline()
 
+        total = filter(lambda t_id: t_id in filtered_trigger_set, triggers_lists)
         filtered_ids = total[page * size: (page + 1) * size]
         triggers = yield self._getTriggersChecks(filtered_ids)
         defer.returnValue((triggers, len(total)))
@@ -833,7 +897,10 @@ def removeTag(self, tag, existing=None):
     @audit
     @defer.inlineCallbacks
     @docstring_parameters(TRIGGER_PREFIX.format("<trigger_id>"),
-                          TRIGGERS, PATTERN_TRIGGERS_PREFIX.format("<pattern>"))
+                          TRIGGERS,
+                          PATTERN_TRIGGERS_PREFIX.format("<pattern>"),
+                          TRIGGER_SWORD_PREFIX.format("<trigger_id>"),
+                          SEARCH_WORD_PREFIX.format("<word>"))
     def removeTrigger(self, trigger_id, existing=None):
         """
         removeTrigger(self, trigger_id)
@@ -842,20 +909,34 @@ def removeTrigger(self, trigger_id, existing=None):
             - Delete key {0}
             - Remove *trigger_id* from set {1}
             - Remove *trigger_id* from set {2}
+	    - Delete key {3}
+            - Remove *trigger_id* from set {4}
 
         :param trigger_id: trigger identity
         :type trigger_id: string
         """
         if existing is not None:
             t = yield self.rc.multi()
+
             yield t.delete(TRIGGER_PREFIX.format(trigger_id))
             yield t.delete(TRIGGER_TAGS_PREFIX.format(trigger_id))
             yield t.srem(TRIGGERS, trigger_id)
+            search_words = yield self.rc.smembers(TRIGGER_SWORD_PREFIX.format(trigger_id))
+            yield t.delete(TRIGGER_SWORD_PREFIX.format(trigger_id))
+
             for tag in existing.get("tags", []):
                 yield t.srem(TAG_TRIGGERS_PREFIX.format(tag), trigger_id)
             for pattern in existing.get("patterns", []):
                 yield t.srem(PATTERN_TRIGGERS_PREFIX.format(pattern), trigger_id)
+            for word in search_words:
+                word_with_prefix = SEARCH_WORD_PREFIX.format(word)
+                yield t.srem(word_with_prefix, trigger_id)
+                have_triggers = (yield t.scard(word_with_prefix)) == 0
+                if not have_triggers:
+                    yield t.delete(word_with_prefix)
+
             yield t.commit()
+
             for pattern in existing.get("patterns", []):
                 count = yield self.rc.scard(PATTERN_TRIGGERS_PREFIX.format(pattern))
                 if count == 0:

moira/tools/search.py

@@ -0,0 +1,13 @@
+from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter, StemmingAnalyzer
+
+
+text_analyzer = StemmingAnalyzer() | LowercaseFilter() | StopFilter()
+pattern_analyzer = RegexTokenizer(r"\w+") | LowercaseFilter()
+
+
+def get_tokens_for(analyzer):
+    return lambda line: [t.text for t in analyzer(line)]
+
+
+get_tokens_for_text = get_tokens_for(text_analyzer)
+get_tokens_for_pattern = get_tokens_for(pattern_analyzer)

requirements.txt

@@ -7,4 +7,5 @@ txredisapi
 ujson
 pyyaml
 fakeredis
-flake8
+flake8
+Whoosh==2.7.4