fixing fqdn searching (#420)

* fixing fqdn searching

* fixing flask vuln

* confidence tweak

cif/store/__init__.py

@@ -207,16 +207,43 @@ def _flush_create_queue(self):
             logger.debug('flushing queue...')
             data = [msg[0] for _, _, msg in self.create_queue[t]['messages']]
             _t = self.store.tokens.write(t)
+
             try:
                 start_time = time.time()
                 logger.info('attempting to insert %d indicators..', len(data))
 
+                # this will raise AuthError if the groups don't match
+                if isinstance(data, dict):
+                    data = [data]
+
+                for i in data:
+                    if not i.get('group'):
+                        i['group'] = 'everyone'
+
+                    if not i.get('provider') or i['provider'] == '':
+                        i['provider'] = _t['username']
+
+                    if i['group'] not in _t['groups']:
+                        raise AuthError('unable to write to %s' % i['group'])
+
+                    if not i.get('tags'):
+                        i['tags'] = ['suspicious']
+
+                    if i.get('message'):
+                        try:
+                            i['message'] = str(b64decode(data['message']))
+                        except (TypeError, binascii.Error) as e:
+                            pass
+
                 n = self.store.indicators.upsert(_t, data)
 
-                #n = len(data)
                 t_time = time.time() - start_time
                 logger.info('actually inserted %d indicators.. took %0.2f seconds (%0.2f/sec)', n, t_time, (n / t_time))
-                rv = {"status": "success", "data": n}
+
+                if n == 0:
+                    rv = {'status': 'failed', 'message': 'invalid indicator'}
+                else:
+                    rv = {"status": "success", "data": n}
 
             except AuthError as e:
                 rv = {'status': 'failed', 'message': 'unauthorized'}
@@ -240,7 +267,7 @@ def handle_indicators_create(self, token, data, id=None, client_id=None, flush=F
         # this will raise AuthError if false
         t = self.store.tokens.write(token)
 
-        if len(data) > 1:
+        if len(data) > 1 or t['username'] == 'admin':
             start_time = time.time()
             logger.info('attempting to insert %d indicators..', len(data))
 
@@ -250,11 +277,17 @@ def handle_indicators_create(self, token, data, id=None, client_id=None, flush=F
 
             for i in data:
                 if not i.get('group'):
-                    raise InvalidIndicator('missing group')
+                    i['group'] = 'everyone'
+
+                if not i.get('provider') or i['provider'] == '':
+                    i['provider'] = t['username']
 
                 if i['group'] not in t['groups']:
                     raise AuthError('unable to write to %s' % i['group'])
 
+                if not i.get('tags'):
+                    i['tags'] = ['suspicious']
+
                 if i.get('message'):
                     try:
                         i['message'] = str(b64decode(data['message']))
@@ -263,15 +296,14 @@ def handle_indicators_create(self, token, data, id=None, client_id=None, flush=F
 
             n = self.store.indicators.upsert(t, data, flush=flush)
 
-            #n = len(data)
             t = time.time() - start_time
             logger.info('actually inserted %d indicators.. took %0.2f seconds (%0.2f/sec)', n, t, (n/t))
 
             return n
 
         data = data[0]
         if data['group'] not in t['groups']:
-            raise AuthError('unauthorized to write to group: %s' % g)
+            raise AuthError('unauthorized to write to group: %s' % data['group'])
 
         if data.get('message'):
             try:

cif/store/sqlite/indicator.py

@@ -13,9 +13,6 @@
 from cifsdk.exceptions import InvalidSearch
 import ipaddress
 from .ip import Ip
-from .fqdn import Fqdn
-from .url import Url
-from .hash import Hash
 from pprint import pprint
 from sqlalchemy.ext.declarative import declarative_base
 import re
@@ -162,7 +159,7 @@ class Fqdn(Base):
     __tablename__ = 'indicators_fqdn'
 
     id = Column(Integer, primary_key=True)
-    fqdn = Column(Fqdn, index=True)
+    fqdn = Column(UnicodeText, index=True)
 
     indicator_id = Column(Integer, ForeignKey('indicators.id', ondelete='CASCADE'))
     indicator = relationship(
@@ -174,7 +171,7 @@ class Url(Base):
     __tablename__ = 'indicators_url'
 
     id = Column(Integer, primary_key=True)
-    url = Column(Url, index=True)
+    url = Column(UnicodeText, index=True)
 
     indicator_id = Column(Integer, ForeignKey('indicators.id', ondelete='CASCADE'))
     indicator = relationship(
@@ -186,7 +183,7 @@ class Hash(Base):
     __tablename__ = 'indicators_hash'
 
     id = Column(Integer, primary_key=True)
-    hash = Column(Hash, index=True)
+    hash = Column(String, index=True)
 
     indicator_id = Column(Integer, ForeignKey('indicators.id', ondelete='CASCADE'))
     indicator = relationship(
@@ -319,7 +316,7 @@ def _filter_indicator(self, filters, s):
         if itype == 'fqdn':
             s = s.join(Fqdn).filter(or_(
                     Fqdn.fqdn.like('%.{}'.format(i)),
-                    Fqdn.fqdn == i)
+                    Fqdn.fqdn == str(i))
             )
             return s
 
@@ -488,7 +485,7 @@ def upsert_indicators(self, s, n, d, token, tmp_added, batch):
             i = s.query(Indicator).options(lazyload('*')).filter_by(
                 provider=d['provider'],
                 itype=d['itype'],
-                indicator=d['indicator'],
+                indicator=d['indicator']
             ).order_by(Indicator.lasttime.desc())
 
             if d.get('rdata'):

requirements.txt

@@ -9,7 +9,7 @@ maxminddb>=1.2.0,<1.3
 geoip2>=2.8.0,<2.9
 pygeoip==0.3.2
 dnspython>=1.15.0,<1.16
-Flask==0.12.2
+Flask==0.12.3
 flask-cors>=3.0,<4.0
 PyYAML>=3.11,<4.0
 SQLAlchemy>=1.0.14,<1.1

test/zsqlite/test_store_sqlite_indicators.py

@@ -40,6 +40,7 @@ def indicator():
         'group': 'everyone',
         'lasttime': arrow.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
         'itype': 'fqdn',
+        'confidence': 6
     }
 
 
@@ -147,7 +148,7 @@ def test_store_sqlite_indicators(store, indicator):
     except InvalidIndicator:
         pass
 
-    assert (x is None or x == 0)
+    assert (x is None or x == 1)
 
     indicator['tags'] = 'malware'
 
@@ -157,7 +158,7 @@ def test_store_sqlite_indicators(store, indicator):
     except InvalidIndicator:
         pass
 
-    assert (x is None or x == 0)
+    assert (x is None or x == 1)
 
     r = store.handle_indicators_delete(t, data=[{
         'indicator': 'example.com',