Release (#377)

* bumping reqs

* fixes #370

* fixes #376

* fixes #371

* cleaning up debug info

* testfix

cif/httpd/__init__.py

@@ -11,6 +11,7 @@
 from flask import Flask, request, session, redirect, url_for, render_template, _request_ctx_stack, send_from_directory, g
 #from flask.ext.session import Session
 from flask_limiter import Limiter
+from flask_cors import CORS
 from flask_limiter.util import get_remote_address
 from flask_bootstrap import Bootstrap
 from os import path
@@ -73,13 +74,15 @@ def proxy_get_remote_address():
 
 app = Flask(__name__)
 Bootstrap(app)
+CORS(app, resources={r"/*": {"origins": "*"}})
 app.secret_key = SECRET_KEY
 
 remote = ROUTER_ADDR
 
 log_level = logging.WARN
 if TRACE == '1':
     log_level = logging.DEBUG
+    logging.getLogger('flask_cors').level = logging.DEBUG
 
 console = logging.StreamHandler()
 logging.getLogger('gunicorn.error').setLevel(log_level)
@@ -100,6 +103,7 @@ def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static'),
                               'favicon.ico', mimetype='image/vnd.microsoft.icon')
 
+
 app.add_url_rule('/u', view_func=IndicatorsUI.as_view('/u'))
 app.add_url_rule('/u/search', view_func=IndicatorsUI.as_view('/u/search'))
 app.add_url_rule('/u/submit', view_func=SubmitUI.as_view('/u/submit'))

cif/httpd/views/help.py

@@ -19,4 +19,4 @@ def get(self):
             'POST /tokens': 'create a token or set of tokens',
             'DELETE /tokens?{username,token}': 'delete a token or set of tokens',
             'PATCH /token': 'update a token'
-        })
+        })

cif/hunter/fqdn_mx.py

@@ -8,6 +8,7 @@
 from pprint import pprint
 import arrow
 
+
 class FqdnMx(object):
 
     def __init__(self):
@@ -38,16 +39,20 @@ def process(self, i, router):
             fqdn.indicator = rr.rstrip('.')
             fqdn.lasttime = arrow.utcnow()
 
+            # 10
+            if re.match('^\d+$', rr):
+                return
 
             try:
                 resolve_itype(fqdn.indicator)
             except InvalidIndicator as e:
-                self.logger.error(fqdn)
-                self.logger.error(e)
+                self.logger.info(fqdn)
+                self.logger.info(e)
             else:
                 fqdn.itype = 'fqdn'
                 fqdn.rdata = i.indicator
                 fqdn.confidence = (fqdn.confidence - 5) if fqdn.confidence >= 5 else 0
                 router.indicators_create(fqdn)
 
+
 Plugin = FqdnMx

cif/store/sqlite/indicator.py

@@ -389,8 +389,12 @@ def _filter_terms(self, filters, s):
 
         return s
 
-    def _filter_groups(self, token, s):
-        groups = token.get('groups', 'everyone')
+    def _filter_groups(self, filters, token, s):
+        if token:
+            groups = token.get('groups', 'everyone')
+        else:
+            groups = filters.get('groups')
+
         if isinstance(groups, str):
             groups = [groups]
 
@@ -408,7 +412,11 @@ def _search(self, filters, token):
 
         s = self._filter_indicator(myfilters, s)
         s = self._filter_terms(myfilters, s)
-        s = self._filter_groups(token, s)
+
+        if myfilters.get('groups'):
+            s = self._filter_groups(myfilters, None, s)
+        else:
+            s = self._filter_groups({}, token, s)
         return s
 
     def search(self, token, filters, limit=500):

cif/store/zelasticsearch/filters.py

@@ -99,8 +99,12 @@ def filter_terms(s, q_filters):
     return s
 
 
-def filter_groups(s, token):
-    groups = token.get('groups', 'everyone')
+def filter_groups(s, q_filters, token=None):
+    if token:
+        groups = token.get('groups', 'everyone')
+    else:
+        groups = q_filters['groups']
+
     if isinstance(groups, basestring):
         groups = [groups]
 
@@ -127,7 +131,10 @@ def filter_build(s, filters, token=None):
     # transform all other filters into term=
     s = filter_terms(s, q_filters)
 
-    if token:
-        s = filter_groups(s, token)
+    if filters.get('groups'):
+        s = filter_groups(s, filters)
+    else:
+        if token:
+            s = filter_groups(s, {}, token=token)
 
     return s

cif/utils/__init__.py

@@ -24,7 +24,7 @@ def resolve_ns(data, t='A', timeout=HUNTER_RESOLVER_TIMEOUT):
 
         if not str(e).startswith('The DNS response does not contain an answer to the question'):
             if not str(e).startswith('None of DNS query names exist'):
-                logger.warn('{} - {}'.format(data, e))
+                logger.info('{} - {}'.format(data, e))
         return []
 
     return resp

requirements.txt

@@ -2,14 +2,15 @@ setuptools>=36
 cython>=0.2
 pyzmq>=16.0.2,<17.0
 csirtg_indicator==0.0.0b24
-cifsdk==3.0.0b3
+cifsdk==3.0.0b4
 Flask-Limiter>=0.9.4,<1.0
 limits>=1.1.1,<1.2
 maxminddb>=1.2.0,<1.3
 geoip2>=2.2.0,<2.3
 pygeoip==0.3.2
 dnspython>=1.15.0,<1.16
 Flask==0.12.2
+flask-cors>=3.0,<4.0
 PyYAML>=3.11,<4.0
 SQLAlchemy>=1.0.14,<1.1
 elasticsearch>=5.3,<5.5
@@ -18,7 +19,7 @@ ujson>=1.35
 html5lib==1.0b8 # bug in csirtg-smrt upstream
 msgpack-python>=0.4.8,<0.5.0
 apwgsdk==0.0.0a6
-csirtg_smrt==0.0.0b6
+csirtg_smrt==0.0.0b7
 csirtg_dnsdb==0.0.0a4
 tornado>=4.4.1,<5.0
 faker==0.7.11

rules/default/bambenek.yml

@@ -1,8 +1,10 @@
+parser: csv
+
 defaults:
   provider: osint.bambenekconsulting.com
   tlp: white
   altid_tlp: white
-  confidence: 9
+  confidence: 8
   tags: botnet
   values:
       - indicator
@@ -11,22 +13,16 @@ defaults:
       - altid
 
 feeds:
-  c2-domainmasterlist:
-    remote: http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt
-    pattern: ^(\S+)\,Domain used by ((?!suppobox)[^,]+)\,([^,]+)\,(\S+)$
-
-  c2-domainmasterlist-suppobox:
-    remote: http://osint.bambenekconsulting.com/feeds/c2-dommasterlist.txt
-    pattern: ^(\S+)\,Domain used by (suppobox)\,([^,]+)\,(\S+)$
-    defaults:
-      confidence: 6.5
+  c2_ipmasterlist_high:
+    remote: http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist-high.txt
 
-  c2-ipmasterlist:
-    remote: http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist.txt
-    pattern: ^(\S+)\,IP used by ((?!suppobox)[^,]+)\,([^,]+)\,(\S+)$
+  c2_domain_masterlist_high:
+    remote: http://osint.bambenekconsulting.com/feeds/c2-dommasterlist-high.txt
 
-  c2-ipmasterlist-suppobox:
-    remote: http://osint.bambenekconsulting.com/feeds/c2-ipmasterlist.txt
-    pattern: ^(\S+)\,IP used by (suppobox C&C)\,([^,]+)\,(\S+)$
+  dga_domains_high:
+    remote: http://osint.bambenekconsulting.com/feeds/dga-feed-high.csv.gz
+    cache: dga-feed-high.csv
     defaults:
-      confidence: 6.5
+      tags:
+        - dga
+        - botnet

test/test_gatherer_asn.py

@@ -25,7 +25,7 @@ def _resolve(i):
     a._resolve_ns = _resolve
     x = a.process(Indicator(indicator='216.90.108.0'))
 
-    if x.asn:
+    if x.asn and x.asn_desc:
         assert x.asn == '23028'
         assert x.asn_desc.startswith('TEAM-CYMRU')
     else:
@@ -41,4 +41,4 @@ def test_gatherer_asn_fast():
     if x.asn:
         assert x['asn'] == 23028
     else:
-        warnings.warn('TC Not Responding...', UserWarning)
+        warnings.warn('TC Not Responding...', UserWarning)

test/zelasticsearch/test_store_elasticsearch_tokens_groups.py

@@ -102,8 +102,12 @@ def test_store_elasticsearch_tokens_groups1(store, token, indicator):
     assert i
 
     i = store.handle_indicators_search(t, {'itype': 'fqdn'})
+    i = json.loads(i)
+    i = [i['_source'] for i in i['hits']['hits']]
     assert len(list(i)) > 0
 
+    pprint(i)
+
     i = store.handle_indicators_search(t, {'indicator': 'example.com'})
     assert len(list(i)) > 0
 
@@ -164,10 +168,49 @@ def test_store_elasticsearch_tokens_groups3(store, indicator):
 
     i = store.handle_indicators_search(t2['token'], {'itype': 'fqdn'})
     i = json.loads(i)
-    #i = [i['_source'] for i in i['hits']['hits']]
     assert len(i) == 0
 
     i = store.handle_indicators_search(t2['token'], {'indicator': 'example.com'})
     i = json.loads(i)
-    #i = [i['_source'] for i in i['hits']['hits']]
     assert len(i) == 0
+
+
+@pytest.mark.skipif(DISABLE_TESTS, reason='need to set CIF_ELASTICSEARCH_TEST=1 to run')
+def test_store_elasticsearch_tokens_groups4(store, indicator):
+    t = store.store.tokens.create({
+        'username': 'test',
+        'groups': ['staff', 'staff2'],
+        'write': True,
+        'read': True
+    })
+
+    i = store.handle_indicators_create(t['token'], {
+        'indicator': 'example.com',
+        'group': 'staff',
+        'provider': 'example.com',
+        'tags': ['test'],
+        'itype': 'fqdn',
+        'lasttime': arrow.utcnow().datetime,
+        'reporttime': arrow.utcnow().datetime
+
+    }, flush=True)
+
+    assert i
+
+    i = store.handle_indicators_create(t['token'], {
+        'indicator': 'example.com',
+        'group': 'staff2',
+        'provider': 'example.com',
+        'tags': ['test'],
+        'itype': 'fqdn',
+        'lasttime': arrow.utcnow().datetime,
+        'reporttime': arrow.utcnow().datetime
+
+    }, flush=True)
+
+    assert i
+
+    i = store.handle_indicators_search(t['token'], {'itype': 'fqdn', 'groups': 'staff'})
+    i = json.loads(i)
+    i = [i['_source'] for i in i['hits']['hits']]
+    assert len(i) == 1

test/zsqlite/test_store_sqlite_tokens.py

@@ -168,3 +168,41 @@ def test_store_sqlite_tokens_groups3(store, indicator):
 
     i = store.store.indicators.search(t2, {'indicator': 'example.com'})
     assert len(list(i)) == 0
+
+
+def test_store_sqlite_tokens_groups4(store, indicator):
+    t = store.store.tokens.create({
+        'username': 'test',
+        'groups': ['staff', 'staff2'],
+        'write': True,
+        'read': True
+    })
+
+    i = store.store.indicators.create(t, {
+        'indicator': 'example.com',
+        'group': 'staff',
+        'provider': 'example.com',
+        'tags': ['test'],
+        'itype': 'fqdn',
+        'lasttime': arrow.utcnow().datetime,
+        'reporttime': arrow.utcnow().datetime
+
+    })
+
+    assert i
+
+    i = store.store.indicators.create(t, {
+        'indicator': 'example.com',
+        'group': 'staff2',
+        'provider': 'example.com',
+        'tags': ['test'],
+        'itype': 'fqdn',
+        'lasttime': arrow.utcnow().datetime,
+        'reporttime': arrow.utcnow().datetime
+
+    })
+
+    assert i
+
+    i = store.store.indicators.search(t['token'], {'itype': 'fqdn', 'groups': 'staff'})
+    assert len(list(i)) == 1