Fix/packaging (#130)

all sorts of packaging fixes

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 kill*.sh
 cif.yml
 cif-router.yml

.travis.yml

@@ -6,14 +6,26 @@ python:
     - 3.4
     - 3.5
 
-before_install:
-    - pip install 'setuptools>=18.5'
-    - pip install pytest-cov
+install:
+  - pip install -r dev_requirements.txt
 
 script:
-    - py.test --cov=cif --cov-fail-under=35
+  - python setup.py test
+  - python setup.py sdist bdist bdist_wheel
+  #- sudo apt-get install build-essential virtualenvwrapper python-virtualenv devscripts debhelper cdbs
+  #- make deb
 
 notifications:
-    email:
-        on_success: never
-        on_failure: never
+  email:
+    on_success: never
+    on_failure: never
+
+deploy:
+  provider: pypi
+  user: wesyoung
+  password:
+    secure: diKbtf7RnRJAo7MdNW3weZDVAfP8j5XYVbcUa6iYBRpQ02zDBhQuZCtmFRms14Ls9nlj2SvBdchdk7WX5R/YbY8j3mI9VtrXdS58foetDPBMQuGJs2dAagpL/AmhssLRqTjKqTX2GXoEgvrKy/HpMiLqGmFKwvd5SlbEWEuC2BNiEL1r+wj/ZlQ36GHGKvNDRnItfwtTZxcDJ62ssdlWItYoRuZyyHEz5PlcYrnhcQHKKpmzzmeRYHHdG2HB42ynr/Plf4/bIjbK1FIMZz3UxZRujblXMkjIzxx/rngPtrUG5M5V28xva0muWrJNnqOvIoOY/A1oOVbysLsCCYR2uh6NcWfLzEFf2PJhjh3y7YZsTVX2rHrvhRKez1Qs+z6cvhvt4AVWRwilMi7TRkwgERhe0YqrRHT1EM5s1OGGN5rSE3Llqa63yvTi4EuMX0GPKZXdfkbihcY0ENexKfu36K3yBMJ39xmtGMKPiUW+aKLo3RZ+XRa+s28GFA5Tju5IYLeBTEz0QkiiovcK0Tl914jDDDE4BVcipL4DuQtF51qGO+9sgo/ppsFWKDFttIBOFYVk2X7UjY73JaXItpQofUKR1v/PRGZ7YSNfUX91bJUKtZ/nQVEysMkflWoqOv6A1296oje9KGE7cqV+8DCzPiFHQrlyLxJqtKfidMXqZC0=
+  on:
+    branch: master
+    tags: true
+    condition: $TRAVIS_PYTHON_VERSION = "3.5"

MANIFEST.in

@@ -1,8 +1,10 @@
 prune packaging
-include README.md COPYING requirements.txt LICENSE
+include README.md COPYING LICENSE
+include requirements.txt dev_requirements.txt
 include Makefile
 include VERSION
-include MANIFEST.ininclude versioneer.py
+include MANIFEST.in
+include versioneer.py
 include 'cif'
 include cif/_version.py
 include Vagrantfile

Makefile

@@ -76,12 +76,12 @@ sdist: clean
 PYINSTALLER = pyinstaller
 PYINSTALLER_OPTS = --distpath=$(PYINSTALLER_DIST_PATH) --workpath=$(PYINSTALLER_WORK_PATH) -y --onefile
 PYINSTALLER_SPEC_DIR = packaging/pyinstaller
-PYINSTALLER_SPECS = cif.spec cif-router.spec cif-smrt.spec cif-httpd.spec
+PYINSTALLER_SPECS = cif-router.spec cif-httpd.spec
 PYINSTALLER_BUILD = pyinstaller-build
 PYINSTALLER_DIST_PATH = dist
 PYINSTALLER_WORK_PATH = temp
 
-pyinstaller:
+binaries:
 	mkdir -p $(PYINSTALLER_BUILD) ;
 	cp -a cif $(PYINSTALLER_BUILD) ;
 	cp -a $(PYINSTALLER_SPEC_DIR)/*.spec $(PYINSTALLER_BUILD)/ ;

README.md

@@ -1,12 +1,12 @@
 # Getting Started
-Current master build status: [![Circle CI](https://circleci.com/gh/csirtgadgets/bearded-avenger.svg?style=svg&circle-token=e08a0a7ec010e17feb6724dbbc93ce584b9868cd)](https://circleci.com/gh/csirtgadgets/bearded-avenger)
+Current master build status: [![Travis](https://travis-ci.org/csirtgadgets/bearded-avenger.svg?branch=master)](https://travis-ci.org/csirtgadgets/bearded-avenger)
 
 ## QuickStart
 This assumes you have a proper Python dev already environment properly configured. If you need help getting started with this, checkout one of our [installation guides](https://github.com/csirtgadgets/bearded-avenger/wiki/Ubuntu14LTS).
 
 ```
-$ tar -zxvf cif-3.x.x.tar.gz
-$ cd cif-3.x.x
+$ git clone https://github.com/csirtgadgets/bearded-avenger.git
+$ cd bearded-avenger
 $ pip install -r requirements.txt
 $ python setup.py develop
 $ mkdir -p log && cp hacking/develop.conf hacking/local.conf

Vagrantfile

@@ -8,6 +8,7 @@ VAGRANTFILE_LOCAL = 'Vagrantfile.local'
 $script = <<SCRIPT
 echo 'yes' | sudo add-apt-repository 'ppa:fkrull/deadsnakes-python2.7'
 sudo apt-get update && sudo apt-get install -y python2.7 python-pip python-dev git libffi-dev libssl-dev sqlite3
+sudo pip install pip --upgrade
 sudo pip install 'setuptools>=11.3' 'ansible>=2.1' versioneer markupsafe
 cd /vagrant/deployment/ubuntu14
 sudo ansible-playbook -i "localhost," -c local vagrant.yml -vv

Vagrantfile_es

@@ -8,9 +8,9 @@ VAGRANTFILE_LOCAL = 'Vagrantfile.local'
 $script = <<SCRIPT
 echo 'yes' | sudo add-apt-repository 'ppa:fkrull/deadsnakes-python2.7'
 sudo apt-get update && sudo apt-get install -y python2.7 python-pip python-dev git libffi-dev libssl-dev sqlite3
-sudo pip install 'setuptools>=11.3' 'ansible>=2.1' versioneer markupsafe
+sudo pip install 'setuptools>=11.3' 'ansible>=2.1,<3.0' versioneer markupsafe
 cd /vagrant/deployment/ubuntu14
-sudo ansible-playbook -i "localhost," -c local elasticsearch.yml -vv
+cd /vagrant/deployment/ubuntu14/ && sudo bash bootstrap_elasticsearch.sh
 SCRIPT
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
@@ -21,7 +21,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.network :forwarded_port, guest: 9200, host:9200
   config.vm.network :forwarded_port, guest: 443, host: 8443
   config.vm.provider :virtualbox do |vb|
-    vb.customize ["modifyvm", :id, "--cpus", "2", "--ioapic", "on", "--memory", "1024" ]
+    vb.customize ["modifyvm", :id, "--cpus", "2", "--ioapic", "on", "--memory", "4096" ]
   end
 
   if File.file?(VAGRANTFILE_LOCAL)

Vagrantfile_prod

@@ -7,8 +7,8 @@ VAGRANTFILE_LOCAL = 'Vagrantfile.local'
 
 $script = <<SCRIPT
 echo 'yes' | sudo add-apt-repository 'ppa:fkrull/deadsnakes-python2.7'
-sudo apt-get update && sudo apt-get install -y python2.7 python-pip python-dev git libffi-dev libssl-dev sqlite3
-sudo pip install 'setuptools>=11.3' 'ansible>=2.1' versioneer markupsafe
+sudo apt-get update && sudo apt-get install -y python2.7 python3 python3-pip python3-dev python-pip python-dev git libffi-dev libssl-dev sqlite3
+sudo pip install 'setuptools>=18.5' 'ansible>=2.1' versioneer markupsafe
 cd /vagrant && sudo bash deployment/easybutton.sh
 SCRIPT
 
@@ -17,10 +17,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.box = 'ubuntu/trusty64'
 
   config.vm.network :forwarded_port, guest: 5000, host: 5000
-  config.vm.network :forwarded_port, guest: 9200, host: 9200
   config.vm.network :forwarded_port, guest: 443, host: 8443
+
   config.vm.provider :virtualbox do |vb|
-    vb.customize ["modifyvm", :id, "--cpus", "2", "--ioapic", "on", "--memory", "1024" ]
+    vb.customize ["modifyvm", :id, "--cpus", "2", "--ioapic", "on", "--memory", "2048" ]
   end
 
   if File.file?(VAGRANTFILE_LOCAL)

Vagrantfile_prod.centos

@@ -6,10 +6,11 @@ VAGRANTFILE_API_VERSION = "2"
 VAGRANTFILE_LOCAL = 'Vagrantfile.local'
 
 $script = <<SCRIPT
+yum -y update
 sudo rpm -iUvh http://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
 yum -y update
-sudo yum install -y gcc python-pip python-devel git libffi-devel openssl-devel
-sudo pip install 'setuptools>=11.3' 'ansible>=2.1' versioneer markupsafe
+sudo yum install -y gcc python-pip python-devel git libffi-devel openssl-devel python-virtualenv python-virtualenvwrapper
+sudo pip install 'setuptools>=18.5' 'ansible>=2.1' versioneer markupsafe
 cd /vagrant
 sudo bash deployment/easybutton.sh
 SCRIPT

cif/gatherer/__init__.py

@@ -4,6 +4,7 @@
 import logging
 import traceback
 import zmq
+import multiprocessing
 
 import cif.gatherer
 from cif.constants import GATHERER_ADDR, GATHERER_SINK_ADDR
@@ -15,17 +16,18 @@
 LINGER = 0
 
 
-class Gatherer(object):
+class Gatherer(multiprocessing.Process):
     def __enter__(self):
         return self
 
     def __exit__(self, type, value, traceback):
         return self
 
     def __init__(self, pull=GATHERER_ADDR, push=GATHERER_SINK_ADDR):
-
+        multiprocessing.Process.__init__(self)
         self.pull = pull
         self.push = push
+        self.exit = multiprocessing.Event()
 
     def _init_plugins(self):
         import pkgutil
@@ -36,6 +38,9 @@ def _init_plugins(self):
             self.gatherers.append(p.Plugin())
             logger.debug('plugin loaded: {}'.format(modname))
 
+    def terminate(self):
+        self.exit.set()
+
     def start(self):
         self._init_plugins()
 
@@ -50,8 +55,16 @@ def start(self):
         push_s.connect(self.push)
         logger.debug('starting Gatherer')
 
-        try:
-            while True:
+        poller = zmq.Poller()
+        poller.register(pull_s)
+
+        while not self.exit.is_set():
+            try:
+                s = dict(poller.poll(1000))
+            except Exception as e:
+                self.logger.error(e)
+                break
+            if pull_s in s:
                 m = pull_s.recv_multipart()
 
                 logger.debug(m)
@@ -80,6 +93,4 @@ def start(self):
                 logger.debug('sending back to router...')
                 push_s.send_multipart([id, null, mtype, token, data.encode('utf-8')])
 
-        except KeyboardInterrupt:
-            logger.info('shutting down gatherer..')
-            return
+        logger.info('shutting down gatherer..')

cif/hunter/__init__.py

@@ -8,26 +8,30 @@
 
 import cif.hunter
 from cifsdk.client.zeromq import ZMQ as Client
+from cifsdk.utils import setup_signals
 from cif.constants import HUNTER_ADDR, ROUTER_ADDR, HUNTER_SINK_ADDR
 from csirtg_indicator import Indicator
+import multiprocessing
 
 logger = logging.getLogger(__name__)
 
 SNDTIMEO = 15000
 ZMQ_HWM = 1000000
 
 
-class Hunter(object):
+class Hunter(multiprocessing.Process):
     def __enter__(self):
         return self
 
     def __exit__(self, type, value, traceback):
         return self
 
     def __init__(self, remote=HUNTER_ADDR, router=ROUTER_ADDR, token=None):
+        multiprocessing.Process.__init__(self)
         self.hunters = remote
         self.router = HUNTER_SINK_ADDR
         self.token = token
+        self.exit = multiprocessing.Event()
 
     def _load_plugins(self):
         import pkgutil
@@ -40,6 +44,9 @@ def _load_plugins(self):
 
         return plugins
 
+    def terminate(self):
+        self.exit.set()
+
     def start(self):
         # TODO - convert this to an async socket
         router = Client(remote=self.router, token=self.token, nowait=True)
@@ -53,13 +60,20 @@ def start(self):
         socket.connect(self.hunters)
         logger.debug('starting hunter')
 
-        try:
-            while True:
-                logger.debug('waiting...')
-                data = socket.recv()
-                logger.debug(data)
+        poller = zmq.Poller()
+        poller.register(socket, zmq.POLLIN)
+
+        while not self.exit.is_set():
+            try:
+                s = dict(poller.poll(1000))
+            except SystemExit or KeyboardInterrupt:
+                break
 
-                data = json.loads(data)
+            if socket in s:
+                data = socket.recv_multipart()
+
+                logger.debug(data)
+                data = json.loads(data[0])
                 if isinstance(data, dict):
                     data = [data]
 
@@ -73,6 +87,3 @@ def start(self):
                             logger.error(e)
                             traceback.print_exc()
                             logger.error('giving up on: {}'.format(d))
-        except KeyboardInterrupt:
-            logger.info('shutting down hunter...')
-            return

cif/p2p.py

@@ -1,5 +1,9 @@
-from pyre import Pyre
-from pyre import zhelper
+try:
+    from pyre import Pyre
+    from pyre import zhelper
+except ImportError as e:
+    raise ImportError('Requires Pyre to run in p2p mode..')
+
 import zmq
 import uuid
 import logging