Merge branch 'release-1.2'

.gitignore

@@ -56,6 +56,9 @@ target/
 fixtures/*
 !fixtures/*.tgz
 !fixtures/*.zip
+tests/fixtures/
+!tests/fixtures/*.tgz
+!tests/fixtures/*.zip
 venv
 conf/tribe.yaml
 .ipynb_checkpoints

.travis.yml

@@ -3,6 +3,7 @@ language: python
 
 python:
   - '2.7'
+  - '3.5'
 
 before_install:
   - pip install nose

Makefile

@@ -13,10 +13,12 @@ PYTHON_BIN := $(VIRTUAL_ENV)/bin
 # Clean build files
 clean:
 	find . -name "*.pyc" -print0 | xargs -0 rm -rf
+	find . -name "__pycache__" -print0 | xargs -0 rm -rf
 	-rm -rf htmlcov
 	-rm -rf .coverage
 	-rm -rf build
 	-rm -rf dist
+	-rm -rf site
 	-rm -rf $(PROJECT).egg-info
 
 # Targets for Coruscate testing

README.md

@@ -114,6 +114,14 @@ The release versions that are sent to the Python package index (PyPI) are also t
 
 The versioning uses a three part version system, "a.b.c" - "a" represents a major release that may not be backwards compatible. "b" is incremented on minor releases that may contain extra features, but are backwards compatible. "c" releases are bug fixes or other micro changes that developers should feel free to immediately update to.
 
+### Version 1.2
+
+* **tag**: [v1.2](https://github.com/DistrictDataLabs/tribe/releases/tag/v1.2)
+* **release**:  Wednesday, June 22, 2016
+* **commit**: [see tag](#)
+
+In this release we have improved some of the handling code to make things a bit more robust with students who work on a variety of operating systems. For example we have added a progress indicator so that something appears to be happening on very large mbox files (and you're not left wondering). Additionally we have added better error handling so one bad email doesn't ruin your day. We also made the library Python 2.7 and Python 3.5 compatible with a better test suite. 
+
 ### Version 1.1.2
 
 * **tag**: [v1.1.2](https://github.com/DistrictDataLabs/tribe/releases/tag/v1.1.2)

docs/about.md

@@ -47,6 +47,14 @@ The release versions that are sent to the Python package index (PyPI) are also t
 
 The versioning uses a three part version system, "a.b.c" - "a" represents a major release that may not be backwards compatible. "b" is incremented on minor releases that may contain extra features, but are backwards compatible. "c" releases are bug fixes or other micro changes that developers should feel free to immediately update to.
 
+### Version 1.2
+
+* **tag**: [v1.2](https://github.com/DistrictDataLabs/tribe/releases/tag/v1.2)
+* **release**:  Wednesday, June 22, 2016
+* **commit**: [see tag](#)
+
+In this release we have improved some of the handling code to make things a bit more robust with students who work on a variety of operating systems. For example we have added a progress indicator so that something appears to be happening on very large mbox files (and you're not left wondering). Additionally we have added better error handling so one bad email doesn't ruin your day. We also made the library Python 2.7 and Python 3.5 compatible with a better test suite. 
+
 ### Version 1.1.2
 
 * **tag**: [v1.1.2](https://github.com/DistrictDataLabs/tribe/releases/tag/v1.1.2)

requirements.txt

@@ -1,39 +1,37 @@
 ## Graphs and Network Analysis
 networkx==1.11
-python-louvain==0.3
+python-louvain==0.5
 
-## Visualization
+## Visualization (uncomment as needed)
 matplotlib==1.5.1
 #pygraphviz==1.3.1
 
 ## Utilities
 confire==0.2.0
-python-dateutil==2.5.0
-unicodecsv==0.14.1
+python-dateutil==2.5.3
+six==1.10.0
+future==0.15.2
 
-## Other Dependencies
-# networkX Dependencies
+## Other Dependencies (installed by above)
 #scipy==0.17.0
-decorator==4.0.9
-# confire Dependencies
-PyYAML==3.11
-six==1.10.0
-# matplotlib Dependencies
-numpy==1.10.4
-pytz==2015.7
-cycler==0.10.0
-pyparsing==2.1.0
+#decorator==4.0.10
+#PyYAML==3.11
+#numpy==1.11.0
+#pytz==2016.4
+#cycler==0.10.0
+#pyparsing==2.1.5
 
 ## Testing (uncomment for development)
 #nose==1.3.7
-#mock==1.3.0
-#coverage==4.0.3
-#funcsigs==0.4
-#pbr==1.8.1
+#mock==2.0.0
+#coverage==4.1
+#funcsigs==1.0.2
+#pbr==1.10.0
+#loremipsum==1.0.5
 
 ## Packaging Requirements
-#Python==2.7.10
-#pip==8.1.0
+#Python==2.7.11
+#pip==8.1.2
 #wheel==0.29.0
-#setuptools==20.2.2
+#setuptools==23.0.0
 #wsgiref==0.1.2

tests/__init__.py

@@ -24,7 +24,7 @@
 ## Module Constants
 ##########################################################################
 
-TEST_VERSION = "1.1.3" ## Also the expected version of the package
+TEST_VERSION = "1.2" ## Also the expected version of the package
 
 ##########################################################################
 ## Initialization Tests

tests/admin_script_tests.py

@@ -0,0 +1,61 @@
+# tests.admin_script_tests
+# Use the subprocess module to execute tribe-admin.py for testing.
+#
+# Author:   Benjamin Bengfort <[email protected]>
+# Created:  Wed Jun 22 15:48:08 2016 -0400
+#
+# Copyright (C) 2016 District Data Labs
+# For license information, see LICENSE.txt
+#
+# ID: admin_script_tests.py [6bf9822] [email protected] $
+
+"""
+Use the subprocess module to execute tribe-admin.py for testing.
+This serves as a form of "integration testing" as well as interface testing.
+"""
+
+##########################################################################
+## Imports
+##########################################################################
+
+import os
+import unittest
+import subprocess
+
+from . import TEST_VERSION
+
+##########################################################################
+## Module Constants and Paths
+##########################################################################
+
+PROJECT  = os.path.join(os.path.dirname(__file__), '..')
+FIXTURES = os.path.join(os.path.dirname(__file__), 'fixtures')
+MBOX     = os.path.join(FIXTURES, "test.mbox")
+ADMIN    = os.path.join(PROJECT, "tribe-admin.py")
+
+
+##########################################################################
+## Admin Tests
+##########################################################################
+
+class TribeAdminTests(unittest.TestCase):
+
+    def test_paths(self):
+        """
+        Assert test paths are available.
+        """
+        for path in (MBOX, ADMIN):
+            if not os.path.exists(path):
+                self.fail("required file {} does not exist!".format(path))
+
+            if not os.path.isfile(path):
+                self.fail("required file {} is not readable!".format(path))
+
+    @unittest.skip("Not python 2.7 compatible for some reason")
+    def test_version(self):
+        """
+        Test that the admin script reports the correct version
+        """
+        output = subprocess.check_output(["python", ADMIN, "--version"])
+        output = output.decode('utf-8')
+        self.assertEqual(output, 'tribe v{}\n'.format(TEST_VERSION))

tests/emails_tests.py

@@ -20,6 +20,7 @@
 import unittest
 
 from tribe.emails import *
+from builtins import str as text
 
 ##########################################################################
 ## EmailAddress Tests
@@ -37,7 +38,7 @@ def test_full_email_parse(self):
         self.assertEqual(email.name, "Benjamin Bengfort")
         self.assertEqual(email.email, "[email protected]")
         self.assertEqual(email.domain, "bengfort.com")
-        self.assertEqual(unicode(email), data)
+        self.assertEqual(text(email), data)
 
     def test_partial_email_parse(self):
         """
@@ -49,4 +50,4 @@ def test_partial_email_parse(self):
         self.assertEqual(email.name, "")
         self.assertEqual(email.email, "[email protected]")
         self.assertEqual(email.domain, "bengfort.com")
-        self.assertEqual(unicode(email), data)
+        self.assertEqual(text(email), data)

tests/extract_tests.py

@@ -0,0 +1,92 @@
+# tests.extract_tests
+# Test the tribe extraction module
+#
+# Author:   Benjamin Bengfort <[email protected]>
+# Created:  Wed Jun 22 16:40:26 2016 -0400
+#
+# Copyright (C) 2016 District Data Labs
+# For license information, see LICENSE.txt
+#
+# ID: extract_tests.py [cc23aed] [email protected] $
+
+"""
+Test the tribe extraction module
+"""
+
+##########################################################################
+## Imports
+##########################################################################
+
+import os
+import json
+import unittest
+import networkx as nx
+
+from datetime import datetime
+from tribe.extract import MBoxReader
+from tribe.emails import EmailMeta, EmailAddress
+from six import string_types
+
+##########################################################################
+## Fixtures
+##########################################################################
+
+FIXTURES = os.path.join(os.path.dirname(__file__), 'fixtures')
+MBOX     = os.path.join(FIXTURES, "test.mbox")
+HEADERS  = os.path.join(FIXTURES, "headers.json")
+
+class MBoxReaderTests(unittest.TestCase):
+    """
+    Testing the mbox reader for email extraction.
+    """
+
+    def setUp(self):
+        self.reader = MBoxReader(MBOX)
+
+    def tearDown(self):
+        self.reader = None
+
+    def test_header_analysis(self):
+        """
+        Test the header analysis functionality
+        """
+        headers  = self.reader.header_analysis()
+        self.assertEqual(len(headers), 25)
+
+        with open(HEADERS, 'r') as f:
+            expected = json.load(f)
+
+        for header, count in expected.items():
+            self.assertEqual(count, headers[header])
+
+    def test_count(self):
+        """
+        Test that the number of emails is expected
+        """
+        self.assertEqual(self.reader.count(), 140)
+        self.assertEqual(self.reader.count(), len(self.reader))
+
+    def test_extract(self):
+        """
+        Make sure that extract does not error
+        """
+        for idx, msg in enumerate(self.reader.extract()):
+
+            # Some simple type checking
+            self.assertIsInstance(msg, EmailMeta)
+            self.assertIsInstance(msg.sender, EmailAddress)
+            self.assertIsInstance(msg.recipients, list)
+            self.assertIsInstance(msg.copied, list)
+            self.assertIsInstance(msg.subject, string_types + (None,))
+            self.assertIsInstance(msg.date, (datetime, None))
+
+        self.assertEqual(idx+1, 140)
+
+    def test_graph_extract(self):
+        """
+        Make sure that extract graph does not error
+        """
+        G = self.reader.extract_graph()
+        self.assertEqual(nx.number_of_nodes(G), 7)
+        self.assertEqual(nx.number_of_edges(G), 6)
+        self.assertFalse(nx.is_directed(G))

tests/fixtures/headers.json

@@ -0,0 +1 @@
+{"Content-Type": 140, "X-Notifications": 2, "X-Account-Notification-Type": 2, "Received": 397, "X-Received": 173, "From": 140, "Feedback-ID": 2, "To": 140, "Subject": 140, "Date": 140, "Message-ID": 140, "X-Priority": 1, "Received-SPF": 87, "X-GM-THRID": 140, "X-Google-DKIM-Signature": 86, "X-Gmail-Labels": 140, "Authentication-Results": 87, "MIME-Version": 140, "DKIM-Signature": 87, "Delivered-To": 87, "Return-Path": 220, "X-Tribe-Message-Count": 140, "X-Automattic-Destination": 1, "x-no-auto-attachment": 3, "X-Gm-Message-State": 86}

tests/fixtures/test.graphml

@@ -0,0 +1,37 @@
+<?xml version='1.0' encoding='utf-8'?>
+<graphml xmlns="http://graphml.graphdrawing.org/xmlns" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://graphml.graphdrawing.org/xmlns http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd">
+  <key attr.name="weight" attr.type="double" for="edge" id="d3" />
+  <key attr.name="mbox" attr.type="string" for="graph" id="d2" />
+  <key attr.name="name" attr.type="string" for="graph" id="d1" />
+  <key attr.name="extracted" attr.type="string" for="graph" id="d0" />
+  <graph edgedefault="undirected">
+    <data key="d0">Wed Jun 22 21:28:08 2016 -0400</data>
+    <data key="d1">Email Network</data>
+    <data key="d2">fixtures/test.mbox</data>
+    <node id="[email protected]" />
+    <node id="[email protected]" />
+    <node id="[email protected]" />
+    <node id="[email protected]" />
+    <node id="[email protected]" />
+    <node id="[email protected]" />
+    <node id="[email protected]" />
+    <edge source="[email protected]" target="[email protected]">
+      <data key="d3">0.5428571428571428</data>
+    </edge>
+    <edge source="[email protected]" target="[email protected]">
+      <data key="d3">0.014285714285714285</data>
+    </edge>
+    <edge source="[email protected]" target="[email protected]">
+      <data key="d3">0.40714285714285714</data>
+    </edge>
+    <edge source="[email protected]" target="[email protected]">
+      <data key="d3">0.007142857142857143</data>
+    </edge>
+    <edge source="[email protected]" target="[email protected]">
+      <data key="d3">0.007142857142857143</data>
+    </edge>
+    <edge source="[email protected]" target="[email protected]">
+      <data key="d3">0.02142857142857143</data>
+    </edge>
+  </graph>
+</graphml>