Make room for chomper

.travis.yml

@@ -0,0 +1,22 @@
+language: python
+services: docker
+branches:
+  only:
+    - master
+before_install:
+ - git branch -vv | grep '^*'
+ - docker build -t gdr .
+ - docker images
+cache:
+  directories:
+    - env/bin
+    - env/lib/python2.7/site-packages
+install:
+ - if [ "${TRAVIS_BRANCH}" = "master" -a "${TRAVIS_PULL_REQUEST}" = "false" ]; then rm -rf env; fi
+ - touch requirements.txt
+ - make env
+script: pytest --cov gdr tests/
+notifications:
+  email: false
+  irc: false
+sudo: false

Makefile

@@ -1,7 +1,12 @@
 env:
 	python2 -m virtualenv env --prompt="[gdr]"
 	pip install -r requirements.txt
-	pip install pytest
+
+	# Uninstall leftovers from bandersnatch's over-eager install_requires, to
+	# avoid test failures. https://bitbucket.org/pypa/bandersnatch/issues/79
+	pip uninstall --yes pytest-capturelog pytest-codecheckers pytest-timeout pytest-cache
+
+	pip install -r requirements/development.txt
 	pip install -e .
 
 clean:

gdr/chomp.py

@@ -0,0 +1,100 @@
+"""Process PyPI artifacts.
+
+The main function is exposed as a console script named `chomp` via setup.py.
+
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import argparse
+import sys
+import os
+import shutil
+import subprocess
+from pkg_resources import parse_requirements
+
+
+def unpack_zip(path, arch_name, dname):
+    arch_name = arch_name.rsplit(".", 1)[0] + ".zip"
+    shutil.copyfile(path, arch_name)
+    cmd = "cd {0} ; unzip {1} >/dev/null 2>&1".format(dname, arch_name)
+    subprocess.check_call(cmd, shell=True)
+    return set(os.listdir(dname)) - set([os.path.basename(arch_name)])
+
+
+def unpack_tar_gz(path, arch_name, dname):
+    shutil.copyfile(path, arch_name)
+    cmd = "cd {0} ; tar -zxvf {1} >/dev/null 2>&1".format(dname, arch_name)
+    subprocess.check_call(cmd, shell=True)
+    return set(os.listdir(dname)) - set([os.path.basename(arch_name)])
+
+
+SEEN = set()
+
+def extract():
+    # do a docker dance
+    # monkeypatch setup.py to hook setup
+    # import setup
+    # run setup.setup
+    # capture relevant info passed to setup.setup
+    # return it!
+    pass
+
+
+def possibly_extract(path):
+
+    # make sure the cleanroom is clean
+    # unpack the artifact into the cleanroom
+    # check the name and version in PKG-INFO (TODO guaranteed?)
+    # if already SEEN, return
+    # extract the data!
+    # serialize the data to SQL
+    # return SQL
+
+    print("{:>42} {}".format(path))
+    arch_name = os.path.join(dname, os.path.basename(path))
+    new_files = []
+
+    if path.endswith('.zip'):
+        extract_zip(path, arch_name, dname)
+    elif path.endswith('.tar.gz'):
+        extract_tgz(path, arch_name, dname)
+    import pdb; pdb.set_trace()
+
+    SQL = ''
+    return SQL
+
+
+def download_artifacts():
+    try:
+        run('cp status status.bak') # store in case we crash
+    except IOError as e:
+
+        run('cp status.bak status') # recover from crash
+    run('rm -rf web todo')      #
+    run('bandersnatch -c conf mirror')  # downloads *all* artifacts for packages which have had
+                                        # any change since `status`
+
+
+def extract_from_artifacts():
+    SQL = ''
+    for root, dirs, files in os.walk('web/packages'):
+        for filename in files:
+            path = os.path.join(root, filename)
+            SQL += possibly_extract(path)
+
+
+def update_database(SQL):
+    pass
+
+
+def parse_args(argv):
+    p = argparse.ArgumentParser()
+    p.add_argument('root', help="path to your bandersnatch root")
+    return p.parse_args(argv)
+
+
+def main(argv=sys.argv):
+    os.chdir(parse_args(argv[1:]).root)
+    download_artifacts()
+    SQL = extract_from_artifacts()
+    update_database(SQL)

requirements.txt

@@ -1,3 +1,4 @@
+<<<<<<< HEAD
 gunicorn==19.6.0 --hash=sha512:0305ac18c016c31d2ab600786f9395a4e2c5eaeb30ee6a45f8d7a84c6e5eb07513704c5a1f13405982a3c184bb7b98c6488793fad86e822529485e03b5b61006
 futures==3.0.5 --hash=sha512:708dad578c83124e799d64ed243a8e223e6b6de71380b39221def0c77a9d8cf79aca916c35cd6cfff240edfa3af43f9d49d7ebc74d21bec5d4312e677c2243d7
 
@@ -8,3 +9,7 @@ dependency_injection==1.1.0 --hash=sha512:6840b28f0ea7dc2263f7ca00bb94a8d88a1df2
 filesystem_tree==1.1.2 --hash=sha512:3e9dbbcc29a1d7d0703df4ab3b8c4361489c18ca1111cbec6908eff6276d990064e45800f0cd91f13d40093e8d27f54d2490b99e587df2b5f1c20f828b9dbab5
 first==2.0.1 --hash=sha512:7f7592c0bc7c1f5aeb7b8bc6e87ced80731071fa411a9e6f847c59be718373c31cbcc8494e1604aacd971ac05c30447d195bf0f350a93a1c0f47e4fc18b1e169
 python_mimeparse==1.6.0 --hash=sha512:0c8fbe7cfe5cea1b2d4f5df04484201238fb3f6f7af3f049136ab86ffd90db8345e72a726dd5ef82eb9b08701ed74802b2ef649df9c676058e6d201bfbf236af
+=======
+-r requirements/webapp.txt
+-r requirements/chomper.txt
+>>>>>>> 2920f55... Start making room for the chomper

requirements/chomper.txt

@@ -0,0 +1,55 @@
+bandersnatch == 1.11 \
+  --hash=sha256:264bdadc783172d34c04c4378bf57e415729df80b9eb10f172c938c7fab613c2
+
+
+# bandersnatch has a ... somewhat overeager install_requires. :-/
+# https://bitbucket.org/pypa/bandersnatch/issues/79
+
+apipkg == 1.4 \
+  --hash=sha256:65d2aa68b28e7d31233bb2ba8eb31cda40e4671f8ac2d6b241e358c9652a74b9
+coverage == 4.2 \
+  --hash=sha256:6d6ca7c7406edc03f50f751a583d0e0c1d64b2c01da8e1b9b40d5961a23c126d \
+  --hash=sha256:4138fc4741a54aaf6dd6d91cebe075a899b7f4a841680638704e4eec803c8268 \
+  --hash=sha256:d12b46247e74c1881768feb3ae48fbbb95e5c883b0c3517acdf86f435560fc61 \
+  --hash=sha256:28a61877d72937d1cc367f09859ddacad99ea26f6fc468c34937e1004db54798 \
+  --hash=sha256:e312776d3ef04632ec742ce2d2b7048b635073e0245e4f44dfe8b08cc50ac656 
+execnet == 1.4.1 \
+  --hash=sha256:d2b909c7945832e1c19cfacd96e78da68bdadc656440cfc7dfe59b766744eb8c
+funcsigs == 1.0.2 \
+  --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca
+mock == 2.0.0 \
+  --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1
+packaging == 16.7 \
+  --hash=sha256:cbbbc487473b4d729cb0d86ea78ceeea22629cb1f31fd6cc717c5235616ed160
+pbr == 1.10.0 \
+  --hash=sha256:f5cf7265a80636ecff66806d13494cbf9d77a3758a65fd8b4d4d4bee81b0c375
+pep8 == 1.7.0 \
+  --hash=sha256:4fc2e478addcf17016657dff30b2d8d611e8341fac19ccf2768802f6635d7b8a
+py == 1.4.31 \
+  --hash=sha256:4a3e4f3000c123835ac39cab5ccc510642153bc47bc1f13e2bbb53039540ae69
+pyflakes == 1.3.0 \
+  --hash=sha256:ad89dafee8ca32282116209a0ca4dff050bdc343af958721d5517d242c1215d5
+pyparsing == 2.1.10 \
+  --hash=sha256:67101d7acee692962f33dd30b5dce079ff532dd9aa99ff48d52a3dad51d2fe84
+pytest == 3.0.3 \
+  --hash=sha256:aea87ca4240b79868ccabd14ba56a12eceb1c1e8076b585fe2fa5602a5874db7
+pytest-cache == 1.0 \
+  --hash=sha256:be7468edd4d3d83f1e844959fd6e3fd28e77a481440a7118d430130ea31b07a9
+pytest-capturelog == 0.7 \
+  --hash=sha256:b6e8d5189b39462109c2188e6b512d6cc7e66d62bb5be65389ed50e96d22000d
+pytest-codecheckers == 0.2 \
+  --hash=sha256:853de10f204865140da2bc173f791c9e13794fc43656e02fffcce23c9999e748
+pytest_cov == 2.4.0 \
+  --hash=sha256:10e37e876f49ddec80d6c83a54b657157f1387ebc0f7755285f8c156130014a1
+pytest_timeout == 1.0.0 \
+  --hash=sha256:d53665f3dad30de23b8377623a5ec9b6f489c4a70aa952c1860febc33dac3c88
+python_dateutil == 2.5.3 \
+  --hash=sha256:598499a75be2e5e18a66f12c00dd47a069de24794effeda4228bfc760f44f527
+requests == 2.11.1 \
+  --hash=sha256:545c4855cd9d7c12671444326337013766f4eea6068c3f0307fb2dc2696d580e
+setuptools == 28.6.1 \
+  --hash=sha256:2b94b5d44914e8ade6c537678c6a0143a1d7038989b2330c8f3c69b6d839e311
+six == 1.10.0 \
+  --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1
+xmlrpc2 == 0.3.1 \
+  --hash=sha256:25a6b911e6f5e3ecbfbda5a93d2d5329b83020d643a067179305def60d751421

requirements/development.txt

@@ -0,0 +1,2 @@
+pytest
+pytest-cov

requirements/webapp.txt

@@ -0,0 +1,8 @@
+gunicorn==19.6.0 --hash=sha512:0305ac18c016c31d2ab600786f9395a4e2c5eaeb30ee6a45f8d7a84c6e5eb07513704c5a1f13405982a3c184bb7b98c6488793fad86e822529485e03b5b61006
+
+algorithm==1.2.0 --hash=sha512:2796195ac336bb18bf0bd932eeb476bcdde0b3fed6a519478317bc3b84fa57d79a35193802175ad63bd6ea6dd524638496cb6f631a91143e65a57d4f16919a06
+aspen==0.42 --hash=sha512:fc5695e2c1605a729481b58809749d3abd82c3a4a7dde88f4fa503ede9f854a9fe87475ed18439b6ee9c19263976a3b34a62de70e0547b4cc251ea30038397a8
+dependency_injection==1.1.0 --hash=sha512:6840b28f0ea7dc2263f7ca00bb94a8d88a1df2a45c6289b1394dd6f3fa9a73d3add8a0aaf52d600611714208099ae8d008e188f2e885d4e16f77d61db742c456
+filesystem_tree==1.1.2 --hash=sha512:3e9dbbcc29a1d7d0703df4ab3b8c4361489c18ca1111cbec6908eff6276d990064e45800f0cd91f13d40093e8d27f54d2490b99e587df2b5f1c20f828b9dbab5
+first==2.0.1 --hash=sha512:7f7592c0bc7c1f5aeb7b8bc6e87ced80731071fa411a9e6f847c59be718373c31cbcc8494e1604aacd971ac05c30447d195bf0f350a93a1c0f47e4fc18b1e169
+python_mimeparse==1.6.0 --hash=sha512:0c8fbe7cfe5cea1b2d4f5df04484201238fb3f6f7af3f049136ab86ffd90db8345e72a726dd5ef82eb9b08701ed74802b2ef649df9c676058e6d201bfbf236af

setup.py

@@ -3,4 +3,5 @@
 
 setup( name='gdr'
      , packages=find_packages()
+     , entry_points = {'console_scripts': ['chomp=gdr.chomp:main']}
       )