diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..4ca2f27 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,22 @@ +language: python +services: docker +branches: + only: + - master +before_install: + - git branch -vv | grep '^*' + - docker build -t gdr . + - docker images +cache: + directories: + - env/bin + - env/lib/python2.7/site-packages +install: + - if [ "${TRAVIS_BRANCH}" = "master" -a "${TRAVIS_PULL_REQUEST}" = "false" ]; then rm -rf env; fi + - touch requirements.txt + - make env +script: pytest --cov gdr tests/ +notifications: + email: false + irc: false +sudo: false diff --git a/Makefile b/Makefile index b4ed292..0bdd497 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,12 @@ env: python2 -m virtualenv env --prompt="[gdr]" pip install -r requirements.txt - pip install pytest + + # Uninstall leftovers from bandersnatch's over-eager install_requires, to + # avoid test failures. https://bitbucket.org/pypa/bandersnatch/issues/79 + pip uninstall --yes pytest-capturelog pytest-codecheckers pytest-timeout pytest-cache + + pip install -r requirements/development.txt pip install -e . clean: diff --git a/gdr.py b/gdr/__init__.py similarity index 100% rename from gdr.py rename to gdr/__init__.py diff --git a/gdr/chomp.py b/gdr/chomp.py new file mode 100644 index 0000000..6cb57aa --- /dev/null +++ b/gdr/chomp.py @@ -0,0 +1,100 @@ +"""Process PyPI artifacts. + +The main function is exposed as a console script named `chomp` via setup.py. + +""" +from __future__ import absolute_import, division, print_function, unicode_literals + +import argparse +import sys +import os +import shutil +import subprocess +from pkg_resources import parse_requirements + + +def unpack_zip(path, arch_name, dname): + arch_name = arch_name.rsplit(".", 1)[0] + ".zip" + shutil.copyfile(path, arch_name) + cmd = "cd {0} ; unzip {1} >/dev/null 2>&1".format(dname, arch_name) + subprocess.check_call(cmd, shell=True) + return set(os.listdir(dname)) - set([os.path.basename(arch_name)]) + + +def unpack_tar_gz(path, arch_name, dname): + shutil.copyfile(path, arch_name) + cmd = "cd {0} ; tar -zxvf {1} >/dev/null 2>&1".format(dname, arch_name) + subprocess.check_call(cmd, shell=True) + return set(os.listdir(dname)) - set([os.path.basename(arch_name)]) + + +SEEN = set() + +def extract(): + # do a docker dance + # monkeypatch setup.py to hook setup + # import setup + # run setup.setup + # capture relevant info passed to setup.setup + # return it! + pass + + +def possibly_extract(path): + + # make sure the cleanroom is clean + # unpack the artifact into the cleanroom + # check the name and version in PKG-INFO (TODO guaranteed?) + # if already SEEN, return + # extract the data! + # serialize the data to SQL + # return SQL + + print("{:>42} {}".format(path)) + arch_name = os.path.join(dname, os.path.basename(path)) + new_files = [] + + if path.endswith('.zip'): + extract_zip(path, arch_name, dname) + elif path.endswith('.tar.gz'): + extract_tgz(path, arch_name, dname) + import pdb; pdb.set_trace() + + SQL = '' + return SQL + + +def download_artifacts(): + try: + run('cp status status.bak') # store in case we crash + except IOError as e: + + run('cp status.bak status') # recover from crash + run('rm -rf web todo') # + run('bandersnatch -c conf mirror') # downloads *all* artifacts for packages which have had + # any change since `status` + + +def extract_from_artifacts(): + SQL = '' + for root, dirs, files in os.walk('web/packages'): + for filename in files: + path = os.path.join(root, filename) + SQL += possibly_extract(path) + + +def update_database(SQL): + pass + + +def parse_args(argv): + p = argparse.ArgumentParser() + p.add_argument('root', help="path to your bandersnatch root") + return p.parse_args(argv) + + +def main(argv=sys.argv): + os.chdir(parse_args(argv[1:]).root) + download_artifacts() + SQL = extract_from_artifacts() + update_database(SQL) diff --git a/requirements.txt b/requirements.txt index 5eb82f4..6186f2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +<<<<<<< HEAD gunicorn==19.6.0 --hash=sha512:0305ac18c016c31d2ab600786f9395a4e2c5eaeb30ee6a45f8d7a84c6e5eb07513704c5a1f13405982a3c184bb7b98c6488793fad86e822529485e03b5b61006 futures==3.0.5 --hash=sha512:708dad578c83124e799d64ed243a8e223e6b6de71380b39221def0c77a9d8cf79aca916c35cd6cfff240edfa3af43f9d49d7ebc74d21bec5d4312e677c2243d7 @@ -8,3 +9,7 @@ dependency_injection==1.1.0 --hash=sha512:6840b28f0ea7dc2263f7ca00bb94a8d88a1df2 filesystem_tree==1.1.2 --hash=sha512:3e9dbbcc29a1d7d0703df4ab3b8c4361489c18ca1111cbec6908eff6276d990064e45800f0cd91f13d40093e8d27f54d2490b99e587df2b5f1c20f828b9dbab5 first==2.0.1 --hash=sha512:7f7592c0bc7c1f5aeb7b8bc6e87ced80731071fa411a9e6f847c59be718373c31cbcc8494e1604aacd971ac05c30447d195bf0f350a93a1c0f47e4fc18b1e169 python_mimeparse==1.6.0 --hash=sha512:0c8fbe7cfe5cea1b2d4f5df04484201238fb3f6f7af3f049136ab86ffd90db8345e72a726dd5ef82eb9b08701ed74802b2ef649df9c676058e6d201bfbf236af +======= +-r requirements/webapp.txt +-r requirements/chomper.txt +>>>>>>> 2920f55... Start making room for the chomper diff --git a/requirements/chomper.txt b/requirements/chomper.txt new file mode 100644 index 0000000..7548375 --- /dev/null +++ b/requirements/chomper.txt @@ -0,0 +1,55 @@ +bandersnatch == 1.11 \ + --hash=sha256:264bdadc783172d34c04c4378bf57e415729df80b9eb10f172c938c7fab613c2 + + +# bandersnatch has a ... somewhat overeager install_requires. :-/ +# https://bitbucket.org/pypa/bandersnatch/issues/79 + +apipkg == 1.4 \ + --hash=sha256:65d2aa68b28e7d31233bb2ba8eb31cda40e4671f8ac2d6b241e358c9652a74b9 +coverage == 4.2 \ + --hash=sha256:6d6ca7c7406edc03f50f751a583d0e0c1d64b2c01da8e1b9b40d5961a23c126d \ + --hash=sha256:4138fc4741a54aaf6dd6d91cebe075a899b7f4a841680638704e4eec803c8268 \ + --hash=sha256:d12b46247e74c1881768feb3ae48fbbb95e5c883b0c3517acdf86f435560fc61 \ + --hash=sha256:28a61877d72937d1cc367f09859ddacad99ea26f6fc468c34937e1004db54798 \ + --hash=sha256:e312776d3ef04632ec742ce2d2b7048b635073e0245e4f44dfe8b08cc50ac656 +execnet == 1.4.1 \ + --hash=sha256:d2b909c7945832e1c19cfacd96e78da68bdadc656440cfc7dfe59b766744eb8c +funcsigs == 1.0.2 \ + --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca +mock == 2.0.0 \ + --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 +packaging == 16.7 \ + --hash=sha256:cbbbc487473b4d729cb0d86ea78ceeea22629cb1f31fd6cc717c5235616ed160 +pbr == 1.10.0 \ + --hash=sha256:f5cf7265a80636ecff66806d13494cbf9d77a3758a65fd8b4d4d4bee81b0c375 +pep8 == 1.7.0 \ + --hash=sha256:4fc2e478addcf17016657dff30b2d8d611e8341fac19ccf2768802f6635d7b8a +py == 1.4.31 \ + --hash=sha256:4a3e4f3000c123835ac39cab5ccc510642153bc47bc1f13e2bbb53039540ae69 +pyflakes == 1.3.0 \ + --hash=sha256:ad89dafee8ca32282116209a0ca4dff050bdc343af958721d5517d242c1215d5 +pyparsing == 2.1.10 \ + --hash=sha256:67101d7acee692962f33dd30b5dce079ff532dd9aa99ff48d52a3dad51d2fe84 +pytest == 3.0.3 \ + --hash=sha256:aea87ca4240b79868ccabd14ba56a12eceb1c1e8076b585fe2fa5602a5874db7 +pytest-cache == 1.0 \ + --hash=sha256:be7468edd4d3d83f1e844959fd6e3fd28e77a481440a7118d430130ea31b07a9 +pytest-capturelog == 0.7 \ + --hash=sha256:b6e8d5189b39462109c2188e6b512d6cc7e66d62bb5be65389ed50e96d22000d +pytest-codecheckers == 0.2 \ + --hash=sha256:853de10f204865140da2bc173f791c9e13794fc43656e02fffcce23c9999e748 +pytest_cov == 2.4.0 \ + --hash=sha256:10e37e876f49ddec80d6c83a54b657157f1387ebc0f7755285f8c156130014a1 +pytest_timeout == 1.0.0 \ + --hash=sha256:d53665f3dad30de23b8377623a5ec9b6f489c4a70aa952c1860febc33dac3c88 +python_dateutil == 2.5.3 \ + --hash=sha256:598499a75be2e5e18a66f12c00dd47a069de24794effeda4228bfc760f44f527 +requests == 2.11.1 \ + --hash=sha256:545c4855cd9d7c12671444326337013766f4eea6068c3f0307fb2dc2696d580e +setuptools == 28.6.1 \ + --hash=sha256:2b94b5d44914e8ade6c537678c6a0143a1d7038989b2330c8f3c69b6d839e311 +six == 1.10.0 \ + --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 +xmlrpc2 == 0.3.1 \ + --hash=sha256:25a6b911e6f5e3ecbfbda5a93d2d5329b83020d643a067179305def60d751421 diff --git a/requirements/development.txt b/requirements/development.txt new file mode 100644 index 0000000..9955dec --- /dev/null +++ b/requirements/development.txt @@ -0,0 +1,2 @@ +pytest +pytest-cov diff --git a/requirements/tests.txt b/requirements/tests.txt new file mode 100644 index 0000000..e69de29 diff --git a/requirements/webapp.txt b/requirements/webapp.txt new file mode 100644 index 0000000..c7ad86f --- /dev/null +++ b/requirements/webapp.txt @@ -0,0 +1,8 @@ +gunicorn==19.6.0 --hash=sha512:0305ac18c016c31d2ab600786f9395a4e2c5eaeb30ee6a45f8d7a84c6e5eb07513704c5a1f13405982a3c184bb7b98c6488793fad86e822529485e03b5b61006 + +algorithm==1.2.0 --hash=sha512:2796195ac336bb18bf0bd932eeb476bcdde0b3fed6a519478317bc3b84fa57d79a35193802175ad63bd6ea6dd524638496cb6f631a91143e65a57d4f16919a06 +aspen==0.42 --hash=sha512:fc5695e2c1605a729481b58809749d3abd82c3a4a7dde88f4fa503ede9f854a9fe87475ed18439b6ee9c19263976a3b34a62de70e0547b4cc251ea30038397a8 +dependency_injection==1.1.0 --hash=sha512:6840b28f0ea7dc2263f7ca00bb94a8d88a1df2a45c6289b1394dd6f3fa9a73d3add8a0aaf52d600611714208099ae8d008e188f2e885d4e16f77d61db742c456 +filesystem_tree==1.1.2 --hash=sha512:3e9dbbcc29a1d7d0703df4ab3b8c4361489c18ca1111cbec6908eff6276d990064e45800f0cd91f13d40093e8d27f54d2490b99e587df2b5f1c20f828b9dbab5 +first==2.0.1 --hash=sha512:7f7592c0bc7c1f5aeb7b8bc6e87ced80731071fa411a9e6f847c59be718373c31cbcc8494e1604aacd971ac05c30447d195bf0f350a93a1c0f47e4fc18b1e169 +python_mimeparse==1.6.0 --hash=sha512:0c8fbe7cfe5cea1b2d4f5df04484201238fb3f6f7af3f049136ab86ffd90db8345e72a726dd5ef82eb9b08701ed74802b2ef649df9c676058e6d201bfbf236af diff --git a/setup.py b/setup.py index 3125b46..a09e783 100644 --- a/setup.py +++ b/setup.py @@ -3,4 +3,5 @@ setup( name='gdr' , packages=find_packages() + , entry_points = {'console_scripts': ['chomp=gdr.chomp:main']} )