Skip to content
This repository has been archived by the owner on Feb 8, 2018. It is now read-only.

Commit

Permalink
Merge branch 'master' into new-project
Browse files Browse the repository at this point in the history
  • Loading branch information
mattbk authored Nov 7, 2016
2 parents bef2850 + bee3823 commit 2b458cd
Show file tree
Hide file tree
Showing 29 changed files with 618 additions and 366 deletions.
10 changes: 8 additions & 2 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ branches:
- master
before_install:
- git branch -vv | grep '^*'
- pwd

# Sometimes ya just halfta ...
- git clone https://github.com/lloyd/yajl.git && cd yajl && git checkout 2.1.0 && ./configure && sudo make install && cd ..

- npm install -g marky-markdown
cache:
directories:
- env/bin
Expand All @@ -14,12 +20,12 @@ install:
- if [ "${TRAVIS_BRANCH}" = "master" -a "${TRAVIS_PULL_REQUEST}" = "false" ]; then rm -rf env; fi
- touch requirements.txt package.json
- make env
- env/bin/pip install --upgrade ijson==2.3.0
before_script:
- echo "DATABASE_URL=dbname=gratipay" | tee -a tests/local.env local.env
- psql -U postgres -c 'CREATE DATABASE "gratipay";'
- if [ "${TRAVIS_BRANCH}" = "master" -a "${TRAVIS_PULL_REQUEST}" = "false" ]; then rm -rfv tests/py/fixtures; fi
script: make bgrun test doc
script: LD_LIBRARY_PATH=/usr/local/lib make bgrun test doc
notifications:
email: false
irc: false
sudo: false
1 change: 0 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ env: requirements.txt requirements.dev.txt setup.py
./env/
$(pip) install --no-index -r requirements.txt
$(pip) install --no-index -r requirements.dev.txt
$(pip) install -e ./
touch env

clean:
Expand Down
34 changes: 34 additions & 0 deletions bin/sync-npm.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/bin/sh
# This is a script to run under the Heroku Scheduler add-on to periodically
# sync our database with the npm registry.

set -e
cd "`dirname $0`/.."

# Install dependencies.
# =====================

# cmake - required by ...
curl https://cmake.org/files/v3.6/cmake-3.6.2-Linux-x86_64.tar.gz > cmake.tgz
echo '5df4b69d9e85093ae78b1070d5cb9f824ce0bdd02528948c3f6a740e240083e5 cmake.tgz' \
| sha256sum -c /dev/stdin --status
tar zxf cmake.tgz
PATH=/app/cmake-3.6.2-Linux-x86_64/bin:$PATH

# yajl
git clone https://github.com/lloyd/yajl.git
cd yajl
git checkout 2.1.0
./configure -p /app/.heroku/python
make install
cd ..

# python
pip install ijson==2.3.0
pip install -e .


# Sync with npm.
# ==============

curl https://registry.npmjs.com/-/all | sync-npm serialize | sync-npm upsert
16 changes: 16 additions & 0 deletions bin/verify-identity.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/usr/bin/env python
"""verify-identity.py <participant_id>, <country_code>
"""
from __future__ import absolute_import, division, print_function, unicode_literals

import sys

from gratipay import wireup
from gratipay.models.participant import Participant
from gratipay.models.country import Country

wireup.db(wireup.env())

participant = Participant.from_id(int(sys.argv[1]))
country = Country.from_code(sys.argv[2])
participant.set_identity_verification(country.id, True)
24 changes: 1 addition & 23 deletions gratipay/billing/payday.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@

import os
import itertools
from multiprocessing.dummy import Pool as ThreadPool

import braintree

Expand All @@ -24,35 +23,14 @@
)
from gratipay.exceptions import NegativeBalance
from gratipay.models import check_db
from gratipay.utils.threaded_map import threaded_map
from psycopg2 import IntegrityError


with open(os.path.join(os.path.dirname(__file__), '../../sql/payday.sql')) as f:
PAYDAY = f.read()


class ExceptionWrapped(Exception): pass


def threaded_map(func, iterable, threads=5):
pool = ThreadPool(threads)
def g(*a, **kw):
# Without this wrapper we get a traceback from inside multiprocessing.
try:
return func(*a, **kw)
except Exception as e:
import traceback
raise ExceptionWrapped(e, traceback.format_exc())
try:
r = pool.map(g, iterable)
except ExceptionWrapped as e:
print(e.args[1])
raise e.args[0]
pool.close()
pool.join()
return r


class NoPayday(Exception):
__str__ = lambda self: "No payday found where one was expected."

Expand Down
Empty file.
65 changes: 65 additions & 0 deletions gratipay/package_managers/readmes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
from __future__ import absolute_import, division, print_function, unicode_literals

import sys

import requests

from gratipay.utils import markdown
from gratipay.utils.threaded_map import threaded_map
from threading import Lock


log_lock = Lock()

def log(*a, **kw):
with log_lock:
print(*a, file=sys.stderr, **kw)


def http_fetch(package_name):
r = requests.get('https://registry.npmjs.com/' + package_name)
if r.status_code != 200:
log(r.status_code, 'for', package_name)
return None
return r.json()


def Syncer(db):
def sync(dirty, fetch=http_fetch):
"""Update all info for one package.
"""
log(dirty.name)
full = fetch(dirty.name)

if not full:
return
elif full['name'] != dirty.name:
log('expected', dirty.name, 'got', full['name'])
return
elif 'readme' not in full:
log('no readme in', full['name'])
return

db.run('''
UPDATE packages
SET readme=%s
, readme_raw=%s
, readme_type=%s
WHERE package_manager=%s
AND name=%s
''', ( markdown.marky(full['readme'])
, full['readme']
, 'x-markdown/npm'
, dirty.package_manager
, dirty.name
))

return sync


def sync_all(db):
dirty = db.all('SELECT package_manager, name FROM packages WHERE readme_raw IS NULL '
'ORDER BY package_manager DESC, name DESC')
threaded_map(Syncer(db), dirty, 4)
146 changes: 146 additions & 0 deletions gratipay/package_managers/sync.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
"""Sync our database with package managers. Just npm for now.
"""
from __future__ import absolute_import, division, print_function, unicode_literals

import argparse
import csv
import sys
import time
import uuid

from gratipay.package_managers import readmes as _readmes


log = lambda *a: print(*a, file=sys.stderr)
NULL = uuid.uuid4().hex


def import_ijson():
import ijson.backends.yajl2_cffi as ijson
return ijson


def arrayize(seq):
"""Given a sequence of str, return a Postgres array literal str.
"""
array = []
for item in seq:
assert type(item) is str
escaped = item.replace(b'\\', b'\\\\').replace(b'"', b'\\"')
quoted = b'"' + escaped + b'"'
array.append(quoted)
joined = b', '.join(array)
return b'{' + joined + b'}'


def serialize_one(out, package):
"""Takes a package and emits a serialization suitable for COPY.
"""
if not package or package['name'].startswith('_'):
log('skipping', package)
return 0

row = ( package['package_manager']
, package['name']
, package['description']
, arrayize(package['emails'])
)

out.writerow(row)
return 1


def serialize(args):
"""Consume raw JSON from the npm registry and spit out CSV for Postgres.
"""
ijson = import_ijson()

path = args.path
parser = ijson.parse(open(path))
start = time.time()
package = None
nprocessed = 0
out = csv.writer(sys.stdout)

def log_stats():
log("processed {} packages in {:3.0f} seconds"
.format(nprocessed, time.time() - start))

for prefix, event, value in parser:

if not prefix and event == b'map_key':

# Flush the current package. We count on the first package being garbage.
processed = serialize_one(out, package)
nprocessed += processed
if processed and not(nprocessed % 1000):
log_stats()

# Start a new package.
package = { 'package_manager': b'npm'
, 'name': value
, 'description': b''
, 'emails': []
}

key = lambda k: package['name'] + b'.' + k

if event == b'string':
assert type(value) is unicode # Who knew? Seems to decode only for `string`.
value = value.encode('utf8')
if prefix == key(b'description'):
package['description'] = value
elif prefix in (key(b'author.email'), key(b'maintainers.item.email')):
package['emails'].append(value)

nprocessed += serialize_one(out, package) # Don't forget the last one!
log_stats()


def upsert(args):
from gratipay import wireup
db = wireup.db(wireup.env())
fp = open(args.path)
with db.get_cursor() as cursor:
assert cursor.connection.encoding == 'UTF8'

# http://tapoueh.org/blog/2013/03/15-batch-update.html
cursor.run("CREATE TEMP TABLE updates (LIKE packages INCLUDING ALL) ON COMMIT DROP")
cursor.copy_expert('COPY updates (package_manager, name, description, emails) '
"FROM STDIN WITH (FORMAT csv, NULL '%s')" % NULL, fp)
cursor.run("""
WITH updated AS (
UPDATE packages p
SET package_manager = u.package_manager
, description = u.description
, emails = u.emails
FROM updates u
WHERE p.name = u.name
RETURNING p.name
)
INSERT INTO packages(package_manager, name, description, emails)
SELECT package_manager, name, description, emails
FROM updates u LEFT JOIN updated USING(name)
WHERE updated.name IS NULL
GROUP BY u.package_manager, u.name, u.description, u.emails
""")


def readmes(args):
from gratipay import wireup
db = wireup.db(wireup.env())
_readmes.sync_all(db)


def parse_args(argv):
p = argparse.ArgumentParser()
p.add_argument('command', choices=['serialize', 'upsert', 'readmes'])
p.add_argument('path', help='the path to the input file', nargs='?', default='/dev/stdin')
return p.parse_args(argv)


def main(argv=sys.argv):
args = parse_args(argv[1:])
globals()[args.command](args)
12 changes: 12 additions & 0 deletions gratipay/utils/markdown.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,21 @@
from subprocess import Popen, PIPE

from markupsafe import Markup
import misaka as m # http://misaka.61924.nl/


def render(markdown):
return Markup(m.html(
markdown,
extensions=m.EXT_AUTOLINK | m.EXT_STRIKETHROUGH | m.EXT_NO_INTRA_EMPHASIS,
render_flags=m.HTML_SKIP_HTML | m.HTML_TOC | m.HTML_SMARTYPANTS | m.HTML_SAFELINK
))


def marky(markdown):
"""Process markdown the same way npm does.
"""
if type(markdown) is unicode:
markdown = markdown.encode('utf8')
marky = Popen(("marky-markdown", "/dev/stdin"), stdin=PIPE, stdout=PIPE)
return Markup(marky.communicate(markdown)[0])
26 changes: 26 additions & 0 deletions gratipay/utils/threaded_map.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
from __future__ import absolute_import, division, print_function, unicode_literals

from multiprocessing.dummy import Pool as ThreadPool


class ExceptionWrapped(Exception):
pass


def threaded_map(func, iterable, threads=5):
pool = ThreadPool(threads)
def g(*a, **kw):
# Without this wrapper we get a traceback from inside multiprocessing.
try:
return func(*a, **kw)
except Exception as e:
import traceback
raise ExceptionWrapped(e, traceback.format_exc())
try:
r = pool.map(g, iterable)
except ExceptionWrapped as e:
print(e.args[1])
raise e.args[0]
pool.close()
pool.join()
return r
Loading

0 comments on commit 2b458cd

Please sign in to comment.