diff --git a/test/corla-test-credentials.psql b/test/corla-test-credentials.psql index e73394db5..5753fdf2d 100644 --- a/test/corla-test-credentials.psql +++ b/test/corla-test-credentials.psql @@ -1,3 +1,4 @@ +delete from administrator; insert into administrator (id, username, full_name, last_login_time, last_logout_time, county_id, type, version) values (-1, 'stateadmin1', 'State Administrator 1', null, null, null, 'STATE', 0), (-2, 'stateadmin2', 'State Administrator 2', null, null, null, 'STATE', 0), diff --git a/test/load_tests/.gitignore b/test/load_tests/.gitignore new file mode 100644 index 000000000..1c1d5687c --- /dev/null +++ b/test/load_tests/.gitignore @@ -0,0 +1,3 @@ +*.out +*.err +*.log diff --git a/test/load_tests/100.csv.gz b/test/load_tests/100.csv.gz new file mode 100644 index 000000000..fa25fae57 Binary files /dev/null and b/test/load_tests/100.csv.gz differ diff --git a/test/load_tests/12_counties_uploads.sh b/test/load_tests/12_counties_uploads.sh new file mode 100755 index 000000000..489fafe07 --- /dev/null +++ b/test/load_tests/12_counties_uploads.sh @@ -0,0 +1,15 @@ +#!/bin/sh +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1 2 3 4; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f large_cvrs.csv -F large_manifest.csv > $c.log & +done & +for c in 5 6 7 8; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done & +for c in 9 10 11 12; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f small_cvrs.csv -F small_manifest.csv > $c.log & +done & +for job in `jobs -p`; do + wait $job +done + diff --git a/test/load_tests/16-cdos.sh b/test/load_tests/16-cdos.sh new file mode 100755 index 000000000..f0d90ff32 --- /dev/null +++ b/test/load_tests/16-cdos.sh @@ -0,0 +1,8 @@ +#!/bin/sh +./main.py -u http://192.168.24.43/api/ reset dos_init > 0.log +for c in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16; do + ./main.py -u http://192.168.24.43/api/ -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done diff --git a/test/load_tests/16.sh b/test/load_tests/16.sh new file mode 100755 index 000000000..131010cda --- /dev/null +++ b/test/load_tests/16.sh @@ -0,0 +1,8 @@ +#!/bin/sh +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done diff --git a/test/load_tests/1_huge_county_cdos.sh b/test/load_tests/1_huge_county_cdos.sh new file mode 100755 index 000000000..94d9d308a --- /dev/null +++ b/test/load_tests/1_huge_county_cdos.sh @@ -0,0 +1,9 @@ +#!/bin/sh +./main.py -u http://192.168.24.43/api/ reset dos_init > 0.log +for c in 1; do + ./main.py -u http://192.168.24.43/api/ -c $c county_setup -f huge_cvrs.csv -F huge_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done + diff --git a/test/load_tests/1_huge_county_upload.sh b/test/load_tests/1_huge_county_upload.sh new file mode 100755 index 000000000..787c9ef61 --- /dev/null +++ b/test/load_tests/1_huge_county_upload.sh @@ -0,0 +1,9 @@ +#!/bin/sh +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f huge_cvrs.csv -F huge_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done + diff --git a/test/load_tests/1_large_county_uploads.sh b/test/load_tests/1_large_county_uploads.sh new file mode 100755 index 000000000..890ce83e9 --- /dev/null +++ b/test/load_tests/1_large_county_uploads.sh @@ -0,0 +1,9 @@ +#!/bin/sh +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f large_cvrs.csv -F large_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done + diff --git a/test/load_tests/2-64-cdos.sh b/test/load_tests/2-64-cdos.sh new file mode 100755 index 000000000..d50390ed8 --- /dev/null +++ b/test/load_tests/2-64-cdos.sh @@ -0,0 +1,20 @@ +#!/bin/sh +time 2-cdos.sh; # 5 5 4 +time 2-cdos.sh; +time 2-cdos.sh; +time 2-cdos.sh; +time 4-cdos.sh; # 8 7 9 +time 4-cdos.sh; +time 4-cdos.sh; +time 8-cdos.sh; # 11 11 13 +time 8-cdos.sh; +time 8-cdos.sh; +time 16-cdos.sh; # 23 22 24 +time 16-cdos.sh; +time 16-cdos.sh; +time 32-cdos.sh; # 67 40 57 +time 32-cdos.sh; +time 32-cdos.sh; +time 64-cdos.sh; # 159 +time 64-cdos.sh; +time 64-cdos.sh; diff --git a/test/load_tests/2-64.sh b/test/load_tests/2-64.sh new file mode 100755 index 000000000..91131b84c --- /dev/null +++ b/test/load_tests/2-64.sh @@ -0,0 +1,20 @@ +#!/bin/sh +time 2.sh; +time 2.sh; +time 2.sh; +time 2.sh; +time 4.sh; +time 4.sh; +time 4.sh; +time 8.sh; +time 8.sh; +time 8.sh; +time 16.sh; +time 16.sh; +time 16.sh; +time 32.sh; +time 32.sh; +time 32.sh; +time 64.sh; +time 64.sh; +time 64.sh; diff --git a/test/load_tests/2-cdos.sh b/test/load_tests/2-cdos.sh new file mode 100755 index 000000000..773e5be9c --- /dev/null +++ b/test/load_tests/2-cdos.sh @@ -0,0 +1,8 @@ +#!/bin/sh +./main.py -u http://192.168.24.43/api/ reset dos_init > 0.log +for c in 1 2; do + ./main.py -u http://192.168.24.43/api/ -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done diff --git a/test/load_tests/2.sh b/test/load_tests/2.sh new file mode 100755 index 000000000..93276f01a --- /dev/null +++ b/test/load_tests/2.sh @@ -0,0 +1,8 @@ +#!/bin/sh +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1 2; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done diff --git a/test/load_tests/20.csv.gz b/test/load_tests/20.csv.gz new file mode 100644 index 000000000..f18e0e37b Binary files /dev/null and b/test/load_tests/20.csv.gz differ diff --git a/test/load_tests/20_counties_uploads.sh b/test/load_tests/20_counties_uploads.sh new file mode 100755 index 000000000..b920ec8d4 --- /dev/null +++ b/test/load_tests/20_counties_uploads.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# argument 1 is delay between simulated user interactions (in seconds) +# argument 2 is delay between county starts (in seconds) +if [ "$#" != "2" ]; then + echo "Usage: upload_script.sh " + echo "See script source for details." + exit +fi +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1 2 3 4 5 6 7 8; do + ./main.py -u http://corla-test.galois.com/api/ -T $1 -c $c county_setup -f small_cvrs.csv -F small_manifest.csv > $c.log & + sleep $((RANDOM % $2 + 1)); +done & +for c in 9 10 11 12 13 14 15 16; do + ./main.py -u http://corla-test.galois.com/api/ -T $1 -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & + sleep $((RANDOM % $2 + 1)); +done & +for c in 17 18; do + ./main.py -u http://corla-test.galois.com/api/ -T $1 -c $c county_setup -f large_cvrs.csv -F large_manifest.csv > $c.log & + sleep $((RANDOM % $2 + 1)); +done & +for c in 19 20; do + ./main.py -u http://corla-test.galois.com/api/ -T 3 -c $c county_setup -f huge_cvrs.csv -F huge_manifest.csv > $c.log & + sleep $((RANDOM % 3 + 1)); +done & +for job in `jobs -p`; do + wait $job +done + diff --git a/test/load_tests/250.csv.gz b/test/load_tests/250.csv.gz new file mode 100644 index 000000000..3225cd4d5 Binary files /dev/null and b/test/load_tests/250.csv.gz differ diff --git a/test/load_tests/30.csv.gz b/test/load_tests/30.csv.gz new file mode 100644 index 000000000..3bf7ddad5 Binary files /dev/null and b/test/load_tests/30.csv.gz differ diff --git a/test/load_tests/300.csv.gz b/test/load_tests/300.csv.gz new file mode 100644 index 000000000..c5c26a691 Binary files /dev/null and b/test/load_tests/300.csv.gz differ diff --git a/test/load_tests/32-cdos.sh b/test/load_tests/32-cdos.sh new file mode 100755 index 000000000..5c00b1cf4 --- /dev/null +++ b/test/load_tests/32-cdos.sh @@ -0,0 +1,8 @@ +#!/bin/sh +./main.py -u http://192.168.24.43/api/ reset dos_init > 0.log +for c in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32; do + ./main.py -u http://192.168.24.43/api/ -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done diff --git a/test/load_tests/32.sh b/test/load_tests/32.sh new file mode 100755 index 000000000..e9dcb764a --- /dev/null +++ b/test/load_tests/32.sh @@ -0,0 +1,8 @@ +#!/bin/sh +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done diff --git a/test/load_tests/4-cdos.sh b/test/load_tests/4-cdos.sh new file mode 100755 index 000000000..bc6b9ebeb --- /dev/null +++ b/test/load_tests/4-cdos.sh @@ -0,0 +1,8 @@ +#!/bin/sh +./main.py -u http://192.168.24.43/api/ reset dos_init > 0.log +for c in 1 2 3 4; do + ./main.py -u http://192.168.24.43/api/ -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done diff --git a/test/load_tests/4.sh b/test/load_tests/4.sh new file mode 100755 index 000000000..82dfb38e7 --- /dev/null +++ b/test/load_tests/4.sh @@ -0,0 +1,8 @@ +#!/bin/sh +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1 2 3 4; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done diff --git a/test/load_tests/4_large_counties_uploads.sh b/test/load_tests/4_large_counties_uploads.sh new file mode 100755 index 000000000..4e4a5f201 --- /dev/null +++ b/test/load_tests/4_large_counties_uploads.sh @@ -0,0 +1,9 @@ +#!/bin/sh +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1 2 3 4; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f large_cvrs.csv -F large_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done + diff --git a/test/load_tests/4_medium_counties_uploads.sh b/test/load_tests/4_medium_counties_uploads.sh new file mode 100755 index 000000000..82dfb38e7 --- /dev/null +++ b/test/load_tests/4_medium_counties_uploads.sh @@ -0,0 +1,8 @@ +#!/bin/sh +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1 2 3 4; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done diff --git a/test/load_tests/50.csv.gz b/test/load_tests/50.csv.gz new file mode 100644 index 000000000..8aec1d318 Binary files /dev/null and b/test/load_tests/50.csv.gz differ diff --git a/test/load_tests/500.csv.gz b/test/load_tests/500.csv.gz new file mode 100644 index 000000000..00ab6fe8c Binary files /dev/null and b/test/load_tests/500.csv.gz differ diff --git a/test/load_tests/58_counties_cdos.sh b/test/load_tests/58_counties_cdos.sh new file mode 100755 index 000000000..5d22e4cbd --- /dev/null +++ b/test/load_tests/58_counties_cdos.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# argument 1 is delay between simulated user interactions (in seconds) +# argument 2 is delay between county starts (in seconds) +if [ "$#" != "2" ]; then + echo "Usage: upload_script.sh " + echo "See script source for details." + exit +fi +./main.py -u http://192.168.24.43/api/ reset dos_init > 0.log +for c in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do + ./main.py -u http://192.168.24.43/api/ -T $1 -c $c county_setup -f 20.csv -F small_manifest.csv > $c.log & + sleep $((RANDOM % $2 + 1)); +done & +for c in 21 22 23 24 25 26 27 28 29 30; do + ./main.py -u http://192.168.24.43/api/ -T $1 -c $c county_setup -f 30.csv -F medium_manifest.csv > $c.log & + sleep $((RANDOM % $2 + 1)); +done & +for c in 31 32 33 34 35 36 37 38 39 40; do + ./main.py -u http://192.168.24.43/api/ -T $1 -c $c county_setup -f 50.csv -F large_manifest.csv > $c.log & + sleep $((RANDOM % $2 + 1)); +done & +for c in 41 42 43 44 45; do + ./main.py -u http://192.168.24.43/api/ -T 3 -c $c county_setup -f 100.csv -F huge_manifest.csv > $c.log & + sleep $((RANDOM % 3 + 1)); +done & +for c in 46 47 48 49; do + ./main.py -u http://192.168.24.43/api/ -T 3 -c $c county_setup -f 250.csv -F huge_manifest.csv > $c.log & + sleep $((RANDOM % 3 + 1)); +done & +for c in 50 51 52 53 54; do + ./main.py -u http://192.168.24.43/api/ -T 3 -c $c county_setup -f 300.csv -F huge_manifest.csv > $c.log & + sleep $((RANDOM % 3 + 1)); +done & +for c in 55 56 57 58; do + ./main.py -u http://192.168.24.43/api/ -T 3 -c $c county_setup -f 500.csv -F huge_manifest.csv > $c.log & + sleep $((RANDOM % 3 + 1)); +done & +for job in `jobs -p`; do + wait $job +done + diff --git a/test/load_tests/58_counties_uploads.sh b/test/load_tests/58_counties_uploads.sh new file mode 100755 index 000000000..cdfa24238 --- /dev/null +++ b/test/load_tests/58_counties_uploads.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# argument 1 is delay between simulated user interactions (in seconds) +# argument 2 is delay between county starts (in seconds) +if [ "$#" != "2" ]; then + echo "Usage: upload_script.sh " + echo "See script source for details." + exit +fi +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do + ./main.py -u http://corla-test.galois.com/api/ -T $1 -c $c county_setup -f 20.csv -F small_manifest.csv > $c.log & + sleep $((RANDOM % $2 + 1)); +done & +for c in 21 22 23 24 25 26 27 28 29 30; do + ./main.py -u http://corla-test.galois.com/api/ -T $1 -c $c county_setup -f 30.csv -F medium_manifest.csv > $c.log & + sleep $((RANDOM % $2 + 1)); +done & +for c in 31 32 33 34 35 36 37 38 39 40; do + ./main.py -u http://corla-test.galois.com/api/ -T $1 -c $c county_setup -f 50.csv -F large_manifest.csv > $c.log & + sleep $((RANDOM % $2 + 1)); +done & +for c in 41 42 43 44 45; do + ./main.py -u http://corla-test.galois.com/api/ -T 3 -c $c county_setup -f 100.csv -F huge_manifest.csv > $c.log & + sleep $((RANDOM % 3 + 1)); +done & +for c in 46 47 48 49; do + ./main.py -u http://corla-test.galois.com/api/ -T 3 -c $c county_setup -f 250.csv -F huge_manifest.csv > $c.log & + sleep $((RANDOM % 3 + 1)); +done & +for c in 50 51 52 53 54; do + ./main.py -u http://corla-test.galois.com/api/ -T 3 -c $c county_setup -f 300.csv -F huge_manifest.csv > $c.log & + sleep $((RANDOM % 3 + 1)); +done & +for c in 55 56 57 58; do + ./main.py -u http://corla-test.galois.com/api/ -T 3 -c $c county_setup -f 500.csv -F huge_manifest.csv > $c.log & + sleep $((RANDOM % 3 + 1)); +done & +for job in `jobs -p`; do + wait $job +done + diff --git a/test/load_tests/5_huge_county_cdos.sh b/test/load_tests/5_huge_county_cdos.sh new file mode 100755 index 000000000..646d93fe9 --- /dev/null +++ b/test/load_tests/5_huge_county_cdos.sh @@ -0,0 +1,9 @@ +#!/bin/sh +./main.py -u http://192.168.24.43/api/ reset dos_init > 0.log +for c in 1 2 3 4 5; do + ./main.py -u http://192.168.24.43/api/ -c $c county_setup -f huge_cvrs.csv -F huge_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done + diff --git a/test/load_tests/5_huge_county_uploads.sh b/test/load_tests/5_huge_county_uploads.sh new file mode 100755 index 000000000..15bedb9a6 --- /dev/null +++ b/test/load_tests/5_huge_county_uploads.sh @@ -0,0 +1,9 @@ +#!/bin/sh +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1 2 3 4 5; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f huge_cvrs.csv -F huge_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done + diff --git a/test/load_tests/64-cdos.sh b/test/load_tests/64-cdos.sh new file mode 100755 index 000000000..988b99676 --- /dev/null +++ b/test/load_tests/64-cdos.sh @@ -0,0 +1,8 @@ +#!/bin/sh +./main.py -u http://192.168.24.43/api/ reset dos_init > 0.log +for c in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64; do + ./main.py -u http://192.168.24.43/api/ -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done diff --git a/test/load_tests/64.sh b/test/load_tests/64.sh new file mode 100755 index 000000000..9f1ea88f2 --- /dev/null +++ b/test/load_tests/64.sh @@ -0,0 +1,8 @@ +#!/bin/sh +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done diff --git a/test/load_tests/6_large_counties_uploads.sh b/test/load_tests/6_large_counties_uploads.sh new file mode 100755 index 000000000..21542de2c --- /dev/null +++ b/test/load_tests/6_large_counties_uploads.sh @@ -0,0 +1,8 @@ +#!/bin/sh +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1 2 3 4 5 6; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f large_cvrs.csv -F large_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done diff --git a/test/load_tests/6_medium_counties_uploads.sh b/test/load_tests/6_medium_counties_uploads.sh new file mode 100755 index 000000000..215bee4e6 --- /dev/null +++ b/test/load_tests/6_medium_counties_uploads.sh @@ -0,0 +1,9 @@ +#!/bin/sh +./main.py -u http://corla-test.galois.com/api reset dos_init > 0.log +for c in 1 2 3 4 5 6; do + ./main.py -u http://corla-test.galois.com/api -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done + diff --git a/test/load_tests/8-cdos.sh b/test/load_tests/8-cdos.sh new file mode 100755 index 000000000..50abe6319 --- /dev/null +++ b/test/load_tests/8-cdos.sh @@ -0,0 +1,8 @@ +#!/bin/sh +./main.py -u http://192.168.24.43/api/ reset dos_init > 0.log +for c in 1 2 3 4 5 6 7 8; do + ./main.py -u http://192.168.24.43/api/ -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done diff --git a/test/load_tests/8.sh b/test/load_tests/8.sh new file mode 100755 index 000000000..30c7d4bca --- /dev/null +++ b/test/load_tests/8.sh @@ -0,0 +1,8 @@ +#!/bin/sh +./main.py -u http://corla-test.galois.com/api/ reset dos_init > 0.log +for c in 1 2 3 4 5 6 7 8; do + ./main.py -u http://corla-test.galois.com/api/ -c $c county_setup -f medium_cvrs.csv -F medium_manifest.csv > $c.log & +done +for job in `jobs -p`; do + wait $job +done diff --git a/test/load_tests/benchmarks.bash b/test/load_tests/benchmarks.bash new file mode 100755 index 000000000..b46e7fce7 --- /dev/null +++ b/test/load_tests/benchmarks.bash @@ -0,0 +1,48 @@ +#!/bin/bash +# Run a series of benchmarks. +# First set the environmental variable URL to the url to use +# For example: $0 mytest "2 4 8" 4 large +# Provide these arguments, of which only the first is mandatory: + +usage="Usage: + $0 testname [series [repeat [cvrs]]]. +e.g. URL=http://example.org/api $0 mytest '2 4 8 16 32 64' 3 medium" + +testname=$1 # Directory to create and save results in +${testname:?"$usage"} +series=${2:-2 4 8 16 32 64} # a list (in quotes) of values for the number of counties to run in parallel +repeat=${3:-3} # Number of times to repeat each element in series +cvrs=${4:-medium} # cvrs to load for each test, + # e.g. "medium" for medium_cvrs.csv and medium_manifest.csv +mkdir -p $testname +cd $testname + +# Save all script output in benchmark, while watching as it comes out +exec > >(tee benchmarks.out) + +echo Test $testname, loading $cvrs on $URL for n=$series + +results="" + +for n in $series; do + for i in $(seq $repeat); do + crtest -u $URL reset dos_init + + start=$(date +%s) + echo Start: $(date -Is), $n in parallel, trial $i + + for c in $(seq $n); do + crtest -u $URL -c $c county_setup -f ../${cvrs}_cvrs.csv -F ../${cvrs}_manifest.csv > $c.log 2> $c.err & + done + + for job in `jobs -p`; do + wait $job + done + end=$(date +%s) + interval=$((end - start)) + results="$results $interval" + echo "Done: $(date -Is) with n=$n, trial $i, after $interval seconds" + done +done + +echo Paste series, repeats, results into spreadsheet: "$series $repeat $results" diff --git a/test/load_tests/bootstrap_cvr.py b/test/load_tests/bootstrap_cvr.py new file mode 100644 index 000000000..ee5936881 --- /dev/null +++ b/test/load_tests/bootstrap_cvr.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +"""Bootstrap a Dominion CVR file +Takes lines from an existing CVR file and bootstraps them to the desired length. + +Bootstrapping is designed to retain the original flavor of CVRs, by picking +each line a random number of times in order to closely approximate the +desired length. + +The initial columns are modified, to follow the standard numbering scheme. +But every CVR is labeled as TabulatorNum 1, BatchId 1. +The CvrNumber and RecordId fields both count from 1 to the overall target number. +An ImprintedId to match is also generated. + +Usage: + + python bootstrap_cvr.py 100 + +generates a 100-CVR file based on the hardcoded D0FILE constant. + +Note: if you don't get the desired length, you'll get a message saying to +try running it again. + +Note: this is a rough approximation. The last few lines may be omitted or be +underrepresented in the sample. + +Adapted from + Bootstrap a very large data file | Tomochika Fujisawa's site + https://tmfujis.wordpress.com/2016/06/24/bootstrap-a-very-large-data-file/ +""" + +import sys +import numpy + +D0FILE="../dominion-2017-CVR_Export_20170310104116.csv" +D0COUNT=165 + +def find_nth(s, x, n=0, overlap=False): + """Find the nth occurrence of the string x in s + From @Mark Byers and @Stefan: https://stackoverflow.com/a/23479065/507544 + """ + l = 1 if overlap else len(x) + i = -l + for c in xrange(n + 1): + i = s.find(x, i + l) + if i < 0: + break + return i + +target = int(sys.argv[1]) + +inHeader = True +cvrNumber = 1 + +for line in open(D0FILE, "r"): + if inHeader: + sys.stdout.write(line) + if line.startswith('"CvrNumber"'): + inHeader = False + continue + + cnt = numpy.random.poisson(lam=1.0 * target/(D0COUNT-1)) #FIXME: hard-code one less than len of assumed input file + + if cnt == 0: + continue + else: + for i in range(cnt): + cvrline = '"%d","1","1","%d","1-1-%d"%s' % \ + (cvrNumber, cvrNumber, cvrNumber, line[find_nth(line, ',', 4):]) + sys.stdout.write(cvrline) + cvrNumber += 1 + if cvrNumber == target + 1: + sys.exit(0) + + +if cvrNumber < target: + sys.stderr.write("Only got %d CVRs. Re-run the command\n" % cvrNumber) diff --git a/test/load_tests/huge_cvrs.csv.gz b/test/load_tests/huge_cvrs.csv.gz new file mode 100644 index 000000000..59f82f66d Binary files /dev/null and b/test/load_tests/huge_cvrs.csv.gz differ diff --git a/test/load_tests/huge_manifest.csv.gz b/test/load_tests/huge_manifest.csv.gz new file mode 100644 index 000000000..55d4a3899 Binary files /dev/null and b/test/load_tests/huge_manifest.csv.gz differ diff --git a/test/load_tests/large_cvrs.csv.gz b/test/load_tests/large_cvrs.csv.gz new file mode 100644 index 000000000..0d4d45490 Binary files /dev/null and b/test/load_tests/large_cvrs.csv.gz differ diff --git a/test/load_tests/large_manifest.csv.gz b/test/load_tests/large_manifest.csv.gz new file mode 100644 index 000000000..4ea536ba5 Binary files /dev/null and b/test/load_tests/large_manifest.csv.gz differ diff --git a/test/load_tests/main.py b/test/load_tests/main.py new file mode 100755 index 000000000..4a4eb95f3 --- /dev/null +++ b/test/load_tests/main.py @@ -0,0 +1,1151 @@ +#!/usr/bin/env python +"""corla_test: drive testing of ColoradoRLA + +Examples. Note that 'crtest' and/or corla_test should be defined +as an alias for main.py. + +# Smoketest: + +./main.py + +# Simple loading of some data, for audit by the web client: + +crtest reset +crtest dos_init +crtest county_setup + +# Test 2-vote overstatement +crtest -l "Distant Loser" + +# 1-vote understatement +crtest -s "22345123451234512345" -p "1 17" + +# 2-vote understatement +crtest -l "Clear Winner" -s "22345123451234512345" -p "1 17" + +# Test ballot-not-found +crtest -C 1 -n "8 15" + +# Simple quick retrievals +crtest -E /audit-board-asm-state +crtest -e /dos-asm-state + +# Display a county or DOS dashboard +crtest -E '/county-dashboard' + +crtest -e '/dos-dashboard' + +# Get csv file with ballot cards to be audited, random sequence numbers +crtest -E '/cvr-to-audit-download?start=0&ballot_count=9&include_duplicates' + +# Get estimate for given counties of how many CVRs have been read in vs total record_count +# 'cvr_export_count' is an estimate of the total processed so far, and +# 'approximate_record_count' is an estimate of the total count in the uploaded file. +crtest -c 1 -c 3 -E '/county-dashboard' | egrep '"id"|filename|cvr_export_count|approximate_record_count' + +# Check whether the /cvr-to-audit responses match the other lists, and also remove audited ballots + +crtest reset dos_init county_setup dos_start +crtest -E "/cvr-to-audit-list?start=0&ballot_count=9&include_duplicates=true" +crtest county_audit +crtest -E "/cvr-to-audit-list?start=0&ballot_count=1&include_duplicates=true" +crtest -E "/cvr-to-audit-list?start=0&ballot_count=9&include_duplicates=true" + +# Request as state user, specifying a county +crtest -e '/cvr-to-audit-download?round=1&county=3&include_audited&include_duplicates' + +crtest -e "/cvr-to-audit-download?county=3&start=0&ballot_count=9&include_audited&include_duplicates" + +# Test two county audits in parallel +( +./main.py reset dos_init +./main.py -c 3 -v 0 county_setup & +./main.py -c 5 -v 2 county_setup & +wait +./main.py dos_start -C 3 +./main.py -c 5 county_audit & +./main.py -c 3 county_audit & +wait +./main.py dos_wrapup +) > multi.out + +# Test two counties, one of which doesn't have a contest to audit +( +date -Is +./main.py reset dos_init +for c in 1 2; do + ./main.py -c $c county_setup & +done +wait +./main.py dos_start +for c in 1 2; do ./main.py -c $c county_audit & +wait +done +date -Is +) 2>&1 | tee 2-counties-one-contest.out + +# Parallel auditing +( +./main.py -C 1 -c 1 -c 2 -c 3 -c 4 -c 5 -c 6 -c 7 -c 8 -c 9 -c 10 -f d0-n1000-fix1.csv reset dos_init county_setup dos_start +wait +for c in 1 2 3 4 5 6 7 8 9 10; do ./main.py -c $c county_audit & done +wait +) | tee parallelaudit.out + +# Parallel uploads of bigger CVRs + +( +date -Is +./main.py reset dos_init +for c in 1 2 3 4 5 6 7 8 9 10; do + ./main.py -c $c -f neal_ignore/d0-n50000.csv county_setup & +done +wait +./main.py dos_start -C 1 +for c in 1 2 3 4 5 6 7 8 9 10; do ./main.py -c $c county_audit & +wait +done +date -Is +) 2>&1 | tee parallelcvr50000x10.out + +# Miscellaneous of specific server endpoint paths + +crtest -e /acvr +crtest -e /contest +crtest -e /contest/id/52253 +crtest -e /contest/county?3 +crtest -E /contest/county?3 -c 3 + +# Not working - minor missing feature: + +crtest -e /acvr/county/3 + +TODO later: + +/hand-count IndicateHandCount + + GET /ballot-manifest (BallotManifestDownload) + GET /acvr/county (ACVRDownloadByCounty) + GET /cvr (CVRDownload) + GET /cvr/id/:id (CVRDownloadByID) + GET /ballot-manifest/county (BallotManifestDownloadByCounty) + GET /cvr/county (CVRDownloadByCounty) + GET /acvr (ACVRDownload) + GET /contest (ContestDownload) + GET /contest/id/:id (ContestDownloadByID) + GET /contest/county (ContestDownloadByCounty) + +Note: zerotest doesn't support POST operations (yet?) +See "File upload via POST request not working: Issue #12" +https://github.com/jjyr/zerotest/issues/12 + +TODO: to help human testers using web client, display CVRs corresponding to selected ACVRs for a given county +""" + +from __future__ import (print_function, division, + absolute_import, unicode_literals) +import sys +import os +import operator +import argparse +from argparse import Namespace +import json +import time +import random +import logging +import hashlib + +import requests + + +__author__ = "Neal McBurnett " +__license__ = "GPLv3+" + + +parser = argparse.ArgumentParser(description='Drive testing for ColoradoRLA auditing.') + +parser.add_argument('-c, --county', dest='counties', metavar='COUNTY', action='append', + type=int, + help='numeric county_id for the given command, e.g. 1 ' + 'for Adams. May be specified multiple times.') +parser.add_argument('-v, --cvr', dest='cvr', type=int, + default=0, + help='predefined cvr filename and hash: integer index to pre-defined array, default 0') +parser.add_argument('-f, --cvrfile', dest='cvrfile', + help='cvr filename, for arbitrary cvrs. Takes precedence over -v. ' + 'Proper hash will be computed.') +parser.add_argument('-F, --manifestfile', dest='manifestfile', + help='manifest filename, for arbitrary manifests. ' + 'Proper hash will be computed.') + +# TODO: allow a way to specify CVRs and contests per county. +parser.add_argument('-C, --contest', dest='contests', metavar='CONTEST', action='append', + type=int, + help='numeric contest_index of contest to use for the given audit commands ' + 'E.g. 0 for first one from the CVRs. May be specified multiple times. ' + '-1 means "audit all contests') +parser.add_argument('-l, --loser', dest='loser', default="UNDERVOTE", + help='Loser to use for -p, default "UNDERVOTE"') +parser.add_argument('-p, --discrepancy-plan', dest='plan', default="2 17", + help='Planned discrepancies. Default is "2 17", i.e. ' + 'Every 17 ACVR upload, upload a possible discrepancy once, ' + 'when the remainder of dividing the upload index is 2. ' + 'Discrepancies thus come with the 3rd of every 17 ACVR uploads.') +parser.add_argument('-P, --discrepancy-end', dest='plan_limit', type=int, default=sys.maxint, + help='Last upload with possible discrepancy is # PLAN_LIMIT') + +parser.add_argument('-n, --notfound-plan', dest='notfound_plan', default="-1 1", + help='Planned rate of ballot-not-found discrepancies. Default is "-1 1", i.e. never') + +parser.add_argument('-R, --rounds', type=int, default=-1, dest='rounds', + help='Set maximum number of rounds. Default is all rounds.') + +parser.add_argument('-r, --risk-limit', type=float, dest='risk_limit', default=0.1, + help='risk limit, e.g. 0.1') +parser.add_argument('-s, --seed', dest='seed', + default='01234567890123456789', + help='random seed to use: 20 or more digts') +parser.add_argument('-u, --url', dest='url', + default='http://localhost:8888', + help='base url of corla server. Defaults to http://localhost:8888. ' + 'Use something like http://example.gov/api when running ' + 'against a full installation.') +parser.add_argument('-e, --dos-endpoint', dest='dos_endpoint', + help='do an HTTP GET from the given endpoint, authenticated as state admin.') +parser.add_argument('-E, --county-endpoint', dest='county_endpoint', + help='do an HTTP GET from the given endpoint, authenticated as a county.') +parser.add_argument('--hand-count', dest='hand_counts', type=int, metavar='CONTEST', action='append', + help='Declare a hand-count for the given numeric contest_index') +parser.add_argument('--download-file', dest='download_file', type=int, metavar='FILE_ID', + help='Just download file with given FILE_ID') + # help='Just list files and download selected ones') +parser.add_argument('-S, --check-audit-size', type=bool, dest='check_audit_size', + help='Check calculations of audit size. Requires rlacalc, psycopg2') + +parser.add_argument('-T, --time-delay', type=float, dest='time_delay', default=0.0, + help='Maximum time to pause before network requests. Default 0.0. ' + 'Actual pauses will be uniformly distributed between 0 and the maximum') +parser.add_argument('-L, --lower-time-delay', type=float, dest='lower_time_delay', default=0.0, + help='Minimum time to pause before network requests. Default 0.0. ' + 'Actual pauses will be uniformly distributed between this and the maximum') + +# TODO: get rid of this and associated old code when /upload-cvr-export and /upload-cvr-export go away +parser.add_argument('-Y, --ye-olde-upload', type=bool, dest='ye_olde_upload', + help='use old file upload protocol') + +parser.add_argument('-t, --trackstates', type=bool, dest='trackstates', + default=False, + help='Show state after most requests') + +parser.add_argument('-d, --debuglevel', type=int, default=logging.WARNING, dest='debuglevel', + help='Set logging level to debuglevel: DEBUG=10, INFO=20,\n WARNING=30 (the default), ERROR=40, CRITICAL=50') + +parser.add_argument('commands', metavar="COMMAND", nargs='*', + help='audit commands to run. May be specified multiple times. ' + 'Possibilities: reset dos_init county_setup dos_start county_audit dos_wrapup') + + +class Pause(object): + """Provide a configurable sleep delay. + Just set Pause.max_pause directly when you want the default to change" + """ + + min_pause = 0.0 + max_pause = 0.0 + + @classmethod + def pause_hook(self, r, *args, **kwargs): + """A hook for a Requests response, which pauses a random amount of time, + between 0.0 and the maximum pause configured for the class + """ + + time.sleep(random.uniform(self.min_pause, self.max_pause)) + + +def requests_retry_session(retries=3, backoff_factor=2, method_whitelist=False, + status_forcelist=(429, 502, 503), session=None): + """Return a Requests session that retries for the given status codes, and + for connection timeouts. + The default of method_whitelist=False means that it retries all HTTP + methods, even POST, which should be OK given the default status_forcelist. + + Inspired by Peter Bengtsson https://www.peterbe.com/plog/best-practice-with-retries-with-requests + + To test 503 error and connection timeout: + crtest -e '' -d 10 -u http://httpbin.org/status/503? + crtest -e '' -d 10 -u http://10.255.255.1/? + """ + + session = session or requests.Session() + session.hooks = dict(response=Pause.pause_hook) # Why isn't this an argument to the constructor? + retry = requests.packages.urllib3.util.retry.Retry( + total=retries, + connect=retries, + read=retries, + status=retries, + method_whitelist=method_whitelist, + status_forcelist=status_forcelist, + backoff_factor=backoff_factor, + raise_on_status=False + ) + adapter = requests.adapters.HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + return session + + +def state_login(ac, s): + "Login as state admin in given requests session" + + path = "/auth-state-admin" + r = s.post(ac.base + path, + data={'username': 'stateadmin1', 'password': '', 'second_factor': ''}) + r = s.post(ac.base + path, + data={'username': 'stateadmin1', 'password': '', 'second_factor': ''}) + ac.logconsole.info("%s %s %s", r, "POST", path) + + +def county_login(ac, s, county_id): + "Login as county admin in given requests session" + + path = "/auth-county-admin" + r = s.post(ac.base + path, + data={'username': 'countyadmin%d' % county_id, 'password': '', 'second_factor': ''}) + r = s.post(ac.base + path, + data={'username': 'countyadmin%d' % county_id, 'password': '', 'second_factor': ''}) + ac.logconsole.info("%s %s %s", r, "POST", path) + + +def test_endpoint_json(ac, s, path, data, show=True): + "Do a generic test of an endpoint that posts the given data to the given path" + + if ac.args.trackstates: + show=True # Make sure we show the action before showing the resulting state + + r = s.post(ac.base + path, json=data) + if r.status_code == 200: + if show: + ac.logconsole.info("%s %s %s", r, "POST", path) + else: + if show: + ac.logconsole.info("%s %s %s %s", r, "POST", path, r.text) + else: + ac.logconsole.info("%s %s %s", r, "POST", path) + + if ac.args.trackstates: + r = test_endpoint_get(ac, ac.state_s, "/dos-asm-state", show=False) + + if 'current_state' in r.json(): + print("DOS: %s" % r.json()['current_state']) + else: + print("smoketest sees no current state", r.text) + + if s != ac.state_s: + print("County: %s" % test_endpoint_get(ac, s, "/audit-board-asm-state", show=False).json()['current_state']) + + return r + + +def test_endpoint_get(ac, s, path, show=True): + "Do a generic test of an endpoint that gets the given path" + + r = s.get(ac.base + path) + if r.status_code == 200: + if show: + ac.logconsole.info("%s %s %s", r, "GET", path) + else: + if show: + ac.logconsole.info("%s %s %s %s", r, "GET", path, r.text) + else: + ac.logconsole.info("%s %s %s", r, "GET", path) + return r + + +def upload_file(ac, s, import_path, filename, sha256): + """Upload the named file, specifying the given sha256 hash. + import_path is either '/import-cvr-export' or '/import-ballot-manifest' + """ + + with open(filename, 'rb') as f: + path = "/upload-file" + payload = {'hash': sha256} + r = s.post(ac.base + path, + files={'file': f}, data=payload) + + if r.status_code != 200: + print(r, "POST", path, r.text) + + logging.debug("%s %s %s" % (r, path, r.text)) + + import_handle = r.json() + + r = test_endpoint_json(ac, s, import_path, import_handle) + if r.status_code != 200: + print(r, "POST", import_path, r.text) + logging.debug("%s %s %s" % (r, import_path, r.text)) + + if import_path == "/import-cvr-export": + while True: + # wait for the verdict on the CVR export + r = test_endpoint_get(ac, s, "/county-dashboard") + dashboard = r.json() + state = dashboard['asm_state'] + imported_count = dashboard.get('cvr_export_count', None) + if 'cvr_export_file' in dashboard: + approximate_record_count = dashboard['cvr_export_file']['approximate_record_count'] + else: + approximate_record_count = -1 + + # more efficient? Do part of the time? + # r = test_endpoint_get(ac, s, "/county-asm-state") + # state = r.json()['current_state'] + + if state in ["CVRS_IMPORTING", "BALLOT_MANIFEST_OK_AND_CVRS_IMPORTING"]: + logging.info("Received about %d of about %d CVRs", + imported_count, approximate_record_count) + time.sleep(30) + else: + print("CVR import complete, state: %s" % state) + # TODO: evaluate whether there was an error + # Importing CVRs always wipes out old ones. So if you start in, e.g., CVRS_OK, + # and an import fails, you'll end up in COUNTY_INITIAL_STATE. + # Otherwise? end up in previous state + break + +""" +TODO: clean this out when ready. + +Alternate approaches that have worked: + r = test_endpoint_bytes(ac, s, import_path, r.text) + r = test_endpoint_json(ac, s, import_path, { "file_id": import_handle['file_id']}) + + print("import_handle: %s" % import_handle) + print("response text: %s" % r.text) +""" + +def download_file(ac, s, file_id, filename): + "Download the previously-uploaded file with the given file_id to the given filename" + + with open(filename, 'wb') as f: + path = "/download-file" + r = s.get(ac.base + path, params={'file_info': json.dumps({'file_id': "%d" % file_id})}) + + if r.status_code != 200: + print(r, "GET", path, r.text) + + logging.debug("%s %s" % (r, path)) + + with open(filename, "wb") as f: + f.write(r.content) + print("file_id %d saved as %s" % (file_id, filename)) + +def upload_cvrs(ac, s, filename, sha256): + "Upload cvrs" + + with open(filename, 'rb') as f: + path = "/upload-cvr-export" + # TODO: make this generic for any county + payload = {'county': '3', 'hash': sha256} + r = s.post(ac.base + path, + files={'cvr_file': f}, data=payload) + print(r, "POST", path, r.text) + + +def upload_manifest(ac, s, filename, sha256): + "Upload manifest" + + with open(filename, 'rb') as f: + path = "/upload-ballot-manifest" + payload = {'county': 'Arapahoe', 'hash': sha256} + r = s.post(ac.base + path, + files={'bmi_file': f}, data=payload) + print(r, "POST", path) + + +def get_county_cvrs(ac, county_id, s): + "Return all cvrs uploaded by a given county" + + path = x + r = s.get("%s/cvr/%d" % (ac.base, county_id)) + if r.status_code != 200: + print(r, "GET", path, r.text) + cvrs = r.json() + + return cvrs + + +def get_cvrs(ac, s): + "Return all cvrs uploaded by any county" + + r = s.get("%s/cvr" % ac.base) + cvrs = r.json() + + return cvrs + + +def publish_ballots_to_audit(seed, cvrs): + """Return lists by county of ballots to audit. + """ + + import sampler + + county_ids = set(cvr['county_id'] for cvr in cvrs) + + ballots_to_audit = [] + for county_id in county_ids: + county_cvrs = sorted( (cvr for cvr in cvrs if cvr['county_id'] == county_id), + key=lambda cvr: "%s-%s-%s" % (cvr['scanner_id'], cvr['batch_id'], cvr['record_id'])) + N = len(county_cvrs) + # n is based on auditing Regent contest. + # TODO: perhaps calculate from margin etc + n = 11 + seed = "01234567890123456789" + + _, new_list = sampler.generate_outputs(n, True, 0, N, seed, False) + + logging.debug("Random selections, N=%d, n=%d, seed=%s: %s" % + (N, n, seed, new_list)) + + selected = [] + for i, cvr in enumerate(county_cvrs): + if i in new_list: + cvr['record_type'] = 'AUDITOR_ENTERED' + selected.append(cvr) + logging.info("Selected cvr %d: id: %d RecordID: %s" % (i, cvr['id'], cvr['imprinted_id'])) + + ballots_to_audit.append([county_id, selected]) + + return ballots_to_audit + + +def compute_hash(filename): + "Compute and return the sha-256 hash of a file" + + BUF_SIZE = 2 ** 18 + + sha256 = hashlib.sha256() + + with open(filename, 'rb') as f: + while True: + data = f.read(BUF_SIZE) + if not data: + break + sha256.update(data) + + return sha256.hexdigest() + + +def upload_files(ac, s): + """Directly upload files, which zerotest doesn't support. + See "File upload via POST request not working: Issue #12" + https://github.com/jjyr/zerotest/issues/12 + """ + + if ac.args.manifestfile is None: + manifestfile = "../e-1/arapahoe-manifest.csv" + hash = "42d409d3394243046cf92e3ce569b7078cba0815d626602d15d0da3e5e844a94" + else: + manifestfile = ac.args.manifestfile + hash = compute_hash(manifestfile) + + if ac.args.ye_olde_upload: + dir_path = os.path.dirname(os.path.realpath(__file__)) + upload_manifest(ac, s, os.path.join(dir_path, manifestfile), hash) + else: + upload_file(ac, s, '/import-ballot-manifest', manifestfile, hash) + + predefined_cvrs = ( + ("../e-1/arapahoe-regent-3-clear-CVR_Export.csv", + "49bd5d56e6107ff6b7381a6f563121e3b1d5d967bba1c29e6ffe31583d646e6d"), + ("../dominion-2017-CVR_Export_20170310104116.csv", + "4e3844b0dabfcea64a499d65bc1cdc00d139cd5cdcaf502f20dd2beaa3d518d2"), + ("../Denver2016Test/CVR_Export_20170804111144.csv", + "1def4aa4c0e1421b4e5adcd4cc18a8d275f709bc07820a37e76e11a038195d02"), + ("../e-1/arapahoe-regent-3-clear-CVR_Export.csv", + "invalid hash"), + ("../e-1/arapahoe-regent-3-clear-CVR_Export.csv", + "00000111111111122222222234444444444456789999999abbbbbbbbbcccddef"), + ) + + if ac.args.cvrfile is None: + cvrfile, hash = predefined_cvrs[ac.args.cvr] + else: + cvrfile = ac.args.cvrfile + hash = compute_hash(cvrfile) + + if ac.args.ye_olde_upload: + upload_cvrs(ac, s, os.path.join(dir_path, cvrfile), hash) + else: + upload_file(ac, s, '/import-cvr-export', cvrfile, hash) + +def get_county_dashboard(ac, county_s, county_id, i=0, acvr={'id': -1}, show=True): + "Get and show useful info about /county-dashboard" + + r = test_endpoint_get(ac, county_s, "/county-dashboard", show=False) + county_dashboard = r.json() + + total_audited = 1 + county_dashboard['audited_ballot_count'] + + if show: + logging.debug("county-dashboard: %s" % r.text) + print("Round %d, county %d, upload %d, prefix %d: aCVR %d; ballots_remaining_in_round: %d, optimistic_ballots_to_audit: %s est %s" % + (ac.round, county_id, total_audited, county_dashboard.get('audited_prefix_length', -1), acvr['id'], # FIXME + county_dashboard['ballots_remaining_in_round'], county_dashboard['optimistic_ballots_to_audit'], county_dashboard['estimated_ballots_to_audit'])) + + + """ Put this back in when estimated_ballots_to_audit makes sense again + print("Round %d, county %d, upload %d, prefix %d: aCVR %d; ballots_remaining_in_round: %d, estimated_ballots_to_audit: %s" % + (ac.round, county_id, total_audited, county_dashboard.get('audited_prefix_length', -1), acvr['id'], # FIXME + county_dashboard['ballots_remaining_in_round'], county_dashboard['estimated_ballots_to_audit'])) + """ + + return county_dashboard + +def server_sequence(): + '''Run thru a given test sequence to explore server ASM transitions. + TODO: needs lots of work to easily handle a full Eulerian traversal + of all transitions in the state graphs. + ''' + + test_get_cvr() + test_get_ballot_manifest() + test_get_contest() + test_get_cvr_county_Arapahoe() + test_get_ballot_manifest_county_Arapahoe() + test_get_contest_id_2() + test_get_contest_county_Arapahoe() + + +def reset(ac): + 'Reset the database, leaving only authentication info' + + r = test_endpoint_json(ac, ac.state_s, "/reset-database", {}) + + +def dos_init(ac): + 'Run initial Dept of State steps: audit definition, risk_limit etc.' + + r = test_endpoint_json(ac, ac.state_s, "/update-audit-info", + { "election_type": "coordinated", + "election_date": "2017-11-09T02:00:00Z", + "public_meeting_date": "2017-11-19T02:00:00Z", + "risk_limit": ac.args.risk_limit } ) + +def county_setup(ac, county_id): + + logging.debug("county setup for county_id %d" % county_id) + + county_s = requests_retry_session() + county_login(ac, county_s, county_id) + + upload_files(ac, county_s) + + r = test_endpoint_get(ac, county_s, "/contest/county?%d" % county_id) + contests = r.json() + + # TODO: get count again: print("Uploaded table of %d CVRs with %d contests" % (, len(contests))) + print("Uploaded CVR table with %d contests" % (len(contests),)) + + # TODO perhaps cleanup - but is it more realistic to just leave sessions open? + # county_s.close() + + +def dos_start(ac): + 'Run DOS steps to start the audit, enabling county auditing to begin: contest selection, seed, etc.' + + r = test_endpoint_get(ac, ac.state_s, "/contest") + contests = r.json() + if len(contests) <= 0: + print("No contests to audit, status_code = %d" % r.status_code) + return + + for i, contest in enumerate(contests): + print("Contest {}: vote for {votes_allowed} in {name}".format(i, **contest)) + + logging.log(5, "Contests: %s" % contests) + + ac.audited_contests = [] + + # -1 is a special value meaning "audit all contests" + if ac.args.contests[0] == -1: + ac.args.contests = range(len(contests)) + + for contest_index in ac.args.contests: + if contest_index >= len(contests): + logging.error("Contest_index %d out of range: only %d contests in election" % + (contest_index, len(contests))) + + ac.audited_contests.append(contests[contest_index]['id']) + r = test_endpoint_json(ac, ac.state_s, "/select-contests", + [{"contest": ac.audited_contests[-1], + "reason": "COUNTY_WIDE_CONTEST", + "audit": "COMPARISON"}]) + + r = test_endpoint_json(ac, ac.state_s, "/random-seed", + {'seed': ac.args.seed}) + + r = test_endpoint_json(ac, ac.state_s, "/start-audit-round", + { "multiplier": 1.0, "use_estimates": True}) + # print(r.text) + # with use_estimates: False: "county_ballots": { "ID": number, "ID": number, ... } + # only the counties listed in it have rounds started + + r = test_endpoint_get(ac, ac.state_s, "/dos-dashboard") + if r.status_code == 200: + dos_dashboard = r.json() + for contest_id, reason in dos_dashboard['audited_contests'].items(): + r = test_endpoint_get(ac, ac.state_s, "/contest/id/%s" % contest_id) + contest = r.json() + print("Audit driver in county {county_id}, contest {id}: vote for {votes_allowed} in {name}".format(**contest)) + for choice in contest['choices']: + print(" %s" % choice['name']) + + for county_id, status in dos_dashboard['county_status'].items(): + if status['estimated_ballots_to_audit'] != 0: + print("County %s has initial sample size of %s sample interpretations, including duplicates" % + (county_id, status['estimated_ballots_to_audit'])) + print("ballots_remaining_in_round %d: %d" % + (ac.round, status['ballots_remaining_in_round'])) + logging.debug("dos-dashboard: %s" % r.text) + +def county_audit(ac, county_id): + 'Audit board uploads ACVRs from a county. Return estimated remaining ballots to audit' + + county_s = requests_retry_session() + county_login(ac, county_s, county_id) + + # Note: we take advantage of a side effect of this also: print where we're at.... + county_dashboard = get_county_dashboard(ac, county_s, county_id, -1) + + if county_dashboard['asm_state'] == "COUNTY_AUDIT_COMPLETE": + return(True) + + audit_board_set = [{"first_name": "Mary", + "last_name": "Doe", + "political_party": "Democrat"}, + {"first_name": "John", + "last_name": "Doe", + "political_party": "Republican"}] + + r = test_endpoint_get(ac, county_s, "/audit-board-asm-state") + if ((r.json()['current_state'] == "WAITING_FOR_ROUND_START_NO_AUDIT_BOARD") or + (r.json()['current_state'] == "ROUND_IN_PROGRESS_NO_AUDIT_BOARD")): + r = test_endpoint_json(ac, county_s, "/audit-board-sign-in", audit_board_set) + + # Print this tool's notion of what should be audited, based on seed etc. + # for auditing the audit. + # TODO or FIXME - doesn't yet match "ballots_to_audit" from the dashboard + # logging.log(5, json.dumps(publish_ballots_to_audit(ac.args.seed, cvrs), indent=2)) + + # r = test_endpoint_get(ac, county_s, "/audit-board-asm-state") + + round = len(county_dashboard['rounds']) + r = test_endpoint_get(ac, county_s, "/cvr-to-audit-download?round=%d" % round) + r = test_endpoint_get(ac, county_s, "/cvr-to-audit-list?round=%d" % round) + selected = r.json() + + print("Retrieved ballots_to_audit, got %d" % len(selected)) + if len(selected) != county_dashboard['ballots_remaining_in_round']: + print("ERROR: got %d CVR ids in ballots_to_audit, but ballots_remaining_in_round is %d in county-dashboard" % + (len(selected), county_dashboard['ballots_remaining_in_round'])) + + # For each of a a bunch of selected cvrs, + # make it into a matching acvr and upload it, watching progress + # TODO: upload the right number of them.... + + if len(selected) < 1: + print("No ballots_to_audit") + + for i in range(len(selected)): + if ac.args.debuglevel >= logging.INFO: + r = test_endpoint_get(ac, ac.state_s, "/dos-dashboard", show=False) + discrepancies = "" + contest_discrepancies = r.json().get('discrepancy_count', {}) + for contest_id, d in contest_discrepancies.iteritems(): + discrepancies += "%s %2d %2d %2d %2d %2d " % (contest_id, d["2"], d["1"], d["0"], d["-1"], d["-2"]) + print(discrepancies) + + if i % 50 == 5: + r = test_endpoint_json(ac, county_s, "/audit-board-sign-out", {}); + r = test_endpoint_get(ac, county_s, "/audit-board-asm-state") + # print(r.text) + r = test_endpoint_json(ac, county_s, "/audit-board-sign-in", audit_board_set) + r = test_endpoint_get(ac, county_s, "/audit-board-asm-state") + # print(r.text) + + r = test_endpoint_get(ac, county_s, "/cvr/id/%d" % selected[i]['db_id'], show=False) + acvr = r.json() + logging.debug("Original CVR: %s" % json.dumps(acvr)) + acvr['record_type'] = 'AUDITOR_ENTERED' + + total_audited = county_dashboard['audited_ballot_count'] + # print("total_audited: %d" % total_audited) + + # Modify the aCVR sometimes. + if (total_audited % ac.discrepancy_cycle == ac.discrepancy_remainder + and total_audited <= ac.args.plan_limit): + + # TODO: use contest info to look for the contests and add votes for losers + + message = "No Discrepancy, contest %d not in this CVR" % ac.audited_contests[0] + for ci in acvr['contest_info']: + if ci['contest'] == ac.audited_contests[0]: + message = "Choice wouldn't change" + if ci['choices'] != ac.false_choices: + message = "Discrepancy: %s in %d, was %s" % (ac.false_choices, ac.audited_contests[0], ci['choices']) + ci['choices'] = ac.false_choices + break + print(message) + + elif False: + # Test: make uploaded cvr not match + # TODO: Decide what API should be for mismatch in contests between CVR and paper + if len(acvr['contest_info']) > 0: + del acvr['contest_info'][0] + + # Either submit a not-found discrepancy, or submit the aCVR + if (total_audited % ac.nf_discrepancy_cycle == ac.nf_discrepancy_remainder + and total_audited <= ac.args.plan_limit): + print('ballot-not-found for %s' % acvr['contest_info']) + r = test_endpoint_json(ac, county_s, "/ballot-not-found", {'id': acvr['id']}) + else: + logging.debug("Submitting aCVR: %s" % json.dumps(acvr)) + test_endpoint_json(ac, county_s, "/upload-audit-cvr", + {'cvr_id': selected[i]['db_id'], 'audit_cvr': acvr}, show=False) + + county_dashboard = get_county_dashboard(ac, county_s, county_id, i, acvr) + if county_dashboard['asm_state'] == "COUNTY_AUDIT_COMPLETE": + break + + r = test_endpoint_json(ac, county_s, "/sign-off-audit-round", audit_board_set) + + remaining = county_dashboard['estimated_ballots_to_audit'] + if remaining <= 0: + print("\nCounty %d Audit completed after %d ballots" % (county_id, total_audited + 1)) + + return(remaining) + + +def download_report(ac, s, path, extension): + "Download and save the given report, adding the given extension" + + r = test_endpoint_get(ac, s, "/%s" % path) + name = "%s.%s" % (path, extension) + with open(name, "wb") as f: + f.write(r.content) + print("/%s report saved as %s" % (path, name)) + + +def county_wrapup(ac, county_id): + 'Audit board summary, wrapup, audit-report' + + county_s = requests_retry_session() + county_login(ac, county_s, county_id) + + county_dashboard = get_county_dashboard(ac, county_s, county_id) + logging.info("county-dashboard: %s" % county_dashboard) + + rounds = len(county_dashboard['rounds']) + + print("Rounds: %s " % json.dumps(county_dashboard['rounds'], indent=2)) + + if (rounds > 1) and county_dashboard['rounds'][rounds - 1]['actual_count'] == 0: + # we didn't actually start the last round + rounds -= 1 + + to_go = county_dashboard['estimated_ballots_to_audit'] + audited = county_dashboard['audited_ballot_count'] + cvr_count = county_dashboard['cvr_export_count'] + + if county_dashboard['asm_state'] == "COUNTY_AUDIT_COMPLETE": + print("\nCounty %d audit complete, ended after %d ballots (of %d exported) and %d rounds, %d to go" % + (county_id, audited, cvr_count, rounds, to_go)) + else: + print("\nCounty %d audit incomplete, ended after %d ballots (of %d exported) and %d rounds, %d to go, state %s" % + (county_id, audited, cvr_count, rounds, to_go, county_dashboard['asm_state'])) + + # TODO: Replaced by audit board sign out? Gone? + # r = test_endpoint_json(ac, county_s, "/intermediate-audit-report", {}) + # and avoid "result": "/intermediate-audit-report attempted to apply illegal event SUBMIT_INTERMEDIATE_AUDIT_REPORT_EVENT from state AUDIT_COMPLETE" + + download_report(ac, county_s, "county-report", "xlsx") + +def dos_wrapup(ac): + + r = test_endpoint_get(ac, ac.state_s, "/dos-dashboard") + logging.info("dos-dashboard: %s" % r.text) + + # r = test_endpoint_json(ac, ac.state_s, "/publish-report", {}) + + download_report(ac, ac.state_s, "state-report", "xlsx") + + +discrepancy_query = """ +-- Retrieve counts of audited ballot cards and each type of discrepancy by contest +-- along with contest ballot counts, outcomes and margins for checking calculations. + +SELECT + contest.name, + contest.id, + contest.winners_allowed, + county_contest_comparison_audit.one_vote_over_count, + county_contest_comparison_audit.one_vote_under_count, + county_contest_comparison_audit.two_vote_over_count, + county_contest_comparison_audit.two_vote_under_count, + county_contest_comparison_audit.id, + county_contest_comparison_audit.audit_reason, + county_contest_comparison_audit.audit_status, + county_contest_comparison_audit.audited_sample_count, + county_contest_comparison_audit.disagreement_count, + county_contest_comparison_audit.estimated_samples_to_audit, + county_contest_comparison_audit.estimated_recalculate_needed, + county_contest_comparison_audit.gamma, + county_contest_comparison_audit.optimistic_recalculate_needed, + county_contest_comparison_audit.optimistic_samples_to_audit, + county_contest_comparison_audit.risk_limit, + contest.county_id, + county_contest_result.min_margin, + county_contest_result.winners, + county_contest_result.losers, + county_contest_result.county_ballot_count, + county_contest_result.contest_ballot_count +FROM + public.county_contest_comparison_audit, + public.contest, + public.county_contest_result +WHERE + county_contest_comparison_audit.contest_id = contest.id AND + county_contest_comparison_audit.contest_result_id = county_contest_result.id +ORDER BY contest.county_id +; +""" + + +def check_audit_size(ac): + """Check the RLA calculations for each contest, i.e. that + optimistic_samples_to_audit matches the rlacalc python implementation (nmin()), + and confirming that audited_sample_count >= nmin() + + It also estimates the round size using the same math that is in + ColoradoRLA 1.0 (called togo_1_0) and checks that against + 'estimated_samples_to_audit'. + + It prints out the calculated values, and also prints an ERROR message if + the checks fail. + + This function acquires data directly from the database via an SQL query, + and requires the psycopg2 and rlacalc python modules. + + The ``-C -1`` option should be used so that ColoradoRLA calculates parameters + for each contest, or else values for 'estimated_samples_to_audit' will be 0 + for the unaudited contests. + """ + + import math + + import rlacalc + import psycopg2 + import psycopg2.extras + + con = psycopg2.connect("dbname='corla'") + + cur = con.cursor(cursor_factory=psycopg2.extras.DictCursor) + cur.execute(discrepancy_query) + rows = cur.fetchall() + + for r in rows: + logging.info("check_audit_size for %s" % r.items()) + + params = {'alpha': float(r['risk_limit']), + 'gamma': float(r['gamma']), + 'margin': r['min_margin'] / r['county_ballot_count'], + 'o1': r['one_vote_over_count'], + 'o2': r['two_vote_over_count'], + 'u1': r['one_vote_under_count'], + 'u2': r['two_vote_under_count'] } + + nmin_size = rlacalc.nmin(**params) + params['audited'] = r['audited_sample_count'] + + if params['audited'] > 0: + togo_size = rlacalc.nminToGo(**params) + togo_1_0 = math.ceil(nmin_size * (1 + (params['o1'] + params['o2']) / params['audited'])) + else: + togo_size = nmin_size + togo_1_0 = nmin_size + + if r['estimated_samples_to_audit'] != togo_1_0: + print("ERROR: r['estimated_samples_to_audit'] %d != togo_1_0 %d]" % + (r['estimated_samples_to_audit'], togo_1_0)) + + print("County {} nmin={:.0f} nminToGo={:.0f} est={} alpha={alpha:.0%} gamma={gamma} margin={margin:.2%}, disc={o2} {o1} {u1} {u2} for contest {}".format( + r['county_id'], nmin_size, togo_size, r['estimated_samples_to_audit'], r['name'], **params)) + + if (r['audit_reason'] != 'OPPORTUNISTIC_BENEFITS' and + r['audit_status'] == 'RISK_LIMIT_ACHIEVED'): + if r['optimistic_samples_to_audit'] != nmin_size: + print("ERROR: r['optimistic_samples_to_audit' %d != nmin_size %d]" % + (r['audited_sample_count'], nmin_size)) + if r['audited_sample_count'] < nmin_size: + print("ERROR: r['audited_sample_count' %d < nmin_size %d]" % + (r['audited_sample_count'], nmin_size)) + +def main(): + # Get unbuffered output + sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) + + # Establist an "audit context", abbreviated ac, for passing state around. + ac = Namespace() + + ac.args = parser.parse_args() + FORMAT = '%(asctime)-15s %(levelname)s %(name)s %(message)s' + logging.basicConfig(stream=sys.stdout, level=ac.args.debuglevel, format=FORMAT) + + # Define a standalone logger to get timestamped results sent to stdout + # creating a nice "print" statement with additional context added in + ac.logconsole = logging.getLogger('console') + ac.logconsole.propagate = False + ac.logconsole.setLevel(logging.INFO) + console = logging.StreamHandler(stream=sys.stdout) + formatter = logging.Formatter(fmt='%(asctime)s.%(msecs)03d %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + console.setFormatter(formatter) + ac.logconsole.addHandler(console) + + # Add a default county here to work around https://bugs.python.org/issue16399 + if ac.args.counties is None: + ac.args.counties = [3] + + # Add a default contest + if ac.args.contests is None: + ac.args.contests = [0] + + # If no commands are listed, enter all of them + if len(ac.args.commands) == 0: + ac.args.commands = ["reset", "dos_init", "county_setup", + "dos_start", "county_audit", "dos_wrapup"] + + fields = [int(f) for f in ac.args.plan.split()] + ac.discrepancy_remainder, ac.discrepancy_cycle = fields + + fields = [int(f) for f in ac.args.notfound_plan.split()] + ac.nf_discrepancy_remainder, ac.nf_discrepancy_cycle = fields + + ac.logconsole.info("Arguments: %s" % ac.args) + + Pause.max_pause = ac.args.time_delay + Pause.min_pause = ac.args.lower_time_delay + # Start off with a pause, since others are inserted at end of requests. + Pause.pause_hook(None) + + ac.round = 1 + + ac.base = ac.args.url + + loser=ac.args.loser + if loser == "UNDERVOTE": + ac.false_choices = [] + else: + ac.false_choices = [loser] + + if ac.args.commands == ['county_setup'] or ac.args.county_endpoint is not None: + # Assuming --trackstates is not on, don't need state session. + # Avoid possible database locking problems with logging in as + # state admin from many clients at once in performance testing. + + logging.info("Skipping state_login()") + ac.state_s = None + else: + ac.state_s = requests_retry_session() + state_login(ac, ac.state_s) + + # These options imply exit after running a single action + if ac.args.county_endpoint is not None: + for county_id in ac.args.counties: + county_s = requests_retry_session() + county_login(ac, county_s, county_id) + + r = test_endpoint_get(ac, county_s, ac.args.county_endpoint) + ac.logconsole.info("%s %s %s %s", r, "GET", ac.args.county_endpoint, r.text) + + sys.exit(0) + + if ac.args.dos_endpoint is not None: + r = test_endpoint_get(ac, ac.state_s, ac.args.dos_endpoint) + print(r, "GET", ac.args.dos_endpoint, r.text) + sys.exit(0) + + if ac.args.hand_counts: + r = test_endpoint_get(ac, ac.state_s, "/contest") + contests = r.json() + + for contest_index in ac.args.hand_counts: + if contest_index >= len(contests): + logging.error("Contest_index %d out of range: only %d contests in election" % + (contest_index, len(contests))) + sys.exit(1) + r = test_endpoint_json(ac, ac.state_s, "/hand-count", + [{"contest": contests[contest_index]['id'], + "reason": "COUNTY_WIDE_CONTEST", + "audit": "HAND_COUNT"}]) + sys.exit(0) + + if ac.args.download_file: + download_file(ac, ac.state_s, ac.args.download_file, "/tmp/testdownload.csv") + sys.exit(0) + + + if "reset" in ac.args.commands: + reset(ac) + + if "dos_init" in ac.args.commands: + dos_init(ac) + + if "county_setup" in ac.args.commands: + for county_id in ac.args.counties: + county_setup(ac, county_id) + + if "dos_start" in ac.args.commands: + dos_start(ac) + + print() + + if "county_audit" in ac.args.commands: + round = 0 + alldone = False + + while (ac.args.rounds == -1) or (round < ac.args.rounds): + if ac.args.check_audit_size: + check_audit_size(ac) + + r = test_endpoint_get(ac, ac.state_s, "/dos-asm-state") + current_state = r.json()['current_state'] + if current_state == "DOS_AUDIT_COMPLETE": + alldone = True + break + elif current_state != "DOS_AUDIT_ONGOING": + print("Not in DOS_AUDIT_ONGOING state, can't audit") + break + round += 1 + print("Start Round %d" % round) + for county_id in ac.args.counties: + # TODO: really needs to track each individual county for being done.... + remaining = county_audit(ac, county_id) + + print() + ac.round += 1 + # Note, may get Illegal transition on ASM... (DOS_AUDIT_COMPLETE, DOS_START_ROUND_EVENT) + r = test_endpoint_json(ac, ac.state_s, "/start-audit-round", + { "multiplier": 1.0, "use_estimates": True}) + + if alldone: + print("State audit complete") + + for county_id in ac.args.counties: + county_wrapup(ac, county_id) + + if "dos_wrapup" in ac.args.commands: + dos_wrapup(ac) + + +if __name__ == "__main__": + main() diff --git a/test/load_tests/medium_cvrs.csv.gz b/test/load_tests/medium_cvrs.csv.gz new file mode 100644 index 000000000..b975c1781 Binary files /dev/null and b/test/load_tests/medium_cvrs.csv.gz differ diff --git a/test/load_tests/medium_manifest.csv.gz b/test/load_tests/medium_manifest.csv.gz new file mode 100644 index 000000000..9aec8a37f Binary files /dev/null and b/test/load_tests/medium_manifest.csv.gz differ diff --git a/test/load_tests/sampler.py b/test/load_tests/sampler.py new file mode 100644 index 000000000..1bce912e8 --- /dev/null +++ b/test/load_tests/sampler.py @@ -0,0 +1,645 @@ +#!/usr/bin/python +# Reference implementation code for pseudo-random sampler +# for election audits or other purposes. +# Written by Ronald L. Rivest +# filename: sampler.py +# url: http://people.csail.mit.edu/rivest/sampler.py +sampler_version = "November 14, 2011" +# +# Relevant to document being produced by an ad-hoc working group chaired +# by Prof. Philip Stark (U.C. Berkeley) regarding election auditing. +# Tested using python version 2.6.7. (see www.python.org) +# (Will not work with Python version 3, e.g. 3.x.y) +# (Note added 2014-09-07: As per a suggestion by Chris Jerdonek, one should +# consider this proposal as based on the use of UTF-8 encoding for strings +# throughout. This comment resolves some potential ambiguities about how +# strings are converted to byte sequences before hashing, and the types of +# strings input by raw_input, etc. See +# https://github.com/cjerdonek/rivest-sampler-tests +# for more discussion and test-cases. +# ) + +""" +This program provides a reference implementation of a recommended procedure +to pick a random sample of a given size from a specified set of integers. + +This program is "open source" (MIT License) and may be freely used in +almost any way whatsoever by others. (Details given below) + +The specified set of integers from which the random sample is drawn +must of the form {a, a+1, ..., b} for some integers a and b, where a +is not greater than b. That is, the integer a is the least integer in +the specified set, and the integer b is the greatest integer in the +specified set. + +The user may request that the sampling be done either "with replacement" +or "without replacement". If sampling is done without replacement, then the +elements of the sample will be distinct---no repetitions will occur. +On the other hand, if sampling is done with replacement, then the +sample may contain repeated elements. The program variable +"with_replacement" is True if sampling is to be done with replacement, +otherwise this program variable is False. + +The size n of the desired sample is an input parameter. Here n must +be a non-negative integer. + +If sampling is done without replacement, then we must have that +n is not larger than b-a+1, the size of the specified set from which +the sample is drawn, otherwise no such sample exists. + +The sampling is not truly random, but is "pseudo-random". The randomness +necessary for the sampling is derived from a "seed value" that is +input to the program. + +The seed value is a text sequence entered on one of more lines by the user. +This seed value should be generated in a truly random manner, such as +by rolling a die many times. In some situations (such as for election +audits), it may be desirable to have different users generate different +portions of the seed, so that no one user can control the random sample +completely. For example, there may be five different users, each of +whom rolls a normal die six times. For example, the users may enter the +following seed values: + seed value >>> 126331 + seed value >>> 563425 + seed value >>> 354643 + seed value >>> 662325 +where user 1 rolled the die six times to obtain 1,2,6,3,3,1, user 2 +rolled the die six times to obtain 5,6,3,4,2,5, and so on. In +this case the complete seed used by the program is + 126331563425354643662325 +Having at least twenty-four random characters or digits in the complete +seed value is recommended practice. For the purposes of election audits, +the seed value should not be determined until after the election is complete +and the audit is about to begin. Note that the seed value is the *only* +source of randomness employed by this program, so that using the same +seed value again (as well as the same n, a, b, and with_replacement values) +will produce exactly the same sample. + +The program prompts for parameters n, a, b, seed, and with_replacement. + +The program also prompts for an "election ID" which is an arbitrary +string of text that is used merely for documentation. + +The requested sample of size n is printed twice: once in the order the +sample elements were generated, and once in sorted into numerically +increasing order. + +The sample produced may also be written to a file, at the user's +request. This output file contains documentation regarding n, a, b, +with_replacement, the election ID, and the seed value, as well as the +sample produced. + +Because of the method used, the sample produced should be a "uniform" +sample. That is, each possible sample of the desired size should +be equally likely. If the sampling is done "without replacement" +(no duplicates allowed), then each integer in the range a...b +(inclusive) should be equally likely to appear in the sample. +The procedure used here would need to be modified if some form +of non-uniform sampling were desired. + +It is also possible to have the program produce an "expanded +sample". By giving the same parameters seed, a, and b, the +same sequence will be produced as were produced on an earlier +run. If you wish a longer sequence to be produced than the +earlier run, the program will prompt for the length of the +earlier run, and the number of new additional elements to +be generated. In this way, support is provided for an +"escalation" of an audit. + +We now give a brief description of the method employed to +produce the sample. + +The cryptographic hash function SHA-256 is used in this program. +This hash function maps arbitrary strings of input text to +"pseudo-random" 256-bit integers. The pseudo-randomness of this +function is of the highest quality: SHA-256 is a U.S. government +standard and has passed the most stringest testing. + +The SHA-256 hash function is used in "counter mode" to obtain +the desired sample. The sample elements are picked one by +one from a..b, with the i-th pick is generated by applying +SHA-256 to the text string obtained by following the seed +by a comma and then the decimal representation of i. This +value reduced modulo (b-a+1) and added to a to obtain a value +in the range a..b. This value is rejected if sampling is done +without replacement and the value obtained is a duplicate of +a previously obtained value. + +We now give a sample transcript of the running of this program. + +$ python sampler.py + +SAMPLER -- pseudo-random sample generation for election auditing or other uses. +Written by Ronald L. Rivest. +Sampler version: November 14, 2011 +Python version: 2.6.7 +Generates a sample of size n of integers from a to b (inclusive) +based on supplied parameters, including seed value(s) +and the specification as to whether sampling with replacement is desired. + +Current date/time: 2011-11-14 20:28:42.477502 + +(1) Enter text describing election id (e.g. name and date), then hit return + This is for documentation purposes only, and does not affect computation. + Example: + Election ID >>> Gotham City Mayor's Race, November 2072 + + Election ID >>> Big Apple City Council, January 20th, 2034 + ------ + Election ID = Big Apple City Council, January 20th, 2034 + +(2) Enter one or more lines of text giving random number seed values. + These are typically decimal numbers, but may be arbitrary strings. + Embedded blanks OK, but initial and trailing blanks ignored. + Using decimal dice to generate these values is good practice. + Having different parties generate different seed values is good practice. + Having at least 24 random digits entered total is good practice. + When finished, enter a blank line. + Example: (USE NEW RANDOM VALUES, NOT THESE ONES) + seed value (or blank line when done) >>> 314525782315 + seed value (or blank line when done) >>> 667241589410 + seed value (or blank line when done) >>> + + seed value (or blank line when done) >>> 3546311 + seed value (or blank line when done) >>> 5561121 + seed value (or blank line when done) >>> 6362461 + seed value (or blank line when done) >>> 5351222 + seed value (or blank line when done) >>> + ------ + Seed = 3546311556112163624615351222 + +(3) Outputs will be in range a to b , inclusive + Enter integer a (start of range) + Example: + a >>> 1 + + a >>> 1 + + Enter integer b (end of range) + Example: + b >>> 213 + + b >>> 876 + ------ + a = 1 , b = 876 + N = 876 (number of integers in set to draw sample from) + +(4) Are duplicates OK (i.e. sample with replacement)? + Example: + Duplicates OK (sample with replacement) (y or n)? >>> n + + Duplicates OK (sample with replacement) (y or n)? >>> n + ------ + Duplicate outputs not allowed (that is sampling 'without replacement') + +(5) Are you now asking for an expanded version of a previously generated sample? + Example: + Is this an expanded version of a previously generated sample? >>> n + + Is this an expanded version of a previously generated sample? >>> n + + How many outputs do you want (integer n)? + Example: + n >>> 43 + + n >>> 47 + ------ + Request is for a new sample of size n = 47 + +(6) Generating output: + 1. 3546311556112163624615351222,1 ==> 740 + 2. 3546311556112163624615351222,2 ==> 180 + 3. 3546311556112163624615351222,3 ==> 264 + 4. 3546311556112163624615351222,4 ==> 789 + 5. 3546311556112163624615351222,5 ==> 238 + 6. 3546311556112163624615351222,6 ==> 448 + 7. 3546311556112163624615351222,7 ==> 272 + 8. 3546311556112163624615351222,8 ==> 611 + 9. 3546311556112163624615351222,9 ==> 761 + 10. 3546311556112163624615351222,10 ==> 208 + 11. 3546311556112163624615351222,11 ==> 596 + 12. 3546311556112163624615351222,12 ==> 88 + 13. 3546311556112163624615351222,13 ==> 160 + 14. 3546311556112163624615351222,14 ==> 113 + 15. 3546311556112163624615351222,15 ==> 766 + 16. 3546311556112163624615351222,16 ==> 427 + 17. 3546311556112163624615351222,17 ==> 184 + 18. 3546311556112163624615351222,18 ==> 816 + 19. 3546311556112163624615351222,19 ==> 653 + 20. 3546311556112163624615351222,20 ==> 411 + 21. 3546311556112163624615351222,21 ==> 779 + 22. 3546311556112163624615351222,22 ==> 331 + 23. 3546311556112163624615351222,23 ==> 339 + 24. 3546311556112163624615351222,24 ==> 487 + 25. 3546311556112163624615351222,25 ==> 594 + 26. 3546311556112163624615351222,26 ==> 235 + 27. 3546311556112163624615351222,27 ==> 65 + 28. 3546311556112163624615351222,28 ==> 527 + 29. 3546311556112163624615351222,29 ==> 821 + 30. 3546311556112163624615351222,30 ==> 490 + 31. 3546311556112163624615351222,31 ==> 461 + 31. 3546311556112163624615351222,32 ==> 611 (duplicate rejected) + 32. 3546311556112163624615351222,33 ==> 251 + 33. 3546311556112163624615351222,34 ==> 471 + 34. 3546311556112163624615351222,35 ==> 414 + 35. 3546311556112163624615351222,36 ==> 174 + 36. 3546311556112163624615351222,37 ==> 567 + 37. 3546311556112163624615351222,38 ==> 300 + 38. 3546311556112163624615351222,39 ==> 134 + 39. 3546311556112163624615351222,40 ==> 144 + 40. 3546311556112163624615351222,41 ==> 357 + 41. 3546311556112163624615351222,42 ==> 786 + 42. 3546311556112163624615351222,43 ==> 792 + 43. 3546311556112163624615351222,44 ==> 218 + 44. 3546311556112163624615351222,45 ==> 550 + 45. 3546311556112163624615351222,46 ==> 787 + 46. 3546311556112163624615351222,47 ==> 537 + 47. 3546311556112163624615351222,48 ==> 197 + + Unsorted list of outputs: + 740 180 264 789 238 448 272 611 761 208 + 596 88 160 113 766 427 184 816 653 411 + 779 331 339 487 594 235 65 527 821 490 + 461 251 471 414 174 567 300 134 144 357 + 786 792 218 550 787 537 197 + + Sorted list of outputs: + 65 88 113 134 144 160 174 180 184 197 + 208 218 235 238 251 264 272 300 331 339 + 357 411 414 427 448 461 471 487 490 527 + 537 550 567 594 596 611 653 740 761 766 + 779 786 787 789 792 816 821 + +(7) Saving results to file if desired. + Example: + Name of output file (or blank line if saving results to file not desired): sample-2072-11-03.txt + + Name of output file (or blank line if saving results to file not desired): BigAppleCityCouncil.txt + Results saved in output file: BigAppleCityCouncil.txt + +Current date/time: 2011-11-14 20:33:11.967061 + +Done. + +Here is the output file generated (filename BigAppleCityCouncil.txt): + +SAMPLER output. +SAMPLER from http://people.csail.mit.edu/rivest/sampler.py +SAMPLER version: November 14, 2011 +Date/Time: 2011-11-14 20:33:11.965916 + +Election ID: Big Apple City Council, January 20th, 2034 +Sample range: a = 1 to b = 876 (inclusive) +Duplicates not allowed (sampling without replacement). +Seed: 3546311556112163624615351222 +Sample of size: n = 47 +Sorted output list: + 65, 88, 113, 134, 144, 160, 174, 180, 184, 197, + 208, 218, 235, 238, 251, 264, 272, 300, 331, 339, + 357, 411, 414, 427, 448, 461, 471, 487, 490, 527, + 537, 550, 567, 594, 596, 611, 653, 740, 761, 766, + 779, 786, 787, 789, 792, 816, 821, + +Done. + +""" + +################################################################################ +## Standard "MIT License" http://www.opensource.org/licenses/mit-license.php ## +################################################################################ +""" +Copyright (c) 2011 Ronald L. Rivest + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +################################################################################ +################################################################################ + +# import library of cryptography hash functions +# This program uses SHA-256 hash function +# For reference, see, e.g. +# http://en.wikipedia.org/wiki/SHA-2 +# http://csrc.nist.gov/publications/fips/fips180-2/fips180-2.pdf +# SHA-256 implemented here as hashlib.sha256 +import hashlib + +# import library of string-related functions +import string + +# import library of date/time routines +import datetime + +# get python version +import platform +python_version = platform.python_version() + +def generate_outputs(n,with_replacement,a,b,seed,skip): + """ + This routine returns two lists: + a list of size 'skip' of "old output values" (i.e., the "previous sample") + a list of size 'n-skip' new output values, + each from the range [a..b] (inclusive). + If 'with_replacement' is False, then no duplicates are produced. + The input 'seed' is an arbitrary string of characters. + The first 'skip' values produced will be skipped. + Typically skip is 0, but if you are expanding a sample, + then skip will be the size of the previously generated + sample. (The same seed etc. should be given.) + An assertion error is raised if with_replacement is False + and n+skip>(b-a+1) since the request is infeasible. + The list produced is effectively a 'random sample' + of the desired size from the universe [a..b], + with each element equally likely to be picked. + """ + # check that input parameters are valid + assert n >= 0 + assert a <= b + N = (b - a + 1) # size of set to draw from + assert (with_replacement or n <= N) + + #initialization + new_output_list = [ ] + old_output_list = [ ] + count = 0 + # printing_wanted = True + printing_wanted = False + if printing_wanted: + print "(6) Generating output:" + + # loop until we have generated the desired sample of size n + while len(old_output_list)+len(new_output_list) < n: + + count = count + 1 + + # hash_input is seed followed by comma followed by decimal rep of count + hash_input = seed + "," + str(count) + # Apply SHA-256, interpreting hex output as hexadecimal integer + # to yield 256-bit integer (a python "long integer") + hash_output = int(hashlib.sha256(hash_input).hexdigest(),16) + # determine "pick" as pseudo-random value in range a to b, inclusive, + # as a function of hash_output + pick = int(a + (hash_output % (b-a+1))) + + if not with_replacement and (pick in new_output_list or pick in old_output_list): + if printing_wanted: + print " %7d."%(len(new_output_list)),hash_input,"==>",pick," (duplicate rejected)" + else: + if len(old_output_list) < skip: + old_output_list.append(pick) + if printing_wanted: + print " %7d."%len(new_output_list),hash_input,"==>",pick," (skipped, since it was in previous sample)" + else: + new_output_list.append(pick) + if printing_wanted: + print " %7d."%len(new_output_list),hash_input,"==>",pick + + return (old_output_list,new_output_list) + +def print_list(L): + """ + Print list L of integers, with at most 10 per line + """ + for i in range(0,len(L),10): + print " ", + for j in range(min(10,len(L)-i)): + print "%7d "%L[i+j], + print + +def write_list_to_file(file,L): + """ + similar to print_list, but writing to given file instead. + """ + number_per_line = 10 + csv_wanted = True # else tabs + for i in range(0,len(L),number_per_line): + file.write(" ") + for j in range(min(number_per_line,len(L)-i)): + file.write("%7d"%L[i+j]) + if csv_wanted: + file.write(", ") + else: + file.write("\t") + file.write("\n") + +def main(): + print + print "SAMPLER -- pseudo-random sample generation for election auditing or other uses." + print "Written by Ronald L. Rivest." + print "Sampler version: ",sampler_version + print "Python version: ",python_version + print "Generates a sample of size n of integers from a to b (inclusive)" + print "based on supplied parameters, including seed value(s)" + print "and the specification as to whether sampling with replacement is desired." + print + print "Current date/time:", datetime.datetime.now().isoformat(" ") + print + + print "(1) Enter text describing election id (e.g. name and date), then hit return" + print " This is for documentation purposes only, and does not affect computation." + print " Example: " + print " Election ID >>> Gotham City Mayor's Race, November 2072" + print + electionid = raw_input(" Election ID >>> ") + print " ------" + print " Election ID =",str(electionid) + print + + print "(2) Enter one or more lines of text giving random number seed values." + print " These are typically decimal numbers, but may be arbitrary strings." + print " Embedded blanks OK, but initial and trailing blanks ignored." + print " Using decimal dice to generate these values is good practice." + print " Having different parties generate different seed values is good practice." + print " Having at least 24 random digits entered total is good practice." + print " When finished, enter a blank line." + print " Example: (USE NEW RANDOM VALUES, NOT THESE ONES)" + print " seed value (or blank line when done) >>> 314525782315" + print " seed value (or blank line when done) >>> 667241589410" + print " seed value (or blank line when done) >>> " + print + seedlist = [ ] + while True: + seed_value = raw_input(" seed value (or blank line when done) >>> ") + seed_value = seed_value.strip() # eliminate initial and trailing blanks + if seed_value == "": + break + seedlist.append(seed_value) + # concatenate all seeds together + seed = string.join(seedlist,"") + print " ------" + print " Seed =",str(seed) + print + + print "(3) Outputs will be in range a to b , inclusive" + print " Enter integer a (start of range)" + print " Example:" + print " a >>> 1" + print + a = int(raw_input(" a >>> ")) + print + print " Enter integer b (end of range)" + print " Example:" + print " b >>> 213" + print + b = int(raw_input(" b >>> ")) + assert (a>> n" + print + with_replacement = raw_input(" Duplicates OK (sample with replacement) (y or n)? >>> ") + with_replacement = string.strip(with_replacement) + with_replacement = string.lower(with_replacement) + assert (with_replacement == 'y' or with_replacement == 'n') + with_replacement = (with_replacement == 'y') + print " ------" + if with_replacement: + print " Duplicate outputs OK (that is, sampling 'with replacement')." + else: + print " Duplicate outputs not allowed (that is sampling 'without replacement')" + print + + print "(5) Are you now asking for an expanded version of a previously generated sample?" + print " Example:" + print " Is this an expanded version of a previously generated sample? >>> n" + print + expanded_sample = raw_input(" Is this an expanded version of a previously generated sample? >>> ") + print + expanded_sample = string.strip(expanded_sample) + expanded_sample = string.lower(expanded_sample) + assert (expanded_sample == 'y' or expanded_sample == 'n') + expanded_sample = (expanded_sample == 'y') + if expanded_sample: + print " What was the size of that previous sample? " + print " Example: " + print " What was the size of the previous sample? >>> 21" + print + skip = raw_input(" What was the size of the previous sample? >>> ") + skip = int(skip) + print + print " How many additional output elements do you now want? " + print " Example: " + print " How many additional output elements do you now want? >>> 25" + print + new_elts = raw_input(" How many additional output elements do you now want? >>> ") + new_elts = int(new_elts) + print + n = skip + new_elts + assert with_replacement or (n <= N) + else: + print " How many outputs do you want (integer n)?" + print " Example: " + print " n >>> 43" + print + n = int(raw_input(" n >>> ")) + assert (n>0) + assert with_replacement or (n <= N) + skip = 0 + new_elts = n + + print " ------" + if expanded_sample: + print " Request is for an expanded sample." + print " Size of previous sample (number of elements to skip now) is %d"%skip + print " Number of new elements to generate is %d"%new_elts + print + else: + print " Request is for a new sample of size n = %d"%n + print + + old_output_list,new_output_list = generate_outputs(n,with_replacement,a,b,seed,skip) + + print + + if len(old_output_list)>0: + print " Unsorted list of outputs in previous sample:" + print_list(old_output_list) + print + print " Sorted list of outputs in previous sample:" + sorted_old_output_list = sorted(old_output_list) + print_list(sorted_old_output_list) + print + print " Unsorted list of new outputs:" + print_list(new_output_list) + print + print " Sorted list of new outputs:" + sorted_new_output_list = sorted(new_output_list) + print_list(sorted_new_output_list) + print + else: + print " Unsorted list of outputs:" + print_list(new_output_list) + print + print " Sorted list of outputs:" + sorted_new_output_list = sorted(new_output_list) + print_list(sorted_new_output_list) + print + + # Write output to a file as well + print "(7) Saving results to file if desired." + print " Example: " + print " Name of output file (or blank line if saving results to file not desired): sample-2072-11-03.txt" + print + filename = raw_input(" Name of output file (or blank line if saving results to file not desired): ") + + if filename != "": + file = open(filename,"w") + file.write("SAMPLER output.\n") + file.write("SAMPLER from http://people.csail.mit.edu/rivest/sampler.py\n") + file.write("SAMPLER version: "+sampler_version+"\n") + file.write("Date/Time: "+datetime.datetime.now().isoformat(" ")+"\n\n") + file.write("Election ID: "+electionid+"\n") + file.write("Sample range: a = %d to b = %d (inclusive)\n"%(a,b)) + if with_replacement: + file.write("Duplicates allowed (sampling with replacement).\n") + else: + file.write("Duplicates not allowed (sampling without replacement).\n") + file.write("Seed: "+seed+"\n") + if skip == 0: + file.write("Sample of size: n = %d\n"%n) + file.write("Sorted output list:\n") + write_list_to_file(file,sorted_new_output_list) + file.write("\n") + else: + file.write("Previous sample of size %d\n"%skip) + write_list_to_file(file,sorted_old_output_list) + file.write("\n") + file.write("New elements in expanded sample (%d of them)\n"%new_elts) + write_list_to_file(file,sorted_new_output_list) + file.write("\n") + file.write("Done.\n") + file.close() + print " Results saved in output file:",filename + print + else: + print " No output file written." + print + + print "Current date/time:", datetime.datetime.now().isoformat(" ") + print + print "Done." + +if __name__ == "__main__": + main() diff --git a/test/load_tests/server_test.py b/test/load_tests/server_test.py new file mode 100644 index 000000000..69c9eebf2 --- /dev/null +++ b/test/load_tests/server_test.py @@ -0,0 +1,78 @@ +# Generated by zerotest +from __future__ import unicode_literals +from zerotest.request import Request +from zerotest.response import Response +from zerotest.response_matcher import ResponseMatcher + + +matcher = ResponseMatcher(ignore_all_headers=True) +verify_ssl = False + + +def test_get_ballot_manifest_county(): + request = Request(headers={u'Accept': u'*/*', u'User-Agent': u'curl/7.35.0'}, host=u'localhost:8888', params=u'3', path=u'/ballot-manifest/county', scheme=u'http', method=u'GET') + + real = Response.from_requests_response(request.send_request(verify=verify_ssl)) + expect = Response(status=200, headers={}, body=u'[\n {\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 825000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "batch_size": 50,\n "storage_location": "Bin 1",\n "id": 196\n },\n {\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 825000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "batch_size": 25,\n "storage_location": "Bin 22",\n "id": 194\n },\n {\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 825000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "batch_size": 20,\n "storage_location": "Bin 1",\n "id": 195\n },\n {\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 825000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "batch_size": 50,\n "storage_location": "Bin 17",\n "id": 198\n },\n {\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 825000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "batch_size": 20,\n "storage_location": "Bin 17",\n "id": 197\n }\n]') + # matcher.match_responses(expect, real) + assert(abs(len(expect.body) - len(real.body)) < 10) + + +def test_get_ballot_manifest(): + request = Request(headers={u'Accept': u'*/*', u'User-Agent': u'curl/7.35.0'}, host=u'localhost:8888', scheme=u'http', method=u'GET', path=u'/ballot-manifest') + + real = Response.from_requests_response(request.send_request(verify=verify_ssl)) + expect = Response(status=200, headers={}, body=u'[\n {\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 825000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "batch_size": 25,\n "storage_location": "Bin 22",\n "id": 194\n },\n {\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 825000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "batch_size": 20,\n "storage_location": "Bin 1",\n "id": 195\n },\n {\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 825000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "batch_size": 50,\n "storage_location": "Bin 1",\n "id": 196\n },\n {\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 825000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "batch_size": 20,\n "storage_location": "Bin 17",\n "id": 197\n },\n {\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 825000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "batch_size": 50,\n "storage_location": "Bin 17",\n "id": 198\n }\n]') + # matcher.match_responses(expect, real) + assert(abs(len(expect.body) - len(real.body)) < 10) + + +def test_get_contest(): + request = Request(headers={u'Accept': u'*/*', u'User-Agent': u'curl/7.35.0'}, host=u'localhost:8888', scheme=u'http', method=u'GET', path=u'/contest') + + real = Response.from_requests_response(request.send_request(verify=verify_ssl)) + expect = Response(status=200, headers={}, body=u'[\n {\n "name": "Regent of the University of Colorado - At Large",\n "description": "",\n "choice_names": [\n "Clear Winner",\n "Distant Loser"\n ],\n "choice_descriptions": {\n "Distant Loser": "REP",\n "Clear Winner": "DEM"\n },\n "votes_allowed": 1,\n "id": 201\n },\n {\n "name": "COUNTY COMMISSIONER DISTRICT 3",\n "description": "",\n "choice_names": [\n "Jeff Baker",\n "Janet Lee Cook"\n ],\n "choice_descriptions": {\n "Janet Lee Cook": "DEM",\n "Jeff Baker": "REP"\n },\n "votes_allowed": 1,\n "id": 202\n },\n {\n "name": "Proposition 107 (Statutory)",\n "description": "",\n "choice_names": [\n "YES",\n "NO"\n ],\n "choice_descriptions": {\n "NO": "",\n "YES": ""\n },\n "votes_allowed": 1,\n "id": 203\n }\n]') + matcher.match_responses(expect, real) + + +def test_get_contest_county(): + request = Request(headers={u'Accept': u'*/*', u'User-Agent': u'curl/7.35.0'}, host=u'localhost:8888', params=u'3', path=u'/contest/county', scheme=u'http', method=u'GET') + + real = Response.from_requests_response(request.send_request(verify=verify_ssl)) + expect = Response(status=200, headers={}, body=u'[\n {\n "name": "Proposition 107 (Statutory)",\n "description": "",\n "choice_names": [\n "YES",\n "NO"\n ],\n "choice_descriptions": {\n "NO": "",\n "YES": ""\n },\n "votes_allowed": 1,\n "id": 203\n },\n {\n "name": "Regent of the University of Colorado - At Large",\n "description": "",\n "choice_names": [\n "Clear Winner",\n "Distant Loser"\n ],\n "choice_descriptions": {\n "Distant Loser": "REP",\n "Clear Winner": "DEM"\n },\n "votes_allowed": 1,\n "id": 201\n },\n {\n "name": "COUNTY COMMISSIONER DISTRICT 3",\n "description": "",\n "choice_names": [\n "Jeff Baker",\n "Janet Lee Cook"\n ],\n "choice_descriptions": {\n "Janet Lee Cook": "DEM",\n "Jeff Baker": "REP"\n },\n "votes_allowed": 1,\n "id": 202\n }\n]') + matcher.match_responses(expect, real) + + +def test_get_contest_id_71(): + request = Request(headers={u'Accept': u'*/*', u'User-Agent': u'curl/7.35.0'}, host=u'localhost:8888', scheme=u'http', method=u'GET', path=u'/contest/id/71') + + real = Response.from_requests_response(request.send_request(verify=verify_ssl)) + expect = Response(status=404, headers={u'content-type': u'text/html;charset=utf-8'}, body=u'

404 Not found

') + matcher.match_responses(expect, real) + + +def test_get_cvr_county(): + request = Request(headers={u'Accept': u'*/*', u'User-Agent': u'curl/7.35.0'}, host=u'localhost:8888', params=u'3', path=u'/cvr/county', scheme=u'http', method=u'GET') + + real = Response.from_requests_response(request.send_request(verify=verify_ssl)) + expect = Response(status=200, headers={}, body=u'[{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "1",\n "imprinted_id": "10-2-1",\n "ballot_type": "3",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 582\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "10",\n "imprinted_id": "10-2-10",\n "ballot_type": "19",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 618\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "11",\n "imprinted_id": "10-2-11",\n "ballot_type": "23",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Jeff Baker"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 621\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "12",\n "imprinted_id": "10-2-12",\n "ballot_type": "12",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 625\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "13",\n "imprinted_id": "10-2-13",\n "ballot_type": "17",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Distant Loser"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Jeff Baker"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 629\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "14",\n "imprinted_id": "10-2-14",\n "ballot_type": "25",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 633\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "15",\n "imprinted_id": "10-2-15",\n "ballot_type": "24",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 636\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "16",\n "imprinted_id": "10-2-16",\n "ballot_type": "1",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 639\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "17",\n "imprinted_id": "10-2-17",\n "ballot_type": "18",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 643\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "18",\n "imprinted_id": "10-2-18",\n "ballot_type": "21",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 646\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "19",\n "imprinted_id": "10-2-19",\n "ballot_type": "22",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 649\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "2",\n "imprinted_id": "10-2-2",\n "ballot_type": "2",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 586\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "20",\n "imprinted_id": "10-2-20",\n "ballot_type": "5",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n },\n {\n "contest": 202,\n "choices": []\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 653\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "21",\n "imprinted_id": "10-2-21",\n "ballot_type": "16",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": []\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 657\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "22",\n "imprinted_id": "10-2-22",\n "ballot_type": "15",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 661\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "23",\n "imprinted_id": "10-2-23",\n "ballot_type": "14",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n },\n {\n "contest": 202,\n "choices": []\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 664\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "24",\n "imprinted_id": "10-2-24",\n "ballot_type": "13",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 668\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "25",\n "imprinted_id": "10-2-25",\n "ballot_type": "10",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 671\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "3",\n "imprinted_id": "10-2-3",\n "ballot_type": "8",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 590\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "4",\n "imprinted_id": "10-2-4",\n "ballot_type": "4",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 594\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "5",\n "imprinted_id": "10-2-5",\n "ballot_type": "6",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 598\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "6",\n "imprinted_id": "10-2-6",\n "ballot_type": "7",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 602\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "7",\n "imprinted_id": "10-2-7",\n "ballot_type": "9",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 606\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "8",\n "imprinted_id": "10-2-8",\n "ballot_type": "11",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 610\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "10",\n "batch_id": "2",\n "record_id": "9",\n "imprinted_id": "10-2-9",\n "ballot_type": "20",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 614\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "1",\n "imprinted_id": "3-800-1",\n "ballot_type": "51",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 204\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "10",\n "imprinted_id": "3-800-10",\n "ballot_type": "55",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 239\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "11",\n "imprinted_id": "3-800-11",\n "ballot_type": "56",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 242\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "12",\n "imprinted_id": "3-800-12",\n "ballot_type": "56",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 222\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "13",\n "imprinted_id": "3-800-13",\n "ballot_type": "57",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 244\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "14",\n "imprinted_id": "3-800-14",\n "ballot_type": "54",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 225\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "15",\n "imprinted_id": "3-800-15",\n "ballot_type": "58",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 228\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "16",\n "imprinted_id": "3-800-16",\n "ballot_type": "58",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 246\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "17",\n "imprinted_id": "3-800-17",\n "ballot_type": "59",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 230\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "18",\n "imprinted_id": "3-800-18",\n "ballot_type": "59",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 232\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "19",\n "imprinted_id": "3-800-19",\n "ballot_type": "60",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n },\n {\n "contest": 202,\n "choices": []\n }\n ],\n "id": 249\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "2",\n "imprinted_id": "3-800-2",\n "ballot_type": "51",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 206\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "20",\n "imprinted_id": "3-800-20",\n "ballot_type": "60",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 252\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "3",\n "imprinted_id": "3-800-3",\n "ballot_type": "52",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 209\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "4",\n "imprinted_id": "3-800-4",\n "ballot_type": "53",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 211\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "5",\n "imprinted_id": "3-800-5",\n "ballot_type": "53",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 235\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "6",\n "imprinted_id": "3-800-6",\n "ballot_type": "53",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 214\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "7",\n "imprinted_id": "3-800-7",\n "ballot_type": "54",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 237\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "8",\n "imprinted_id": "3-800-8",\n "ballot_type": "54",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 217\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "3",\n "batch_id": "800",\n "record_id": "9",\n "imprinted_id": "3-800-9",\n "ballot_type": "55",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 220\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "1",\n "imprinted_id": "4-1200-1",\n "ballot_type": "26",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Jeff Baker"\n ]\n }\n ],\n "id": 305\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "10",\n "imprinted_id": "4-1200-10",\n "ballot_type": "30",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 332\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "11",\n "imprinted_id": "4-1200-11",\n "ballot_type": "31",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Distant Loser"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Jeff Baker"\n ]\n }\n ],\n "id": 431\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "12",\n "imprinted_id": "4-1200-12",\n "ballot_type": "31",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 335\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "13",\n "imprinted_id": "4-1200-13",\n "ballot_type": "32",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Distant Loser"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Jeff Baker"\n ]\n }\n ],\n "id": 338\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "14",\n "imprinted_id": "4-1200-14",\n "ballot_type": "32",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 341\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "15",\n "imprinted_id": "4-1200-15",\n "ballot_type": "33",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Distant Loser"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Jeff Baker"\n ]\n }\n ],\n "id": 344\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "16",\n "imprinted_id": "4-1200-16",\n "ballot_type": "33",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 347\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "17",\n "imprinted_id": "4-1200-17",\n "ballot_type": "34",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Distant Loser"\n ]\n }\n ],\n "id": 434\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "18",\n "imprinted_id": "4-1200-18",\n "ballot_type": "34",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 350\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "19",\n "imprinted_id": "4-1200-19",\n "ballot_type": "35",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Distant Loser"\n ]\n }\n ],\n "id": 353\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "2",\n "imprinted_id": "4-1200-2",\n "ballot_type": "26",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 308\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "20",\n "imprinted_id": "4-1200-20",\n "ballot_type": "35",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 355\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "21",\n "imprinted_id": "4-1200-21",\n "ballot_type": "36",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 358\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "22",\n "imprinted_id": "4-1200-22",\n "ballot_type": "36",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 360\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "23",\n "imprinted_id": "4-1200-23",\n "ballot_type": "37",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Jeff Baker"\n ]\n }\n ],\n "id": 363\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "24",\n "imprinted_id": "4-1200-24",\n "ballot_type": "37",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 366\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "25",\n "imprinted_id": "4-1200-25",\n "ballot_type": "38",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Distant Loser"\n ]\n }\n ],\n "id": 369\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "26",\n "imprinted_id": "4-1200-26",\n "ballot_type": "38",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 371\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "27",\n "imprinted_id": "4-1200-27",\n "ballot_type": "40",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 374\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "28",\n "imprinted_id": "4-1200-28",\n "ballot_type": "40",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 376\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "29",\n "imprinted_id": "4-1200-29",\n "ballot_type": "41",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 379\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "3",\n "imprinted_id": "4-1200-3",\n "ballot_type": "27",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Jeff Baker"\n ]\n }\n ],\n "id": 311\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "30",\n "imprinted_id": "4-1200-30",\n "ballot_type": "41",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 381\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "31",\n "imprinted_id": "4-1200-31",\n "ballot_type": "42",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 384\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "32",\n "imprinted_id": "4-1200-32",\n "ballot_type": "42",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 386\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "33",\n "imprinted_id": "4-1200-33",\n "ballot_type": "43",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 389\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "34",\n "imprinted_id": "4-1200-34",\n "ballot_type": "43",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 391\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "35",\n "imprinted_id": "4-1200-35",\n "ballot_type": "44",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 394\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "36",\n "imprinted_id": "4-1200-36",\n "ballot_type": "44",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 396\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "37",\n "imprinted_id": "4-1200-37",\n "ballot_type": "45",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 399\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "38",\n "imprinted_id": "4-1200-38",\n "ballot_type": "45",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 401\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "39",\n "imprinted_id": "4-1200-39",\n "ballot_type": "46",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 404\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "4",\n "imprinted_id": "4-1200-4",\n "ballot_type": "27",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 314\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "40",\n "imprinted_id": "4-1200-40",\n "ballot_type": "46",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 436\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "41",\n "imprinted_id": "4-1200-41",\n "ballot_type": "47",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 406\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "42",\n "imprinted_id": "4-1200-42",\n "ballot_type": "47",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 408\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "43",\n "imprinted_id": "4-1200-43",\n "ballot_type": "48",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 411\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "44",\n "imprinted_id": "4-1200-44",\n "ballot_type": "48",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 413\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "45",\n "imprinted_id": "4-1200-45",\n "ballot_type": "49",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 416\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "46",\n "imprinted_id": "4-1200-46",\n "ballot_type": "49",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 418\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "47",\n "imprinted_id": "4-1200-47",\n "ballot_type": "50",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 421\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "48",\n "imprinted_id": "4-1200-48",\n "ballot_type": "50",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 423\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "49",\n "imprinted_id": "4-1200-49",\n "ballot_type": "39",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 426\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "5",\n "imprinted_id": "4-1200-5",\n "ballot_type": "28",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Distant Loser"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n }\n ],\n "id": 317\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "50",\n "imprinted_id": "4-1200-50",\n "ballot_type": "39",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 428\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "6",\n "imprinted_id": "4-1200-6",\n "ballot_type": "28",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 320\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "7",\n "imprinted_id": "4-1200-7",\n "ballot_type": "29",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Distant Loser"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Jeff Baker"\n ]\n }\n ],\n "id": 323\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "8",\n "imprinted_id": "4-1200-8",\n "ballot_type": "29",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 326\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "4",\n "batch_id": "1200",\n "record_id": "9",\n "imprinted_id": "4-1200-9",\n "ballot_type": "30",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Distant Loser"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Jeff Baker"\n ]\n }\n ],\n "id": 329\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "1",\n "imprinted_id": "5-1600-1",\n "ballot_type": "51",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 255\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "10",\n "imprinted_id": "5-1600-10",\n "ballot_type": "45",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 271\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "11",\n "imprinted_id": "5-1600-11",\n "ballot_type": "45",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 296\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "12",\n "imprinted_id": "5-1600-12",\n "ballot_type": "45",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 274\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "13",\n "imprinted_id": "5-1600-13",\n "ballot_type": "45",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Distant Loser"\n ]\n }\n ],\n "id": 298\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "14",\n "imprinted_id": "5-1600-14",\n "ballot_type": "45",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 277\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "15",\n "imprinted_id": "5-1600-15",\n "ballot_type": "51",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 300\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "16",\n "imprinted_id": "5-1600-16",\n "ballot_type": "51",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 302\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "17",\n "imprinted_id": "5-1600-17",\n "ballot_type": "51",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 280\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "18",\n "imprinted_id": "5-1600-18",\n "ballot_type": "51",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 282\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "19",\n "imprinted_id": "5-1600-19",\n "ballot_type": "51",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Distant Loser"\n ]\n }\n ],\n "id": 285\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "2",\n "imprinted_id": "5-1600-2",\n "ballot_type": "51",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 257\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "20",\n "imprinted_id": "5-1600-20",\n "ballot_type": "51",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 287\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "3",\n "imprinted_id": "5-1600-3",\n "ballot_type": "51",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 290\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "4",\n "imprinted_id": "5-1600-4",\n "ballot_type": "51",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 260\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "5",\n "imprinted_id": "5-1600-5",\n "ballot_type": "51",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Distant Loser"\n ]\n }\n ],\n "id": 263\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "6",\n "imprinted_id": "5-1600-6",\n "ballot_type": "51",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 265\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "7",\n "imprinted_id": "5-1600-7",\n "ballot_type": "45",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 292\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "8",\n "imprinted_id": "5-1600-8",\n "ballot_type": "45",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 268\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "5",\n "batch_id": "1600",\n "record_id": "9",\n "imprinted_id": "5-1600-9",\n "ballot_type": "45",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 294\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "1",\n "imprinted_id": "9-3200-1",\n "ballot_type": "3",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n }\n ],\n "id": 439\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "10",\n "imprinted_id": "9-3200-10",\n "ballot_type": "6",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 460\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "11",\n "imprinted_id": "9-3200-11",\n "ballot_type": "7",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n }\n ],\n "id": 463\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "12",\n "imprinted_id": "9-3200-12",\n "ballot_type": "7",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 466\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "13",\n "imprinted_id": "9-3200-13",\n "ballot_type": "10",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 469\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "14",\n "imprinted_id": "9-3200-14",\n "ballot_type": "10",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n }\n ],\n "id": 472\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "15",\n "imprinted_id": "9-3200-15",\n "ballot_type": "13",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 475\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "16",\n "imprinted_id": "9-3200-16",\n "ballot_type": "13",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 478\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "17",\n "imprinted_id": "9-3200-17",\n "ballot_type": "14",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 557\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "18",\n "imprinted_id": "9-3200-18",\n "ballot_type": "14",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n },\n {\n "contest": 202,\n "choices": []\n }\n ],\n "id": 480\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "19",\n "imprinted_id": "9-3200-19",\n "ballot_type": "15",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 483\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "2",\n "imprinted_id": "9-3200-2",\n "ballot_type": "3",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 442\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "20",\n "imprinted_id": "9-3200-20",\n "ballot_type": "15",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 486\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "21",\n "imprinted_id": "9-3200-21",\n "ballot_type": "16",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 488\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "22",\n "imprinted_id": "9-3200-22",\n "ballot_type": "16",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": []\n }\n ],\n "id": 491\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "23",\n "imprinted_id": "9-3200-23",\n "ballot_type": "5",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 494\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "24",\n "imprinted_id": "9-3200-24",\n "ballot_type": "5",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n },\n {\n "contest": 202,\n "choices": []\n }\n ],\n "id": 497\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "25",\n "imprinted_id": "9-3200-25",\n "ballot_type": "22",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 500\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "26",\n "imprinted_id": "9-3200-26",\n "ballot_type": "22",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n }\n ],\n "id": 503\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "27",\n "imprinted_id": "9-3200-27",\n "ballot_type": "21",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 506\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "28",\n "imprinted_id": "9-3200-28",\n "ballot_type": "21",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 560\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "29",\n "imprinted_id": "9-3200-29",\n "ballot_type": "18",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 509\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "3",\n "imprinted_id": "9-3200-3",\n "ballot_type": "2",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n }\n ],\n "id": 445\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "30",\n "imprinted_id": "9-3200-30",\n "ballot_type": "18",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 511\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "31",\n "imprinted_id": "9-3200-31",\n "ballot_type": "9",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n }\n ],\n "id": 514\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "32",\n "imprinted_id": "9-3200-32",\n "ballot_type": "9",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 562\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "33",\n "imprinted_id": "9-3200-33",\n "ballot_type": "11",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n }\n ],\n "id": 517\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "34",\n "imprinted_id": "9-3200-34",\n "ballot_type": "11",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 520\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "35",\n "imprinted_id": "9-3200-35",\n "ballot_type": "17",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Distant Loser"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Jeff Baker"\n ]\n }\n ],\n "id": 565\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "36",\n "imprinted_id": "9-3200-36",\n "ballot_type": "17",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 523\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "37",\n "imprinted_id": "9-3200-37",\n "ballot_type": "12",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n }\n ],\n "id": 568\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "38",\n "imprinted_id": "9-3200-38",\n "ballot_type": "12",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 526\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "39",\n "imprinted_id": "9-3200-39",\n "ballot_type": "23",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Jeff Baker"\n ]\n }\n ],\n "id": 529\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "4",\n "imprinted_id": "9-3200-4",\n "ballot_type": "2",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 551\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "40",\n "imprinted_id": "9-3200-40",\n "ballot_type": "23",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 571\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "41",\n "imprinted_id": "9-3200-41",\n "ballot_type": "19",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 574\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "42",\n "imprinted_id": "9-3200-42",\n "ballot_type": "19",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 532\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "43",\n "imprinted_id": "9-3200-43",\n "ballot_type": "20",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n }\n ],\n "id": 535\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "44",\n "imprinted_id": "9-3200-44",\n "ballot_type": "20",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 576\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "45",\n "imprinted_id": "9-3200-45",\n "ballot_type": "25",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n }\n ],\n "id": 538\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "46",\n "imprinted_id": "9-3200-46",\n "ballot_type": "25",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "NO"\n ]\n }\n ],\n "id": 540\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "47",\n "imprinted_id": "9-3200-47",\n "ballot_type": "24",\n "contest_info": [\n {\n "contest": 201,\n "choices": []\n }\n ],\n "id": 543\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "48",\n "imprinted_id": "9-3200-48",\n "ballot_type": "24",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 579\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "49",\n "imprinted_id": "9-3200-49",\n "ballot_type": "1",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n }\n ],\n "id": 545\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "5",\n "imprinted_id": "9-3200-5",\n "ballot_type": "8",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n }\n ],\n "id": 448\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "50",\n "imprinted_id": "9-3200-50",\n "ballot_type": "1",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 548\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "6",\n "imprinted_id": "9-3200-6",\n "ballot_type": "8",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": [\n "YES"\n ]\n }\n ],\n "id": 451\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "7",\n "imprinted_id": "9-3200-7",\n "ballot_type": "4",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n }\n ],\n "id": 454\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "8",\n "imprinted_id": "9-3200-8",\n "ballot_type": "4",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 203,\n "choices": []\n }\n ],\n "id": 554\n},{\n "record_type": "UPLOADED",\n "timestamp": {\n "seconds": 1502605767,\n "nanos": 944000000\n },\n "county_id": 3,\n "scanner_id": "9",\n "batch_id": "3200",\n "record_id": "9",\n "imprinted_id": "9-3200-9",\n "ballot_type": "6",\n "contest_info": [\n {\n "contest": 201,\n "choices": [\n "Clear Winner"\n ]\n },\n {\n "contest": 202,\n "choices": [\n "Janet Lee Cook"\n ]\n }\n ],\n "id": 457\n}]') + # matcher.match_responses(expect, real) + assert(abs(len(expect.body) - len(real.body)) < 10) + + +def test_get_acvr_county(): + request = Request(headers={u'Accept': u'*/*', u'User-Agent': u'curl/7.35.0'}, host=u'localhost:8888', params=u'3', path=u'/acvr/county', scheme=u'http', method=u'GET') + + real = Response.from_requests_response(request.send_request(verify=verify_ssl)) + expect = Response(status=200, headers={}, body=u'[]') + matcher.match_responses(expect, real) + + +def test_get_acvr(): + request = Request(headers={u'Accept': u'*/*', u'User-Agent': u'curl/7.35.0'}, host=u'localhost:8888', scheme=u'http', method=u'GET', path=u'/acvr') + + real = Response.from_requests_response(request.send_request(verify=verify_ssl)) + expect = Response(status=200, headers={}, body=u'[]') + matcher.match_responses(expect, real) + + diff --git a/test/load_tests/small_cvrs.csv.gz b/test/load_tests/small_cvrs.csv.gz new file mode 100644 index 000000000..8768b800a Binary files /dev/null and b/test/load_tests/small_cvrs.csv.gz differ diff --git a/test/load_tests/small_manifest.csv.gz b/test/load_tests/small_manifest.csv.gz new file mode 100644 index 000000000..496e2ab28 Binary files /dev/null and b/test/load_tests/small_manifest.csv.gz differ diff --git a/test/load_tests/util.py b/test/load_tests/util.py new file mode 100755 index 000000000..4ff047cb6 --- /dev/null +++ b/test/load_tests/util.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +""" +Utility functions for testing ColoradoRLA. + +For testing, convert CVRs for ballots to be audited into aCVRs for automatic entry. + +Implement publish_ballots_to_audit, or at least a stub. + +get_cvrs(): Get all cvrs + +""" + +from __future__ import print_function +import sys +import logging +import json +import requests +import sampler + +#class CVR(object): # TODO possibility. Why would we do this? + # access fields as attributes + # define key + # print it + # filter by county + # various sort orders + +def get_cvrs(baseurl="http://localhost:8888"): + "Return all cvrs uploaded by any county" + + r = requests.get("%s/cvr" % baseurl) + cvrs = r.json() + + return cvrs + + +def publish_ballots_to_audit(seed, n, N, cvrs, manifest): + """Return lists by county of ballots to audit. + """ + + county_ids = set(cvr['county_id'] for cvr in cvrs) + + ballots_to_audit = [] + for county_id in county_ids: + county_cvrs = sorted( (cvr for cvr in cvrs if cvr['county_id'] == county_id), + key=lambda cvr: "%s-%s-%s" % (cvr['scanner_id'], cvr['batch_id'], cvr['record_id'])) + N = len(county_cvrs) + # n is based on auditing Regent contest. + # TODO: perhaps calculate from margin etc + n = 11 + seed = "01234567890123456789" + + _, new_list = sampler.generate_outputs(n, True, 0, N, seed, False) + + logging.debug("Random selections, N=%d, n=%d, seed=%s: %s" % + (N, n, seed, new_list)) + + selected = [] + for i, cvr in enumerate(county_cvrs): + if i in new_list: + cvr['record_type'] = 'AUDITOR_ENTERED' + selected.append(cvr) + logging.debug("selected cvr %d: %s" % (i, cvr['imprinted_id'])) + + ballots_to_audit.append([county_id, selected]) + + return ballots_to_audit + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG) + + seed = n = N = manifest = None + + cvrs = get_cvrs() + print(json.dumps(publish_ballots_to_audit(seed, n, N, cvrs, manifest), indent=2))