diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..ba6ba0a --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,14 @@ +name: Lint +on: [push, pull_request] +jobs: + build: + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - run: pip install --upgrade flake8 isort + - run: flake8 . + - run: isort . diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c02b271 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=61.2"] +build-backend = "setuptools.build_meta" diff --git a/representatives/admin.py b/representatives/admin.py index d4310f6..c489174 100644 --- a/representatives/admin.py +++ b/representatives/admin.py @@ -2,33 +2,39 @@ from django.contrib import admin, messages -from representatives.models import RepresentativeSet, Representative, Election, Candidate, app_settings +from representatives.models import Candidate, Election, Representative, RepresentativeSet, app_settings +@admin.register(RepresentativeSet) class RepresentativeSetAdmin(admin.ModelAdmin): actions = ['update_from_data_source'] list_display = ['name', 'last_import_time', 'last_import_successful', 'enabled'] list_filter = ['last_import_successful', 'enabled'] + @admin.action( + description="Update from data source" + ) def update_from_data_source(self, request, queryset): for individual_set in queryset: try: count = individual_set.update_from_data_source() except Exception: - messages.error(request, "Couldn't update individuals in %s: %s" % (individual_set, traceback.format_exc())) + messages.error(request, f"Couldn't update individuals in {individual_set}: {traceback.format_exc()}") continue if count is False: - messages.error(request, "Couldn't update individuals in %s." % individual_set) + messages.error(request, f"Couldn't update individuals in {individual_set}.") else: - message = "Updated %s individuals in %s." % (count, individual_set) + message = f"Updated {count} individuals in {individual_set}." no_boundaries = individual_set.individuals.filter(boundary='').values_list('name', flat=True) if no_boundaries: - messages.warning(request, message + " %d match no boundary (%s)." % (len(no_boundaries), ', '.join(no_boundaries))) + messages.warning( + request, message + f" {len(no_boundaries)} match no boundary ({', '.join(no_boundaries)})." + ) else: messages.success(request, message) - update_from_data_source.short_description = "Update from data source" +@admin.register(Representative) class RepresentativeAdmin(admin.ModelAdmin): list_display = ['name', 'representative_set', 'district_name', 'elected_office', 'boundary'] list_filter = ['representative_set'] @@ -41,8 +47,6 @@ class CandidateAdmin(admin.ModelAdmin): search_fields = ['name', 'district_name', 'elected_office'] -admin.site.register(RepresentativeSet, RepresentativeSetAdmin) -admin.site.register(Representative, RepresentativeAdmin) if app_settings.ENABLE_CANDIDATES: admin.site.register(Election, RepresentativeSetAdmin) admin.site.register(Candidate, CandidateAdmin) diff --git a/representatives/management/commands/updaterepresentatives.py b/representatives/management/commands/updaterepresentatives.py index bc45e8a..cee7f11 100644 --- a/representatives/management/commands/updaterepresentatives.py +++ b/representatives/management/commands/updaterepresentatives.py @@ -1,9 +1,9 @@ -import logging import itertools +import logging from django.core.management.base import BaseCommand -from representatives.models import RepresentativeSet, Election +from representatives.models import Election, RepresentativeSet log = logging.getLogger(__name__) @@ -12,9 +12,13 @@ class Command(BaseCommand): help = 'Updates representatives from sources.' def handle(self, *args, **options): - for representative_set in itertools.chain(RepresentativeSet.objects.filter(enabled=True), Election.objects.filter(enabled=True)): + for representative_set in itertools.chain( + RepresentativeSet.objects.filter(enabled=True), Election.objects.filter(enabled=True) + ): try: representative_set.update_from_data_source() except Exception: log.error("Couldn't update representatives in %s." % representative_set) - representative_set.__class__.objects.filter(pk=representative_set.pk).update(last_import_successful=False) + representative_set.__class__.objects.filter(pk=representative_set.pk).update( + last_import_successful=False + ) diff --git a/representatives/migrations/0001_initial.py b/representatives/migrations/0001_initial.py index 47dfc43..9943adb 100644 --- a/representatives/migrations/0001_initial.py +++ b/representatives/migrations/0001_initial.py @@ -1,7 +1,5 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals +from django.db import migrations, models -from django.db import models, migrations class JSONField(models.TextField): """Mocks jsonfield 0.92's column-type behaviour""" @@ -9,7 +7,7 @@ def db_type(self, connection): if connection.vendor == 'postgresql' and connection.pg_version >= 90300: return 'json' else: - return super(JSONField, self).db_type(connection) + return super().db_type(connection) class Migration(migrations.Migration): @@ -110,13 +108,13 @@ class Migration(migrations.Migration): migrations.AddField( model_name='representative', name='representative_set', - field=models.ForeignKey(related_name='individuals', to='representatives.RepresentativeSet'), + field=models.ForeignKey(on_delete=models.CASCADE, related_name='individuals', to='representatives.RepresentativeSet'), preserve_default=True, ), migrations.AddField( model_name='candidate', name='election', - field=models.ForeignKey(related_name='individuals', to='representatives.Election'), + field=models.ForeignKey(on_delete=models.CASCADE, related_name='individuals', to='representatives.Election'), preserve_default=True, ), ] diff --git a/representatives/migrations/0002_auto_20141129_1450.py b/representatives/migrations/0002_auto_20141129_1450.py index 6490425..541cb0a 100644 --- a/representatives/migrations/0002_auto_20141129_1450.py +++ b/representatives/migrations/0002_auto_20141129_1450.py @@ -1,7 +1,5 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals +from django.db import migrations, models -from django.db import models, migrations class JSONField(models.TextField): """Mocks jsonfield 0.92's column-type behaviour""" @@ -9,7 +7,7 @@ def db_type(self, connection): if connection.vendor == 'postgresql' and connection.pg_version >= 90300: return 'json' else: - return super(JSONField, self).db_type(connection) + return super().db_type(connection) class Migration(migrations.Migration): diff --git a/representatives/migrations/0003_auto_20170214_1237.py b/representatives/migrations/0003_auto_20170214_1237.py index 8bc890e..85c94af 100644 --- a/representatives/migrations/0003_auto_20170214_1237.py +++ b/representatives/migrations/0003_auto_20170214_1237.py @@ -1,16 +1,15 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.9.11 on 2017-02-14 12:37 -from __future__ import unicode_literals from django.db import migrations, models + class JSONField(models.TextField): """Mocks jsonfield 0.92's column-type behaviour""" def db_type(self, connection): if connection.vendor == 'postgresql' and connection.pg_version >= 90300: return 'json' else: - return super(JSONField, self).db_type(connection) + return super().db_type(connection) class Migration(migrations.Migration): diff --git a/representatives/migrations/0004_switch_to_django_jsonfield.py b/representatives/migrations/0004_switch_to_django_jsonfield.py index 49ba4fe..9ef2de6 100644 --- a/representatives/migrations/0004_switch_to_django_jsonfield.py +++ b/representatives/migrations/0004_switch_to_django_jsonfield.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.10.5 on 2017-02-23 20:05 -from __future__ import unicode_literals import django.contrib.postgres.fields.jsonb from django.db import migrations diff --git a/representatives/models.py b/representatives/models.py index cddc7c4..6509727 100644 --- a/representatives/models.py +++ b/representatives/models.py @@ -1,4 +1,3 @@ -# coding: utf-8 import datetime import json import logging @@ -8,13 +7,12 @@ from urllib.parse import urljoin from urllib.request import urlopen -from django.contrib.postgres.fields import JSONField +from appconf import AppConf from django.db import models, transaction +from django.db.models import JSONField from django.template.defaultfilters import slugify from django.urls import reverse -from appconf import AppConf - from representatives.utils import boundary_url_to_name logger = logging.getLogger(__name__) @@ -39,15 +37,20 @@ class MyAppConf(AppConf): class BaseRepresentativeSet(models.Model): - name = models.CharField(max_length=300, + name = models.CharField( + max_length=300, + unique=True, help_text="The name of the political body, e.g. House of Commons", - unique=True) + ) data_url = models.URLField(help_text="URL to a JSON array of individuals within this set") data_about_url = models.URLField(blank=True, help_text="URL to information about the scraper used to gather data") last_import_time = models.DateTimeField(blank=True, null=True) - last_import_successful = models.NullBooleanField(blank=True, null=True) - boundary_set = models.CharField(max_length=300, blank=True, - help_text="Name of the boundary set on the boundaries API, e.g. federal-electoral-districts") + last_import_successful = models.BooleanField(blank=True, null=True) + boundary_set = models.CharField( + blank=True, + max_length=300, + help_text="Name of the boundary set on the boundaries API, e.g. federal-electoral-districts", + ) slug = models.SlugField(max_length=300, unique=True, db_index=True) enabled = models.BooleanField(default=True, blank=True, db_index=True) @@ -60,7 +63,7 @@ def __str__(self): def save(self, *args, **kwargs): if not self.slug: self.slug = slugify(self.name) - return super(BaseRepresentativeSet, self).save(*args, **kwargs) + return super().save(*args, **kwargs) @property def boundary_set_url(self): @@ -120,25 +123,36 @@ def update_from_data_source(self): self.individuals.all().delete() boundaries = self.get_list_of_boundaries() - boundary_names = dict(( - (get_comparison_string(b['name']), b['url']) for b in boundaries - )) - boundary_ids = dict(( - (b.get('external_id'), b['url']) for b in boundaries - )) - url_to_name = dict(( - (b['url'], b['name']) for b in boundaries - )) - url_to_id = dict(( - (b['url'], b.get('external_id')) for b in boundaries - )) + boundary_names = { + get_comparison_string(b['name']): b['url'] for b in boundaries + } + boundary_ids = { + b.get('external_id'): b['url'] for b in boundaries + } + url_to_name = { + b['url']: b['name'] for b in boundaries + } + url_to_id = { + b['url']: b.get('external_id') for b in boundaries + } for source_rep in data: rep = self.create_child() - for fieldname in ('name', 'district_name', 'elected_office', - 'source_url', 'first_name', 'last_name', 'party_name', - 'email', 'url', 'personal_url', 'photo_url', 'district_id', - 'gender'): + for fieldname in ( + 'name', + 'district_name', + 'elected_office', + 'source_url', + 'first_name', + 'last_name', + 'party_name', + 'email', + 'url', + 'personal_url', + 'photo_url', + 'district_id', + 'gender', + ): if source_rep.get(fieldname) is not None: setattr(rep, fieldname, source_rep[fieldname]) for json_fieldname in ('offices', 'extra'): @@ -146,7 +160,7 @@ def update_from_data_source(self): try: setattr(rep, json_fieldname, json.loads(source_rep.get(json_fieldname))) except ValueError: - raise Exception("Invalid JSON in %s: %s" % (json_fieldname, source_rep.get(json_fieldname))) + raise Exception(f"Invalid JSON in {json_fieldname}: {source_rep.get(json_fieldname)}") if isinstance(getattr(rep, json_fieldname), list): for d in getattr(rep, json_fieldname): if isinstance(d, dict): @@ -161,7 +175,7 @@ def update_from_data_source(self): rep.incumbent = False if not source_rep.get('name'): - rep.name = ' '.join([component for component in [source_rep.get('first_name'), source_rep.get('last_name')] if component]) + rep.name = ' '.join([c for c in [source_rep.get('first_name'), source_rep.get('last_name')] if c]) if not source_rep.get('first_name') and not source_rep.get('last_name'): (rep.first_name, rep.last_name) = split_name(rep.name) @@ -176,7 +190,9 @@ def update_from_data_source(self): boundary_url = boundary_names.get(get_comparison_string(rep.district_name)) if not boundary_url: - logger.warning("%s: Couldn't find district boundary %s in %s" % (self.slug, rep.district_name, self.boundary_set)) + logger.warning( + "%s: Couldn't find district boundary %s in %s", self.slug, rep.district_name, self.boundary_set + ) else: rep.boundary = boundary_url_to_name(boundary_url) if not rep.district_name: @@ -197,13 +213,13 @@ def create_child(self): return Representative(representative_set=self) def get_absolute_url(self): - return reverse('representatives_representative_set_detail', - kwargs={'slug': self.slug}) + return reverse('representatives_representative_set_detail', kwargs={'slug': self.slug}) def as_dict(self): - r = super(RepresentativeSet, self).as_dict() + r = super().as_dict() r['related']['representatives_url'] = reverse( - 'representatives_representative_list', kwargs={'set_slug': self.slug}) + 'representatives_representative_list', kwargs={'set_slug': self.slug} + ) return r @@ -214,11 +230,10 @@ def create_child(self): return Candidate(election=self) def get_absolute_url(self): - return reverse('representatives_election_detail', - kwargs={'slug': self.slug}) + return reverse('representatives_election_detail', kwargs={'slug': self.slug}) def as_dict(self): - r = super(Election, self).as_dict() + r = super().as_dict() r['election_date'] = str(self.election_date) if self.election_date else None r['related']['candidates_url'] = reverse( 'representatives_candidate_list', kwargs={'set_slug': self.slug}) @@ -234,7 +249,7 @@ def update_from_data_source(self): self.save() self.individuals.all().delete() return False - return super(Election, self).update_from_data_source() + return super().update_from_data_source() class BaseRepresentative(models.Model): @@ -242,38 +257,53 @@ class BaseRepresentative(models.Model): district_name = models.CharField(max_length=300) elected_office = models.CharField(max_length=200) source_url = models.URLField(max_length=2048) - boundary = models.CharField(max_length=300, blank=True, db_index=True, - help_text="e.g. federal-electoral-districts/outremont") - first_name = models.CharField(max_length=200, blank=True) - last_name = models.CharField(max_length=200, blank=True) - party_name = models.CharField(max_length=200, blank=True) + boundary = models.CharField( + blank=True, + max_length=300, + db_index=True, + help_text="e.g. federal-electoral-districts/outremont", + ) + first_name = models.CharField(blank=True, max_length=200) + last_name = models.CharField(blank=True, max_length=200) + party_name = models.CharField(blank=True, max_length=200) email = models.EmailField(blank=True) url = models.URLField(blank=True, max_length=2048) personal_url = models.URLField(blank=True, max_length=2048) photo_url = models.URLField(blank=True, max_length=2048) - district_id = models.CharField(max_length=200, blank=True) - gender = models.CharField(max_length=1, blank=True, choices=( - ('F', 'Female'), - ('M', 'Male'))) - offices = JSONField(default=[]) - extra = JSONField(default={}) + district_id = models.CharField(blank=True, max_length=200) + gender = models.CharField(blank=True, max_length=1, choices=(('F', 'Female'), ('M', 'Male'))) + offices = JSONField(default=list) + extra = JSONField(default=dict) class Meta: abstract = True def __str__(self): - return "%s (%s for %s)" % ( - self.name, self.elected_office, self.district_name) + return f"{self.name} ({self.elected_office} for {self.district_name})" @property def boundary_url(self): return '/boundaries/%s/' % self.boundary if self.boundary else '' def as_dict(self): - r = dict(((f, getattr(self, f)) for f in - ('name', 'district_name', 'elected_office', 'source_url', - 'first_name', 'last_name', 'party_name', 'email', 'url', 'personal_url', - 'photo_url', 'gender', 'offices', 'extra'))) + r = { + f: getattr(self, f) for f in ( + 'name', + 'district_name', + 'elected_office', + 'source_url', + 'first_name', + 'last_name', + 'party_name', + 'email', + 'url', + 'personal_url', + 'photo_url', + 'gender', + 'offices', + 'extra', + ) + } set_obj = getattr(self, self.set_name) r[self.set_name + '_name'] = set_obj.name r['related'] = { @@ -289,17 +319,17 @@ def get_dicts(reps): class Representative(BaseRepresentative): - representative_set = models.ForeignKey(RepresentativeSet, related_name='individuals') + representative_set = models.ForeignKey(RepresentativeSet, on_delete=models.CASCADE, related_name='individuals') set_name = 'representative_set' class Candidate(BaseRepresentative): - election = models.ForeignKey(Election, related_name='individuals') - incumbent = models.NullBooleanField(blank=True) + election = models.ForeignKey(Election, on_delete=models.CASCADE, related_name='individuals') + incumbent = models.BooleanField(blank=True, null=True) set_name = 'election' def as_dict(self): - r = super(Candidate, self).as_dict() + r = super().as_dict() r['incumbent'] = self.incumbent return r diff --git a/representatives/urls.py b/representatives/urls.py index c22eb75..10ae039 100644 --- a/representatives/urls.py +++ b/representatives/urls.py @@ -1,25 +1,44 @@ -from django.conf.urls import url +from django.urls import path, re_path from representatives.models import app_settings from representatives.views import ( - RepresentativeSetListView, RepresentativeSetDetailView, RepresentativeListView, - ElectionListView, ElectionDetailView, CandidateListView) + CandidateListView, + ElectionDetailView, + ElectionListView, + RepresentativeListView, + RepresentativeSetDetailView, + RepresentativeSetListView, +) urlpatterns = [ - url(r'^representatives/$', RepresentativeListView.as_view()), - url(r'^representatives/(?P[\w_-]+)/$', RepresentativeListView.as_view(), name='representatives_representative_list'), - url(r'^boundaries/(?P[\w_-]+/[\w_-]+)/representatives/', RepresentativeListView.as_view()), - url(r'^representative-sets/$', RepresentativeSetListView.as_view()), - url(r'^representative-sets/(?P[\w_-]+)/$', RepresentativeSetDetailView.as_view(), - name='representatives_representative_set_detail'), + path('representatives/', RepresentativeListView.as_view()), + re_path( + r'^representatives/(?P[\w_-]+)/$', + RepresentativeListView.as_view(), + name='representatives_representative_list', + ), + re_path(r'^boundaries/(?P[\w_-]+/[\w_-]+)/representatives/', RepresentativeListView.as_view()), + path('representative-sets/', RepresentativeSetListView.as_view()), + re_path( + r'^representative-sets/(?P[\w_-]+)/$', + RepresentativeSetDetailView.as_view(), + name='representatives_representative_set_detail' + ), ] if app_settings.ENABLE_CANDIDATES: urlpatterns += [ - url(r'^candidates/$', CandidateListView.as_view()), - url(r'^candidates/(?P[\w_-]+)/$', CandidateListView.as_view(), name='representatives_candidate_list'), - url(r'^boundaries/(?P[\w_-]+/[\w_-]+)/candidates/$', CandidateListView.as_view()), - url(r'^elections/$', ElectionListView.as_view()), - url(r'^elections/(?P[\w_-]+)/$', ElectionDetailView.as_view(), - name='representatives_election_detail'), + path('candidates/', CandidateListView.as_view()), + re_path( + r'^candidates/(?P[\w_-]+)/$', + CandidateListView.as_view(), + name='representatives_candidate_list', + ), + re_path(r'^boundaries/(?P[\w_-]+/[\w_-]+)/candidates/$', CandidateListView.as_view()), + path('elections/', ElectionListView.as_view()), + re_path( + r'^elections/(?P[\w_-]+)/$', + ElectionDetailView.as_view(), + name='representatives_election_detail' + ), ] diff --git a/representatives/views.py b/representatives/views.py index 89974fb..05302f9 100644 --- a/representatives/views.py +++ b/representatives/views.py @@ -3,10 +3,10 @@ from urllib.parse import urlencode from urllib.request import urlopen -from boundaries.base_views import ModelListView, ModelDetailView, BadRequest +from boundaries.base_views import BadRequest, ModelDetailView, ModelListView from boundaries.models import Boundary -from representatives.models import RepresentativeSet, Representative, Election, Candidate, app_settings +from representatives.models import Candidate, Election, Representative, RepresentativeSet, app_settings from representatives.utils import boundary_url_to_name @@ -28,7 +28,7 @@ class RepresentativeListView(ModelListView): filterable_fields = ('name', 'first_name', 'last_name', 'gender', 'district_name', 'elected_office', 'party_name') def get_qs(self, request, slug=None, set_slug=None): - qs = super(RepresentativeListView, self).get_qs(request) + qs = super().get_qs(request) if slug: qs = qs.filter(boundary=slug) elif set_slug: @@ -36,7 +36,7 @@ def get_qs(self, request, slug=None, set_slug=None): return qs.select_related(self.model.set_name) def filter(self, request, qs): - qs = super(RepresentativeListView, self).filter(request, qs) + qs = super().filter(request, qs) if 'districts' in request.GET: qs = qs.filter(boundary__in=request.GET['districts'].split(',')) @@ -44,11 +44,14 @@ def filter(self, request, qs): if 'point' in request.GET: if app_settings.RESOLVE_POINT_REQUESTS_OVER_HTTP: url = app_settings.BOUNDARYSERVICE_URL + 'boundaries/?' + urlencode({'contains': request.GET['point']}) - boundaries = [boundary_url_to_name(boundary['url']) for boundary in json.loads(urlopen(url).read().decode())['objects']] + boundaries = [ + boundary_url_to_name(boundary['url']) + for boundary in json.loads(urlopen(url).read().decode())['objects'] + ] else: try: latitude, longitude = re.sub(r'[^\d.,-]', '', request.GET['point']).split(',') - wkt = 'POINT(%s %s)' % (longitude, latitude) + wkt = f'POINT({longitude} {latitude})' boundaries = Boundary.objects.filter(shape__contains=wkt).values_list('set_id', 'slug') except ValueError: raise BadRequest("Invalid latitude,longitude '%s' provided." % request.GET['point']) @@ -62,7 +65,7 @@ class RepresentativeSetListView(ModelListView): model = RepresentativeSet def get_qs(self, request): - qs = super(RepresentativeSetListView, self).get_qs(request) + qs = super().get_qs(request) return qs.filter(enabled=True) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..e7c532d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,24 @@ +[metadata] +name = represent-representatives +version = 0.3 +license = MIT +description = A web API for elected officials tied to electoral districts, packaged as a Django app. +url = https://github.com/opennorth/represent-reps +classifiers = + Programming Language :: Python :: 3 + License :: OSI Approved :: MIT License + Framework :: Django + +[options] +packages = find: +install_requires = + django-appconf + represent-boundaries + +[isort] +line_length = 119 +profile = black + +[flake8] +max-line-length = 119 +exclude = representatives/migrations diff --git a/setup.py b/setup.py deleted file mode 100644 index 0d1fa15..0000000 --- a/setup.py +++ /dev/null @@ -1,25 +0,0 @@ -from setuptools import setup - -setup( - name="represent-representatives", - version="0.3", - description="A web API for elected officials tied to electoral districts, packaged as a Django app.", - url="https://github.com/opennorth/represent-reps", - license="MIT", - packages=[ - 'representatives', - 'representatives.management', - 'representatives.management.commands', - 'representatives.migrations', - ], - install_requires=[ - 'django-appconf', - 'represent-boundaries', - ], - classifiers=[ - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'License :: OSI Approved :: MIT License', - 'Framework :: Django', - ], -) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index e700552..0000000 --- a/tox.ini +++ /dev/null @@ -1,5 +0,0 @@ -[flake8] -ignore=E128,E501 -# E128 continuation line under-indented for visual indent -# E501 line too long (X > 79 characters) -exclude = representatives/migrations,representatives/south_migrations