From ab75ae18081e0b0e38e485e14a6a2098b8846df5 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Sat, 9 Dec 2023 22:23:54 -0500 Subject: [PATCH 01/11] chore: Run pyupgrade --py310-plus **/*.py --- representatives/admin.py | 4 +- representatives/migrations/0001_initial.py | 5 +-- .../migrations/0002_auto_20141129_1450.py | 5 +-- .../migrations/0003_auto_20170214_1237.py | 4 +- .../0004_switch_to_django_jsonfield.py | 2 - representatives/models.py | 45 +++++++++---------- representatives/views.py | 8 ++-- 7 files changed, 31 insertions(+), 42 deletions(-) diff --git a/representatives/admin.py b/representatives/admin.py index d4310f6..1eed447 100644 --- a/representatives/admin.py +++ b/representatives/admin.py @@ -15,12 +15,12 @@ def update_from_data_source(self, request, queryset): try: count = individual_set.update_from_data_source() except Exception: - messages.error(request, "Couldn't update individuals in %s: %s" % (individual_set, traceback.format_exc())) + messages.error(request, "Couldn't update individuals in {}: {}".format(individual_set, traceback.format_exc())) continue if count is False: messages.error(request, "Couldn't update individuals in %s." % individual_set) else: - message = "Updated %s individuals in %s." % (count, individual_set) + message = "Updated {} individuals in {}.".format(count, individual_set) no_boundaries = individual_set.individuals.filter(boundary='').values_list('name', flat=True) if no_boundaries: messages.warning(request, message + " %d match no boundary (%s)." % (len(no_boundaries), ', '.join(no_boundaries))) diff --git a/representatives/migrations/0001_initial.py b/representatives/migrations/0001_initial.py index 47dfc43..db8ce96 100644 --- a/representatives/migrations/0001_initial.py +++ b/representatives/migrations/0001_initial.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - from django.db import models, migrations class JSONField(models.TextField): @@ -9,7 +6,7 @@ def db_type(self, connection): if connection.vendor == 'postgresql' and connection.pg_version >= 90300: return 'json' else: - return super(JSONField, self).db_type(connection) + return super().db_type(connection) class Migration(migrations.Migration): diff --git a/representatives/migrations/0002_auto_20141129_1450.py b/representatives/migrations/0002_auto_20141129_1450.py index 6490425..81469e7 100644 --- a/representatives/migrations/0002_auto_20141129_1450.py +++ b/representatives/migrations/0002_auto_20141129_1450.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - from django.db import models, migrations class JSONField(models.TextField): @@ -9,7 +6,7 @@ def db_type(self, connection): if connection.vendor == 'postgresql' and connection.pg_version >= 90300: return 'json' else: - return super(JSONField, self).db_type(connection) + return super().db_type(connection) class Migration(migrations.Migration): diff --git a/representatives/migrations/0003_auto_20170214_1237.py b/representatives/migrations/0003_auto_20170214_1237.py index 8bc890e..c1cd342 100644 --- a/representatives/migrations/0003_auto_20170214_1237.py +++ b/representatives/migrations/0003_auto_20170214_1237.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.9.11 on 2017-02-14 12:37 -from __future__ import unicode_literals from django.db import migrations, models @@ -10,7 +8,7 @@ def db_type(self, connection): if connection.vendor == 'postgresql' and connection.pg_version >= 90300: return 'json' else: - return super(JSONField, self).db_type(connection) + return super().db_type(connection) class Migration(migrations.Migration): diff --git a/representatives/migrations/0004_switch_to_django_jsonfield.py b/representatives/migrations/0004_switch_to_django_jsonfield.py index 49ba4fe..9ef2de6 100644 --- a/representatives/migrations/0004_switch_to_django_jsonfield.py +++ b/representatives/migrations/0004_switch_to_django_jsonfield.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- # Generated by Django 1.10.5 on 2017-02-23 20:05 -from __future__ import unicode_literals import django.contrib.postgres.fields.jsonb from django.db import migrations diff --git a/representatives/models.py b/representatives/models.py index cddc7c4..3285a82 100644 --- a/representatives/models.py +++ b/representatives/models.py @@ -1,4 +1,3 @@ -# coding: utf-8 import datetime import json import logging @@ -60,7 +59,7 @@ def __str__(self): def save(self, *args, **kwargs): if not self.slug: self.slug = slugify(self.name) - return super(BaseRepresentativeSet, self).save(*args, **kwargs) + return super().save(*args, **kwargs) @property def boundary_set_url(self): @@ -120,18 +119,18 @@ def update_from_data_source(self): self.individuals.all().delete() boundaries = self.get_list_of_boundaries() - boundary_names = dict(( - (get_comparison_string(b['name']), b['url']) for b in boundaries - )) - boundary_ids = dict(( - (b.get('external_id'), b['url']) for b in boundaries - )) - url_to_name = dict(( - (b['url'], b['name']) for b in boundaries - )) - url_to_id = dict(( - (b['url'], b.get('external_id')) for b in boundaries - )) + boundary_names = { + get_comparison_string(b['name']): b['url'] for b in boundaries + } + boundary_ids = { + b.get('external_id'): b['url'] for b in boundaries + } + url_to_name = { + b['url']: b['name'] for b in boundaries + } + url_to_id = { + b['url']: b.get('external_id') for b in boundaries + } for source_rep in data: rep = self.create_child() @@ -146,7 +145,7 @@ def update_from_data_source(self): try: setattr(rep, json_fieldname, json.loads(source_rep.get(json_fieldname))) except ValueError: - raise Exception("Invalid JSON in %s: %s" % (json_fieldname, source_rep.get(json_fieldname))) + raise Exception("Invalid JSON in {}: {}".format(json_fieldname, source_rep.get(json_fieldname))) if isinstance(getattr(rep, json_fieldname), list): for d in getattr(rep, json_fieldname): if isinstance(d, dict): @@ -176,7 +175,7 @@ def update_from_data_source(self): boundary_url = boundary_names.get(get_comparison_string(rep.district_name)) if not boundary_url: - logger.warning("%s: Couldn't find district boundary %s in %s" % (self.slug, rep.district_name, self.boundary_set)) + logger.warning("{}: Couldn't find district boundary {} in {}".format(self.slug, rep.district_name, self.boundary_set)) else: rep.boundary = boundary_url_to_name(boundary_url) if not rep.district_name: @@ -201,7 +200,7 @@ def get_absolute_url(self): kwargs={'slug': self.slug}) def as_dict(self): - r = super(RepresentativeSet, self).as_dict() + r = super().as_dict() r['related']['representatives_url'] = reverse( 'representatives_representative_list', kwargs={'set_slug': self.slug}) return r @@ -218,7 +217,7 @@ def get_absolute_url(self): kwargs={'slug': self.slug}) def as_dict(self): - r = super(Election, self).as_dict() + r = super().as_dict() r['election_date'] = str(self.election_date) if self.election_date else None r['related']['candidates_url'] = reverse( 'representatives_candidate_list', kwargs={'set_slug': self.slug}) @@ -234,7 +233,7 @@ def update_from_data_source(self): self.save() self.individuals.all().delete() return False - return super(Election, self).update_from_data_source() + return super().update_from_data_source() class BaseRepresentative(models.Model): @@ -262,7 +261,7 @@ class Meta: abstract = True def __str__(self): - return "%s (%s for %s)" % ( + return "{} ({} for {})".format( self.name, self.elected_office, self.district_name) @property @@ -270,10 +269,10 @@ def boundary_url(self): return '/boundaries/%s/' % self.boundary if self.boundary else '' def as_dict(self): - r = dict(((f, getattr(self, f)) for f in + r = {f: getattr(self, f) for f in ('name', 'district_name', 'elected_office', 'source_url', 'first_name', 'last_name', 'party_name', 'email', 'url', 'personal_url', - 'photo_url', 'gender', 'offices', 'extra'))) + 'photo_url', 'gender', 'offices', 'extra')} set_obj = getattr(self, self.set_name) r[self.set_name + '_name'] = set_obj.name r['related'] = { @@ -299,7 +298,7 @@ class Candidate(BaseRepresentative): set_name = 'election' def as_dict(self): - r = super(Candidate, self).as_dict() + r = super().as_dict() r['incumbent'] = self.incumbent return r diff --git a/representatives/views.py b/representatives/views.py index 89974fb..c36a7f0 100644 --- a/representatives/views.py +++ b/representatives/views.py @@ -28,7 +28,7 @@ class RepresentativeListView(ModelListView): filterable_fields = ('name', 'first_name', 'last_name', 'gender', 'district_name', 'elected_office', 'party_name') def get_qs(self, request, slug=None, set_slug=None): - qs = super(RepresentativeListView, self).get_qs(request) + qs = super().get_qs(request) if slug: qs = qs.filter(boundary=slug) elif set_slug: @@ -36,7 +36,7 @@ def get_qs(self, request, slug=None, set_slug=None): return qs.select_related(self.model.set_name) def filter(self, request, qs): - qs = super(RepresentativeListView, self).filter(request, qs) + qs = super().filter(request, qs) if 'districts' in request.GET: qs = qs.filter(boundary__in=request.GET['districts'].split(',')) @@ -48,7 +48,7 @@ def filter(self, request, qs): else: try: latitude, longitude = re.sub(r'[^\d.,-]', '', request.GET['point']).split(',') - wkt = 'POINT(%s %s)' % (longitude, latitude) + wkt = 'POINT({} {})'.format(longitude, latitude) boundaries = Boundary.objects.filter(shape__contains=wkt).values_list('set_id', 'slug') except ValueError: raise BadRequest("Invalid latitude,longitude '%s' provided." % request.GET['point']) @@ -62,7 +62,7 @@ class RepresentativeSetListView(ModelListView): model = RepresentativeSet def get_qs(self, request): - qs = super(RepresentativeSetListView, self).get_qs(request) + qs = super().get_qs(request) return qs.filter(enabled=True) From d43aa9290dd9fa88f769910d74eb2c17e6fef220 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Sat, 9 Dec 2023 22:29:35 -0500 Subject: [PATCH 02/11] chore: Run django-upgrade --target-version 4.2 --- representatives/admin.py | 8 +++++--- representatives/migrations/0001_initial.py | 4 ++-- representatives/models.py | 10 +++++----- representatives/urls.py | 22 +++++++++++----------- 4 files changed, 23 insertions(+), 21 deletions(-) diff --git a/representatives/admin.py b/representatives/admin.py index 1eed447..14ea399 100644 --- a/representatives/admin.py +++ b/representatives/admin.py @@ -5,11 +5,15 @@ from representatives.models import RepresentativeSet, Representative, Election, Candidate, app_settings +@admin.register(RepresentativeSet) class RepresentativeSetAdmin(admin.ModelAdmin): actions = ['update_from_data_source'] list_display = ['name', 'last_import_time', 'last_import_successful', 'enabled'] list_filter = ['last_import_successful', 'enabled'] + @admin.action( + description="Update from data source" + ) def update_from_data_source(self, request, queryset): for individual_set in queryset: try: @@ -26,9 +30,9 @@ def update_from_data_source(self, request, queryset): messages.warning(request, message + " %d match no boundary (%s)." % (len(no_boundaries), ', '.join(no_boundaries))) else: messages.success(request, message) - update_from_data_source.short_description = "Update from data source" +@admin.register(Representative) class RepresentativeAdmin(admin.ModelAdmin): list_display = ['name', 'representative_set', 'district_name', 'elected_office', 'boundary'] list_filter = ['representative_set'] @@ -41,8 +45,6 @@ class CandidateAdmin(admin.ModelAdmin): search_fields = ['name', 'district_name', 'elected_office'] -admin.site.register(RepresentativeSet, RepresentativeSetAdmin) -admin.site.register(Representative, RepresentativeAdmin) if app_settings.ENABLE_CANDIDATES: admin.site.register(Election, RepresentativeSetAdmin) admin.site.register(Candidate, CandidateAdmin) diff --git a/representatives/migrations/0001_initial.py b/representatives/migrations/0001_initial.py index db8ce96..b33cf96 100644 --- a/representatives/migrations/0001_initial.py +++ b/representatives/migrations/0001_initial.py @@ -107,13 +107,13 @@ class Migration(migrations.Migration): migrations.AddField( model_name='representative', name='representative_set', - field=models.ForeignKey(related_name='individuals', to='representatives.RepresentativeSet'), + field=models.ForeignKey(on_delete=models.CASCADE, related_name='individuals', to='representatives.RepresentativeSet'), preserve_default=True, ), migrations.AddField( model_name='candidate', name='election', - field=models.ForeignKey(related_name='individuals', to='representatives.Election'), + field=models.ForeignKey(on_delete=models.CASCADE, related_name='individuals', to='representatives.Election'), preserve_default=True, ), ] diff --git a/representatives/models.py b/representatives/models.py index 3285a82..309dd6b 100644 --- a/representatives/models.py +++ b/representatives/models.py @@ -7,7 +7,7 @@ from urllib.parse import urljoin from urllib.request import urlopen -from django.contrib.postgres.fields import JSONField +from django.db.models import JSONField from django.db import models, transaction from django.template.defaultfilters import slugify from django.urls import reverse @@ -44,7 +44,7 @@ class BaseRepresentativeSet(models.Model): data_url = models.URLField(help_text="URL to a JSON array of individuals within this set") data_about_url = models.URLField(blank=True, help_text="URL to information about the scraper used to gather data") last_import_time = models.DateTimeField(blank=True, null=True) - last_import_successful = models.NullBooleanField(blank=True, null=True) + last_import_successful = models.BooleanField(blank=True, null=True) boundary_set = models.CharField(max_length=300, blank=True, help_text="Name of the boundary set on the boundaries API, e.g. federal-electoral-districts") slug = models.SlugField(max_length=300, unique=True, db_index=True) @@ -288,13 +288,13 @@ def get_dicts(reps): class Representative(BaseRepresentative): - representative_set = models.ForeignKey(RepresentativeSet, related_name='individuals') + representative_set = models.ForeignKey(RepresentativeSet, on_delete=models.CASCADE, related_name='individuals') set_name = 'representative_set' class Candidate(BaseRepresentative): - election = models.ForeignKey(Election, related_name='individuals') - incumbent = models.NullBooleanField(blank=True) + election = models.ForeignKey(Election, on_delete=models.CASCADE, related_name='individuals') + incumbent = models.BooleanField(blank=True, null=True) set_name = 'election' def as_dict(self): diff --git a/representatives/urls.py b/representatives/urls.py index c22eb75..a8f495d 100644 --- a/representatives/urls.py +++ b/representatives/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls import url +from django.urls import path, re_path from representatives.models import app_settings from representatives.views import ( @@ -6,20 +6,20 @@ ElectionListView, ElectionDetailView, CandidateListView) urlpatterns = [ - url(r'^representatives/$', RepresentativeListView.as_view()), - url(r'^representatives/(?P[\w_-]+)/$', RepresentativeListView.as_view(), name='representatives_representative_list'), - url(r'^boundaries/(?P[\w_-]+/[\w_-]+)/representatives/', RepresentativeListView.as_view()), - url(r'^representative-sets/$', RepresentativeSetListView.as_view()), - url(r'^representative-sets/(?P[\w_-]+)/$', RepresentativeSetDetailView.as_view(), + path('representatives/', RepresentativeListView.as_view()), + re_path(r'^representatives/(?P[\w_-]+)/$', RepresentativeListView.as_view(), name='representatives_representative_list'), + re_path(r'^boundaries/(?P[\w_-]+/[\w_-]+)/representatives/', RepresentativeListView.as_view()), + path('representative-sets/', RepresentativeSetListView.as_view()), + re_path(r'^representative-sets/(?P[\w_-]+)/$', RepresentativeSetDetailView.as_view(), name='representatives_representative_set_detail'), ] if app_settings.ENABLE_CANDIDATES: urlpatterns += [ - url(r'^candidates/$', CandidateListView.as_view()), - url(r'^candidates/(?P[\w_-]+)/$', CandidateListView.as_view(), name='representatives_candidate_list'), - url(r'^boundaries/(?P[\w_-]+/[\w_-]+)/candidates/$', CandidateListView.as_view()), - url(r'^elections/$', ElectionListView.as_view()), - url(r'^elections/(?P[\w_-]+)/$', ElectionDetailView.as_view(), + path('candidates/', CandidateListView.as_view()), + re_path(r'^candidates/(?P[\w_-]+)/$', CandidateListView.as_view(), name='representatives_candidate_list'), + re_path(r'^boundaries/(?P[\w_-]+/[\w_-]+)/candidates/$', CandidateListView.as_view()), + path('elections/', ElectionListView.as_view()), + re_path(r'^elections/(?P[\w_-]+)/$', ElectionDetailView.as_view(), name='representatives_election_detail'), ] From feddca4f6906d7bce115600446af6b8da16552fe Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Sat, 9 Dec 2023 23:10:51 -0500 Subject: [PATCH 03/11] chore: Fix fields.E010 warning --- representatives/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/representatives/models.py b/representatives/models.py index 309dd6b..c576b0f 100644 --- a/representatives/models.py +++ b/representatives/models.py @@ -254,8 +254,8 @@ class BaseRepresentative(models.Model): gender = models.CharField(max_length=1, blank=True, choices=( ('F', 'Female'), ('M', 'Male'))) - offices = JSONField(default=[]) - extra = JSONField(default={}) + offices = JSONField(default=list) + extra = JSONField(default=dict) class Meta: abstract = True From 606e04fd5b072ddc1b0c87daf02a963ba596f1ee Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Sat, 9 Dec 2023 23:42:28 -0500 Subject: [PATCH 04/11] chore: Convert setup.py to setup.cfg --- pyproject.toml | 3 +++ setup.cfg | 16 ++++++++++++++++ setup.py | 25 ------------------------- 3 files changed, 19 insertions(+), 25 deletions(-) create mode 100644 pyproject.toml create mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c02b271 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=61.2"] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3d50191 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,16 @@ +[metadata] +name = represent-representatives +version = 0.3 +license = MIT +description = A web API for elected officials tied to electoral districts, packaged as a Django app. +url = https://github.com/opennorth/represent-reps +classifiers = + Programming Language :: Python :: 3 + License :: OSI Approved :: MIT License + Framework :: Django + +[options] +packages = find: +install_requires = + django-appconf + represent-boundaries diff --git a/setup.py b/setup.py deleted file mode 100644 index 0d1fa15..0000000 --- a/setup.py +++ /dev/null @@ -1,25 +0,0 @@ -from setuptools import setup - -setup( - name="represent-representatives", - version="0.3", - description="A web API for elected officials tied to electoral districts, packaged as a Django app.", - url="https://github.com/opennorth/represent-reps", - license="MIT", - packages=[ - 'representatives', - 'representatives.management', - 'representatives.management.commands', - 'representatives.migrations', - ], - install_requires=[ - 'django-appconf', - 'represent-boundaries', - ], - classifiers=[ - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'License :: OSI Approved :: MIT License', - 'Framework :: Django', - ], -) From a8b16ad0855ebf5cc4cb6487ca7c262464cc2c4b Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Sat, 9 Dec 2023 23:45:15 -0500 Subject: [PATCH 05/11] chore: Move tox.ini into setup.cfg --- setup.cfg | 8 ++++++++ tox.ini | 5 ----- 2 files changed, 8 insertions(+), 5 deletions(-) delete mode 100644 tox.ini diff --git a/setup.cfg b/setup.cfg index 3d50191..a136a6f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,3 +14,11 @@ packages = find: install_requires = django-appconf represent-boundaries + +[flake8] +exclude = representatives/migrations +extend-ignore = + # E128 continuation line under-indented for visual indent + E128, + # E501 line too long (X > 79 characters) + E501 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index e700552..0000000 --- a/tox.ini +++ /dev/null @@ -1,5 +0,0 @@ -[flake8] -ignore=E128,E501 -# E128 continuation line under-indented for visual indent -# E501 line too long (X > 79 characters) -exclude = representatives/migrations,representatives/south_migrations From 062ccc361269e7d19c51faa1386058bd580f7518 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Sat, 9 Dec 2023 23:56:18 -0500 Subject: [PATCH 06/11] chore: isort --- representatives/admin.py | 8 +++++++- .../management/commands/updaterepresentatives.py | 4 ++-- representatives/migrations/0001_initial.py | 3 ++- representatives/migrations/0002_auto_20141129_1450.py | 3 ++- representatives/migrations/0003_auto_20170214_1237.py | 1 + representatives/models.py | 5 ++--- representatives/urls.py | 9 +++++++-- representatives/views.py | 10 ++++++++-- setup.cfg | 3 +++ 9 files changed, 34 insertions(+), 12 deletions(-) diff --git a/representatives/admin.py b/representatives/admin.py index 14ea399..c304348 100644 --- a/representatives/admin.py +++ b/representatives/admin.py @@ -2,7 +2,13 @@ from django.contrib import admin, messages -from representatives.models import RepresentativeSet, Representative, Election, Candidate, app_settings +from representatives.models import ( + Candidate, + Election, + Representative, + RepresentativeSet, + app_settings, +) @admin.register(RepresentativeSet) diff --git a/representatives/management/commands/updaterepresentatives.py b/representatives/management/commands/updaterepresentatives.py index bc45e8a..4623b74 100644 --- a/representatives/management/commands/updaterepresentatives.py +++ b/representatives/management/commands/updaterepresentatives.py @@ -1,9 +1,9 @@ -import logging import itertools +import logging from django.core.management.base import BaseCommand -from representatives.models import RepresentativeSet, Election +from representatives.models import Election, RepresentativeSet log = logging.getLogger(__name__) diff --git a/representatives/migrations/0001_initial.py b/representatives/migrations/0001_initial.py index b33cf96..9943adb 100644 --- a/representatives/migrations/0001_initial.py +++ b/representatives/migrations/0001_initial.py @@ -1,4 +1,5 @@ -from django.db import models, migrations +from django.db import migrations, models + class JSONField(models.TextField): """Mocks jsonfield 0.92's column-type behaviour""" diff --git a/representatives/migrations/0002_auto_20141129_1450.py b/representatives/migrations/0002_auto_20141129_1450.py index 81469e7..541cb0a 100644 --- a/representatives/migrations/0002_auto_20141129_1450.py +++ b/representatives/migrations/0002_auto_20141129_1450.py @@ -1,4 +1,5 @@ -from django.db import models, migrations +from django.db import migrations, models + class JSONField(models.TextField): """Mocks jsonfield 0.92's column-type behaviour""" diff --git a/representatives/migrations/0003_auto_20170214_1237.py b/representatives/migrations/0003_auto_20170214_1237.py index c1cd342..85c94af 100644 --- a/representatives/migrations/0003_auto_20170214_1237.py +++ b/representatives/migrations/0003_auto_20170214_1237.py @@ -2,6 +2,7 @@ from django.db import migrations, models + class JSONField(models.TextField): """Mocks jsonfield 0.92's column-type behaviour""" def db_type(self, connection): diff --git a/representatives/models.py b/representatives/models.py index c576b0f..85828c0 100644 --- a/representatives/models.py +++ b/representatives/models.py @@ -7,13 +7,12 @@ from urllib.parse import urljoin from urllib.request import urlopen -from django.db.models import JSONField +from appconf import AppConf from django.db import models, transaction +from django.db.models import JSONField from django.template.defaultfilters import slugify from django.urls import reverse -from appconf import AppConf - from representatives.utils import boundary_url_to_name logger = logging.getLogger(__name__) diff --git a/representatives/urls.py b/representatives/urls.py index a8f495d..0372606 100644 --- a/representatives/urls.py +++ b/representatives/urls.py @@ -2,8 +2,13 @@ from representatives.models import app_settings from representatives.views import ( - RepresentativeSetListView, RepresentativeSetDetailView, RepresentativeListView, - ElectionListView, ElectionDetailView, CandidateListView) + CandidateListView, + ElectionDetailView, + ElectionListView, + RepresentativeListView, + RepresentativeSetDetailView, + RepresentativeSetListView, +) urlpatterns = [ path('representatives/', RepresentativeListView.as_view()), diff --git a/representatives/views.py b/representatives/views.py index c36a7f0..e792d66 100644 --- a/representatives/views.py +++ b/representatives/views.py @@ -3,10 +3,16 @@ from urllib.parse import urlencode from urllib.request import urlopen -from boundaries.base_views import ModelListView, ModelDetailView, BadRequest +from boundaries.base_views import BadRequest, ModelDetailView, ModelListView from boundaries.models import Boundary -from representatives.models import RepresentativeSet, Representative, Election, Candidate, app_settings +from representatives.models import ( + Candidate, + Election, + Representative, + RepresentativeSet, + app_settings, +) from representatives.utils import boundary_url_to_name diff --git a/setup.cfg b/setup.cfg index a136a6f..18ccec1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,6 +15,9 @@ install_requires = django-appconf represent-boundaries +[isort] +profile = black + [flake8] exclude = representatives/migrations extend-ignore = From 434238a2437939a2fea75305d8a8714f6c0693b3 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Sat, 9 Dec 2023 23:56:26 -0500 Subject: [PATCH 07/11] ci: Add lint workflow --- .github/workflows/lint.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..ebeed86 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,13 @@ +name: Lint +on: [push, pull_request] +jobs: + build: + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - run: flake8 . + - run: isort . From ebd5ddc45100fd46b5d01355bec5659f960c415f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Sun, 10 Dec 2023 00:22:00 -0500 Subject: [PATCH 08/11] chore: Fix E128 --- representatives/models.py | 83 +++++++++++++++++++++++++++------------ representatives/urls.py | 12 ++++-- setup.cfg | 2 - 3 files changed, 65 insertions(+), 32 deletions(-) diff --git a/representatives/models.py b/representatives/models.py index 85828c0..1add873 100644 --- a/representatives/models.py +++ b/representatives/models.py @@ -37,15 +37,20 @@ class MyAppConf(AppConf): class BaseRepresentativeSet(models.Model): - name = models.CharField(max_length=300, + name = models.CharField( + max_length=300, + unique=True, help_text="The name of the political body, e.g. House of Commons", - unique=True) + ) data_url = models.URLField(help_text="URL to a JSON array of individuals within this set") data_about_url = models.URLField(blank=True, help_text="URL to information about the scraper used to gather data") last_import_time = models.DateTimeField(blank=True, null=True) last_import_successful = models.BooleanField(blank=True, null=True) - boundary_set = models.CharField(max_length=300, blank=True, - help_text="Name of the boundary set on the boundaries API, e.g. federal-electoral-districts") + boundary_set = models.CharField( + blank=True, + max_length=300, + help_text="Name of the boundary set on the boundaries API, e.g. federal-electoral-districts", + ) slug = models.SlugField(max_length=300, unique=True, db_index=True) enabled = models.BooleanField(default=True, blank=True, db_index=True) @@ -133,10 +138,21 @@ def update_from_data_source(self): for source_rep in data: rep = self.create_child() - for fieldname in ('name', 'district_name', 'elected_office', - 'source_url', 'first_name', 'last_name', 'party_name', - 'email', 'url', 'personal_url', 'photo_url', 'district_id', - 'gender'): + for fieldname in ( + 'name', + 'district_name', + 'elected_office', + 'source_url', + 'first_name', + 'last_name', + 'party_name', + 'email', + 'url', + 'personal_url', + 'photo_url', + 'district_id', + 'gender', + ): if source_rep.get(fieldname) is not None: setattr(rep, fieldname, source_rep[fieldname]) for json_fieldname in ('offices', 'extra'): @@ -195,13 +211,13 @@ def create_child(self): return Representative(representative_set=self) def get_absolute_url(self): - return reverse('representatives_representative_set_detail', - kwargs={'slug': self.slug}) + return reverse('representatives_representative_set_detail', kwargs={'slug': self.slug}) def as_dict(self): r = super().as_dict() r['related']['representatives_url'] = reverse( - 'representatives_representative_list', kwargs={'set_slug': self.slug}) + 'representatives_representative_list', kwargs={'set_slug': self.slug} + ) return r @@ -212,8 +228,7 @@ def create_child(self): return Candidate(election=self) def get_absolute_url(self): - return reverse('representatives_election_detail', - kwargs={'slug': self.slug}) + return reverse('representatives_election_detail', kwargs={'slug': self.slug}) def as_dict(self): r = super().as_dict() @@ -240,19 +255,21 @@ class BaseRepresentative(models.Model): district_name = models.CharField(max_length=300) elected_office = models.CharField(max_length=200) source_url = models.URLField(max_length=2048) - boundary = models.CharField(max_length=300, blank=True, db_index=True, - help_text="e.g. federal-electoral-districts/outremont") - first_name = models.CharField(max_length=200, blank=True) - last_name = models.CharField(max_length=200, blank=True) - party_name = models.CharField(max_length=200, blank=True) + boundary = models.CharField( + blank=True, + max_length=300, + db_index=True, + help_text="e.g. federal-electoral-districts/outremont", + ) + first_name = models.CharField(blank=True, max_length=200) + last_name = models.CharField(blank=True, max_length=200) + party_name = models.CharField(blank=True, max_length=200) email = models.EmailField(blank=True) url = models.URLField(blank=True, max_length=2048) personal_url = models.URLField(blank=True, max_length=2048) photo_url = models.URLField(blank=True, max_length=2048) - district_id = models.CharField(max_length=200, blank=True) - gender = models.CharField(max_length=1, blank=True, choices=( - ('F', 'Female'), - ('M', 'Male'))) + district_id = models.CharField(blank=True, max_length=200) + gender = models.CharField(blank=True, max_length=1, choices=(('F', 'Female'), ('M', 'Male'))) offices = JSONField(default=list) extra = JSONField(default=dict) @@ -268,10 +285,24 @@ def boundary_url(self): return '/boundaries/%s/' % self.boundary if self.boundary else '' def as_dict(self): - r = {f: getattr(self, f) for f in - ('name', 'district_name', 'elected_office', 'source_url', - 'first_name', 'last_name', 'party_name', 'email', 'url', 'personal_url', - 'photo_url', 'gender', 'offices', 'extra')} + r = { + f: getattr(self, f) for f in ( + 'name', + 'district_name', + 'elected_office', + 'source_url', + 'first_name', + 'last_name', + 'party_name', + 'email', + 'url', + 'personal_url', + 'photo_url', + 'gender', + 'offices', + 'extra', + ) + } set_obj = getattr(self, self.set_name) r[self.set_name + '_name'] = set_obj.name r['related'] = { diff --git a/representatives/urls.py b/representatives/urls.py index 0372606..addbb50 100644 --- a/representatives/urls.py +++ b/representatives/urls.py @@ -15,8 +15,10 @@ re_path(r'^representatives/(?P[\w_-]+)/$', RepresentativeListView.as_view(), name='representatives_representative_list'), re_path(r'^boundaries/(?P[\w_-]+/[\w_-]+)/representatives/', RepresentativeListView.as_view()), path('representative-sets/', RepresentativeSetListView.as_view()), - re_path(r'^representative-sets/(?P[\w_-]+)/$', RepresentativeSetDetailView.as_view(), - name='representatives_representative_set_detail'), + re_path( + r'^representative-sets/(?P[\w_-]+)/$', RepresentativeSetDetailView.as_view(), + name='representatives_representative_set_detail' + ), ] if app_settings.ENABLE_CANDIDATES: @@ -25,6 +27,8 @@ re_path(r'^candidates/(?P[\w_-]+)/$', CandidateListView.as_view(), name='representatives_candidate_list'), re_path(r'^boundaries/(?P[\w_-]+/[\w_-]+)/candidates/$', CandidateListView.as_view()), path('elections/', ElectionListView.as_view()), - re_path(r'^elections/(?P[\w_-]+)/$', ElectionDetailView.as_view(), - name='representatives_election_detail'), + re_path( + r'^elections/(?P[\w_-]+)/$', ElectionDetailView.as_view(), + name='representatives_election_detail' + ), ] diff --git a/setup.cfg b/setup.cfg index 18ccec1..bdd9d69 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,7 +21,5 @@ profile = black [flake8] exclude = representatives/migrations extend-ignore = - # E128 continuation line under-indented for visual indent - E128, # E501 line too long (X > 79 characters) E501 From cef4ac83ea6e9946fbec4bd9ff41e7385720ee3f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Sun, 10 Dec 2023 00:36:10 -0500 Subject: [PATCH 09/11] chore: Fix E501 --- representatives/admin.py | 10 ++++++---- .../commands/updaterepresentatives.py | 8 ++++++-- representatives/models.py | 11 ++++++----- representatives/urls.py | 18 ++++++++++++++---- representatives/views.py | 7 +++++-- setup.cfg | 4 +--- 6 files changed, 38 insertions(+), 20 deletions(-) diff --git a/representatives/admin.py b/representatives/admin.py index c304348..888aa34 100644 --- a/representatives/admin.py +++ b/representatives/admin.py @@ -25,15 +25,17 @@ def update_from_data_source(self, request, queryset): try: count = individual_set.update_from_data_source() except Exception: - messages.error(request, "Couldn't update individuals in {}: {}".format(individual_set, traceback.format_exc())) + messages.error(request, f"Couldn't update individuals in {individual_set}: {traceback.format_exc()}") continue if count is False: - messages.error(request, "Couldn't update individuals in %s." % individual_set) + messages.error(request, f"Couldn't update individuals in {individual_set}.") else: - message = "Updated {} individuals in {}.".format(count, individual_set) + message = f"Updated {count} individuals in {individual_set}." no_boundaries = individual_set.individuals.filter(boundary='').values_list('name', flat=True) if no_boundaries: - messages.warning(request, message + " %d match no boundary (%s)." % (len(no_boundaries), ', '.join(no_boundaries))) + messages.warning( + request, message + f" {len(no_boundaries)} match no boundary ({', '.join(no_boundaries)})." + ) else: messages.success(request, message) diff --git a/representatives/management/commands/updaterepresentatives.py b/representatives/management/commands/updaterepresentatives.py index 4623b74..cee7f11 100644 --- a/representatives/management/commands/updaterepresentatives.py +++ b/representatives/management/commands/updaterepresentatives.py @@ -12,9 +12,13 @@ class Command(BaseCommand): help = 'Updates representatives from sources.' def handle(self, *args, **options): - for representative_set in itertools.chain(RepresentativeSet.objects.filter(enabled=True), Election.objects.filter(enabled=True)): + for representative_set in itertools.chain( + RepresentativeSet.objects.filter(enabled=True), Election.objects.filter(enabled=True) + ): try: representative_set.update_from_data_source() except Exception: log.error("Couldn't update representatives in %s." % representative_set) - representative_set.__class__.objects.filter(pk=representative_set.pk).update(last_import_successful=False) + representative_set.__class__.objects.filter(pk=representative_set.pk).update( + last_import_successful=False + ) diff --git a/representatives/models.py b/representatives/models.py index 1add873..6509727 100644 --- a/representatives/models.py +++ b/representatives/models.py @@ -160,7 +160,7 @@ def update_from_data_source(self): try: setattr(rep, json_fieldname, json.loads(source_rep.get(json_fieldname))) except ValueError: - raise Exception("Invalid JSON in {}: {}".format(json_fieldname, source_rep.get(json_fieldname))) + raise Exception(f"Invalid JSON in {json_fieldname}: {source_rep.get(json_fieldname)}") if isinstance(getattr(rep, json_fieldname), list): for d in getattr(rep, json_fieldname): if isinstance(d, dict): @@ -175,7 +175,7 @@ def update_from_data_source(self): rep.incumbent = False if not source_rep.get('name'): - rep.name = ' '.join([component for component in [source_rep.get('first_name'), source_rep.get('last_name')] if component]) + rep.name = ' '.join([c for c in [source_rep.get('first_name'), source_rep.get('last_name')] if c]) if not source_rep.get('first_name') and not source_rep.get('last_name'): (rep.first_name, rep.last_name) = split_name(rep.name) @@ -190,7 +190,9 @@ def update_from_data_source(self): boundary_url = boundary_names.get(get_comparison_string(rep.district_name)) if not boundary_url: - logger.warning("{}: Couldn't find district boundary {} in {}".format(self.slug, rep.district_name, self.boundary_set)) + logger.warning( + "%s: Couldn't find district boundary %s in %s", self.slug, rep.district_name, self.boundary_set + ) else: rep.boundary = boundary_url_to_name(boundary_url) if not rep.district_name: @@ -277,8 +279,7 @@ class Meta: abstract = True def __str__(self): - return "{} ({} for {})".format( - self.name, self.elected_office, self.district_name) + return f"{self.name} ({self.elected_office} for {self.district_name})" @property def boundary_url(self): diff --git a/representatives/urls.py b/representatives/urls.py index addbb50..10ae039 100644 --- a/representatives/urls.py +++ b/representatives/urls.py @@ -12,11 +12,16 @@ urlpatterns = [ path('representatives/', RepresentativeListView.as_view()), - re_path(r'^representatives/(?P[\w_-]+)/$', RepresentativeListView.as_view(), name='representatives_representative_list'), + re_path( + r'^representatives/(?P[\w_-]+)/$', + RepresentativeListView.as_view(), + name='representatives_representative_list', + ), re_path(r'^boundaries/(?P[\w_-]+/[\w_-]+)/representatives/', RepresentativeListView.as_view()), path('representative-sets/', RepresentativeSetListView.as_view()), re_path( - r'^representative-sets/(?P[\w_-]+)/$', RepresentativeSetDetailView.as_view(), + r'^representative-sets/(?P[\w_-]+)/$', + RepresentativeSetDetailView.as_view(), name='representatives_representative_set_detail' ), ] @@ -24,11 +29,16 @@ if app_settings.ENABLE_CANDIDATES: urlpatterns += [ path('candidates/', CandidateListView.as_view()), - re_path(r'^candidates/(?P[\w_-]+)/$', CandidateListView.as_view(), name='representatives_candidate_list'), + re_path( + r'^candidates/(?P[\w_-]+)/$', + CandidateListView.as_view(), + name='representatives_candidate_list', + ), re_path(r'^boundaries/(?P[\w_-]+/[\w_-]+)/candidates/$', CandidateListView.as_view()), path('elections/', ElectionListView.as_view()), re_path( - r'^elections/(?P[\w_-]+)/$', ElectionDetailView.as_view(), + r'^elections/(?P[\w_-]+)/$', + ElectionDetailView.as_view(), name='representatives_election_detail' ), ] diff --git a/representatives/views.py b/representatives/views.py index e792d66..b995fe2 100644 --- a/representatives/views.py +++ b/representatives/views.py @@ -50,11 +50,14 @@ def filter(self, request, qs): if 'point' in request.GET: if app_settings.RESOLVE_POINT_REQUESTS_OVER_HTTP: url = app_settings.BOUNDARYSERVICE_URL + 'boundaries/?' + urlencode({'contains': request.GET['point']}) - boundaries = [boundary_url_to_name(boundary['url']) for boundary in json.loads(urlopen(url).read().decode())['objects']] + boundaries = [ + boundary_url_to_name(boundary['url']) + for boundary in json.loads(urlopen(url).read().decode())['objects'] + ] else: try: latitude, longitude = re.sub(r'[^\d.,-]', '', request.GET['point']).split(',') - wkt = 'POINT({} {})'.format(longitude, latitude) + wkt = f'POINT({longitude} {latitude})' boundaries = Boundary.objects.filter(shape__contains=wkt).values_list('set_id', 'slug') except ValueError: raise BadRequest("Invalid latitude,longitude '%s' provided." % request.GET['point']) diff --git a/setup.cfg b/setup.cfg index bdd9d69..95cada6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,7 +19,5 @@ install_requires = profile = black [flake8] +max-line-length = 119 exclude = representatives/migrations -extend-ignore = - # E501 line too long (X > 79 characters) - E501 From 8dc470706f73671e065a9cf6e4742453a7a6bc95 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Sun, 10 Dec 2023 00:39:54 -0500 Subject: [PATCH 10/11] chore: Set isort line_length --- representatives/admin.py | 8 +------- representatives/views.py | 8 +------- setup.cfg | 1 + 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/representatives/admin.py b/representatives/admin.py index 888aa34..c489174 100644 --- a/representatives/admin.py +++ b/representatives/admin.py @@ -2,13 +2,7 @@ from django.contrib import admin, messages -from representatives.models import ( - Candidate, - Election, - Representative, - RepresentativeSet, - app_settings, -) +from representatives.models import Candidate, Election, Representative, RepresentativeSet, app_settings @admin.register(RepresentativeSet) diff --git a/representatives/views.py b/representatives/views.py index b995fe2..05302f9 100644 --- a/representatives/views.py +++ b/representatives/views.py @@ -6,13 +6,7 @@ from boundaries.base_views import BadRequest, ModelDetailView, ModelListView from boundaries.models import Boundary -from representatives.models import ( - Candidate, - Election, - Representative, - RepresentativeSet, - app_settings, -) +from representatives.models import Candidate, Election, Representative, RepresentativeSet, app_settings from representatives.utils import boundary_url_to_name diff --git a/setup.cfg b/setup.cfg index 95cada6..e7c532d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,6 +16,7 @@ install_requires = represent-boundaries [isort] +line_length = 119 profile = black [flake8] From 8a03db0c071ee4a866fb87c9c7c45657eecd26f6 Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Sun, 10 Dec 2023 01:11:49 -0500 Subject: [PATCH 11/11] ci: Install flake8, isort --- .github/workflows/lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ebeed86..ba6ba0a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -9,5 +9,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.10' + - run: pip install --upgrade flake8 isort - run: flake8 . - run: isort .