diff --git a/crowdsourcer/management/commands/import_volunteers.py b/crowdsourcer/management/commands/import_volunteers.py index 4eb1b94..76bd541 100644 --- a/crowdsourcer/management/commands/import_volunteers.py +++ b/crowdsourcer/management/commands/import_volunteers.py @@ -7,7 +7,14 @@ import pandas as pd -from crowdsourcer.models import Assigned, PublicAuthority, Section +from crowdsourcer.models import ( + Assigned, + Marker, + MarkingSession, + PublicAuthority, + ResponseType, + Section, +) YELLOW = "\033[33m" RED = "\033[31m" @@ -18,10 +25,12 @@ class Command(BaseCommand): help = "import volunteers" - volunteer_file = settings.BASE_DIR / "data" / "volunteers.xlsx" + volunteer_file = settings.BASE_DIR / "data" / "volunteers.csv" + response_type = "First Mark" section_map = { "Buildings": "Buildings & Heating", + "Buildings and Heating": "Buildings & Heating", "Planning": "Planning & Land Use", "Gov & Finance": "Governance & Finance", "Waste & Food": "Waste Reduction & Food", @@ -30,6 +39,7 @@ class Command(BaseCommand): num_council_map = { "scorecards_volunteering": 6, + "scorecards_assessor": 6, "local_climate_policy_programme": 15, } @@ -42,11 +52,46 @@ class Command(BaseCommand): "assigned_section", ] + usecols = [ + "First name", + "Last name", + "Email", + "Council Area", + "Type of Volunteering", + "Assigned Section", + ] + def add_arguments(self, parser): parser.add_argument( "-q", "--quiet", action="store_true", help="Silence progress bars." ) + parser.add_argument( + "--file", + action="store", + required=True, + help="CSV file containing the assignments", + ) + + parser.add_argument( + "--col_names", + action="store", + help="CSV file containing use_cols and col_names args (one column for each)", + ) + + parser.add_argument( + "--response_type", + action="store", + help="Stage to assign markers to", + ) + + parser.add_argument( + "--session", + action="store", + required=True, + help="Marking session to use assignements with", + ) + parser.add_argument( "--add_users", action="store_true", help="add users to database" ) @@ -55,21 +100,51 @@ def add_arguments(self, parser): "--make_assignments", action="store_true", help="assign councils to users" ) - def handle(self, quiet: bool = False, *args, **options): - df = pd.read_excel( - self.volunteer_file, - usecols=[ - "First name", - "Last name", - "Email", - "Council Area", - "Type of Volunteering", - "Assigned Section", - ], - sheet_name="Volunteer Recruitment Cohort 2", + def get_df(self, filename): + df = pd.read_csv( + filename, + usecols=self.usecols, ) df.columns = self.column_names + return df + + def set_cols(self, col_names): + if col_names is not None: + cols = pd.read_csv(col_names) + self.usecols = cols.use_cols + self.column_names = cols.col_names + + def get_assignment_count(self, user_type): + user_type = user_type.lower().replace(" ", "_") + + num_councils = self.num_council_map.get(user_type) + return num_councils + + def handle( + self, + quiet: bool = False, + file: str = None, + session: str = None, + col_names: str = None, + response_type: str = None, + *args, + **options, + ): + if file is None: + file = self.volunteer_file + + self.set_cols(col_names) + + df = self.get_df(file) + + if response_type is None: + response_type = self.response_type + + session = MarkingSession.objects.get(label=session) + rt = ResponseType.objects.get(type=response_type) + + bad_councils = [] for index, row in df.iterrows(): if pd.isna(row["email"]): continue @@ -82,8 +157,15 @@ def handle(self, quiet: bool = False, *args, **options): self.stdout.write(f"{YELLOW}No user type for {row['email']}{NOBOLD}") continue + num_councils = self.get_assignment_count(user_type) + if num_councils is None: + self.stdout.write( + f"{YELLOW}Don't know how many councils to assign for {row['email']}{NOBOLD}" + ) + continue + if options["add_users"] is True: - u, created = User.objects.update_or_create( + u, _ = User.objects.update_or_create( username=row["email"], defaults={ "email": row["email"], @@ -92,6 +174,13 @@ def handle(self, quiet: bool = False, *args, **options): }, ) u.save() + m, _ = Marker.objects.update_or_create( + user=u, + defaults={ + "response_type": rt, + }, + ) + m.marking_session.set([session]) else: try: u = User.objects.get(username=row["email"]) @@ -106,7 +195,7 @@ def handle(self, quiet: bool = False, *args, **options): title = self.section_map.get( row["assigned_section"], row["assigned_section"] ) - s = Section.objects.get(title=title) + s = Section.objects.get(title=title, marking_session=session) except Section.DoesNotExist: self.stdout.write( f"{RED}could not assign section for {row['email']}, no section {title}{NOBOLD}" @@ -134,6 +223,7 @@ def handle(self, quiet: bool = False, *args, **options): ) if len(councils) > 0 and own_council.count() == 0: + bad_councils.append((row["council_area"], row["email"])) self.stdout.write( f"{RED}Bad council: {row['council_area']} (f{row['email']}){NOBOLD}" ) @@ -148,11 +238,13 @@ def handle(self, quiet: bool = False, *args, **options): own_council_list = list(own_council.values_list("id", flat=True)) assigned_councils = assigned_councils + own_council_list - num_councils = self.num_council_map[user_type] - - councils_to_assign = PublicAuthority.objects.exclude( + councils_to_assign = PublicAuthority.objects.filter( + marking_session=session + ).exclude( Q(id__in=assigned_councils) | Q(type="COMB") | Q(do_not_mark=True) - )[:num_councils] + )[ + :num_councils + ] if councils_to_assign.count() == 0: self.stdout.write( @@ -162,11 +254,13 @@ def handle(self, quiet: bool = False, *args, **options): if options["make_assignments"] is True: for council in councils_to_assign: a, created = Assigned.objects.update_or_create( - user=u, section=s, authority=council + user=u, section=s, authority=council, marking_session=session ) - council_count = PublicAuthority.objects.filter(do_not_mark=False).count() - for section in Section.objects.all(): + council_count = PublicAuthority.objects.filter( + marking_session=session, do_not_mark=False + ).count() + for section in Section.objects.filter(marking_session=session).all(): assigned = Assigned.objects.filter(section=section).count() if assigned != council_count: self.stdout.write( @@ -191,3 +285,8 @@ def handle(self, quiet: bool = False, *args, **options): self.stdout.write( f"{YELLOW}Dry run, no assignments made, call with --make_assignments to make them{NOBOLD}" ) + + if len(bad_councils): + self.stdout.write("Bad councils are:") + for c in bad_councils: + self.stdout.write(f"{YELLOW}{c[0]}, {c[1]}{NOBOLD}") diff --git a/crowdsourcer/tests/data/volunteers.csv b/crowdsourcer/tests/data/volunteers.csv new file mode 100644 index 0000000..f2b476f --- /dev/null +++ b/crowdsourcer/tests/data/volunteers.csv @@ -0,0 +1,2 @@ +First name,Last name,Email,Council Area,Type of Volunteering,Assigned Section +First,Last,first_last@example.org,Aberdeenshire Council,Scorecards Volunteering,Transport diff --git a/crowdsourcer/tests/data/volunteers_multiple.csv b/crowdsourcer/tests/data/volunteers_multiple.csv new file mode 100644 index 0000000..b40414a --- /dev/null +++ b/crowdsourcer/tests/data/volunteers_multiple.csv @@ -0,0 +1,3 @@ +First name,Last name,Email,Council Area,Type of Volunteering,Assigned Section +First,Last,first_last@example.org,Aberdeenshire Council,Scorecards Volunteering,Transport +Primary,Secondary,primary_secondary@example.org,Aberdeen City Council,Scorecards Volunteering,Transport diff --git a/crowdsourcer/tests/test_import_volunteers_script.py b/crowdsourcer/tests/test_import_volunteers_script.py new file mode 100644 index 0000000..f46f6e1 --- /dev/null +++ b/crowdsourcer/tests/test_import_volunteers_script.py @@ -0,0 +1,354 @@ +import pathlib +from io import StringIO + +from django.contrib.auth.models import User +from django.core.management import call_command +from django.test import TestCase + +from crowdsourcer.models import ( + Assigned, + Marker, + MarkingSession, + PublicAuthority, + ResponseType, + Section, +) + + +class BaseCommandTestCase(TestCase): + def call_command(self, command, *args, **kwargs): + out = StringIO() + call_command( + command, + *args, + stdout=out, + stderr=StringIO(), + **kwargs, + ) + return out.getvalue() + + +class AssignVolunteers(BaseCommandTestCase): + fixtures = [ + "authorities.json", + "basics.json", + "questions.json", + "options.json", + ] + + def add_extra_councils(self): + councils = [ + ("East Borsetshire", "E90001"), + ("West Borsetshire", "E90002"), + ("North Borsetshire", "E90003"), + ("South Borsetshire", "E90004"), + ("Upper Borsetshire", "E90005"), + ("Lower Borsetshire", "E90006"), + ("Mid Borsetshire", "E90007"), + ("Old Borsetshire", "E90008"), + ] + + ms = MarkingSession.objects.get(label="Default") + for council in councils: + a = PublicAuthority.objects.create( + name=council[0], unique_id=council[1], type="DIS", questiongroup_id=2 + ) + a.marking_session.set([ms]) + + def test_no_add_users(self): + data_file = pathlib.Path(__file__).parent.resolve() / "data" / "volunteers.csv" + + self.assertEquals(User.objects.count(), 0) + self.assertEquals(Marker.objects.count(), 0) + self.assertEquals(Assigned.objects.count(), 0) + self.call_command( + "import_volunteers", + session="Default", + file=data_file, + ) + self.assertEquals(User.objects.count(), 0) + self.assertEquals(Marker.objects.count(), 0) + self.assertEquals(Assigned.objects.count(), 0) + + def test_no_add_assignments(self): + data_file = pathlib.Path(__file__).parent.resolve() / "data" / "volunteers.csv" + + self.assertEquals(User.objects.count(), 0) + self.assertEquals(Marker.objects.count(), 0) + self.assertEquals(Assigned.objects.count(), 0) + self.call_command( + "import_volunteers", + session="Default", + file=data_file, + add_users=True, + ) + self.assertEquals(User.objects.count(), 1) + self.assertEquals(Marker.objects.count(), 1) + self.assertEquals(Assigned.objects.count(), 0) + + def test_basic_run(self): + data_file = pathlib.Path(__file__).parent.resolve() / "data" / "volunteers.csv" + + self.assertEquals(User.objects.count(), 0) + self.assertEquals(Marker.objects.count(), 0) + self.assertEquals(Assigned.objects.count(), 0) + self.call_command( + "import_volunteers", + session="Default", + file=data_file, + add_users=True, + make_assignments=True, + ) + self.assertEquals(User.objects.count(), 1) + self.assertEquals(Marker.objects.count(), 1) + self.assertEquals(Assigned.objects.count(), 2) + self.assertEquals( + Assigned.objects.filter( + marking_session__label="Default", section__title="Transport" + ).count(), + 2, + ) + + u = User.objects.all()[0] + m = Marker.objects.all()[0] + self.assertEquals(u.username, "first_last@example.org") + self.assertEquals(m.response_type.type, "First Mark") + + councils = [a.authority.name for a in Assigned.objects.all()] + self.assertTrue("Aberdeenshire Council" not in councils) + self.assertTrue("Aberdeen City Council" in councils) + self.assertTrue("Adur District Council" in councils) + + def test_multi_councils(self): + data_file = ( + pathlib.Path(__file__).parent.resolve() + / "data" + / "volunteers_two_councils.csv" + ) + + self.assertEquals(Assigned.objects.count(), 0) + self.call_command( + "import_volunteers", + session="Default", + file=data_file, + add_users=True, + make_assignments=True, + ) + self.assertEquals(Assigned.objects.count(), 1) + + councils = [a.authority.name for a in Assigned.objects.all()] + self.assertTrue("Aberdeenshire Council" not in councils) + self.assertTrue("Aberdeen City Council" not in councils) + self.assertTrue("Adur District Council" in councils) + + def test_assignment_limits(self): + data_file = pathlib.Path(__file__).parent.resolve() / "data" / "volunteers.csv" + self.add_extra_councils() + + self.assertEquals(Assigned.objects.count(), 0) + self.call_command( + "import_volunteers", + session="Default", + file=data_file, + add_users=True, + make_assignments=True, + ) + self.assertEquals(User.objects.count(), 1) + self.assertEquals(Marker.objects.count(), 1) + self.assertEquals(Assigned.objects.count(), 6) + self.assertEquals( + Assigned.objects.filter( + marking_session__label="Default", section__title="Transport" + ).count(), + 6, + ) + + def test_alt_columns(self): + data_file = ( + pathlib.Path(__file__).parent.resolve() / "data" / "volunteers_alt.csv" + ) + col_file = ( + pathlib.Path(__file__).parent.resolve() / "data" / "volunteer_cols.csv" + ) + + self.assertEquals(User.objects.count(), 0) + self.assertEquals(Marker.objects.count(), 0) + self.assertEquals(Assigned.objects.count(), 0) + self.call_command( + "import_volunteers", + session="Default", + file=data_file, + col_names=col_file, + add_users=True, + make_assignments=True, + ) + self.assertEquals(User.objects.count(), 1) + self.assertEquals(Marker.objects.count(), 1) + self.assertEquals(Assigned.objects.count(), 2) + self.assertEquals( + Assigned.objects.filter( + marking_session__label="Default", section__title="Transport" + ).count(), + 2, + ) + + def test_multiple_assignments(self): + data_file = ( + pathlib.Path(__file__).parent.resolve() / "data" / "volunteers_multiple.csv" + ) + self.add_extra_councils() + + Section.objects.exclude(title="Transport").delete() + + self.assertEquals(Assigned.objects.count(), 0) + out = self.call_command( + "import_volunteers", + session="Default", + file=data_file, + add_users=True, + make_assignments=True, + ) + self.assertEquals(User.objects.count(), 2) + self.assertEquals(Marker.objects.count(), 2) + self.assertEquals(Assigned.objects.count(), 11) + self.assertEquals( + Assigned.objects.filter( + marking_session__label="Default", section__title="Transport" + ).count(), + 11, + ) + + self.assertEquals( + Assigned.objects.filter( + marking_session__label="Default", + user__username="first_last@example.org", + ).count(), + 6, + ) + + self.assertEquals( + Assigned.objects.filter( + marking_session__label="Default", + user__username="primary_secondary@example.org", + ).count(), + 5, + ) + + self.assertRegex(out, r"All councils and sections assigned") + self.assertRegex(out, r"2/2 users assigned marking") + + def test_skip_bad_own_council(self): + data_file = ( + pathlib.Path(__file__).parent.resolve() / "data" / "volunteers_multiple.csv" + ) + self.add_extra_councils() + + PublicAuthority.objects.get(name="Aberdeen City Council").delete() + + self.assertEquals(Assigned.objects.count(), 0) + out = self.call_command( + "import_volunteers", + session="Default", + file=data_file, + add_users=True, + make_assignments=True, + ) + self.assertEquals(User.objects.count(), 2) + self.assertEquals(Marker.objects.count(), 2) + self.assertEquals(Assigned.objects.count(), 6) + + self.assertEquals( + Assigned.objects.filter( + marking_session__label="Default", + user__username="primary_secondary@example.org", + ).count(), + 0, + ) + + self.assertRegex(out, r"Bad council: Aberdeen City Council") + + def test_skip_assignments_if_existing(self): + data_file = pathlib.Path(__file__).parent.resolve() / "data" / "volunteers.csv" + + self.call_command( + "import_volunteers", + session="Default", + file=data_file, + add_users=True, + ) + + u = User.objects.get(username="first_last@example.org") + s = Section.objects.get(title="Transport", marking_session__label="Default") + a = PublicAuthority.objects.get(name="Aberdeen City Council") + rt = ResponseType.objects.get(type="First Mark") + ms = MarkingSession.objects.get(label="Default") + + Assigned.objects.create( + user=u, + section=s, + authority=a, + response_type=rt, + marking_session=ms, + ) + + self.assertEquals(Assigned.objects.count(), 1) + out = self.call_command( + "import_volunteers", + session="Default", + file=data_file, + add_users=True, + make_assignments=True, + ) + self.assertEquals(Assigned.objects.count(), 1) + self.assertRegex(out, r"Existing assignments: first_last@example.org") + + def test_not_all_councils_warning(self): + data_file = pathlib.Path(__file__).parent.resolve() / "data" / "volunteers.csv" + + self.add_extra_councils() + + out = self.call_command( + "import_volunteers", + session="Default", + file=data_file, + add_users=True, + make_assignments=True, + ) + self.assertEquals(Assigned.objects.count(), 6) + self.assertRegex(out, r"Not all councils assigned for Transport \(6/11\)") + + def test_not_all_users_warning(self): + data_file = pathlib.Path(__file__).parent.resolve() / "data" / "volunteers.csv" + + PublicAuthority.objects.get(name="Aberdeen City Council").delete() + PublicAuthority.objects.get(name="Adur District Council").delete() + + out = self.call_command( + "import_volunteers", + session="Default", + file=data_file, + add_users=True, + make_assignments=True, + ) + self.assertEquals(Assigned.objects.count(), 0) + self.assertRegex( + out, r"No councils left in Transport for first_last@example.org" + ) + + def test_bad_section_warning(self): + data_file = pathlib.Path(__file__).parent.resolve() / "data" / "volunteers.csv" + + Section.objects.filter(title="Transport").delete() + + out = self.call_command( + "import_volunteers", + session="Default", + file=data_file, + add_users=True, + make_assignments=True, + ) + self.assertEquals(Assigned.objects.count(), 0) + self.assertRegex( + out, + r"could not assign section for first_last@example.org, no section Transport", + )