From f4728ca28fa0876ee21dfbdda2ce9f28a2fadeb8 Mon Sep 17 00:00:00 2001 From: Varsha Menon Date: Fri, 1 Mar 2024 16:31:46 -0500 Subject: [PATCH] feat: add course staff mgmt command --- edx_exams/apps/core/management/__init__.py | 0 .../apps/core/management/commands/__init__.py | 0 .../commands/bulk_add_course_staff.py | 98 +++++++++++++++++++ .../core/management/commands/test/__init__.py | 0 .../test/test_bulk_add_course_staff.py | 86 ++++++++++++++++ requirements/base.in | 1 + requirements/base.txt | 2 + requirements/common_constraints.txt | 2 + requirements/dev.txt | 2 + requirements/doc.txt | 2 + requirements/production.txt | 2 + requirements/quality.txt | 2 + requirements/test.txt | 2 + requirements/validation.txt | 4 + 14 files changed, 203 insertions(+) create mode 100644 edx_exams/apps/core/management/__init__.py create mode 100644 edx_exams/apps/core/management/commands/__init__.py create mode 100644 edx_exams/apps/core/management/commands/bulk_add_course_staff.py create mode 100644 edx_exams/apps/core/management/commands/test/__init__.py create mode 100644 edx_exams/apps/core/management/commands/test/test_bulk_add_course_staff.py diff --git a/edx_exams/apps/core/management/__init__.py b/edx_exams/apps/core/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/edx_exams/apps/core/management/commands/__init__.py b/edx_exams/apps/core/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/edx_exams/apps/core/management/commands/bulk_add_course_staff.py b/edx_exams/apps/core/management/commands/bulk_add_course_staff.py new file mode 100644 index 00000000..e8c72f1b --- /dev/null +++ b/edx_exams/apps/core/management/commands/bulk_add_course_staff.py @@ -0,0 +1,98 @@ +"""Management command to populate the CourseStaffRole model and related User objects, from LMS, using CSV""" +import logging +import time + +import unicodecsv +from django.core.management.base import BaseCommand +from django.db import transaction + +from edx_exams.apps.core.models import CourseStaffRole, User + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + """ + Management command to add Course Staff (and User) in batches from CSV + """ + help = """ + Add Course Staff in bulk from CSV. + Expects that the data will be provided in a csv file format with the first row + being the header and columns being: username, email, role, course_id. + Example: + $ ... bulk_add_course_staff --csv_path=foo.csv + """ + + def add_arguments(self, parser): + parser.add_argument( + '-p', '--csv_path', + metavar='csv_path', + dest='csv_path', + required=False, + help='Path to CSV file.') + parser.add_argument( + '--batch_size', + type=int, + default=200, + dest='batch_size', + help='Batch size') + parser.add_argument( + '--batch_delay', + type=float, + default=1.0, + dest='batch_delay', + help='Time delay in each iteration') + + @transaction.atomic + def handle(self, *args, **options): + """ + The main logic and entry point of the management command + """ + csv_path = options['csv_path'] + batch_size = options['batch_size'] + batch_delay = options['batch_delay'] + + if csv_path: + with open(csv_path, 'rb') as csv_file: + self.add_course_staff_from_csv(csv_file, batch_size, batch_delay) + + logger.info('Bulk add course staff complete!') + + def add_course_staff_from_csv(self, csv_file, batch_size, batch_delay): + """ + Add the given set of course staff provided in csv + """ + reader = list(unicodecsv.DictReader(csv_file)) + users_to_create = [] + users_existing = {u.username for u in User.objects.filter(username__in=[r.get('username') for r in reader])} + for row in reader: + # if not User.objects.filter(username=row.get('username')).exists(): + # users_to_create.append(row) + if row.get('username') not in users_existing: + users_to_create.append(row) + + # bulk create users + for i in range(0, len(users_to_create), batch_size): + User.objects.bulk_create( + User( + username=user.get('username'), + email=user.get('email'), + ) + for user in users_to_create[i:i + batch_size] + ) + time.sleep(batch_delay) + + # bulk create course staff + for i in range(0, len(reader), batch_size): + CourseStaffRole.objects.bulk_create( + CourseStaffRole( + user=User.objects.get( + username=row.get('username'), + email=row.get('email'), + ), + course_id=row.get('course_id'), + role=row.get('role'), + ) + for row in reader[i:i + batch_size] + ) + time.sleep(batch_delay) diff --git a/edx_exams/apps/core/management/commands/test/__init__.py b/edx_exams/apps/core/management/commands/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/edx_exams/apps/core/management/commands/test/test_bulk_add_course_staff.py b/edx_exams/apps/core/management/commands/test/test_bulk_add_course_staff.py new file mode 100644 index 00000000..da9229d3 --- /dev/null +++ b/edx_exams/apps/core/management/commands/test/test_bulk_add_course_staff.py @@ -0,0 +1,86 @@ +""" Tests for bulk_add_course_staff management command """ +from tempfile import NamedTemporaryFile + +from django.core.management import call_command +from django.test import TestCase + +from edx_exams.apps.core.models import CourseStaffRole +from edx_exams.apps.core.test_utils.factories import UserFactory + + +class TestBulkAddCourseStaff(TestCase): + """ Test bulk_add_course_staff management command """ + + def setUp(self): + super().setUp() + self.command = 'bulk_add_course_staff' + self.success_log_message = 'Bulk add course staff complete!' + + # create existing user + self.user = UserFactory.create( + username='amy', + is_active=True, + is_staff=True, + ) + self.user.email = 'amy@pond.com' + self.user.save() + + self.course_id = 'course-v1:edx+test+f19' + + def _write_test_csv(self, csv, lines): + """ Write a test csv file with the lines provided """ + csv.write(b'username,email,role,course_id\n') + for line in lines: + csv.write(line.encode()) + csv.seek(0) + return csv + + def test_empty_csv(self): + lines = [] + with NamedTemporaryFile() as csv: + csv = self._write_test_csv(csv, lines) + call_command(self.command, f'--csv_path={csv.name}') + + def test_add_course_staff_with_existing_user(self): + lines = ['amy,amy@pond.com,staff,course-v1:edx+test+f19\n'] + with NamedTemporaryFile() as csv: + csv = self._write_test_csv(csv, lines) + call_command(self.command, f'--csv_path={csv.name}') + assert CourseStaffRole.objects.filter(user=self.user.id).exists() + + def test_add_course_staff_with_new_user(self): + lines = ['pam,pam@pond.com,staff,course-v1:edx+test+f20\n'] + with NamedTemporaryFile() as csv: + csv = self._write_test_csv(csv, lines) + call_command(self.command, f'--csv_path={csv.name}') + assert CourseStaffRole.objects.filter(course_id='course-v1:edx+test+f20').count() == 1 + + def test_add_course_staff_with_not_default_batch_size(self): + lines = ['pam,pam@pond.com,staff,course-v1:edx+test+f20\n', + 'sam,sam@pond.com,staff,course-v1:edx+test+f20\n'] + with NamedTemporaryFile() as csv: + csv = self._write_test_csv(csv, lines) + call_command(self.command, f'--csv_path={csv.name}', '--batch_size=1') + assert CourseStaffRole.objects.filter(course_id='course-v1:edx+test+f20').count() == 2 + + def test_add_course_staff_with_not_default_batch_delay(self): + lines = ['pam,pam@pond.com,staff,course-v1:edx+test+f20\n', + 'sam,sam@pond.com,staff,course-v1:edx+test+f20\n'] + with NamedTemporaryFile() as csv: + csv = self._write_test_csv(csv, lines) + call_command(self.command, f'--csv_path={csv.name}', '--batch_size=1', '--batch_delay=2') + assert CourseStaffRole.objects.filter(course_id='course-v1:edx+test+f20').count() == 2 + + def test_num_queries_correct(self): + """ + Expect the number of queries to be 5 + 1 * number of lines + - 2 for savepoint/release savepoint, 1 to get existing usernames, + - 1 to bulk create users, 1 to bulk create course role + - 1 for each user (to get user) + """ + num_lines = 20 + lines = [f'pam{i},pam{i}@pond.com,staff,course-v1:edx+test+f20\n' for i in range(num_lines)] + with NamedTemporaryFile() as csv: + csv = self._write_test_csv(csv, lines) + with self.assertNumQueries(5 + num_lines): + call_command(self.command, f'--csv_path={csv.name}') diff --git a/requirements/base.in b/requirements/base.in index 06cf9408..bb4c94ca 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -25,3 +25,4 @@ mysqlclient openedx-events pytz pymemcache +unicodecsv diff --git a/requirements/base.txt b/requirements/base.txt index 37b1563e..2536a9a2 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -333,6 +333,8 @@ typing-extensions==4.9.0 # via # asgiref # edx-opaque-keys +unicodecsv==0.14.1 + # via -r requirements/base.in uritemplate==4.1.1 # via # coreapi diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index 6f2af3dc..8c26d0cf 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -1,4 +1,6 @@ + + # A central location for most common version constraints # (across edx repos) for pip-installation. # diff --git a/requirements/dev.txt b/requirements/dev.txt index 624c59ad..683ad54e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -704,6 +704,8 @@ typing-extensions==4.9.0 # faker # pylint # rich +unicodecsv==0.14.1 + # via -r requirements/validation.txt uritemplate==4.1.1 # via # -r requirements/validation.txt diff --git a/requirements/doc.txt b/requirements/doc.txt index ca9a9c3b..24346401 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -683,6 +683,8 @@ typing-extensions==4.9.0 # faker # pylint # rich +unicodecsv==0.14.1 + # via -r requirements/test.txt uritemplate==4.1.1 # via # -r requirements/test.txt diff --git a/requirements/production.txt b/requirements/production.txt index 4bfb5d40..2a06eca0 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -462,6 +462,8 @@ typing-extensions==4.9.0 # -r requirements/base.txt # asgiref # edx-opaque-keys +unicodecsv==0.14.1 + # via -r requirements/base.txt uritemplate==4.1.1 # via # -r requirements/base.txt diff --git a/requirements/quality.txt b/requirements/quality.txt index fa2ff26f..e26f3248 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -640,6 +640,8 @@ typing-extensions==4.9.0 # faker # pylint # rich +unicodecsv==0.14.1 + # via -r requirements/test.txt uritemplate==4.1.1 # via # -r requirements/test.txt diff --git a/requirements/test.txt b/requirements/test.txt index 4b2293ee..a9c7b381 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -554,6 +554,8 @@ typing-extensions==4.9.0 # edx-opaque-keys # faker # pylint +unicodecsv==0.14.1 + # via -r requirements/base.txt uritemplate==4.1.1 # via # -r requirements/base.txt diff --git a/requirements/validation.txt b/requirements/validation.txt index e8aeacd1..70a4a011 100644 --- a/requirements/validation.txt +++ b/requirements/validation.txt @@ -819,6 +819,10 @@ typing-extensions==4.9.0 # faker # pylint # rich +unicodecsv==0.14.1 + # via + # -r requirements/quality.txt + # -r requirements/test.txt uritemplate==4.1.1 # via # -r requirements/quality.txt