Skip to content

Commit

Permalink
feat: product_type = degree in management command for SFMC Course Cat…
Browse files Browse the repository at this point in the history 
…alog POC' (#4419)
  • Loading branch information
@hamza-56
hamza-56 authored Sep 4, 2024
1 parent 249cc1b commit ec8fc50
Show file tree
Hide file tree
Showing 2 changed files with 221 additions and 34 deletions.
65 changes: 49 additions & 16 deletions course_discovery/apps/course_metadata/management/commands/populate_product_catalog.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from django.db.models import Prefetch

from course_discovery.apps.course_metadata.gspread_client import GspreadClient
from course_discovery.apps.course_metadata.models import Course, CourseType, SubjectTranslation
from course_discovery.apps.course_metadata.models import Course, CourseType, Program, SubjectTranslation

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -73,7 +73,7 @@ def get_products(self, product_type, product_source):
'verified', 'spoc-verified-audit'
]

if (product_type := product_type.lower()) in ['executive_education', 'bootcamp', 'ocm_course']:
if product_type in ['executive_education', 'bootcamp', 'ocm_course']:
queryset = Course.objects.available()

if product_type == 'ocm_course':
Expand All @@ -100,6 +100,24 @@ def get_products(self, product_type, product_source):
'subjects',
subject_translations
)
elif product_type == 'degree':
queryset = Program.objects.marketable().exclude(degree__isnull=True).select_related('partner', 'type')

if product_source:
queryset = queryset.filter(product_source__slug=product_source)

subject_translations = Prefetch(
'courses__subjects__translations',
queryset=SubjectTranslation.objects.filter(language_code='es'),
to_attr='spanish_translations'
)

return queryset.prefetch_related(
'authoring_organizations',
'courses__subjects',
'courses__course_runs',
subject_translations,
)
else:
# Return empty queryset if invalid product type specified
return Course.objects.none()
Expand All @@ -112,28 +130,42 @@ def write_csv_header(self, output_csv):
writer.writeheader()
return writer

def get_transformed_data(self, product):
def get_transformed_data(self, product, product_type):
"""
Transforms the product data for product's catalog
"""
authoring_orgs = product.authoring_organizations.all()
return {

data = {
"UUID": str(product.uuid),
"Title": product.title,
"Organizations Name": ", ".join(org.name for org in authoring_orgs),
"Organizations Logo": ", ".join(
org.logo_image.url for org in authoring_orgs if org.logo_image
),
"Organizations Logo": ", ".join(org.logo_image.url for org in authoring_orgs if org.logo_image),
"Organizations Abbr": ", ".join(org.key for org in authoring_orgs),
"Languages": product.languages_codes,
"Subjects": ", ".join(subject.name for subject in product.subjects.all()),
"Subjects Spanish": ", ".join(
translation.name for subject in product.subjects.all()
for translation in subject.spanish_translations
),
"Marketing URL": product.marketing_url,
"Marketing Image": (product.image.url if product.image else ""),
}
if product_type in ['executive_education', 'bootcamp', 'ocm_course']:
data.update({
"Subjects": ", ".join(subject.name for subject in product.subjects.all()),
"Subjects Spanish": ", ".join(
translation.name for subject in product.subjects.all()
for translation in subject.spanish_translations
),
"Languages": product.languages_codes,
"Marketing Image": product.image.url if product.image else "",
})
elif product_type == 'degree':
data.update({
"Subjects": ", ".join(subject.name for subject in product.subjects),
"Subjects Spanish": ", ".join(
translation.name for subject in product.subjects
for translation in subject.spanish_translations
),
"Languages": ", ".join(language.code for language in product.languages),
"Marketing Image": product.card_image.url if product.card_image else "",
})

return data

def handle(self, *args, **options):
product_type = options.get('product_type')
Expand All @@ -152,6 +184,7 @@ def handle(self, *args, **options):
gspread_client = GspreadClient()

try:
product_type = product_type.lower()
products = self.get_products(product_type, product_source)
if not products.exists():
raise CommandError('No products found for the given criteria.')
Expand All @@ -163,15 +196,15 @@ def handle(self, *args, **options):
output_writer = self.write_csv_header(output_file)
for product in products:
try:
output_writer.writerow(self.get_transformed_data(product))
output_writer.writerow(self.get_transformed_data(product, product_type))
except Exception as e: # pylint: disable=broad-exception-caught
logger.error(f"Error writing product {product.uuid} to CSV: {str(e)}")
continue

logger.info(f'Populated {products_count} {product_type}s to {output_csv}')

elif gspread_client_flag:
csv_data = [self.get_transformed_data(product) for product in products]
csv_data = [self.get_transformed_data(product, product_type) for product in products]
gspread_client.write_data(
PRODUCT_CATALOG_CONFIG,
self.CATALOG_CSV_HEADERS,
Expand Down
190 changes: 172 additions & 18 deletions ...discovery/apps/course_metadata/management/commands/tests/test_populate_product_catalog.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,12 @@
from django.core.management import CommandError, call_command
from django.test import TestCase

from course_discovery.apps.course_metadata.choices import CourseRunStatus
from course_discovery.apps.course_metadata.choices import CourseRunStatus, ProgramStatus
from course_discovery.apps.course_metadata.management.commands.populate_product_catalog import Command
from course_discovery.apps.course_metadata.models import Course, CourseType
from course_discovery.apps.course_metadata.models import Course, CourseType, ProgramType
from course_discovery.apps.course_metadata.tests.factories import (
CourseFactory, CourseRunFactory, CourseTypeFactory, PartnerFactory, SeatFactory, SourceFactory
CourseFactory, CourseRunFactory, CourseTypeFactory, DegreeFactory, PartnerFactory, ProgramTypeFactory, SeatFactory,
SourceFactory
)


Expand All @@ -37,12 +38,20 @@ def setUp(self):
self.course_run_2 = CourseRunFactory.create_batch(
2, course=Course.objects.all()[1]
)
self.program_type = ProgramTypeFactory.create(slug=ProgramType.MICROMASTERS)
self.degrees = DegreeFactory.create_batch(
2,
product_source=self.source,
partner=self.partner,
additional_metadata=None,
type=self.program_type,
)

def test_populate_product_catalog(self):
"""
Test populate_product_catalog command and verify data has been populated successfully
"""
with NamedTemporaryFile(mode="w", delete=False) as output_csv:
with NamedTemporaryFile() as output_csv:
call_command(
"populate_product_catalog",
product_type="ocm_course",
Expand All @@ -51,19 +60,138 @@ def test_populate_product_catalog(self):
gspread_client_flag=False,
)

with open(output_csv.name, "r") as output_csv_file:
csv_reader = csv.DictReader(output_csv_file)
for row in csv_reader:
self.assertIn("UUID", row)
self.assertIn("Title", row)
self.assertIn("Organizations Name", row)
self.assertIn("Organizations Logo", row)
self.assertIn("Organizations Abbr", row)
self.assertIn("Languages", row)
self.assertIn("Subjects", row)
self.assertIn("Subjects Spanish", row)
self.assertIn("Marketing URL", row)
self.assertIn("Marketing Image", row)
with open(output_csv.name, "r") as output_csv_file:
csv_reader = csv.DictReader(output_csv_file)
for row in csv_reader:
self.assertIn("UUID", row)
self.assertIn("Title", row)
self.assertIn("Organizations Name", row)
self.assertIn("Organizations Logo", row)
self.assertIn("Organizations Abbr", row)
self.assertIn("Languages", row)
self.assertIn("Subjects", row)
self.assertIn("Subjects Spanish", row)
self.assertIn("Marketing URL", row)
self.assertIn("Marketing Image", row)

def test_populate_product_catalog_for_degrees(self):
"""
Test populate_product_catalog command for all degree products and verify data has been populated successfully.
"""
with NamedTemporaryFile() as output_csv:
call_command(
"populate_product_catalog",
product_type="degree",
output_csv=output_csv.name,
product_source="edx",
gspread_client_flag=False,
)

with open(output_csv.name, "r") as output_csv_file:
csv_reader = csv.DictReader(output_csv_file)
rows = list(csv_reader)
self.assertEqual(len(rows), len(self.degrees))

for degree in self.degrees:
with self.subTest(degree=degree):
matching_rows = [row for row in rows if row["UUID"] == str(degree.uuid)]
self.assertEqual(len(matching_rows), 1)

row = matching_rows[0]
self.assertEqual(row["UUID"], str(degree.uuid))
self.assertEqual(row["Title"], degree.title)
self.assertIn("Organizations Name", row)
self.assertIn("Organizations Logo", row)
self.assertIn("Organizations Abbr", row)
self.assertIn("Languages", row)
self.assertIn("Subjects", row)
self.assertIn("Subjects Spanish", row)
self.assertIn("Marketing URL", row)
self.assertIn("Marketing Image", row)

def test_populate_product_catalog_excludes_non_marketable_degrees(self):
"""
Test that the populate_product_catalog command excludes non-marketable degrees, which are defined as:
- Degrees with status not set to ProgramStatus.Active
- Degrees with an empty marketing_slug
"""
non_marketable_degrees = [
DegreeFactory.create(
product_source=self.source,
partner=self.partner,
additional_metadata=None,
type=self.program_type,
status=ProgramStatus.Unpublished,
marketing_slug="valid-slug-1",
title="Non-Marketable Degree - Unpublished Status"
),
DegreeFactory.create(
product_source=self.source,
partner=self.partner,
additional_metadata=None,
type=self.program_type,
status=ProgramStatus.Retired,
marketing_slug="valid-slug-2",
title="Non-Marketable Degree - Retired Status"
),
DegreeFactory.create(
product_source=self.source,
partner=self.partner,
additional_metadata=None,
type=self.program_type,
status=ProgramStatus.Deleted,
marketing_slug="valid-slug-3",
title="Non-Marketable Degree - Deleted Status"
),
DegreeFactory.create(
product_source=self.source,
partner=self.partner,
additional_metadata=None,
type=self.program_type,
status=ProgramStatus.Active,
marketing_slug="",
title="Non-Marketable Degree - Empty Slug"
)
]

marketable_degree = DegreeFactory.create(
product_source=self.source,
partner=self.partner,
additional_metadata=None,
type=self.program_type,
status=ProgramStatus.Active,
marketing_slug="valid-marketing-slug",
title="Marketable Degree"
)

with NamedTemporaryFile() as output_csv:
call_command(
"populate_product_catalog",
product_type="degree",
output_csv=output_csv.name,
product_source="edx",
gspread_client_flag=False,
)

with open(output_csv.name, "r") as output_csv_file:
csv_reader = csv.DictReader(output_csv_file)
rows = list(csv_reader)

# Check that non-marketable degrees are not in the CSV
for degree in non_marketable_degrees:
with self.subTest(degree=degree):
matching_rows = [
row for row in rows if row["UUID"] == str(degree.uuid)
]
self.assertEqual(len(matching_rows), 0,
f"Non-marketable degree '{degree.title}' should not be in the CSV")

# Check that the marketable degree is in the CSV
matching_rows = [
row for row in rows if row["UUID"] == str(marketable_degree.uuid)
]
self.assertEqual(len(matching_rows), 1,
f"Marketable degree '{marketable_degree.title}' should be in the CSV")

@mock.patch(
"course_discovery.apps.course_metadata.management.commands.populate_product_catalog.Command.get_products"
Expand Down Expand Up @@ -114,7 +242,7 @@ def test_get_transformed_data(self):
product = self.courses[0]
command = Command()
product_authoring_orgs = product.authoring_organizations.all()
transformed_prod_data = command.get_transformed_data(product)
transformed_prod_data = command.get_transformed_data(product, "ocm_course")
assert transformed_prod_data == {
"UUID": str(product.uuid),
"Title": product.title,
Expand All @@ -140,6 +268,32 @@ def test_get_transformed_data(self):
"Marketing Image": (product.image.url if product.image else ""),
}

def test_get_transformed_data_for_degree(self):
"""
Verify get_transformed_data method is working correctly for degree
"""
product = self.degrees[0]
command = Command()
product_authoring_orgs = product.authoring_organizations.all()
transformed_prod_data = command.get_transformed_data(product, "degree")
assert transformed_prod_data == {
"UUID": str(product.uuid),
"Title": product.title,
"Organizations Name": ", ".join(org.name for org in product_authoring_orgs),
"Organizations Logo": ", ".join(
org.logo_image.url for org in product_authoring_orgs if org.logo_image
),
"Organizations Abbr": ", ".join(org.key for org in product_authoring_orgs),
"Languages": ", ".join(language.code for language in product.languages),
"Subjects": ", ".join(subject.name for subject in product.subjects),
"Subjects Spanish": ", ".join(
translation.name for subject in product.subjects
for translation in subject.spanish_translations
),
"Marketing URL": product.marketing_url,
"Marketing Image": product.card_image.url if product.card_image else "",
}

@mock.patch('course_discovery.apps.course_metadata.management.commands.populate_product_catalog.GspreadClient')
@mock.patch(
'course_discovery.apps.course_metadata.management.commands.populate_product_catalog.Command.get_products'
Expand Down

0 comments on commit ec8fc50

Please sign in to comment.